Spaces:
Running
Running
| /** | |
| * Converts modlogs between text and SQLite; also modernizes old-format modlogs | |
| * @author Annika | |
| * @author jetou | |
| */ | |
| if (!global.Config) { | |
| let hasSQLite = true; | |
| try { | |
| require.resolve('better-sqlite3'); | |
| } catch { | |
| console.warn(`Warning: the modlog conversion script is running without a SQLite library.`); | |
| hasSQLite = false; | |
| } | |
| global.Config = { | |
| nofswriting: false, | |
| usesqlitemodlog: hasSQLite, | |
| usesqlite: hasSQLite, | |
| } as any; | |
| } | |
| import type * as DatabaseType from 'better-sqlite3'; | |
| import type {ModlogEntry} from '../../server/modlog'; | |
| import {FS} from '../../lib'; | |
| import {IPTools} from '../../server/ip-tools'; | |
| const Database = Config.usesqlite ? require('better-sqlite3') : null; | |
| const {Modlog} = require('../../server/modlog'); | |
| type ModlogFormat = 'txt' | 'sqlite'; | |
| /** The number of modlog entries to write to the database on each transaction */ | |
| const ENTRIES_TO_BUFFER = 7500; | |
| const ALTS_REGEX = /\(.*?'s (lock|mut|bann|blacklist)ed alts: (.*)\)/; | |
| const AUTOCONFIRMED_REGEX = /\(.*?'s ac account: (.*)\)/; | |
| const IP_ONLY_ACTIONS = new Set([ | |
| 'SHAREDIP', 'UNSHAREDIP', 'UNLOCKIP', 'UNLOCKRANGE', 'RANGEBAN', 'RANGELOCK', | |
| ]); | |
| export function parseBrackets(line: string, openingBracket: '(' | '[', greedy?: boolean) { | |
| const brackets = { | |
| '(': ')', | |
| '[': ']', | |
| }; | |
| const bracketOpenIndex = line.indexOf(openingBracket); | |
| const bracketCloseIndex = greedy ? line.lastIndexOf(brackets[openingBracket]) : line.indexOf(brackets[openingBracket]); | |
| if (bracketCloseIndex < 0 || bracketOpenIndex < 0) return ''; | |
| return line.slice(bracketOpenIndex + 1, bracketCloseIndex); | |
| } | |
| function toID(text: any): ID { | |
| return (text && typeof text === "string" ? text : "").toLowerCase().replace(/[^a-z0-9]+/g, "") as ID; | |
| } | |
| export function modernizeLog(line: string, nextLine?: string): string | undefined { | |
| // first we save and remove the timestamp and the roomname | |
| const prefix = line.match(/\[.+?\] \(.+?\) /i)?.[0]; | |
| if (!prefix) return; | |
| if (ALTS_REGEX.test(line) || AUTOCONFIRMED_REGEX.test(line)) return; | |
| line = line.replace(prefix, ''); | |
| // handle duplicate room bug | |
| if (line.startsWith('(')) line = line.replace(/\([a-z0-9-]*\) /, ''); | |
| if (line.startsWith('(') && line.endsWith(')')) { | |
| line = line.slice(1, -1); | |
| } | |
| const getAlts = () => { | |
| let alts = ''; | |
| nextLine?.replace(ALTS_REGEX, (_a, _b, rawAlts) => { | |
| if (rawAlts) alts = `alts: [${rawAlts.split(',').map(toID).join('], [')}] `; | |
| return ''; | |
| }); | |
| return alts; | |
| }; | |
| const getAutoconfirmed = () => { | |
| let autoconfirmed = ''; | |
| nextLine?.replace(AUTOCONFIRMED_REGEX, (_a, rawAutoconfirmed) => { | |
| if (rawAutoconfirmed) autoconfirmed = `ac: [${toID(rawAutoconfirmed)}] `; | |
| return ''; | |
| }); | |
| return autoconfirmed; | |
| }; | |
| // Special cases | |
| if (line.startsWith('SCAV ')) { | |
| line = line.replace(/: (\[room: .*?\]) by (.*)/, (match, roominfo, rest) => `: by ${rest} ${roominfo}`); | |
| } | |
| line = line.replace( | |
| /(GIVEAWAY WIN|GTS FINISHED): ([A-Za-z0-9].*?)(won|has finished)/, | |
| (match, action, user) => `${action}: [${toID(user)}]:` | |
| ); | |
| if (line.includes(':')) { | |
| const possibleModernAction = line.slice(0, line.indexOf(':')).trim(); | |
| if (possibleModernAction === possibleModernAction.toUpperCase()) { | |
| if (possibleModernAction.includes('[')) { | |
| // for corrupted lines | |
| const [drop, ...keep] = line.split('['); | |
| process.stderr.write(`Ignoring malformed line: ${drop}\n`); | |
| return modernizeLog(keep.join('')); | |
| } | |
| if (/\(.+\) by [a-z0-9]{1,19}$/.test(line) && !['OLD MODLOG', 'NOTE'].includes(possibleModernAction)) { | |
| // weird reason formatting | |
| const reason = parseBrackets(line, '(', true); | |
| return `${prefix}${line.replace(` (${reason})`, '')}: ${reason}`; | |
| } | |
| // Log is already modernized | |
| return `${prefix}${line}`; | |
| } | |
| } | |
| if (/\[(the|a)poll\] was (started|ended) by/.test(line)) { | |
| const actionTaker = toID(line.slice(line.indexOf(' by ') + ' by '.length)); | |
| const isEnding = line.includes('was ended by'); | |
| return `${prefix}POLL${isEnding ? ' END' : ''}: by ${actionTaker}`; | |
| } | |
| if (/User (.*?) won the game of (.*?) mode trivia/.test(line)) { | |
| return `${prefix}TRIVIAGAME: by unknown: ${line}`; | |
| } | |
| const modernizerTransformations: {[k: string]: (log: string) => string} = { | |
| 'notes: ': (log) => { | |
| const [actionTaker, ...rest] = line.split(' notes: '); | |
| return `NOTE: by ${toID(actionTaker)}: ${rest.join('')}`; | |
| }, | |
| ' declared': (log) => { | |
| let newAction = 'DECLARE'; | |
| let oldAction = ' declared'; | |
| if (log.includes(' globally declared')) { | |
| oldAction = ' globally declared'; | |
| newAction = 'GLOBALDECLARE'; | |
| } | |
| if (log.includes('(chat level)')) { | |
| oldAction += ' (chat level)'; | |
| newAction = `CHATDECLARE`; | |
| } | |
| const actionTakerName = toID(log.slice(0, log.lastIndexOf(oldAction))); | |
| log = log.slice(actionTakerName.length); | |
| log = log.slice(oldAction.length); | |
| log = log.replace(/^\s?:/, '').trim(); | |
| return `${newAction}: by ${actionTakerName}: ${log}`; | |
| }, | |
| 'changed the roomdesc to: ': (log) => { | |
| const actionTaker = parseBrackets(log, '['); | |
| log = log.slice(actionTaker.length + 3); | |
| log = log.slice('changed the roomdesc to: '.length + 1, -2); | |
| return `ROOMDESC: by ${actionTaker}: to "${log}"`; | |
| }, | |
| 'roomevent titled "': (log) => { | |
| let action; | |
| if (log.includes(' added a roomevent titled "')) { | |
| action = 'added a'; | |
| } else if (log.includes(' removed a roomevent titled "')) { | |
| action = 'removed a'; | |
| } else { | |
| action = 'edited the'; | |
| } | |
| const actionTakerName = log.slice(0, log.lastIndexOf(` ${action} roomevent titled "`)); | |
| log = log.slice(actionTakerName.length + 1); | |
| const eventName = log.slice(` ${action} roomevent titled `.length, -2); | |
| return `ROOMEVENT: by ${toID(actionTakerName)}: ${action.split(' ')[0]} "${eventName}"`; | |
| }, | |
| 'set modchat to ': (log) => { | |
| const actionTaker = parseBrackets(log, '['); | |
| log = log.slice(actionTaker.length + 3); | |
| log = log.slice('set modchat to '.length); | |
| return `MODCHAT: by ${actionTaker}: to ${log}`; | |
| }, | |
| 'set modjoin to ': (log) => { | |
| const actionTakerName = log.slice(0, log.lastIndexOf(' set')); | |
| log = log.slice(actionTakerName.length + 1); | |
| log = log.slice('set modjoin to '.length); | |
| const rank = log.startsWith('sync') ? 'sync' : log.replace('.', ''); | |
| return `MODJOIN${rank === 'sync' ? ' SYNC' : ''}: by ${toID(actionTakerName)}${rank !== 'sync' ? `: ${rank}` : ``}`; | |
| }, | |
| 'turned off modjoin': (log) => { | |
| const actionTakerName = log.slice(0, log.lastIndexOf(' turned off modjoin')); | |
| return `MODJOIN: by ${toID(actionTakerName)}: OFF`; | |
| }, | |
| 'changed the roomintro': (log) => { | |
| const isDeletion = /deleted the (staff|room)intro/.test(log); | |
| const isRoomintro = log.includes('roomintro'); | |
| const actionTaker = toID(log.slice(0, log.indexOf(isDeletion ? 'deleted' : 'changed'))); | |
| return `${isDeletion ? 'DELETE' : ''}${isRoomintro ? 'ROOM' : 'STAFF'}INTRO: by ${actionTaker}`; | |
| }, | |
| 'deleted the roomintro': (log) => modernizerTransformations['changed the roomintro'](log), | |
| 'changed the staffintro': (log) => modernizerTransformations['changed the roomintro'](log), | |
| 'deleted the staffintro': (log) => modernizerTransformations['changed the roomintro'](log), | |
| 'created a tournament in': (log) => { | |
| const actionTaker = parseBrackets(log, '['); | |
| log = log.slice(actionTaker.length + 3); | |
| log = log.slice(24, -8); | |
| return `TOUR CREATE: by ${actionTaker}: ${log}`; | |
| }, | |
| 'was disqualified from the tournament by': (log) => { | |
| const disqualified = parseBrackets(log, '['); | |
| log = log.slice(disqualified.length + 3); | |
| log = log.slice('was disqualified from the tournament by'.length); | |
| return `TOUR DQ: [${toID(disqualified)}] by ${toID(log)}`; | |
| }, | |
| 'The tournament auto disqualify timeout was set to': (log) => { | |
| const byIndex = log.indexOf(' by '); | |
| const actionTaker = log.slice(byIndex + ' by '.length); | |
| const length = log.slice('The tournament auto disqualify timeout was set to'.length, byIndex); | |
| return `TOUR AUTODQ: by ${toID(actionTaker)}: ${length.trim()}`; | |
| }, | |
| ' was blacklisted from ': (log) => { | |
| const isName = log.includes(' was nameblacklisted from '); | |
| const banned = toID(log.slice(0, log.indexOf(` was ${isName ? 'name' : ''}blacklisted from `))); | |
| log = log.slice(log.indexOf(' by ') + ' by '.length); | |
| let reason, ip; | |
| if (/\(.*\)/.test(log)) { | |
| reason = parseBrackets(log, '('); | |
| if (/\[.*\]/.test(log)) ip = parseBrackets(log, '['); | |
| log = log.slice(0, log.indexOf('(')); | |
| } | |
| const actionTaker = toID(log); | |
| return `${isName ? 'NAME' : ''}BLACKLIST: [${banned}] ${getAutoconfirmed()}${getAlts()}${ip ? `[${ip}] ` : ``}by ${actionTaker}${reason ? `: ${reason}` : ``}`; | |
| }, | |
| ' was nameblacklisted from ': (log) => modernizerTransformations[' was blacklisted from '](log), | |
| ' was banned from room ': (log) => { | |
| const banned = toID(log.slice(0, log.indexOf(' was banned from room '))); | |
| log = log.slice(log.indexOf(' by ') + ' by '.length); | |
| let reason, ip; | |
| if (/\(.*\)/.test(log)) { | |
| reason = parseBrackets(log, '('); | |
| if (/\[.*\]/.test(log)) ip = parseBrackets(log, '['); | |
| log = log.slice(0, log.indexOf('(')); | |
| } | |
| const actionTaker = toID(log); | |
| return `ROOMBAN: [${banned}] ${getAutoconfirmed()}${getAlts()}${ip ? `[${ip}] ` : ``}by ${actionTaker}${reason ? `: ${reason}` : ``}`; | |
| }, | |
| ' was muted by ': (log) => { | |
| let muted = ''; | |
| let isHour = false; | |
| [muted, log] = log.split(' was muted by '); | |
| muted = toID(muted); | |
| let reason, ip; | |
| if (/\(.*\)/.test(log)) { | |
| reason = parseBrackets(log, '('); | |
| if (/\[.*\]/.test(log)) ip = parseBrackets(log, '['); | |
| log = log.slice(0, log.indexOf('(')); | |
| } | |
| let actionTaker = toID(log); | |
| if (actionTaker.endsWith('for1hour')) { | |
| isHour = true; | |
| actionTaker = actionTaker.replace(/^(.*)(for1hour)$/, (match, staff) => staff) as ID; | |
| } | |
| return `${isHour ? 'HOUR' : ''}MUTE: [${muted}] ${getAutoconfirmed()}${getAlts()}${ip ? `[${ip}] ` : ``}by ${actionTaker}${reason ? `: ${reason}` : ``}`; | |
| }, | |
| ' was locked from talking ': (log) => { | |
| const isWeek = log.includes(' was locked from talking for a week '); | |
| const locked = toID(log.slice(0, log.indexOf(' was locked from talking '))); | |
| log = log.slice(log.indexOf(' by ') + ' by '.length); | |
| let reason, ip; | |
| if (/\(.*\)/.test(log)) { | |
| reason = parseBrackets(log, '('); | |
| if (/\[.*\]/.test(log)) ip = parseBrackets(log, '['); | |
| log = log.slice(0, log.indexOf('(')); | |
| } | |
| const actionTaker = toID(log); | |
| return `${isWeek ? 'WEEK' : ''}LOCK: [${locked}] ${getAutoconfirmed()}${getAlts()}${ip ? `[${ip}] ` : ``}by ${actionTaker}${reason ? `: ${reason}` : ``}`; | |
| }, | |
| ' was banned ': (log) => { | |
| if (log.includes(' was banned from room ')) return modernizerTransformations[' was banned from room '](log); | |
| const banned = toID(log.slice(0, log.indexOf(' was banned '))); | |
| log = log.slice(log.indexOf(' by ') + ' by '.length); | |
| let reason, ip; | |
| if (/\(.*\)/.test(log)) { | |
| reason = parseBrackets(log, '('); | |
| if (/\[.*\]/.test(log)) ip = parseBrackets(log, '['); | |
| log = log.slice(0, log.indexOf('(')); | |
| } | |
| const actionTaker = toID(log); | |
| return `BAN: [${banned}] ${getAutoconfirmed()}${getAlts()}${ip ? `[${ip}] ` : ``}by ${actionTaker}${reason ? `: ${reason}` : ``}`; | |
| }, | |
| 'was promoted to ': (log) => { | |
| const isDemotion = log.includes('was demoted to '); | |
| const userid = toID(log.split(' was ')[0]); | |
| if (!userid) { | |
| throw new Error(`Ignoring malformed line: ${prefix}${log}`); | |
| } | |
| log = log.slice(userid.length + 3); | |
| log = log.slice(`was ${isDemotion ? 'demoted' : 'promoted'} to `.length); | |
| let rank = log.slice(0, log.indexOf(' by')).replace(/ /, '').toUpperCase(); | |
| log = log.slice(`${rank} by `.length); | |
| if (!rank.startsWith('ROOM')) rank = `GLOBAL ${rank}`; | |
| const actionTaker = parseBrackets(log, '['); | |
| return `${rank}: [${userid}] by ${actionTaker}${isDemotion ? ': (demote)' : ''}`; | |
| }, | |
| 'was demoted to ': (log) => modernizerTransformations['was promoted to '](log), | |
| 'was appointed Room Owner by ': (log) => { | |
| const userid = parseBrackets(log, '['); | |
| log = log.slice(userid.length + 3); | |
| log = log.slice('was appointed Room Owner by '.length); | |
| const actionTaker = parseBrackets(log, '['); | |
| return `ROOMOWNER: [${userid}] by ${actionTaker}`; | |
| }, | |
| ' claimed this ticket': (log) => { | |
| const actions: {[k: string]: string} = { | |
| ' claimed this ticket': 'TICKETCLAIM', | |
| ' closed this ticket': 'TICKETCLOSE', | |
| ' deleted this ticket': 'TICKETDELETE', | |
| }; | |
| for (const oldAction in actions) { | |
| if (log.includes(oldAction)) { | |
| const actionTaker = toID(log.slice(0, log.indexOf(oldAction))); | |
| return `${actions[oldAction]}: by ${actionTaker}`; | |
| } | |
| } | |
| return log; | |
| }, | |
| 'This ticket is now claimed by ': (log) => { | |
| const claimer = toID(log.slice(log.indexOf(' by ') + ' by '.length)); | |
| return `TICKETCLAIM: by ${claimer}`; | |
| }, | |
| ' is no longer interested in this ticket': (log) => { | |
| const abandoner = toID(log.slice(0, log.indexOf(' is no longer interested in this ticket'))); | |
| return `TICKETABANDON: by ${abandoner}`; | |
| }, | |
| ' opened a new ticket': (log) => { | |
| const opener = toID(log.slice(0, log.indexOf(' opened a new ticket'))); | |
| const problem = log.slice(log.indexOf(' Issue: ') + ' Issue: '.length).trim(); | |
| return `TICKETOPEN: by ${opener}: ${problem}`; | |
| }, | |
| ' closed this ticket': (log) => modernizerTransformations[' claimed this ticket'](log), | |
| ' deleted this ticket': (log) => modernizerTransformations[' claimed this ticket'](log), | |
| 'This ticket is no longer claimed': () => 'TICKETUNCLAIM', | |
| ' has been caught attempting a hunt with ': (log) => { | |
| const index = log.indexOf(' has been caught attempting a hunt with '); | |
| const user = toID(log.slice(0, index)); | |
| log = log.slice(index + ' has been caught attempting a hunt with '.length); | |
| log = log.replace('. The user has also', '; has also').replace('.', ''); | |
| return `SCAV CHEATER: [${user}]: caught attempting a hunt with ${log}`; | |
| }, | |
| 'made this room hidden': (log) => { | |
| const user = toID(log.slice(0, log.indexOf(' made this room hidden'))); | |
| return `HIDDENROOM: by ${user}`; | |
| }, | |
| 'The tournament auto start timer was set to ': (log) => { | |
| log = log.slice('The tournament auto start timer was set to'.length); | |
| const [length, setter] = log.split(' by ').map(toID); | |
| return `TOUR AUTOSTART: by ${setter}: ${length}`; | |
| }, | |
| 'The tournament auto disqualify timer was set to ': (log) => { | |
| log = log.slice('The tournament auto disqualify timer was set to'.length); | |
| const [length, setter] = log.split(' by ').map(toID); | |
| return `TOUR AUTODQ: by ${setter}: ${length}`; | |
| }, | |
| " set the tournament's banlist to ": (log) => { | |
| const [setter, banlist] = log.split(` set the tournament's banlist to `); | |
| return `TOUR BANLIST: by ${toID(setter)}: ${banlist.slice(0, -1)}`; // remove trailing . from banlist | |
| }, | |
| " set the tournament's custom rules to": (log) => { | |
| const [setter, rules] = log.split(` set the tournament's custom rules to `); | |
| return `TOUR RULES: by ${toID(setter)}: ${rules.slice(0, -1)}`; | |
| }, | |
| '[agameofhangman] was started by ': (log) => `HANGMAN: by ${toID(log.slice('[agameofhangman] was started by '.length))}`, | |
| '[agameofunowas] created by ': (log) => `UNO CREATE: by ${toID(log.slice('[agameofunowas] created by '.length))}`, | |
| '[thetournament] was set to autostart': (log) => { | |
| const [, user] = log.split(' by '); | |
| return `TOUR AUTOSTART: by ${toID(user)}: when playercap is reached`; | |
| }, | |
| '[thetournament] was set to allow scouting': (log) => { | |
| const [, user] = log.split(' by '); | |
| return `TOUR SCOUT: by ${toID(user)}: allow`; | |
| }, | |
| '[thetournament] was set to disallow scouting': (log) => { | |
| const [, user] = log.split(' by '); | |
| return `TOUR SCOUT: by ${toID(user)}: disallow`; | |
| }, | |
| }; | |
| for (const oldAction in modernizerTransformations) { | |
| if (line.includes(oldAction)) { | |
| try { | |
| return prefix + modernizerTransformations[oldAction](line); | |
| } catch (err: any) { | |
| if (Config.nofswriting) throw err; | |
| process.stderr.write(`${err.message}\n`); | |
| } | |
| } | |
| } | |
| return `${prefix}${line}`; | |
| } | |
| export function parseModlog(raw: string, nextLine?: string, isGlobal = false): ModlogEntry | undefined { | |
| let line = modernizeLog(raw); | |
| if (!line) return; | |
| const timestamp = parseBrackets(line, '['); | |
| line = line.slice(timestamp.length + 3); | |
| const [roomID, ...bonus] = parseBrackets(line, '(').split(' '); | |
| const log: ModlogEntry = { | |
| action: 'NULL', | |
| roomID, | |
| visualRoomID: '', | |
| userid: null, | |
| autoconfirmedID: null, | |
| alts: [], | |
| ip: null, | |
| isGlobal, | |
| loggedBy: null, | |
| note: '', | |
| time: Math.floor(new Date(timestamp).getTime()) || 0, | |
| }; | |
| if (bonus.length) log.visualRoomID = `${log.roomID} ${bonus.join(' ')}`; | |
| line = line.slice((log.visualRoomID || log.roomID).length + 3); | |
| const actionColonIndex = line.indexOf(':'); | |
| const action = line.slice(0, actionColonIndex); | |
| if (action !== action.toUpperCase()) { | |
| // no action (probably an old-format log that slipped past the modernizer) | |
| log.action = 'OLD MODLOG'; | |
| log.loggedBy = 'unknown' as ID; | |
| log.note = line.trim(); | |
| return log; | |
| } else { | |
| log.action = action; | |
| if (log.action === 'OLD MODLOG') { | |
| log.loggedBy = 'unknown' as ID; | |
| log.note = line.slice(line.indexOf('by unknown: ') + 'by unknown :'.length).trim(); | |
| return log; | |
| } | |
| line = line.slice(actionColonIndex + 2); | |
| } | |
| if (line[0] === '[') { | |
| if (!IP_ONLY_ACTIONS.has(log.action)) { | |
| const userid = toID(parseBrackets(line, '[')); | |
| log.userid = userid; | |
| line = line.slice(userid.length + 3).trim(); | |
| if (line.startsWith('ac:')) { | |
| line = line.slice(3).trim(); | |
| const ac = parseBrackets(line, '['); | |
| log.autoconfirmedID = toID(ac); | |
| line = line.slice(ac.length + 3).trim(); | |
| } | |
| if (line.startsWith('alts:')) { | |
| line = line.slice(5).trim(); | |
| const alts = new Set<ID>(); // we need to weed out duplicate alts | |
| let alt = parseBrackets(line, '['); | |
| do { | |
| if (alt.includes(', ')) { | |
| // old alt format | |
| for (const trueAlt of alt.split(', ')) { | |
| alts.add(toID(trueAlt)); | |
| } | |
| line = line.slice(line.indexOf(`[${alt}],`) + `[${alt}],`.length).trim(); | |
| if (!line.startsWith('[')) line = `[${line}`; | |
| } else { | |
| if (IPTools.ipRegex.test(alt)) break; | |
| alts.add(toID(alt)); | |
| line = line.slice(line.indexOf(`[${alt}],`) + `[${alt}],`.length).trim(); | |
| if (alt.includes('[') && !line.startsWith('[')) line = `[${line}`; | |
| } | |
| alt = parseBrackets(line, '['); | |
| } while (alt); | |
| log.alts = [...alts]; | |
| } | |
| } | |
| if (line[0] === '[') { | |
| log.ip = parseBrackets(line, '['); | |
| line = line.slice(log.ip.length + 3).trim(); | |
| } | |
| } | |
| let regex = /\bby .*:/; | |
| let actionTakerIndex = regex.exec(line)?.index; | |
| if (actionTakerIndex === undefined) { | |
| actionTakerIndex = line.indexOf('by '); | |
| regex = /\bby .*/; | |
| } | |
| if (actionTakerIndex !== -1) { | |
| const colonIndex = line.indexOf(': '); | |
| const actionTaker = line.slice(actionTakerIndex + 3, colonIndex > actionTakerIndex ? colonIndex : undefined); | |
| if (toID(actionTaker).length < 19) { | |
| log.loggedBy = toID(actionTaker) || null; | |
| if (colonIndex > actionTakerIndex) line = line.slice(colonIndex); | |
| line = line.replace(regex, ' '); | |
| } | |
| } | |
| if (line) log.note = line.replace(/^\s?:\s?/, '').trim(); | |
| return log; | |
| } | |
| export function rawifyLog(log: ModlogEntry) { | |
| let result = `[${new Date(log.time || Date.now()).toJSON()}] (${(log.visualRoomID || log.roomID || 'global').replace(/^global-/, '')}) ${log.action}`; | |
| if (log.userid) result += `: [${log.userid}]`; | |
| if (log.autoconfirmedID) result += ` ac: [${log.autoconfirmedID}]`; | |
| if (log.alts.length) result += ` alts: [${log.alts.join('], [')}]`; | |
| if (log.ip) { | |
| if (!log.userid) result += `:`; | |
| result += ` [${log.ip}]`; | |
| } | |
| if (log.loggedBy) result += `${result.endsWith(']') ? '' : ':'} by ${log.loggedBy}`; | |
| if (log.note) result += `: ${log.note}`; | |
| return result + `\n`; | |
| } | |
| export class ModlogConverterSQLite { | |
| readonly databaseFile: string; | |
| readonly textLogDir: string; | |
| readonly isTesting: {files: Map<string, string>, db: DatabaseType.Database} | null = null; | |
| readonly newestAllowedTimestamp?: number; | |
| constructor( | |
| databaseFile: string, textLogDir: string, | |
| isTesting?: DatabaseType.Database, newestAllowedTimestamp?: number | |
| ) { | |
| this.databaseFile = databaseFile; | |
| this.textLogDir = textLogDir; | |
| if (isTesting || Config.nofswriting) { | |
| this.isTesting = {files: new Map<string, string>(), db: isTesting || new Database(':memory:')}; | |
| } | |
| this.newestAllowedTimestamp = newestAllowedTimestamp; | |
| } | |
| async toTxt() { | |
| const database = this.isTesting?.db || new Database(this.databaseFile, {fileMustExist: true}); | |
| const roomids = database.prepare('SELECT DISTINCT roomid FROM modlog').all(); | |
| const globalEntries = []; | |
| for (const {roomid} of roomids) { | |
| if (!Config.nofswriting) console.log(`Reading ${roomid}...`); | |
| const results = database.prepare( | |
| `SELECT *, (SELECT group_concat(userid, ',') FROM alts WHERE alts.modlog_id = modlog.modlog_id) as alts ` + | |
| `FROM modlog WHERE roomid = ? ORDER BY timestamp ASC` | |
| ).all(roomid); | |
| const trueRoomID = roomid.replace(/^global-/, ''); | |
| let entriesLogged = 0; | |
| let entries: string[] = []; | |
| const insertEntries = async () => { | |
| if (roomid === 'global') return; | |
| entriesLogged += entries.length; | |
| if (!Config.nofswriting && (entriesLogged % ENTRIES_TO_BUFFER === 0 || entriesLogged < ENTRIES_TO_BUFFER)) { | |
| process.stdout.clearLine(0); | |
| process.stdout.cursorTo(0); | |
| process.stdout.write(`Wrote ${entriesLogged} entries from '${trueRoomID}'`); | |
| } | |
| await this.writeFile(`${this.textLogDir}/modlog_${trueRoomID}.txt`, entries.join('')); | |
| entries = []; | |
| }; | |
| for (const result of results) { | |
| if (this.newestAllowedTimestamp && result.timestamp > this.newestAllowedTimestamp) break; | |
| const entry: ModlogEntry = { | |
| action: result.action, | |
| roomID: result.roomid?.replace(/^global-/, ''), | |
| visualRoomID: result.visual_roomid, | |
| userid: result.userid, | |
| autoconfirmedID: result.autoconfirmed_userid, | |
| alts: result.alts?.split(','), | |
| ip: result.ip, | |
| isGlobal: result.roomid?.startsWith('global-') || result.roomid === 'global' || result.is_global, | |
| loggedBy: result.action_taker_userid, | |
| note: result.note, | |
| time: result.timestamp, | |
| }; | |
| const rawLog = rawifyLog(entry); | |
| entries.push(rawLog); | |
| if (entry.isGlobal) { | |
| globalEntries.push(rawLog); | |
| } | |
| if (entries.length === ENTRIES_TO_BUFFER) await insertEntries(); | |
| } | |
| await insertEntries(); | |
| if (entriesLogged) process.stdout.write('\n'); | |
| } | |
| if (!Config.nofswriting) console.log(`Writing the global modlog...`); | |
| await this.writeFile(`${this.textLogDir}/modlog_global.txt`, globalEntries.join('')); | |
| } | |
| async writeFile(path: string, text: string) { | |
| if (this.isTesting) { | |
| const old = this.isTesting.files.get(path); | |
| return this.isTesting.files.set(path, `${old || ''}${text}`); | |
| } | |
| return FS(path).append(text); | |
| } | |
| } | |
| export class ModlogConverterTxt { | |
| readonly databaseFile: string; | |
| readonly modlog: typeof Modlog; | |
| readonly newestAllowedTimestamp?: number; | |
| readonly textLogDir: string; | |
| readonly isTesting: {files: Map<string, string>, ml?: typeof Modlog} | null = null; | |
| constructor( | |
| databaseFile: string, | |
| textLogDir: string, | |
| isTesting?: Map<string, string>, | |
| newestAllowedTimestamp?: number | |
| ) { | |
| this.databaseFile = databaseFile; | |
| this.textLogDir = textLogDir; | |
| if (isTesting || Config.nofswriting) { | |
| this.isTesting = { | |
| files: isTesting || new Map<string, string>(), | |
| }; | |
| } | |
| this.modlog = new Modlog( | |
| this.isTesting ? ':memory:' : this.databaseFile, | |
| // wait 15 seconds for DB to no longer be busy - this is important since I'm trying to do | |
| // a no-downtime transfer of text -> SQLite | |
| {sqliteOptions: {timeout: 15000}}, | |
| ); | |
| this.newestAllowedTimestamp = newestAllowedTimestamp; | |
| } | |
| async toSQLite() { | |
| await this.modlog.readyPromise; | |
| const files = this.isTesting ? [...this.isTesting.files.keys()] : await FS(this.textLogDir).readdir(); | |
| // Read global modlog first to avoid inserting duplicate data to database | |
| if (files.includes('modlog_global.txt')) { | |
| files.splice(files.indexOf('modlog_global.txt'), 1); | |
| files.unshift('modlog_global.txt'); | |
| } | |
| // we don't want to insert global modlog entries twice, so we keep track of global ones | |
| // and don't reinsert them | |
| /** roomid:list of modlog entry strings */ | |
| const globalEntries: {[k: string]: string[]} = {}; | |
| for (const file of files) { | |
| if (file === 'README.md') continue; | |
| const roomid = file.slice(7, -4); | |
| const lines = this.isTesting ? | |
| this.isTesting.files.get(file)?.split('\n') || [] : | |
| FS(`${this.textLogDir}/${file}`).createReadStream().byLine(); | |
| let entriesLogged = 0; | |
| let lastLine = undefined; | |
| let entries: ModlogEntry[] = []; | |
| const insertEntries = async () => { | |
| await this.modlog.writeSQL(entries); | |
| entriesLogged += entries.length; | |
| if (!Config.nofswriting) { | |
| process.stdout.clearLine(0); | |
| process.stdout.cursorTo(0); | |
| process.stdout.write(`Inserted ${entriesLogged} entries from '${roomid}'`); | |
| } | |
| entries = []; | |
| }; | |
| for await (const line of lines) { | |
| const entry = parseModlog(line, lastLine, roomid === 'global'); | |
| lastLine = line; | |
| if (!entry) continue; | |
| if (this.newestAllowedTimestamp && entry.time > this.newestAllowedTimestamp) break; | |
| if (roomid !== 'global' && globalEntries[entry.roomID]?.includes(line)) { | |
| // this is a global modlog entry that has already been inserted | |
| continue; | |
| } | |
| if (entry.isGlobal) { | |
| if (!globalEntries[entry.roomID]) globalEntries[entry.roomID] = []; | |
| globalEntries[entry.roomID].push(line); | |
| } | |
| entries.push(entry); | |
| if (entries.length === ENTRIES_TO_BUFFER) await insertEntries(); | |
| } | |
| delete globalEntries[roomid]; | |
| await insertEntries(); | |
| if (entriesLogged) process.stdout.write('\n'); | |
| } | |
| return this.modlog.database; | |
| } | |
| } | |
| export class ModlogConverterTest { | |
| readonly inputDir: string; | |
| readonly outputDir: string; | |
| constructor(inputDir: string, outputDir: string) { | |
| this.inputDir = inputDir; | |
| this.outputDir = outputDir; | |
| } | |
| async toTxt() { | |
| const files = await FS(this.inputDir).readdir(); | |
| // Read global modlog last to avoid inserting duplicate data to database | |
| if (files.includes('modlog_global.txt')) { | |
| files.splice(files.indexOf('modlog_global.txt'), 1); | |
| files.push('modlog_global.txt'); | |
| } | |
| const globalEntries = []; | |
| for (const file of files) { | |
| if (file === 'README.md') continue; | |
| const roomid = file.slice(7, -4); | |
| let entriesLogged = 0; | |
| let lastLine = undefined; | |
| let entries: string[] = []; | |
| const insertEntries = async () => { | |
| if (roomid === 'global') return; | |
| entriesLogged += entries.length; | |
| if (!Config.nofswriting && (entriesLogged % ENTRIES_TO_BUFFER === 0 || entriesLogged < ENTRIES_TO_BUFFER)) { | |
| process.stdout.clearLine(0); | |
| process.stdout.cursorTo(0); | |
| process.stdout.write(`Wrote ${entriesLogged} entries from '${roomid}'`); | |
| } | |
| await FS(`${this.outputDir}/modlog_${roomid}.txt`).append(entries.join('')); | |
| entries = []; | |
| }; | |
| const readStream = FS(`${this.inputDir}/${file}`).createReadStream(); | |
| for await (const line of readStream.byLine()) { | |
| const entry = parseModlog(line, lastLine, roomid === 'global'); | |
| lastLine = line; | |
| if (!entry) continue; | |
| const rawLog = rawifyLog(entry); | |
| if (roomid !== 'global') entries.push(rawLog); | |
| if (entry.isGlobal) { | |
| globalEntries.push(rawLog); | |
| } | |
| if (entries.length === ENTRIES_TO_BUFFER) await insertEntries(); | |
| } | |
| await insertEntries(); | |
| if (entriesLogged) process.stdout.write('\n'); | |
| } | |
| if (!Config.nofswriting) console.log(`Writing the global modlog...`); | |
| await FS(`${this.outputDir}/modlog_global.txt`).append(globalEntries.join('')); | |
| } | |
| } | |
| export const ModlogConverter = { | |
| async convert( | |
| from: ModlogFormat, to: ModlogFormat, databasePath: string, | |
| textLogDirectoryPath: string, outputLogPath?: string, newestAllowedTimestamp?: number, | |
| ) { | |
| if (from === 'txt' && to === 'txt' && outputLogPath) { | |
| const converter = new ModlogConverterTest(textLogDirectoryPath, outputLogPath); | |
| await converter.toTxt(); | |
| console.log("\nDone!"); | |
| process.exit(); | |
| } else if (from === 'sqlite' && to === 'txt') { | |
| const converter = new ModlogConverterSQLite(databasePath, textLogDirectoryPath, undefined, newestAllowedTimestamp); | |
| await converter.toTxt(); | |
| console.log("\nDone!"); | |
| process.exit(); | |
| } else if (from === 'txt' && to === 'sqlite') { | |
| const converter = new ModlogConverterTxt(databasePath, textLogDirectoryPath, undefined, newestAllowedTimestamp); | |
| await converter.toSQLite(); | |
| console.log("\nDone!"); | |
| process.exit(); | |
| } | |
| }, | |
| }; | |