Commit f1cbd75f authored by nanahira's avatar nanahira

finish smbreader

parent 198d1278
Pipeline #864 passed with stages
in 5 minutes and 42 seconds
......@@ -2,8 +2,7 @@ import { Config, loadConfig } from "./config";
import SMB from "@marsaud/smb2";
import mysql from "promise-mysql";
import moment from "moment";
import iconv from "iconv-lite";
import HTML from "posthtml-parser";
import { Message, readChatBuffer } from "./parse-buffer";
export class SMBReader {
......@@ -42,33 +41,72 @@ export class SMBReader {
") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci");
console.error(`${this.logPrefix}Initialization finished.`);
}
private getFilePath(filename: string) {
return this.config.pathPrefix ? `${this.config.pathPrefix}\\${filename}` : filename;
}
private flatternMessages(messages: Message[]): string {
return messages.map(m => JSON.stringify(m)).join("|");
}
async run() {
console.error(`${this.logPrefix}Reading file list...`);
let filelist = (await this.smb.readdir(this.config.pathPrefix)).filter(m => m.match(/^chat_.*\.html$/));
const files = filelist.map(filename => {
const dateString = filename.match(/^chat_(.*)\.html$/)[1].replace(/-/g, ":");
const date = moment(dateString).utcOffset("+08:00", true)
return {
name: filename,
date
}
});
files.sort((f1, f2) => {
return f1.date.unix() - f2.date.unix();
});
for (let file of files) {
const testIndex: any[] = await this.db.query("select * from `filesRead` where filename = ?", [file.name]);
if (testIndex.length) {
console.error(`${this.logPrefix}File ${file.name} has already been read. Skipping.`);
continue;
}
console.error(`${this.logPrefix}Reading file ${file.name}...`);
const fileMetadata = await this.db.query("insert into `filesRead` set ?", {
filename: file.name,
date: file.date.format("YYYY-MM-DD HH:mm:ss")
try {
console.error(`${this.logPrefix}Reading file list...`);
let filelist = (await this.smb.readdir(this.config.pathPrefix)).filter(m => m.match(/^chat_.*\.html$/));
const files = filelist.map(filename => {
const dateString = filename.match(/^chat_(.*)\.html$/)[1].replace(/-/g, ":");
const date = moment(dateString).utcOffset("+08:00", true)
return {
name: filename,
date
}
});
const insertedFileID: number = fileMetadata.insertId;
files.sort((f1, f2) => {
return f1.date.unix() - f2.date.unix();
});
for (let file of files) {
const filePath = this.getFilePath(file.name);
try {
const testIndex: any[] = await this.db.query("select * from `filesRead` where filename = ?", [file.name]);
if (testIndex.length) {
console.error(`${this.logPrefix}File ${filePath} has already been read. Skipping.`);
continue;
}
console.error(`${this.logPrefix}Reading file ${filePath}...`);
const fileMetadata = await this.db.query("insert into `filesRead` set ?", {
filename: file.name,
date: file.date.format("YYYY-MM-DD HH:mm:ss")
});
const insertedFileID: number = fileMetadata.insertId;
const previousFilenameArray: any[] = await this.db.query("select filename from `filesRead` where fileid = ?", [insertedFileID - 1]);
let currentMessages = readChatBuffer((await this.smb.readFile(filePath)) as Buffer);
if (previousFilenameArray.length && files.find(f => f.name === previousFilenameArray[0].filename)) {
const previousFilePath = this.getFilePath(previousFilenameArray[0].filename);
console.error(`${this.logPrefix}Comparing file ${filePath} with old file ${previousFilePath}.`)
try {
const previousMessages = readChatBuffer((await this.smb.readFile(previousFilePath)) as Buffer);
const flattenedPreviousMessages = this.flatternMessages(previousMessages);
const flattenedCurrentMessages = this.flatternMessages(currentMessages);
if (flattenedCurrentMessages.startsWith(flattenedPreviousMessages)) {
const sliceCount = previousMessages.length;
console.error(`${this.logPrefix}Slicing the first ${sliceCount} messages of file ${filePath}.`);
currentMessages = currentMessages.slice(sliceCount);
}
} catch (e) {
console.error(`${this.logPrefix}Failed comparing file ${filePath} with old file ${previousFilePath}. Skipping:`, e.toString());
}
}
for (let message of currentMessages) {
console.error(`${this.logPrefix}Recording message from file ${insertedFileID} ${filePath}:`, JSON.stringify(message));
await this.db.query("insert into `messages` set ?", {
fileid: insertedFileID,
...message
});
}
} catch (e) {
console.error(`${this.logPrefix}Errored reading file ${filePath}:`, e.toString());
}
}
} catch (e) {
console.error(`${this.logPrefix}Read error:`, e.toString());
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment