finish smbreader

src/smbreader.ts

@@ -2,8 +2,7 @@ import { Config, loadConfig } from "./config";
 import SMB from "@marsaud/smb2";
 import mysql from "promise-mysql";
 import moment from "moment";
-import iconv from "iconv-lite";
-import HTML from "posthtml-parser";
+import { Message, readChatBuffer } from "./parse-buffer";
 
 
 export class SMBReader {
@@ -42,33 +41,72 @@ export class SMBReader {
 			") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci");
 		console.error(`${this.logPrefix}Initialization finished.`);
 	}
+	private getFilePath(filename: string) {
+		return this.config.pathPrefix ? `${this.config.pathPrefix}\\${filename}` : filename;
+	}
+	private flatternMessages(messages: Message[]): string {
+		return messages.map(m => JSON.stringify(m)).join("|");
+	}
 	async run() {
-		console.error(`${this.logPrefix}Reading file list...`);
-		let filelist = (await this.smb.readdir(this.config.pathPrefix)).filter(m => m.match(/^chat_.*\.html$/));
-		const files = filelist.map(filename => {
-			const dateString = filename.match(/^chat_(.*)\.html$/)[1].replace(/-/g, ":");
-			const date = moment(dateString).utcOffset("+08:00", true)
-			return {
-				name: filename,
-				date
-			}
-		});
-		files.sort((f1, f2) => {
-			return f1.date.unix() - f2.date.unix();
-		});
-		for (let file of files) {
-			const testIndex: any[] = await this.db.query("select * from `filesRead` where filename = ?", [file.name]);
-			if (testIndex.length) {
-				console.error(`${this.logPrefix}File ${file.name} has already been read. Skipping.`);
-				continue;
-			}
-			console.error(`${this.logPrefix}Reading file ${file.name}...`);
-			const fileMetadata = await this.db.query("insert into `filesRead` set ?", {
-				filename: file.name,
-				date: file.date.format("YYYY-MM-DD HH:mm:ss")
+		try {
+			console.error(`${this.logPrefix}Reading file list...`);
+			let filelist = (await this.smb.readdir(this.config.pathPrefix)).filter(m => m.match(/^chat_.*\.html$/));
+			const files = filelist.map(filename => {
+				const dateString = filename.match(/^chat_(.*)\.html$/)[1].replace(/-/g, ":");
+				const date = moment(dateString).utcOffset("+08:00", true)
+				return {
+					name: filename,
+					date
+				}
 			});
-			const insertedFileID: number = fileMetadata.insertId;
-
+			files.sort((f1, f2) => {
+				return f1.date.unix() - f2.date.unix();
+			});
+			for (let file of files) {
+				const filePath = this.getFilePath(file.name);
+				try {
+					const testIndex: any[] = await this.db.query("select * from `filesRead` where filename = ?", [file.name]);
+					if (testIndex.length) {
+						console.error(`${this.logPrefix}File ${filePath} has already been read. Skipping.`);
+						continue;
+					}
+					console.error(`${this.logPrefix}Reading file ${filePath}...`);
+					const fileMetadata = await this.db.query("insert into `filesRead` set ?", {
+						filename: file.name,
+						date: file.date.format("YYYY-MM-DD HH:mm:ss")
+					});
+					const insertedFileID: number = fileMetadata.insertId;
+					const previousFilenameArray: any[] = await this.db.query("select filename from `filesRead` where fileid = ?", [insertedFileID - 1]);
+					let currentMessages = readChatBuffer((await this.smb.readFile(filePath)) as Buffer);
+					if (previousFilenameArray.length && files.find(f => f.name === previousFilenameArray[0].filename)) {
+						const previousFilePath = this.getFilePath(previousFilenameArray[0].filename);
+						console.error(`${this.logPrefix}Comparing file ${filePath} with old file ${previousFilePath}.`)
+						try {
+							const previousMessages = readChatBuffer((await this.smb.readFile(previousFilePath)) as Buffer);
+							const flattenedPreviousMessages = this.flatternMessages(previousMessages);
+							const flattenedCurrentMessages = this.flatternMessages(currentMessages);
+							if (flattenedCurrentMessages.startsWith(flattenedPreviousMessages)) {
+								const sliceCount = previousMessages.length;
+								console.error(`${this.logPrefix}Slicing the first ${sliceCount} messages of file ${filePath}.`);
+								currentMessages = currentMessages.slice(sliceCount);
+							}
+						} catch (e) {
+							console.error(`${this.logPrefix}Failed comparing file ${filePath} with old file ${previousFilePath}. Skipping:`, e.toString());
+						}
+					}
+					for (let message of currentMessages) {
+						console.error(`${this.logPrefix}Recording message from file ${insertedFileID} ${filePath}:`, JSON.stringify(message));
+						await this.db.query("insert into `messages` set ?", {
+							fileid: insertedFileID,
+							...message
+						});
+					}
+				} catch (e) {
+					console.error(`${this.logPrefix}Errored reading file ${filePath}:`, e.toString());
+				}
+			}
+		} catch (e) {
+			console.error(`${this.logPrefix}Read error:`, e.toString());
 		}
 	}
 }