finish fetch part

Dockerfile

@@ -6,4 +6,6 @@ RUN npm ci
 COPY . ./
 RUN npm run build
 
-CMD ["npm", "run", "start"]
+ENTRYPOINT ["npm", "run"]
+
+CMD "fetch"

config.example.yaml

@@ -9,4 +9,4 @@ smbConfig:
   domain: WORKGROUP
   password: my_password
 pathPrefix: "."
-fetchCron: "0 1 * * * *"
+fetchInterval: 30000

fetch.ts

+import { read } from "fs";
+import { loadConfig } from "./src/config";
+import { SMBReader } from "./src/smbreader";
+import Q from "q";
+
+async function main() {
+	const config = await loadConfig();
+	const reader = new SMBReader("Fetcher");
+	await reader.init();
+	while (true) {
+		await reader.run();
+		await Q.delay(config.fetchInterval);
+	}
+}
+main();

package-lock.json

@@ -18,6 +18,14 @@
       "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.32.tgz",
       "integrity": "sha512-dIOxFfI0C+jz89g6lQ+TqhGgPQ0MxSnh/E4xuC0blhFtyW269+mPG5QeLgbdwst/LvdP8o1y0o/Gz5EHXLec/g=="
     },
+    "@types/bunyan": {
+      "version": "1.8.6",
+      "resolved": "https://registry.npmjs.org/@types/bunyan/-/bunyan-1.8.6.tgz",
+      "integrity": "sha512-YiozPOOsS6bIuz31ilYqR5SlLif4TBWsousN2aCWLi5233nZSX19tFbcQUPdR7xJ8ypPyxkCGNxg0CIV5n9qxQ==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/mysql": {
       "version": "2.15.15",
       "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.15.tgz",
@@ -31,11 +39,22 @@
       "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz",
       "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA=="
     },
+    "@types/q": {
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.4.tgz",
+      "integrity": "sha512-1HcDas8SEj4z1Wc696tH56G8OlRaH/sqZOynNNB+HF0WOeXPaxTtbYzJY2oEfiUxjSKjhCKr+MvR7dCHcEelug=="
+    },
     "@types/underscore": {
       "version": "1.10.24",
       "resolved": "https://registry.npmjs.org/@types/underscore/-/underscore-1.10.24.tgz",
       "integrity": "sha512-T3NQD8hXNW2sRsSbLNjF/aBo18MyJlbw0lSpQHB/eZZtScPdexN4HSa8cByYwTw9Wy7KuOFr81mlDQcQQaZ79w=="
     },
+    "balanced-match": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
+      "optional": true
+    },
     "bignumber.js": {
       "version": "9.0.0",
       "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
@@ -46,6 +65,33 @@
       "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
       "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
     },
+    "brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "optional": true,
+      "requires": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "bunyan": {
+      "version": "1.8.14",
+      "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.14.tgz",
+      "integrity": "sha512-LlahJUxXzZLuw/hetUQJmRgZ1LF6+cr5TPpRj6jf327AsiIq2jhYEH4oqUUkVKTor+9w2BT3oxVwhzE5lw9tcg==",
+      "requires": {
+        "dtrace-provider": "~0.8",
+        "moment": "^2.19.3",
+        "mv": "~2",
+        "safe-json-stringify": "~1"
+      }
+    },
+    "concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "optional": true
+    },
     "core-util-is": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
@@ -94,11 +140,33 @@
         "domelementtype": "1"
       }
     },
+    "dtrace-provider": {
+      "version": "0.8.8",
+      "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz",
+      "integrity": "sha512-b7Z7cNtHPhH9EJhNNbbeqTcXB8LGFFZhq1PGgEvpeHlzd36bhbdTWoE/Ba/YguqpBSlAPKnARWhVlhunCMwfxg==",
+      "optional": true,
+      "requires": {
+        "nan": "^2.14.0"
+      }
+    },
     "entities": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
       "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
     },
+    "glob": {
+      "version": "6.0.4",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
+      "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=",
+      "optional": true,
+      "requires": {
+        "inflight": "^1.0.4",
+        "inherits": "2",
+        "minimatch": "2 || 3",
+        "once": "^1.3.0",
+        "path-is-absolute": "^1.0.0"
+      }
+    },
     "htmlparser2": {
       "version": "3.10.1",
       "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
@@ -120,6 +188,16 @@
         "safer-buffer": ">= 2.1.2 < 3.0.0"
       }
     },
+    "inflight": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
+      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "optional": true,
+      "requires": {
+        "once": "^1.3.0",
+        "wrappy": "1"
+      }
+    },
     "inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
@@ -130,11 +208,46 @@
       "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
       "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
     },
+    "minimatch": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+      "optional": true,
+      "requires": {
+        "brace-expansion": "^1.1.7"
+      }
+    },
+    "minimist": {
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
+      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
+      "optional": true
+    },
+    "mkdirp": {
+      "version": "0.5.5",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
+      "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
+      "optional": true,
+      "requires": {
+        "minimist": "^1.2.5"
+      }
+    },
     "moment": {
       "version": "2.29.0",
       "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.0.tgz",
       "integrity": "sha512-z6IJ5HXYiuxvFTI6eiQ9dm77uE0gyy1yXNApVHqTcnIKfY9tIwEjlzsZ6u1LQXvVgKeTnv9Xm7NDvJ7lso3MtA=="
     },
+    "mv": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
+      "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=",
+      "optional": true,
+      "requires": {
+        "mkdirp": "~0.5.1",
+        "ncp": "~2.0.0",
+        "rimraf": "~2.4.0"
+      }
+    },
     "mysql": {
       "version": "2.18.1",
       "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
@@ -175,11 +288,38 @@
         }
       }
     },
+    "nan": {
+      "version": "2.14.1",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
+      "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==",
+      "optional": true
+    },
+    "ncp": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
+      "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=",
+      "optional": true
+    },
     "ntlm": {
       "version": "0.1.3",
       "resolved": "https://registry.npmjs.org/ntlm/-/ntlm-0.1.3.tgz",
       "integrity": "sha1-O4FOvFMKHmzXEtzwz1kBVZMRlcE="
     },
+    "once": {
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
+      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "optional": true,
+      "requires": {
+        "wrappy": "1"
+      }
+    },
+    "path-is-absolute": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+      "optional": true
+    },
     "posthtml-parser": {
       "version": "0.5.0",
       "resolved": "https://registry.npmjs.org/posthtml-parser/-/posthtml-parser-0.5.0.tgz",
@@ -204,6 +344,11 @@
         "mysql": "^2.18.1"
       }
     },
+    "q": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
+      "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc="
+    },
     "readable-stream": {
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -214,11 +359,26 @@
         "util-deprecate": "^1.0.1"
       }
     },
+    "rimraf": {
+      "version": "2.4.5",
+      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
+      "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=",
+      "optional": true,
+      "requires": {
+        "glob": "^6.0.1"
+      }
+    },
     "safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
       "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
     },
+    "safe-json-stringify": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz",
+      "integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==",
+      "optional": true
+    },
     "safer-buffer": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
@@ -252,6 +412,12 @@
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
       "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
     },
+    "wrappy": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+      "optional": true
+    },
     "yaml": {
       "version": "1.10.0",
       "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz",

package.json

@@ -5,8 +5,8 @@
   "main": "build/index.js",
   "scripts": {
     "build": "tsc",
-    "fetch": "node build/src/run.js",
-    "start": "node build/src/run.js"
+    "fetch": "node build/fetch.js",
+    "start": "npm run fetch"
   },
   "repository": {
     "type": "git",
@@ -16,12 +16,16 @@
   "license": "AGPL-3.0",
   "dependencies": {
     "@marsaud/smb2": "^0.17.1",
+    "@types/bunyan": "^1.8.6",
     "@types/node": "^14.11.2",
+    "@types/q": "^1.5.4",
     "@types/underscore": "^1.10.24",
+    "bunyan": "^1.8.14",
     "iconv-lite": "^0.6.2",
     "moment": "^2.29.0",
     "posthtml-parser": "^0.5.0",
     "promise-mysql": "^4.1.3",
+    "q": "^1.5.1",
     "typescript": "^4.0.3",
     "underscore": "^1.11.0",
     "yaml": "^1.10.0"

src/config.ts

@@ -18,7 +18,7 @@ export interface Config {
 	MySQLConfig: PoolConfig;
 	smbConfig: ISMB2Options;
 	pathPrefix: string;
-	fetchCron: string;
+	fetchInterval: number;
 }
 
 export async function loadConfig(): Promise<Config> {

src/smbreader.ts

@@ -3,23 +3,28 @@ import SMB from "@marsaud/smb2";
 import mysql from "promise-mysql";
 import moment from "moment";
 import { Message, readChatBuffer } from "./parse-buffer";
-
+import bunyan from "bunyan";
 
 export class SMBReader {
 	logPrefix: string;
-	constructor(logPrefix: string) {
+	logLevel: bunyan.LogLevel;
+	constructor(logPrefix: string, logLevel?: bunyan.LogLevel) {
 		this.logPrefix = logPrefix;
+		this.logLevel = logLevel;
 	}
 	config: Config;
 	smb: SMB;
 	db: mysql.Pool;
+	log: bunyan;
 	async init() {
-		console.error(`${this.logPrefix}Reading config...`);
+		this.log = bunyan.createLogger({ name: this.logPrefix, level: this.logLevel || "info" });
+		this.log.info(`Start initializing.`);
+		this.log.debug(`Reading config...`);
 		this.config = await loadConfig();
 		this.smb = new SMB(this.config.smbConfig);
-		console.error(`${this.logPrefix}Connecting to database...`);
+		this.log.debug(`Connecting to database...`);
 		this.db = await mysql.createPool(this.config.MySQLConfig);
-		console.error(`${this.logPrefix}Creating table...`);
+		this.log.debug(`Creating table...`);
 		await this.db.query("CREATE TABLE IF NOT EXISTS `filesRead` (\n" +
 			"  `fileid` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT,\n" +
 			"  `date` datetime UNIQUE NOT NULL,\n" +
@@ -39,7 +44,7 @@ export class SMBReader {
 			"  KEY (channel(4)),\n" +
 			"  INDEX (speaker(16))\n" +
 			") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci");
-		console.error(`${this.logPrefix}Initialization finished.`);
+		this.log.info(`Initialization finished.`);
 	}
 	private getFilePath(filename: string) {
 		return this.config.pathPrefix ? `${this.config.pathPrefix}\\${filename}` : filename;
@@ -48,8 +53,9 @@ export class SMBReader {
 		return messages.map(m => JSON.stringify(m)).join("|");
 	}
 	async run() {
+		this.log.info(`Started.`);
 		try {
-			console.error(`${this.logPrefix}Reading file list...`);
+			this.log.info(`Reading file list...`);
 			let filelist = (await this.smb.readdir(this.config.pathPrefix)).filter(m => m.match(/^chat_.*\.html$/));
 			const files = filelist.map(filename => {
 				const dateString = filename.match(/^chat_(.*)\.html$/)[1].replace(/-/g, ":");
@@ -67,10 +73,10 @@ export class SMBReader {
 				try {
 					const testIndex: any[] = await this.db.query("select * from `filesRead` where filename = ?", [file.name]);
 					if (testIndex.length) {
-						console.error(`${this.logPrefix}File ${filePath} has already been read. Skipping.`);
+						this.log.info(`File ${filePath} has already been read. Skipping.`);
 						continue;
 					}
-					console.error(`${this.logPrefix}Reading file ${filePath}...`);
+					this.log.info(`Reading file ${filePath}...`);
 					const fileMetadata = await this.db.query("insert into `filesRead` set ?", {
 						filename: file.name,
 						date: file.date.format("YYYY-MM-DD HH:mm:ss")
@@ -80,22 +86,22 @@ export class SMBReader {
 					let currentMessages = readChatBuffer((await this.smb.readFile(filePath)) as Buffer);
 					if (previousFilenameArray.length && files.find(f => f.name === previousFilenameArray[0].filename)) { // compare with old file
 						const previousFilePath = this.getFilePath(previousFilenameArray[0].filename);
-						console.error(`${this.logPrefix}Comparing file ${filePath} with old file ${previousFilePath}.`)
+						this.log.debug(`Comparing file ${filePath} with old file ${previousFilePath}.`)
 						try {
 							const previousMessages = readChatBuffer((await this.smb.readFile(previousFilePath)) as Buffer);
 							const flattenedPreviousMessages = this.flatternMessages(previousMessages);
 							const flattenedCurrentMessages = this.flatternMessages(currentMessages);
 							if (flattenedCurrentMessages.startsWith(flattenedPreviousMessages)) {
 								const sliceCount = previousMessages.length;
-								console.error(`${this.logPrefix}Slicing the first ${sliceCount} messages of file ${filePath}.`);
+								this.log.debug(`Slicing the first ${sliceCount} messages of file ${filePath}.`);
 								currentMessages = currentMessages.slice(sliceCount);
 							}
 						} catch (e) {
-							console.error(`${this.logPrefix}Failed comparing file ${filePath} with old file ${previousFilePath}. Skipping:`, e.toString());
+							this.log.warn(`Failed comparing file ${filePath} with old file ${previousFilePath}. Skipping:`, e.toString());
 						}
 					}
 					for (let message of currentMessages) {
-						console.error(`${this.logPrefix}Recording message from file ${insertedFileID} ${filePath}:`, JSON.stringify(message));
+						this.log.debug(`Recording message from file ${insertedFileID} ${filePath}:`, JSON.stringify(message));
 						await this.db.query("insert into `messages` set ?", {
 							fileid: insertedFileID,
 							...message
@@ -105,18 +111,19 @@ export class SMBReader {
 					if (deleteFilenameArray.length && files.find(f => f.name === deleteFilenameArray[0].filename)) { // delete file with id - 2 because useless
 						const deleteFilePath = this.getFilePath(deleteFilenameArray[0].filename);
 						try {
-							console.error(`${this.logPrefix}Deleting file ${deleteFilePath}.`);
+							this.log.debug(`Deleting file ${deleteFilePath}.`);
 							await this.smb.unlink(deleteFilePath);
 						} catch (e) {
-							console.error(`${this.logPrefix}Failed deleting file ${deleteFilePath}:`, e.toString());
+							this.log.debug(`Failed deleting file ${deleteFilePath}:`, e.toString());
 						}
 					}
 				} catch (e) {
-					console.error(`${this.logPrefix}Errored reading file ${filePath}:`, e.toString());
+					this.log.warn(`Errored reading file ${filePath}:`, e.toString());
 				}
 			}
 		} catch (e) {
-			console.error(`${this.logPrefix}Read error:`, e.toString());
+			this.log.warn(`Read error:`, e.toString());
 		}
+		this.log.info(`Finished.`);
 	}
 }

test/smb.ts

@@ -2,7 +2,7 @@ import { read } from "fs";
 import { SMBReader } from "../src/smbreader";
 
 async function main() {
-	const reader = new SMBReader("Test: ");
+	const reader = new SMBReader("Test", "debug");
 	await reader.init();
 	await reader.run();
 	process.exit();