Skip to content

T
tx3-message-reader
Commit 4f66fd00 authored by nanahira's avatar nanahira
Browse files
Options

finish fetch part

parent cca0b175
Pipeline #875 passed with stages
      in 6 minutes and 31 seconds
      Hide whitespace changes
      Inline Side-by-side
      Showing with 217 additions and 23 deletions
      Dockerfile
      View file @ 4f66fd00
      ...@@ -6,4 +6,6 @@ RUN npm ci ...@@ -6,4 +6,6 @@ RUN npm ci
      COPY . ./ COPY . ./
      RUN npm run build RUN npm run build
      CMD ["npm", "run", "start"] ENTRYPOINT ["npm", "run"]
      CMD "fetch"
      config.example.yaml
      View file @ 4f66fd00
      ...@@ -9,4 +9,4 @@ smbConfig: ...@@ -9,4 +9,4 @@ smbConfig:
      domain: WORKGROUP domain: WORKGROUP
      password: my_password password: my_password
      pathPrefix: "." pathPrefix: "."
      fetchCron: "0 1 * * * *" fetchInterval: 30000
      fetch.ts 0 → 100644
      View file @ 4f66fd00
      import { read } from "fs";
      import { loadConfig } from "./src/config";
      import { SMBReader } from "./src/smbreader";
      import Q from "q";
      async function main() {
      const config = await loadConfig();
      const reader = new SMBReader("Fetcher");
      await reader.init();
      while (true) {
      await reader.run();
      await Q.delay(config.fetchInterval);
      }
      }
      main();
      package-lock.json
      View file @ 4f66fd00
      ...@@ -18,6 +18,14 @@ ...@@ -18,6 +18,14 @@
      "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.32.tgz", "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.32.tgz",
      "integrity": "sha512-dIOxFfI0C+jz89g6lQ+TqhGgPQ0MxSnh/E4xuC0blhFtyW269+mPG5QeLgbdwst/LvdP8o1y0o/Gz5EHXLec/g==" "integrity": "sha512-dIOxFfI0C+jz89g6lQ+TqhGgPQ0MxSnh/E4xuC0blhFtyW269+mPG5QeLgbdwst/LvdP8o1y0o/Gz5EHXLec/g=="
      }, },
      "@types/bunyan": {
      "version": "1.8.6",
      "resolved": "https://registry.npmjs.org/@types/bunyan/-/bunyan-1.8.6.tgz",
      "integrity": "sha512-YiozPOOsS6bIuz31ilYqR5SlLif4TBWsousN2aCWLi5233nZSX19tFbcQUPdR7xJ8ypPyxkCGNxg0CIV5n9qxQ==",
      "requires": {
      "@types/node": "*"
      }
      },
      "@types/mysql": { "@types/mysql": {
      "version": "2.15.15", "version": "2.15.15",
      "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.15.tgz", "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.15.tgz",
      ...@@ -31,11 +39,22 @@ ...@@ -31,11 +39,22 @@
      "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz",
      "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==" "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA=="
      }, },
      "@types/q": {
      "version": "1.5.4",
      "resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.4.tgz",
      "integrity": "sha512-1HcDas8SEj4z1Wc696tH56G8OlRaH/sqZOynNNB+HF0WOeXPaxTtbYzJY2oEfiUxjSKjhCKr+MvR7dCHcEelug=="
      },
      "@types/underscore": { "@types/underscore": {
      "version": "1.10.24", "version": "1.10.24",
      "resolved": "https://registry.npmjs.org/@types/underscore/-/underscore-1.10.24.tgz", "resolved": "https://registry.npmjs.org/@types/underscore/-/underscore-1.10.24.tgz",
      "integrity": "sha512-T3NQD8hXNW2sRsSbLNjF/aBo18MyJlbw0lSpQHB/eZZtScPdexN4HSa8cByYwTw9Wy7KuOFr81mlDQcQQaZ79w==" "integrity": "sha512-T3NQD8hXNW2sRsSbLNjF/aBo18MyJlbw0lSpQHB/eZZtScPdexN4HSa8cByYwTw9Wy7KuOFr81mlDQcQQaZ79w=="
      }, },
      "balanced-match": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
      "optional": true
      },
      "bignumber.js": { "bignumber.js": {
      "version": "9.0.0", "version": "9.0.0",
      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
      ...@@ -46,6 +65,33 @@ ...@@ -46,6 +65,33 @@
      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
      }, },
      "brace-expansion": {
      "version": "1.1.11",
      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
      "optional": true,
      "requires": {
      "balanced-match": "^1.0.0",
      "concat-map": "0.0.1"
      }
      },
      "bunyan": {
      "version": "1.8.14",
      "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.14.tgz",
      "integrity": "sha512-LlahJUxXzZLuw/hetUQJmRgZ1LF6+cr5TPpRj6jf327AsiIq2jhYEH4oqUUkVKTor+9w2BT3oxVwhzE5lw9tcg==",
      "requires": {
      "dtrace-provider": "~0.8",
      "moment": "^2.19.3",
      "mv": "~2",
      "safe-json-stringify": "~1"
      }
      },
      "concat-map": {
      "version": "0.0.1",
      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
      "optional": true
      },
      "core-util-is": { "core-util-is": {
      "version": "1.0.2", "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
      ...@@ -94,11 +140,33 @@ ...@@ -94,11 +140,33 @@
      "domelementtype": "1" "domelementtype": "1"
      } }
      }, },
      "dtrace-provider": {
      "version": "0.8.8",
      "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz",
      "integrity": "sha512-b7Z7cNtHPhH9EJhNNbbeqTcXB8LGFFZhq1PGgEvpeHlzd36bhbdTWoE/Ba/YguqpBSlAPKnARWhVlhunCMwfxg==",
      "optional": true,
      "requires": {
      "nan": "^2.14.0"
      }
      },
      "entities": { "entities": {
      "version": "1.1.2", "version": "1.1.2",
      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
      "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
      }, },
      "glob": {
      "version": "6.0.4",
      "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
      "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=",
      "optional": true,
      "requires": {
      "inflight": "^1.0.4",
      "inherits": "2",
      "minimatch": "2 || 3",
      "once": "^1.3.0",
      "path-is-absolute": "^1.0.0"
      }
      },
      "htmlparser2": { "htmlparser2": {
      "version": "3.10.1", "version": "3.10.1",
      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
      ...@@ -120,6 +188,16 @@ ...@@ -120,6 +188,16 @@
      "safer-buffer": ">= 2.1.2 < 3.0.0" "safer-buffer": ">= 2.1.2 < 3.0.0"
      } }
      }, },
      "inflight": {
      "version": "1.0.6",
      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
      "optional": true,
      "requires": {
      "once": "^1.3.0",
      "wrappy": "1"
      }
      },
      "inherits": { "inherits": {
      "version": "2.0.4", "version": "2.0.4",
      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
      ...@@ -130,11 +208,46 @@ ...@@ -130,11 +208,46 @@
      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
      }, },
      "minimatch": {
      "version": "3.0.4",
      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
      "optional": true,
      "requires": {
      "brace-expansion": "^1.1.7"
      }
      },
      "minimist": {
      "version": "1.2.5",
      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
      "optional": true
      },
      "mkdirp": {
      "version": "0.5.5",
      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
      "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
      "optional": true,
      "requires": {
      "minimist": "^1.2.5"
      }
      },
      "moment": { "moment": {
      "version": "2.29.0", "version": "2.29.0",
      "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.0.tgz", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.0.tgz",
      "integrity": "sha512-z6IJ5HXYiuxvFTI6eiQ9dm77uE0gyy1yXNApVHqTcnIKfY9tIwEjlzsZ6u1LQXvVgKeTnv9Xm7NDvJ7lso3MtA==" "integrity": "sha512-z6IJ5HXYiuxvFTI6eiQ9dm77uE0gyy1yXNApVHqTcnIKfY9tIwEjlzsZ6u1LQXvVgKeTnv9Xm7NDvJ7lso3MtA=="
      }, },
      "mv": {
      "version": "2.1.1",
      "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
      "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=",
      "optional": true,
      "requires": {
      "mkdirp": "~0.5.1",
      "ncp": "~2.0.0",
      "rimraf": "~2.4.0"
      }
      },
      "mysql": { "mysql": {
      "version": "2.18.1", "version": "2.18.1",
      "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz", "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
      ...@@ -175,11 +288,38 @@ ...@@ -175,11 +288,38 @@
      } }
      } }
      }, },
      "nan": {
      "version": "2.14.1",
      "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
      "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==",
      "optional": true
      },
      "ncp": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
      "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=",
      "optional": true
      },
      "ntlm": { "ntlm": {
      "version": "0.1.3", "version": "0.1.3",
      "resolved": "https://registry.npmjs.org/ntlm/-/ntlm-0.1.3.tgz", "resolved": "https://registry.npmjs.org/ntlm/-/ntlm-0.1.3.tgz",
      "integrity": "sha1-O4FOvFMKHmzXEtzwz1kBVZMRlcE=" "integrity": "sha1-O4FOvFMKHmzXEtzwz1kBVZMRlcE="
      }, },
      "once": {
      "version": "1.4.0",
      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
      "optional": true,
      "requires": {
      "wrappy": "1"
      }
      },
      "path-is-absolute": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
      "optional": true
      },
      "posthtml-parser": { "posthtml-parser": {
      "version": "0.5.0", "version": "0.5.0",
      "resolved": "https://registry.npmjs.org/posthtml-parser/-/posthtml-parser-0.5.0.tgz", "resolved": "https://registry.npmjs.org/posthtml-parser/-/posthtml-parser-0.5.0.tgz",
      ...@@ -204,6 +344,11 @@ ...@@ -204,6 +344,11 @@
      "mysql": "^2.18.1" "mysql": "^2.18.1"
      } }
      }, },
      "q": {
      "version": "1.5.1",
      "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
      "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc="
      },
      "readable-stream": { "readable-stream": {
      "version": "3.6.0", "version": "3.6.0",
      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
      ...@@ -214,11 +359,26 @@ ...@@ -214,11 +359,26 @@
      "util-deprecate": "^1.0.1" "util-deprecate": "^1.0.1"
      } }
      }, },
      "rimraf": {
      "version": "2.4.5",
      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
      "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=",
      "optional": true,
      "requires": {
      "glob": "^6.0.1"
      }
      },
      "safe-buffer": { "safe-buffer": {
      "version": "5.2.1", "version": "5.2.1",
      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
      }, },
      "safe-json-stringify": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz",
      "integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==",
      "optional": true
      },
      "safer-buffer": { "safer-buffer": {
      "version": "2.1.2", "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
      ...@@ -252,6 +412,12 @@ ...@@ -252,6 +412,12 @@
      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
      }, },
      "wrappy": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
      "optional": true
      },
      "yaml": { "yaml": {
      "version": "1.10.0", "version": "1.10.0",
      "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz", "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz",
      ......
      package.json
      View file @ 4f66fd00
      ...@@ -5,8 +5,8 @@ ...@@ -5,8 +5,8 @@
      "main": "build/index.js", "main": "build/index.js",
      "scripts": { "scripts": {
      "build": "tsc", "build": "tsc",
      "fetch": "node build/src/run.js", "fetch": "node build/fetch.js",
      "start": "node build/src/run.js" "start": "npm run fetch"
      }, },
      "repository": { "repository": {
      "type": "git", "type": "git",
      ...@@ -16,12 +16,16 @@ ...@@ -16,12 +16,16 @@
      "license": "AGPL-3.0", "license": "AGPL-3.0",
      "dependencies": { "dependencies": {
      "@marsaud/smb2": "^0.17.1", "@marsaud/smb2": "^0.17.1",
      "@types/bunyan": "^1.8.6",
      "@types/node": "^14.11.2", "@types/node": "^14.11.2",
      "@types/q": "^1.5.4",
      "@types/underscore": "^1.10.24", "@types/underscore": "^1.10.24",
      "bunyan": "^1.8.14",
      "iconv-lite": "^0.6.2", "iconv-lite": "^0.6.2",
      "moment": "^2.29.0", "moment": "^2.29.0",
      "posthtml-parser": "^0.5.0", "posthtml-parser": "^0.5.0",
      "promise-mysql": "^4.1.3", "promise-mysql": "^4.1.3",
      "q": "^1.5.1",
      "typescript": "^4.0.3", "typescript": "^4.0.3",
      "underscore": "^1.11.0", "underscore": "^1.11.0",
      "yaml": "^1.10.0" "yaml": "^1.10.0"
      ......
      src/config.ts
      View file @ 4f66fd00
      ...@@ -18,7 +18,7 @@ export interface Config { ...@@ -18,7 +18,7 @@ export interface Config {
      MySQLConfig: PoolConfig; MySQLConfig: PoolConfig;
      smbConfig: ISMB2Options; smbConfig: ISMB2Options;
      pathPrefix: string; pathPrefix: string;
      fetchCron: string; fetchInterval: number;
      } }
      export async function loadConfig(): Promise<Config> { export async function loadConfig(): Promise<Config> {
      ......
      src/smbreader.ts
      View file @ 4f66fd00
      ...@@ -3,23 +3,28 @@ import SMB from "@marsaud/smb2"; ...@@ -3,23 +3,28 @@ import SMB from "@marsaud/smb2";
      import mysql from "promise-mysql"; import mysql from "promise-mysql";
      import moment from "moment"; import moment from "moment";
      import { Message, readChatBuffer } from "./parse-buffer"; import { Message, readChatBuffer } from "./parse-buffer";
      import bunyan from "bunyan";
      export class SMBReader { export class SMBReader {
      logPrefix: string; logPrefix: string;
      constructor(logPrefix: string) { logLevel: bunyan.LogLevel;
      constructor(logPrefix: string, logLevel?: bunyan.LogLevel) {
      this.logPrefix = logPrefix; this.logPrefix = logPrefix;
      this.logLevel = logLevel;
      } }
      config: Config; config: Config;
      smb: SMB; smb: SMB;
      db: mysql.Pool; db: mysql.Pool;
      log: bunyan;
      async init() { async init() {
      console.error(`${this.logPrefix}Reading config...`); this.log = bunyan.createLogger({ name: this.logPrefix, level: this.logLevel || "info" });
      this.log.info(`Start initializing.`);
      this.log.debug(`Reading config...`);
      this.config = await loadConfig(); this.config = await loadConfig();
      this.smb = new SMB(this.config.smbConfig); this.smb = new SMB(this.config.smbConfig);
      console.error(`${this.logPrefix}Connecting to database...`); this.log.debug(`Connecting to database...`);
      this.db = await mysql.createPool(this.config.MySQLConfig); this.db = await mysql.createPool(this.config.MySQLConfig);
      console.error(`${this.logPrefix}Creating table...`); this.log.debug(`Creating table...`);
      await this.db.query("CREATE TABLE IF NOT EXISTS `filesRead` (\n" + await this.db.query("CREATE TABLE IF NOT EXISTS `filesRead` (\n" +
      " `fileid` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT,\n" + " `fileid` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT,\n" +
      " `date` datetime UNIQUE NOT NULL,\n" + " `date` datetime UNIQUE NOT NULL,\n" +
      ...@@ -39,7 +44,7 @@ export class SMBReader { ...@@ -39,7 +44,7 @@ export class SMBReader {
      " KEY (channel(4)),\n" + " KEY (channel(4)),\n" +
      " INDEX (speaker(16))\n" + " INDEX (speaker(16))\n" +
      ") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci"); ") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci");
      console.error(`${this.logPrefix}Initialization finished.`); this.log.info(`Initialization finished.`);
      } }
      private getFilePath(filename: string) { private getFilePath(filename: string) {
      return this.config.pathPrefix ? `${this.config.pathPrefix}\\${filename}` : filename; return this.config.pathPrefix ? `${this.config.pathPrefix}\\${filename}` : filename;
      ...@@ -48,8 +53,9 @@ export class SMBReader { ...@@ -48,8 +53,9 @@ export class SMBReader {
      return messages.map(m => JSON.stringify(m)).join("|"); return messages.map(m => JSON.stringify(m)).join("|");
      } }
      async run() { async run() {
      this.log.info(`Started.`);
      try { try {
      console.error(`${this.logPrefix}Reading file list...`); this.log.info(`Reading file list...`);
      let filelist = (await this.smb.readdir(this.config.pathPrefix)).filter(m => m.match(/^chat_.*\.html$/)); let filelist = (await this.smb.readdir(this.config.pathPrefix)).filter(m => m.match(/^chat_.*\.html$/));
      const files = filelist.map(filename => { const files = filelist.map(filename => {
      const dateString = filename.match(/^chat_(.*)\.html$/)[1].replace(/-/g, ":"); const dateString = filename.match(/^chat_(.*)\.html$/)[1].replace(/-/g, ":");
      ...@@ -67,10 +73,10 @@ export class SMBReader { ...@@ -67,10 +73,10 @@ export class SMBReader {
      try { try {
      const testIndex: any[] = await this.db.query("select * from `filesRead` where filename = ?", [file.name]); const testIndex: any[] = await this.db.query("select * from `filesRead` where filename = ?", [file.name]);
      if (testIndex.length) { if (testIndex.length) {
      console.error(`${this.logPrefix}File ${filePath} has already been read. Skipping.`); this.log.info(`File ${filePath} has already been read. Skipping.`);
      continue; continue;
      } }
      console.error(`${this.logPrefix}Reading file ${filePath}...`); this.log.info(`Reading file ${filePath}...`);
      const fileMetadata = await this.db.query("insert into `filesRead` set ?", { const fileMetadata = await this.db.query("insert into `filesRead` set ?", {
      filename: file.name, filename: file.name,
      date: file.date.format("YYYY-MM-DD HH:mm:ss") date: file.date.format("YYYY-MM-DD HH:mm:ss")
      ...@@ -80,22 +86,22 @@ export class SMBReader { ...@@ -80,22 +86,22 @@ export class SMBReader {
      let currentMessages = readChatBuffer((await this.smb.readFile(filePath)) as Buffer); let currentMessages = readChatBuffer((await this.smb.readFile(filePath)) as Buffer);
      if (previousFilenameArray.length && files.find(f => f.name === previousFilenameArray[0].filename)) { // compare with old file if (previousFilenameArray.length && files.find(f => f.name === previousFilenameArray[0].filename)) { // compare with old file
      const previousFilePath = this.getFilePath(previousFilenameArray[0].filename); const previousFilePath = this.getFilePath(previousFilenameArray[0].filename);
      console.error(`${this.logPrefix}Comparing file ${filePath} with old file ${previousFilePath}.`) this.log.debug(`Comparing file ${filePath} with old file ${previousFilePath}.`)
      try { try {
      const previousMessages = readChatBuffer((await this.smb.readFile(previousFilePath)) as Buffer); const previousMessages = readChatBuffer((await this.smb.readFile(previousFilePath)) as Buffer);
      const flattenedPreviousMessages = this.flatternMessages(previousMessages); const flattenedPreviousMessages = this.flatternMessages(previousMessages);
      const flattenedCurrentMessages = this.flatternMessages(currentMessages); const flattenedCurrentMessages = this.flatternMessages(currentMessages);
      if (flattenedCurrentMessages.startsWith(flattenedPreviousMessages)) { if (flattenedCurrentMessages.startsWith(flattenedPreviousMessages)) {
      const sliceCount = previousMessages.length; const sliceCount = previousMessages.length;
      console.error(`${this.logPrefix}Slicing the first ${sliceCount} messages of file ${filePath}.`); this.log.debug(`Slicing the first ${sliceCount} messages of file ${filePath}.`);
      currentMessages = currentMessages.slice(sliceCount); currentMessages = currentMessages.slice(sliceCount);
      } }
      } catch (e) { } catch (e) {
      console.error(`${this.logPrefix}Failed comparing file ${filePath} with old file ${previousFilePath}. Skipping:`, e.toString()); this.log.warn(`Failed comparing file ${filePath} with old file ${previousFilePath}. Skipping:`, e.toString());
      } }
      } }
      for (let message of currentMessages) { for (let message of currentMessages) {
      console.error(`${this.logPrefix}Recording message from file ${insertedFileID} ${filePath}:`, JSON.stringify(message)); this.log.debug(`Recording message from file ${insertedFileID} ${filePath}:`, JSON.stringify(message));
      await this.db.query("insert into `messages` set ?", { await this.db.query("insert into `messages` set ?", {
      fileid: insertedFileID, fileid: insertedFileID,
      ...message ...message
      ...@@ -105,18 +111,19 @@ export class SMBReader { ...@@ -105,18 +111,19 @@ export class SMBReader {
      if (deleteFilenameArray.length && files.find(f => f.name === deleteFilenameArray[0].filename)) { // delete file with id - 2 because useless if (deleteFilenameArray.length && files.find(f => f.name === deleteFilenameArray[0].filename)) { // delete file with id - 2 because useless
      const deleteFilePath = this.getFilePath(deleteFilenameArray[0].filename); const deleteFilePath = this.getFilePath(deleteFilenameArray[0].filename);
      try { try {
      console.error(`${this.logPrefix}Deleting file ${deleteFilePath}.`); this.log.debug(`Deleting file ${deleteFilePath}.`);
      await this.smb.unlink(deleteFilePath); await this.smb.unlink(deleteFilePath);
      } catch (e) { } catch (e) {
      console.error(`${this.logPrefix}Failed deleting file ${deleteFilePath}:`, e.toString()); this.log.debug(`Failed deleting file ${deleteFilePath}:`, e.toString());
      } }
      } }
      } catch (e) { } catch (e) {
      console.error(`${this.logPrefix}Errored reading file ${filePath}:`, e.toString()); this.log.warn(`Errored reading file ${filePath}:`, e.toString());
      } }
      } }
      } catch (e) { } catch (e) {
      console.error(`${this.logPrefix}Read error:`, e.toString()); this.log.warn(`Read error:`, e.toString());
      } }
      this.log.info(`Finished.`);
      } }
      } }
      test/smb.ts
      View file @ 4f66fd00
      ...@@ -2,7 +2,7 @@ import { read } from "fs"; ...@@ -2,7 +2,7 @@ import { read } from "fs";
      import { SMBReader } from "../src/smbreader"; import { SMBReader } from "../src/smbreader";
      async function main() { async function main() {
      const reader = new SMBReader("Test: "); const reader = new SMBReader("Test", "debug");
      await reader.init(); await reader.init();
      await reader.run(); await reader.run();
      process.exit(); process.exit();
      ......
      Markdown is supported
      0% or
      You are about to add 0 people to the discussion. Proceed with caution.
      Finish editing this message first!
      Please register or to comment