add cron

Dockerfile

 FROM node:buster-slim
-RUN apt update && apt -y install cron python3 && rm -rf /var/lib/apt/lists/*
+RUN apt update && apt -y install python3 && rm -rf /var/lib/apt/lists/*
 WORKDIR /usr/src/app
 COPY ./package*.json ./
 RUN npm ci
 COPY . ./
 RUN npm run build
 
-RUN echo '0 0 * * * bash -c "cd /usr/src/app && npm run fetch"' > /etc/cron.d/tx3 && \
-    crontab /etc/cron.d/tx3
-
-CMD ["bash", "-c", "npm run fetch && cron -f"]
+CMD ["npm", "run", "start"]

config.example.yaml

@@ -11,4 +11,4 @@ proxy:
   proxySource:
     - "http://www.89ip.cn/tqdl.html?api=1&num=9999"
     - "http://www.66ip.cn/mo.php?tqsl=9999"
-  timeout: 10000
+  timeout: 10000
\ No newline at end of file

package-lock.json

@@ -9,6 +9,15 @@
       "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.32.tgz",
       "integrity": "sha512-dIOxFfI0C+jz89g6lQ+TqhGgPQ0MxSnh/E4xuC0blhFtyW269+mPG5QeLgbdwst/LvdP8o1y0o/Gz5EHXLec/g=="
     },
+    "@types/cron": {
+      "version": "1.7.2",
+      "resolved": "https://registry.npmjs.org/@types/cron/-/cron-1.7.2.tgz",
+      "integrity": "sha512-AEpNLRcsVSc5AdseJKNHpz0d4e8+ow+abTaC0fKDbAU86rF1evoFF0oC2fV9FdqtfVXkG2LKshpLTJCFOpyvTg==",
+      "requires": {
+        "@types/node": "*",
+        "moment": ">=2.14.0"
+      }
+    },
     "@types/mysql": {
       "version": "2.15.14",
       "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.14.tgz",
@@ -58,6 +67,14 @@
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
     },
+    "cron": {
+      "version": "1.8.2",
+      "resolved": "https://registry.npmjs.org/cron/-/cron-1.8.2.tgz",
+      "integrity": "sha512-Gk2c4y6xKEO8FSAUTklqtfSr7oTq0CiPQeLBG5Fl0qoXpZyMcj1SG59YL+hqq04bu6/IuEA7lMkYDAplQNKkyg==",
+      "requires": {
+        "moment-timezone": "^0.5.x"
+      }
+    },
     "debug": {
       "version": "3.1.0",
       "resolved": "https://registry.npm.taobao.org/debug/download/debug-3.1.0.tgz",
@@ -108,6 +125,14 @@
       "resolved": "https://registry.npmjs.org/moment/-/moment-2.27.0.tgz",
       "integrity": "sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ=="
     },
+    "moment-timezone": {
+      "version": "0.5.31",
+      "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.31.tgz",
+      "integrity": "sha512-+GgHNg8xRhMXfEbv81iDtrVeTcWt0kWmTEY1XQK14dICTXnWJnT0dxdlPspwqF3keKMVPXwayEsk1DI0AA/jdA==",
+      "requires": {
+        "moment": ">= 2.9.0"
+      }
+    },
     "ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npm.taobao.org/ms/download/ms-2.0.0.tgz",

package.json

@@ -5,7 +5,8 @@
   "main": "build/fetcher.js",
   "scripts": {
     "build": "./node_modules/.bin/tsc",
-    "fetch": "node build/src/run.js"
+    "fetch": "node build/src/run.js",
+    "start": "node build/src/run.js cron"
   },
   "repository": {
     "type": "git",
@@ -21,10 +22,12 @@
   },
   "homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme",
   "dependencies": {
+    "@types/cron": "^1.7.2",
     "@types/node": "^14.0.14",
     "@types/underscore": "^1.10.2",
     "@types/yaml": "^1.9.7",
     "axios": "^0.19.2",
+    "cron": "^1.8.2",
     "moment": "^2.27.0",
     "mysql-promise": "^5.0.0",
     "posthtml-parser": "^0.4.2",

src/fetcher.ts

@@ -121,7 +121,7 @@ export class Tx3Fetcher {
 			resPromises.push(this.fetchListFromSchoolAndServer(school, server));
 		}
 		const result = _.flatten(await Promise.all(resPromises));
-		console.log(`Fetched user list with ${result.length} users from server ${server}.`);
+		console.log(`Fetched user list from server ${server}. ${result.length} users found.`);
 		return result;
 	}
 	async fetchListFromSchoolAndServer(school: number, server: string): Promise<PlayerRow[]> {
@@ -134,7 +134,9 @@ export class Tx3Fetcher {
 			}
 			res.push(list);
 		}
-		return _.flatten(res);
+		const ret = _.flatten(res);
+		console.log(`Fetched users from server ${server} with school ${school}.`);
+		return ret;
 	}
 	async fetchList(school: number, server: string, page: number): Promise<PlayerRow[]> {
 		console.log(`Fetching user list from server ${server} with school ${school} page ${page}.`);
@@ -153,7 +155,9 @@ export class Tx3Fetcher {
 				}
 			});
 			const playerRows = parsePlayerRows(content);
-			if(this.db) {
+			console.log(`Fetched user list from server ${server} with school ${school} page ${page}. ${playerRows.length} users found.`);
+			if (this.db) {
+				//await Promise.all(playerRows.map(m => this.db.query("delete from userdata where url = ? and date = ?", [m.url, this.curDate])));
 				await Promise.all(playerRows.map(m => this.db.query("insert into userdata set ?", {
 					date: this.curDate,
 					...m

src/run.ts

@@ -2,19 +2,25 @@ import {Tx3Fetcher, servers, Config} from "./fetcher";
 import fs from "fs";
 import _ from "underscore";
 import yaml from "yaml";
+import { CronJob } from "cron";
 
-let fetcher;
+let config: Config;
 
-async function runServer(server: string) {
+async function loadConfig() {
+	config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
+}
+
+async function runServer(fetcher: Tx3Fetcher, server: string) {
 	const users = await fetcher.fetchListFromServer(server);
 	await fs.promises.writeFile(`./output/servers/${server}.json`, JSON.stringify({
-		date: fetcher.date,
+		date: fetcher.curDate,
 		data: users
 	}, null, 2));
 	return users;
 }
 
-async function main() {
+async function run() {
+	console.log(`Fetch started.`);
 	try {
 		await fs.promises.access("./output/servers");
 	} catch (e) {
@@ -22,23 +28,37 @@ async function main() {
 			recursive: true
 		});
 	}
-	const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
-	fetcher = new Tx3Fetcher(config);
+	const fetcher = new Tx3Fetcher(config);
 	await fetcher.init();
 	if (config.server) {
-		await Promise.all(config.server.map(runServer));
+		await Promise.all(config.server.map(server => {
+			return runServer(fetcher, server)
+		}));
 	} else {
-		const userListWithServer = await Promise.all(servers.map(runServer));
+		const userListWithServer = await Promise.all(servers.map(server => {
+			return runServer(fetcher, server)
+		}));
 		const allServersList: any = {};
 		for (let i = 0; i < servers.length;++i) {
 			allServersList[servers[i]] = userListWithServer[i];
 		}
 		await fs.promises.writeFile(`./output/all.json`, JSON.stringify({
-			date: fetcher.date,
+			date: fetcher.curDate,
 			data: allServersList
 		}, null, 2));
 	}
 	console.log("Finished.");
-	process.exit();
+}
+
+async function main() {
+	await loadConfig();
+	if (process.argv[2] === "cron") {
+		const job = new CronJob("0 0 1 * * *", run, null, true, "Asia/Shanghai");
+		job.start();
+	} else {
+		await run();
+		process.exit();
+	}
+	
 }
 main();