add migrate

package-lock.json

@@ -18,6 +18,14 @@
         "moment": ">=2.14.0"
       }
     },
+    "@types/csv-parse": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/@types/csv-parse/-/csv-parse-1.2.2.tgz",
+      "integrity": "sha512-k33tLtRKTQxf7hQfMlkWoS2TQYsnpk1ibZN+rzbuCkeBs8m23nHTeDTF1wb/e7/MSLdtgCzqu3oM1I101kd6yw==",
+      "requires": {
+        "csv-parse": "*"
+      }
+    },
     "@types/mysql": {
       "version": "2.15.14",
       "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.14.tgz",
@@ -75,6 +83,11 @@
         "moment-timezone": "^0.5.x"
       }
     },
+    "csv-parse": {
+      "version": "4.10.1",
+      "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.10.1.tgz",
+      "integrity": "sha512-gdDJVchi0oSLIcYXz1H/VSgLE6htHDqJyFsRU/vTkQgmVOZ3S0IR2LXnNbWUYG7VD76dYVwdfBLyx8AX9+An8A=="
+    },
     "debug": {
       "version": "3.1.0",
       "resolved": "https://registry.npm.taobao.org/debug/download/debug-3.1.0.tgz",

package.json

@@ -23,11 +23,13 @@
   "homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme",
   "dependencies": {
     "@types/cron": "^1.7.2",
+    "@types/csv-parse": "^1.2.2",
     "@types/node": "^14.0.14",
     "@types/underscore": "^1.10.2",
     "@types/yaml": "^1.9.7",
     "axios": "^0.19.2",
     "cron": "^1.8.2",
+    "csv-parse": "^4.10.1",
     "moment": "^2.27.0",
     "mysql-promise": "^5.0.0",
     "posthtml-parser": "^0.4.2",

src/migrate.ts

+import mysql from "promise-mysql";
+import moment from "moment";
+import fs from "fs";
+import _csv_parse from "csv-parse";
+import util from 'util';
+import { Config } from "./fetcher";
+import { PlayerRow, PlayerRowDated } from "./playerlist";
+import _ from "underscore";
+import yaml from "yaml";
+
+const parse_csv: (input: Buffer | string, options?: _csv_parse.Options) => Promise<any[]> = util.promisify(_csv_parse);
+let config: Config;
+let db: mysql.Pool;
+
+const serverAreaCache = new Map<string, string>();
+async function getServerAreaFromServer(server: string) {
+	if (serverAreaCache.has(server)) {
+		return serverAreaCache.get(server);
+	} else {
+		const [res] = await db.query("select serverArea from userdata where server = ? limit 1", server);
+		if (!res) {
+			return "none";
+		}
+		const serverArea = res.serverArea;
+		serverAreaCache.set(server, serverArea);
+		return serverArea;
+	}
+}
+
+async function readSingleRecord(col: string[], offset: number, base: PlayerRow): Promise<PlayerRowDated> {
+	let pointer = offset;
+	const newRecord: PlayerRowDated = {
+		date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"),
+		...(_.clone(base))
+	};
+	newRecord.name = col[pointer++];
+	newRecord.server = col[pointer++];
+	newRecord.serverArea = await getServerAreaFromServer(newRecord.server);
+	const _region = col[pointer++];
+	newRecord.region = _region.length ? _region : "none";
+	newRecord.category = col[pointer++];
+	newRecord.equip = parseInt(col[pointer++]);
+	newRecord.totalScore = newRecord.equip + newRecord.score;
+	return newRecord;
+}
+
+let leftCount: number;
+async function readColumn(col: string[]): Promise<void> {
+	const recordCount = parseInt(col[0]);
+	const url = `/bang/role/${col[3]}`;
+	console.error(`Reading column ${url}.`);
+	let [base] = await db.query("select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by date asc limit 1", url) as PlayerRow[];
+	if (!base) {
+		console.error(`Base record of ${url} not found. Using default values.`);
+		base = {
+			url,
+			rank: 500,
+			name: null,
+			server: null,
+			serverArea: null,
+			category: null,
+			level: 80,
+			region: "none",
+			score: 0,
+			equip: 0,
+			totalScore: 0
+		}
+	}
+	for (let i = 0; i < recordCount; ++i) {
+		const offset = 4 + (i * 6);
+		if (!col[offset].length) {
+			continue;
+		}
+		const record = await readSingleRecord(col, offset, base);
+		const sql = "insert into userdata set ?";
+		console.log(sql, JSON.stringify(record), JSON.stringify(await db.query(sql, record)));
+	}
+	console.error(`Read column ${url}. ${--leftCount} columns left.`);
+}
+
+async function loadCsv(path: string): Promise<string[][]> {
+	const data = await fs.promises.readFile(path);
+	return await parse_csv(data, {
+		trim: true
+	});
+}
+
+async function main() {
+	console.error("Started.");
+	const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
+	db = await mysql.createPool(config.MySQLConfig);
+	const data = await loadCsv(process.argv[2]);
+	leftCount = data.length;
+	//await Promise.all(data.map(col => readColumn(col)));
+	for (let col of data) {
+		await readColumn(col);
+	}
+	console.error("Finished.");
+	process.exit();
+}
+main();

src/playerlist.ts

@@ -16,6 +16,10 @@ export interface PlayerRow {
 	totalScore: number;
 }
 
+export interface PlayerRowDated extends PlayerRow {
+	date: string;
+}
+
 function getPlayerRowFromTree(tree: HTML.Tree): PlayerRow {
 	const nodes = getContinuousNodes(tree, [1], 0, 2, 10);
 	return {