add mysql

5ebca9cd · nanahira · cfb001c8 · 5ebca9cd · 5ebca9cd · 5ebca9cd
Commit 5ebca9cd authored Jul 02, 2020 by nanahira
13 changed files
--- a/.gitignore
+++ b/.gitignore
@@ -105,3 +105,4 @@ dist

 /build
 /output
+/config.yaml
--- a/.idea/.gitignore
+++ b/.idea/.gitignore
+# Default ignored files
+/shelf/
+/workspace.xml
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# Editor-based HTTP Client requests
+/httpRequests/
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="JavaScriptSettings">
+    <option name="languageLevel" value="ES6" />
+  </component>
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_14" project-jdk-name="14" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>
\ No newline at end of file
--- a/.idea/modules.xml
+++ b/.idea/modules.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/tx3-bang-reader.iml" filepath="$PROJECT_DIR$/.idea/tx3-bang-reader.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
--- a/.idea/tx3-bang-reader.iml
+++ b/.idea/tx3-bang-reader.iml
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
\ No newline at end of file
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file
--- a/config.example.yaml
+++ b/config.example.yaml
+outDir: ./output
+useMySQL: false
+MySQLConfig:
+  host: mysql
+  user: tx3
+  database: tx3
+  password: my_password
+proxy:
+  useProxy: false
+  proxySource:
+    - "http://www.89ip.cn/tqdl.html?api=1&num=9999"
+    - "http://www.66ip.cn/mo.php?tqsl=9999"
+  timeout: 10000
\ No newline at end of file
--- a/package-lock.json
+++ b/package-lock.json
@@ -4,6 +4,19 @@
  "lockfileVersion": 1,
  "requires": true,
  "dependencies": {
+    "@types/bluebird": {
+      "version": "3.5.32",
+      "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.32.tgz",
+      "integrity": "sha512-dIOxFfI0C+jz89g6lQ+TqhGgPQ0MxSnh/E4xuC0blhFtyW269+mPG5QeLgbdwst/LvdP8o1y0o/Gz5EHXLec/g=="
+    },
+    "@types/mysql": {
+      "version": "2.15.14",
+      "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.14.tgz",
+      "integrity": "sha512-YyNcxZZ4qlb9v41lu8Qfo+quX8eUqz4mr/6Nx7sclc2pUemOvjU7xrpuVave0/sR/VdtDPTXZOWQLC/kthxrHg==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
    "@types/node": {
      "version": "14.0.14",
      "resolved": "https://registry.npm.taobao.org/@types/node/download/@types/node-14.0.14.tgz?cache=0&sync_timestamp=1592989417189&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2F%40types%2Fnode%2Fdownload%2F%40types%2Fnode-14.0.14.tgz",
@@ -14,6 +27,14 @@
      "resolved": "https://registry.npm.taobao.org/@types/underscore/download/@types/underscore-1.10.2.tgz?cache=0&sync_timestamp=1593310825578&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2F%40types%2Funderscore%2Fdownload%2F%40types%2Funderscore-1.10.2.tgz",
      "integrity": "sha1-JCYVPgkmRu155Y8VMvQprq+mmk4="
    },
+    "@types/yaml": {
+      "version": "1.9.7",
+      "resolved": "https://registry.npmjs.org/@types/yaml/-/yaml-1.9.7.tgz",
+      "integrity": "sha512-8WMXRDD1D+wCohjfslHDgICd2JtMATZU8CkhH8LVJqcJs6dyYj5TGptzP8wApbmEullGBSsCEzzap73DQ1HJaA==",
+      "requires": {
+        "yaml": "*"
+      }
+    },
    "axios": {
      "version": "0.19.2",
      "resolved": "https://registry.npm.taobao.org/axios/download/axios-0.19.2.tgz",
@@ -22,6 +43,21 @@
        "follow-redirects": "1.5.10"
      }
    },
+    "bignumber.js": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz",
+      "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A=="
+    },
+    "bluebird": {
+      "version": "3.7.2",
+      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz",
+      "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
+    },
+    "core-util-is": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
+    },
    "debug": {
      "version": "3.1.0",
      "resolved": "https://registry.npm.taobao.org/debug/download/debug-3.1.0.tgz",
@@ -62,11 +98,70 @@
      "resolved": "https://registry.npm.taobao.org/inherits/download/inherits-2.0.4.tgz",
      "integrity": "sha1-D6LGT5MpF8NDOg3tVTY6rjdBa3w="
    },
+    "isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+    },
+    "moment": {
+      "version": "2.27.0",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.27.0.tgz",
+      "integrity": "sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ=="
+    },
    "ms": {
      "version": "2.0.0",
      "resolved": "https://registry.npm.taobao.org/ms/download/ms-2.0.0.tgz",
      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
    },
+    "mysql": {
+      "version": "2.18.1",
+      "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz",
+      "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==",
+      "requires": {
+        "bignumber.js": "9.0.0",
+        "readable-stream": "2.3.7",
+        "safe-buffer": "5.1.2",
+        "sqlstring": "2.3.1"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "2.3.7",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
+          "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.3",
+            "isarray": "~1.0.0",
+            "process-nextick-args": "~2.0.0",
+            "safe-buffer": "~5.1.1",
+            "string_decoder": "~1.1.1",
+            "util-deprecate": "~1.0.1"
+          }
+        },
+        "safe-buffer": {
+          "version": "5.1.2",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+          "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+        },
+        "string_decoder": {
+          "version": "1.1.1",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+          "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+          "requires": {
+            "safe-buffer": "~5.1.0"
+          }
+        }
+      }
+    },
+    "mysql-promise": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/mysql-promise/-/mysql-promise-5.0.0.tgz",
+      "integrity": "sha512-TJDFvUuhNGp5L8q5P1ZOGvksON4pY2LYLQpkfETRDguLOITzDCF8039nAgxtMb/dMma0ZvP9WV2GaM5slAQfAg==",
+      "requires": {
+        "bluebird": "^3.5.2",
+        "mysql": "^2.16.0"
+      }
+    },
    "posthtml-parser": {
      "version": "0.4.2",
      "resolved": "https://registry.npm.taobao.org/posthtml-parser/download/posthtml-parser-0.4.2.tgz",
@@ -117,6 +212,22 @@
        }
      }
    },
+    "process-nextick-args": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+    },
+    "promise-mysql": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/promise-mysql/-/promise-mysql-4.1.3.tgz",
+      "integrity": "sha512-TSqIBjeflVLK8LDv6dszWfhn+NJggwf5e8o3k8OdUe4XHiJRiX/caDoX8ohNeIk4xsT0b60PBHlGk8J5wUj/0Q==",
+      "requires": {
+        "@types/bluebird": "^3.5.26",
+        "@types/mysql": "^2.15.2",
+        "bluebird": "^3.5.1",
+        "mysql": "^2.18.1"
+      }
+    },
    "querystring": {
      "version": "0.2.0",
      "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
@@ -137,6 +248,11 @@
      "resolved": "https://registry.npm.taobao.org/safe-buffer/download/safe-buffer-5.2.1.tgz",
      "integrity": "sha1-Hq+fqb2x/dTsdfWPnNtOa3gn7sY="
    },
+    "sqlstring": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
+      "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A="
+    },
    "string_decoder": {
      "version": "1.3.0",
      "resolved": "https://registry.npm.taobao.org/string_decoder/download/string_decoder-1.3.0.tgz",
@@ -159,6 +275,11 @@
      "version": "1.0.2",
      "resolved": "https://registry.npm.taobao.org/util-deprecate/download/util-deprecate-1.0.2.tgz",
      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
+    },
+    "yaml": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz",
+      "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg=="
    }
  }
 }
--- a/package.json
+++ b/package.json
@@ -2,7 +2,7 @@
  "name": "tx3-bang-reader",
  "version": "1.0.0",
  "description": "Read TX3 bang and parse it",
-  "main": "build/run.js",
+  "main": "build/fetcher.js",
  "scripts": {
    "build": "./node_modules/.bin/tsc",
    "fetch": "node build/src/run.js"
@@ -23,10 +23,15 @@
  "dependencies": {
    "@types/node": "^14.0.14",
    "@types/underscore": "^1.10.2",
+    "@types/yaml": "^1.9.7",
    "axios": "^0.19.2",
+    "moment": "^2.27.0",
+    "mysql-promise": "^5.0.0",
    "posthtml-parser": "^0.4.2",
+    "promise-mysql": "^4.1.3",
    "querystring": "^0.2.0",
    "typescript": "^3.9.5",
-    "underscore": "^1.10.2"
+    "underscore": "^1.10.2",
+    "yaml": "^1.10.0"
  }
 }
--- a/src/fetcher.ts
+++ b/src/fetcher.ts
 import axios from "axios";
 import _ from "underscore";
 import { User } from "./user";
-import { ProxyFetcher } from "./proxy";
+import {ProxyConfig, ProxyFetcher} from "./proxy";
 import { PlayerRow, parsePlayerRows } from "./playerlist";
 import qs from "querystring";
+import mysql from "promise-mysql";
+import moment from "moment";

 export const servers = [
 	"东方明珠",
@@ -37,13 +39,45 @@ export const servers = [
 	"锦绣花朝"
 ]

+export interface Config {
+	outDir: string;
+	useMySQL: boolean,
+	MySQLConfig: mysql.PoolConfig,
+	proxy: ProxyConfig
+}
 export class Tx3Fetcher {
+	config: Config;
 	proxyFetcher: ProxyFetcher;
-	constructor() {
-		this.proxyFetcher = new ProxyFetcher();
+	db: mysql.Pool;
+	curDate: string;
+	constructor(config: Config) {
+		this.config = config;
+		this.proxyFetcher = new ProxyFetcher(config.proxy);
 	}
-	async initProxies() {
-		await this.proxyFetcher.initProxies();
+	async init() {
+		this.curDate = moment().format("YYYY-MM-DD");
+		if(this.config.useMySQL) {
+			this.db = await mysql.createPool(this.config.MySQLConfig);
+			await this.db.query("CREATE TABLE IF NOT EXISTS `userdata` (\n" +
+				"  `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" +
+				"  `date` date NOT NULL DEFAULT current_timestamp(),\n" +
+				"  `url` varchar(50) COLLATE utf8_unicode_ci NOT NULL,\n" +
+				"  `rank` int(11) UNSIGNED NOT NULL,\n" +
+				"  `name` varchar(7) COLLATE utf8_unicode_ci NOT NULL,\n" +
+				"  `category` varchar(5) COLLATE utf8_unicode_ci NOT NULL,\n" +
+				"  `serverArea` varchar(4) COLLATE utf8_unicode_ci NOT NULL,\n" +
+				"  `server` varchar(4) COLLATE utf8_unicode_ci NOT NULL,\n" +
+				"  `level` tinyint(4) UNSIGNED NOT NULL,\n" +
+				"  `region` varchar(7) COLLATE utf8_unicode_ci,\n" +
+				"  `score` int(11) UNSIGNED NOT NULL,\n" +
+				"  `equip` int(11) UNSIGNED NOT NULL,\n" +
+				"  `totalScore` int(11) UNSIGNED NOT NULL,\n" +
+				"  PRIMARY KEY (`id`)\n" +
+				") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci");
+		}
+		if(this.config.proxy.useProxy) {
+			await this.proxyFetcher.initProxies();
+		}
 	}
 	async fetchAll(): Promise<Map<string, User[]>> {
 		const res = new Map<string, User[]>();
@@ -118,6 +152,12 @@ export class Tx3Fetcher {
 				}
 			});
 			const playerRows = parsePlayerRows(content);
+			if(this.db) {
+				await Promise.all(playerRows.map(m => this.db.query("insert into userdata set ?", {
+					date: this.curDate,
+					...m
+				})));
+			}
 			return playerRows;
 		} catch(e) {
 			console.error(`Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`);

--- a/src/proxy.ts
+++ b/src/proxy.ts
 import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios";

-const NO_PROXY = true;
-
-const proxySourceList = [
-	"http://www.89ip.cn/tqdl.html?api=1&num=9999", "http://www.66ip.cn/mo.php?tqsl=9999"
-]
+//const proxySourceList = [
+//	"http://www.89ip.cn/tqdl.html?api=1&num=9999", "http://www.66ip.cn/mo.php?tqsl=9999"
+//]

 //for (let i = 1; i <= 2000; ++i) {
 //	proxySourceList.push(`http://www.xiladaili.com/http/${i}`);
 //}

+export interface ProxyConfig {
+	useProxy: boolean,
+	proxySource: string[],
+	timeout: number
+}
+
 const agentList = [
 	'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
 	'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
@@ -36,7 +40,7 @@ async function testProxy(proxy: AxiosProxyConfig) {
 		headers: {
 			"User-Agent": agentList[4]
 		},
-		timeout: 1000,
+		timeout: this.config.timeout,
 	});
 	return proxy;
 }
@@ -67,12 +71,14 @@ async function filterProxies(proxies: AxiosProxyConfig[]) {
 export class ProxyFetcher {
 	proxies: AxiosProxyConfig[];
 	counter: number;
-	constructor() {
+	config: ProxyConfig;
+	constructor(config: ProxyConfig) {
+		this.config = config;
 		this.proxies = [];
 		this.counter = 0;
 	}
 	async initProxiesFrom(url: string) {
-		if (NO_PROXY) {
+		if (!this.config.useProxy) {
 			return;
 		}
 		console.log(`Fetching proxies from ${url}.`)
@@ -99,30 +105,30 @@ export class ProxyFetcher {
 		}
 	}
 	async initProxies() {
-		await Promise.all(proxySourceList.map((m) => {
+		await Promise.all(this.config.proxySource.map((m) => {
 			return this.initProxiesFrom(m);
 		}));
 	}
 	async getWithProxy(url: string, options: AxiosRequestConfig) {
 		while (true) {
-			if (!NO_PROXY && !this.proxies.length) {
+			if (this.config.useProxy && !this.proxies.length) {
 				await this.initProxies();
 			}
-			const proxyIndex = NO_PROXY ? null : (++this.counter) % this.proxies.length;
+			const proxyIndex = !this.config.useProxy ? null : (++this.counter) % this.proxies.length;
 			//const proxyIndex = 0;
-			const proxy = NO_PROXY ? null : this.proxies[proxyIndex];
+			const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex];
 			try {
 				const data = (await axios.get(url, {
 					proxy,
 					headers: {
 						"User-Agent": agentList[this.counter % agentList.length]
 					},
-					timeout: 30000,
+					timeout: this.config.timeout,
 					...options
 				})).data;
 				return data;
 			} catch (e) {
-				if (!NO_PROXY) {
+				if (this.config.useProxy) {
 					this.proxies.splice(proxyIndex, 1);
 				}
 				console.error(`Failed fetching data from ${url}: ${e.toString()} ${this.proxies.length} proxies left.`)

--- a/src/run.ts
+++ b/src/run.ts
-import { Tx3Fetcher, servers } from "./fetcher";
+import {Tx3Fetcher, servers, Config} from "./fetcher";
 import fs from "fs";
 import _ from "underscore";
+import yaml from "yaml";

-const fetcher = new Tx3Fetcher();
+let fetcher;

 async function runServer(server: string) {
 	const users = await fetcher.fetchListFromServer(server);
@@ -18,7 +19,9 @@ async function main() {
 			recursive: true
 		});
 	}
-	await fetcher.initProxies();
+	const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
+	fetcher = new Tx3Fetcher(config);
+	await fetcher.init();
 	if (process.env.SERVER) {
 		await runServer(servers[parseInt(process.env.SERVER)]);
 		return;

--- a/tests/test-proxy.ts
+++ b/tests/test-proxy.ts
 import { ProxyFetcher } from "../src/proxy";
 async function main() { 
-	const fetcher = new ProxyFetcher();
+	const fetcher = new ProxyFetcher({
+		useProxy: true,
+		proxySource: [
+			"http://www.89ip.cn/tqdl.html?api=1&num=9999",
+			"http://www.66ip.cn/mo.php?tqsl=9999"
+		],
+		timeout: 10000
+	});
 	await fetcher.initProxies();
 	console.log(await fetcher.getWithProxy("https://mycard.moe", {}));
 }