proxy fetcher

.gitignore

@@ -104,3 +104,4 @@ dist
 .tern-port
 
 /build
+/output

package-lock.json

@@ -44,24 +44,6 @@
       "resolved": "https://registry.npm.taobao.org/domelementtype/download/domelementtype-2.0.1.tgz",
       "integrity": "sha1-H4vf6R9aeAYydOgDtL3O326U+U0="
     },
-    "domhandler": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-3.0.0.tgz",
-      "integrity": "sha512-eKLdI5v9m67kbXQbJSNn1zjh0SDzvzWVWtX+qEI3eMjZw8daH9k8rlj1FZY9memPwjiskQFbe7vHVVJIAqoEhw==",
-      "requires": {
-        "domelementtype": "^2.0.1"
-      }
-    },
-    "domutils": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npm.taobao.org/domutils/download/domutils-2.1.0.tgz?cache=0&sync_timestamp=1589052559692&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdomutils%2Fdownload%2Fdomutils-2.1.0.tgz",
-      "integrity": "sha1-et4yAa9DcD/eFUlS46ho60tjXxY=",
-      "requires": {
-        "dom-serializer": "^0.2.1",
-        "domelementtype": "^2.0.1",
-        "domhandler": "^3.0.0"
-      }
-    },
     "entities": {
       "version": "2.0.3",
       "resolved": "https://registry.npm.taobao.org/entities/download/entities-2.0.3.tgz",
@@ -75,17 +57,6 @@
         "debug": "=3.1.0"
       }
     },
-    "htmlparser2": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npm.taobao.org/htmlparser2/download/htmlparser2-4.1.0.tgz",
-      "integrity": "sha1-mk7xYfLkYl6/ffvmwKL1LRilnng=",
-      "requires": {
-        "domelementtype": "^2.0.1",
-        "domhandler": "^3.0.0",
-        "domutils": "^2.0.0",
-        "entities": "^2.0.0"
-      }
-    },
     "inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npm.taobao.org/inherits/download/inherits-2.0.4.tgz",
@@ -146,6 +117,11 @@
         }
       }
     },
+    "querystring": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
+      "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
+    },
     "readable-stream": {
       "version": "3.6.0",
       "resolved": "https://registry.npm.taobao.org/readable-stream/download/readable-stream-3.6.0.tgz?cache=0&sync_timestamp=1581624324274&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Freadable-stream%2Fdownload%2Freadable-stream-3.6.0.tgz",

package.json

@@ -24,9 +24,8 @@
     "@types/node": "^14.0.14",
     "@types/underscore": "^1.10.2",
     "axios": "^0.19.2",
-    "domhandler": "^3.0.0",
-    "htmlparser2": "^4.1.0",
     "posthtml-parser": "^0.4.2",
+    "querystring": "^0.2.0",
     "typescript": "^3.9.5",
     "underscore": "^1.10.2"
   }

src/proxy.ts

+import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios";
+
+export class ProxyFetcher {
+	proxies: AxiosProxyConfig[];
+	counter: number;
+	constructor() {
+		this.proxies = [];
+		this.counter = 0;
+	}
+	async initProxiesFrom(url: string) {
+		while (true) {
+			try {
+				const proxyPage: string = (await axios.get(url, {
+					responseType: "document"
+				})).data;
+				const proxyMatches: string[] = proxyPage.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g);
+				for (let proxyString of proxyMatches) {
+					const [host, _port] = proxyString.split(":");
+					const port = parseInt(_port);
+					this.proxies.push({ host, port });
+				}
+				console.error(`Got ${proxyMatches.length} proxies from ${url}.`);
+				return;
+			} catch (e) {
+				console.error(`Failed fetching proxy list from ${url}: ${e.toString()}`)
+			}
+		}
+	}
+	async initProxies() {
+		await Promise.all(["http://www.89ip.cn/tqdl.html?api=1&num=9999", "http://www.66ip.cn/mo.php?tqsl=9999"].map((m) => {
+			return this.initProxiesFrom(m);
+		}));
+	}
+	async getWithProxy(url: string, options: AxiosRequestConfig) {
+		while (true) {
+			if (!this.proxies.length) {
+				await this.initProxies();
+			}
+			const proxyIndex = (++this.counter) % this.proxies.length;
+			const proxy = this.proxies[proxyIndex];
+			try {
+				const data = (await axios.get(url, {
+					proxy,
+					...options
+				})).data;
+				return data;
+			} catch (e) {
+				this.proxies.splice(proxyIndex, 1);
+				console.error(`Failed fetching data from ${url}: ${e.toString()}`)
+			}
+		}
+	}
+}

tests/test-list.ts

+import { User } from "../src/user";
+import fs from "fs";
+import { parsePlayerRows } from "../src/playerlist";
+
+async function main() {
+	let html = await fs.promises.readFile("./tests/playerlist.html", "utf-8");
+	console.log(parsePlayerRows(html));
+	html = await fs.promises.readFile("./tests/playerlist-null.html", "utf-8");
+	console.log(parsePlayerRows(html));
+}
+main();

tests/test-proxy.ts

+import { ProxyFetcher } from "../src/proxy";
+async function main() { 
+	const fetcher = new ProxyFetcher();
+	await fetcher.initProxies();
+	console.log(await fetcher.getWithProxy("https://mycard.moe", {}));
+}
+main();