Skip to content

T
tx3-bang-reader
Commit 593d03fb authored by nanahira's avatar nanahira
Browse files
Options

rework

parent 85d2aa62
Pipeline #9850 failed with stages
        in 1 minute and 33 seconds
        Hide whitespace changes
        Inline Side-by-side
        Showing with 12316 additions and 1266 deletions
        .eslintignore 0 → 100644
        View file @ 593d03fb
        webpack.config.js
        dist/*
        build/*
        *.js
        .eslintrc.js 0 → 100644
        View file @ 593d03fb
        module.exports = {
        parser: '@typescript-eslint/parser',
        parserOptions: {
        project: 'tsconfig.json',
        sourceType: 'module',
        },
        plugins: ['@typescript-eslint/eslint-plugin'],
        extends: [
        'plugin:@typescript-eslint/recommended',
        'plugin:prettier/recommended',
        ],
        root: true,
        env: {
        node: true,
        jest: true,
        },
        ignorePatterns: ['.eslintrc.js'],
        rules: {
        '@typescript-eslint/interface-name-prefix': 'off',
        '@typescript-eslint/explicit-function-return-type': 'off',
        '@typescript-eslint/explicit-module-boundary-types': 'off',
        '@typescript-eslint/no-explicit-any': 'off',
        },
        };
        .gitlab-ci.yml
        View file @ 593d03fb
        stages: stages:
        - install
        - build - build
        - deploy - deploy
        variables: variables:
        GIT_DEPTH: "1" GIT_DEPTH: "1"
        CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
        CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest
        docker: npm_ci:
        stage: install
        tags:
        - linux
        script:
        - npm ci
        artifacts:
        paths:
        - node_modules
        .build_base:
        stage: build stage: build
        tags: tags:
        - docker - linux
        before_script: dependencies:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - npm_ci
        build:
        extends:
        - .build_base
        script: script:
        - docker build --pull -t $CONTAINER_TEST_IMAGE . - npm run build
        - docker push $CONTAINER_TEST_IMAGE artifacts:
        paths:
        - dist/
        deploy_latest: unit-test:
        stage: deploy extends:
        tags: - .build_base
        - docker
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        script: script:
        - docker pull $CONTAINER_TEST_IMAGE - npm run test
        - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
        - docker push $CONTAINER_RELEASE_IMAGE
        only:
        - master
        deploy_tag: deploy_npm:
        stage: deploy stage: deploy
        tags: dependencies:
        - docker - build
        variables: tags:
        CONTAINER_TAG_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG - linux
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        script: script:
        - docker pull $CONTAINER_TEST_IMAGE - apt update;apt -y install coreutils
        - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_TAG_IMAGE - echo $NPMRC | base64 --decode > ~/.npmrc
        - docker push $CONTAINER_TAG_IMAGE - npm publish . || true
        only: only:
        - tags - master
        .idea/inspectionProfiles/Project_Default.xml 0 → 100644
        View file @ 593d03fb
        <component name="InspectionProjectProfileManager">
        <profile version="1.0">
        <option name="myName" value="Project Default" />
        <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
        </profile>
        </component>
        \ No newline at end of file
        .idea/jsLinters/eslint.xml 0 → 100644
        View file @ 593d03fb
        <?xml version="1.0" encoding="UTF-8"?>
        <project version="4">
        <component name="EslintConfiguration">
        <option name="fix-on-save" value="true" />
        </component>
        </project>
        \ No newline at end of file
        .npmignore 0 → 100644
        View file @ 593d03fb
        /install-npm.sh
        .git*
        /data
        /output
        /config.yaml
        .idea
        .dockerignore
        Dockerfile
        /src
        /coverage
        /tests
        /dist/tests
        Dockerfile
        View file @ 593d03fb
        FROM node:bullseye-slim FROM node:bullseye-slim as base
        RUN apt update && apt -y install python3 && rm -rf /var/lib/apt/lists/* LABEL Author="Nanahira <nanahira@momobako.com>"
        RUN apt update && apt -y install python3 build-essential && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/log/*
        WORKDIR /usr/src/app WORKDIR /usr/src/app
        COPY ./package*.json ./ COPY ./package*.json ./
        RUN npm ci
        FROM base as builder
        RUN npm ci && npm cache clean --force
        COPY . ./ COPY . ./
        RUN npm run build RUN npm run build
        CMD ["npm", "run", "start"] FROM base
        ENV NODE_ENV production
        RUN npm ci && npm cache clean --force
        COPY --from=builder /usr/src/app/dist ./dist
        CMD [ "npm", "start" ]
        package-lock.json
        View file @ 593d03fb
        This source diff could not be displayed because it is too large. You can view the blob instead.
        package.json
        View file @ 593d03fb
        ...@@ -2,12 +2,11 @@ ...@@ -2,12 +2,11 @@
        "name": "tx3-bang-reader", "name": "tx3-bang-reader",
        "version": "1.0.0", "version": "1.0.0",
        "description": "Read TX3 bang and parse it", "description": "Read TX3 bang and parse it",
        "main": "build/src/run.js", "main": "dist/run.js",
        "scripts": { "scripts": {
        "build": "./node_modules/.bin/tsc", "build": "tsc",
        "pack": "mkdir dist ; ./node_modules/.bin/pkg --out-path dist .", "fetch": "node dist/src/run.js",
        "fetch": "node build/src/run.js", "start": "node dist/src/run.js cron"
        "start": "node build/src/run.js cron"
        }, },
        "repository": { "repository": {
        "type": "git", "type": "git",
        ...@@ -22,17 +21,10 @@ ...@@ -22,17 +21,10 @@
        "url": "https://github.com/purerosefallen/tx3-bang-reader/issues" "url": "https://github.com/purerosefallen/tx3-bang-reader/issues"
        }, },
        "homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme", "homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme",
        "bin": "build/src/run.js", "bin": "dist/src/run.js",
        "pkg": {
        "scripts": [
        "build/src/*.js"
        ],
        "assets": []
        },
        "dependencies": { "dependencies": {
        "@types/cron": "^1.7.2", "@types/cron": "^1.7.2",
        "@types/csv-parse": "^1.2.2", "@types/csv-parse": "^1.2.2",
        "@types/node": "^14.0.14",
        "@types/underscore": "^1.10.2", "@types/underscore": "^1.10.2",
        "@types/yaml": "^1.9.7", "@types/yaml": "^1.9.7",
        "axios": "^0.19.2", "axios": "^0.19.2",
        ...@@ -44,8 +36,21 @@ ...@@ -44,8 +36,21 @@
        "posthtml-parser": "^0.4.2", "posthtml-parser": "^0.4.2",
        "promise-mysql": "^4.1.3", "promise-mysql": "^4.1.3",
        "querystring": "^0.2.0", "querystring": "^0.2.0",
        "typescript": "^3.9.5",
        "underscore": "^1.10.2", "underscore": "^1.10.2",
        "yaml": "^1.10.0" "yaml": "^1.10.0"
        },
        "devDependencies": {
        "@types/jest": "^27.4.0",
        "@types/node": "^17.0.18",
        "@typescript-eslint/eslint-plugin": "^4.33.0",
        "@typescript-eslint/parser": "^4.33.0",
        "eslint": "^7.32.0",
        "eslint-config-prettier": "^8.3.0",
        "eslint-plugin-prettier": "^3.4.1",
        "jest": "^27.5.1",
        "prettier": "^2.5.1",
        "rimraf": "^3.0.2",
        "ts-jest": "^27.1.3",
        "typescript": "^4.5.5"
        } }
        } }
        src/fetcher.ts
        View file @ 593d03fb
        import axios from "axios";
        import _ from "underscore"; import _ from "underscore";
        import { User } from "./user"; import { User } from "./user";
        import {ProxyConfig, ProxyFetcher} from "./proxy"; import { ProxyConfig, ProxyFetcher } from "./proxy";
        import { PlayerRow, parsePlayerRows } from "./playerlist"; import { parsePlayerRows, PlayerRow } from "./playerlist";
        import qs from "querystring"; import qs from "querystring";
        import mysql from "promise-mysql"; import mysql from "promise-mysql";
        import moment from "moment"; import moment from "moment";
        export const servers = [ export const servers = [
        "东方明珠", "东方明珠",
        "紫禁之巅", "紫禁之巅",
        "一纪山海", "一纪山海",
        "剑心问道", "剑心问道",
        "天与秋光", "天与秋光",
        "梦回山海", "梦回山海",
        "绝代风华", "绝代风华",
        "鼎立山河", "鼎立山河",
        "天府之国", "天府之国",
        "天下无双", "天下无双",
        "情动大荒", "情动大荒",
        "琉璃月", "琉璃月",
        "齐鲁天下", "齐鲁天下",
        "剑舞香江", "剑舞香江",
        "白云山", "白云山",
        "瘦西湖", "瘦西湖",
        "逐鹿中原", "逐鹿中原",
        "黄鹤楼", "黄鹤楼",
        "武夷九曲", "武夷九曲",
        "上善若水", "上善若水",
        "君临天下", "君临天下",
        "气壮山河", "气壮山河",
        "飞龙在天", "飞龙在天",
        "烽火关东", "烽火关东",
        "盛世长安", "盛世长安",
        ];
        ]
        export interface Config { export interface Config {
        outDir: string; outDir: string;
        server: string[]; server: string[];
        useMySQL: boolean, useMySQL: boolean;
        MySQLConfig: mysql.PoolConfig, MySQLConfig: mysql.PoolConfig;
        proxy: ProxyConfig, proxy: ProxyConfig;
        cronString: string; cronString: string;
        } }
        export class Tx3Fetcher { export class Tx3Fetcher {
        config: Config; config: Config;
        proxyFetcher: ProxyFetcher; proxyFetcher: ProxyFetcher;
        db: mysql.Pool; db: mysql.Pool;
        curDate: string; curDate: string;
        constructor(config: Config) { constructor(config: Config) {
        this.config = config; this.config = config;
        this.proxyFetcher = new ProxyFetcher(config.proxy); this.proxyFetcher = new ProxyFetcher(config.proxy);
        } }
        async init() { async init() {
        this.curDate = moment().format("YYYY-MM-DD HH:mm:ss"); this.curDate = moment().format("YYYY-MM-DD HH:mm:ss");
        if(this.config.useMySQL) { if (this.config.useMySQL) {
        this.db = await mysql.createPool(this.config.MySQLConfig); this.db = await mysql.createPool(this.config.MySQLConfig);
        await this.db.query("CREATE TABLE IF NOT EXISTS `userdata` (\n" + await this.db.query(
        " `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" + "CREATE TABLE IF NOT EXISTS `userdata` (\n" +
        " `date` datetime NOT NULL DEFAULT current_timestamp(),\n" + " `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" +
        " `url` varchar(50) COLLATE utf8_unicode_ci NOT NULL,\n" + " `date` datetime NOT NULL DEFAULT current_timestamp(),\n" +
        " `rank` int(11) UNSIGNED NOT NULL,\n" + " `url` varchar(50) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `name` varchar(7) COLLATE utf8_unicode_ci NOT NULL,\n" + " `rank` int(11) UNSIGNED NOT NULL,\n" +
        " `category` varchar(5) COLLATE utf8_unicode_ci NOT NULL,\n" + " `name` varchar(7) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `serverArea` varchar(4) COLLATE utf8_unicode_ci NOT NULL,\n" + " `category` varchar(5) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `server` varchar(4) COLLATE utf8_unicode_ci NOT NULL,\n" + " `serverArea` varchar(4) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `level` tinyint(4) UNSIGNED NOT NULL,\n" + " `server` varchar(4) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `region` varchar(7) COLLATE utf8_unicode_ci NOT NULL,\n" + " `level` tinyint(4) UNSIGNED NOT NULL,\n" +
        " `score` int(11) UNSIGNED NOT NULL,\n" + " `region` varchar(7) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `equip` int(11) UNSIGNED NOT NULL,\n" + " `score` int(11) UNSIGNED NOT NULL,\n" +
        " `totalScore` int(11) UNSIGNED NOT NULL,\n" + " `equip` int(11) UNSIGNED NOT NULL,\n" +
        " PRIMARY KEY (`id`),\n" + " `totalScore` int(11) UNSIGNED NOT NULL,\n" +
        " INDEX (date),\n" + " PRIMARY KEY (`id`),\n" +
        " INDEX (name(7)),\n" + " INDEX (date),\n" +
        " INDEX (url(50)),\n" + " INDEX (name(7)),\n" +
        " INDEX (category(5)),\n" + " INDEX (url(50)),\n" +
        " INDEX (serverArea(4)),\n" + " INDEX (category(5)),\n" +
        " INDEX (server(4)),\n" + " INDEX (serverArea(4)),\n" +
        " INDEX (region(7)),\n" + " INDEX (server(4)),\n" +
        " INDEX (equip)\n" + " INDEX (region(7)),\n" +
        ") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci"); " INDEX (equip)\n" +
        console.log(`Removing existing records of ${this.curDate}.`); ") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci"
        //await this.db.query("delete from userdata where date = ?", this.curDate); );
        } console.log(`Removing existing records of ${this.curDate}.`);
        if(this.config.proxy.useProxy) { //await this.db.query("delete from userdata where date = ?", this.curDate);
        await this.proxyFetcher.initProxies(); }
        } if (this.config.proxy.useProxy) {
        } await this.proxyFetcher.initProxies();
        async fetchAll(): Promise<Map<string, User[]>> { }
        const res = new Map<string, User[]>(); }
        const userLists = await Promise.all(servers.map(server => this.fetchUsersFromServer(server))); async fetchAll(): Promise<Map<string, User[]>> {
        for (let i = 1; i < userLists.length; ++i){ const res = new Map<string, User[]>();
        res.set(servers[i], userLists[i]) const userLists = await Promise.all(
        } servers.map((server) => this.fetchUsersFromServer(server))
        return res; );
        } for (let i = 1; i < userLists.length; ++i) {
        async fetchUsersFromServer(server: string): Promise<User[]> { res.set(servers[i], userLists[i]);
        console.log(`Fetching user list from server ${server}.`); }
        const resPromises: Promise<User[]>[] = []; return res;
        for (let school = 1; school < 12; ++school) { }
        resPromises.push(this.fetchUsersFromSchoolAndServer(school, server)); async fetchUsersFromServer(server: string): Promise<User[]> {
        } console.log(`Fetching user list from server ${server}.`);
        const result = _.flatten(await Promise.all(resPromises)); const resPromises: Promise<User[]>[] = [];
        console.log(`Fetched user list with ${result.length} users from server ${server}.`); for (let school = 1; school < 12; ++school) {
        return result; resPromises.push(this.fetchUsersFromSchoolAndServer(school, server));
        } }
        async fetchUsersFromSchoolAndServer(school: number, server: string): Promise<User[]> { const result = _.flatten(await Promise.all(resPromises));
        console.log(`Fetching users from server ${server} with school ${school}.`); console.log(
        const res: User[][] = []; `Fetched user list with ${result.length} users from server ${server}.`
        for (let page = 1; page <= 25; ++page) { );
        const list = await this.fetchUsers(school, server, page); return result;
        if (!list.length) { }
        break; async fetchUsersFromSchoolAndServer(
        } school: number,
        res.push(list); server: string
        } ): Promise<User[]> {
        return _.flatten(res); console.log(`Fetching users from server ${server} with school ${school}.`);
        } const res: User[][] = [];
        async fetchUsers(school: number, server: string, page: number): Promise<User[]> { for (let page = 1; page <= 25; ++page) {
        const playerRows = await this.fetchList(school, server, page); const list = await this.fetchUsers(school, server, page);
        return await Promise.all(playerRows.map(row => this.fetchUser(row))); if (!list.length) {
        } break;
        async fetchListFromServer(server: string): Promise<PlayerRow[]> { }
        console.log(`Fetching user list from server ${server}.`); res.push(list);
        const resPromises: Promise<PlayerRow[]>[] = []; }
        for (let school = 1; school < 12; ++school) { return _.flatten(res);
        resPromises.push(this.fetchListFromSchoolAndServer(school, server)); }
        } async fetchUsers(
        const result = _.flatten(await Promise.all(resPromises)); school: number,
        console.log(`Fetched user list from server ${server}. ${result.length} users found.`); server: string,
        return result; page: number
        } ): Promise<User[]> {
        async fetchListFromSchoolAndServer(school: number, server: string): Promise<PlayerRow[]> { const playerRows = await this.fetchList(school, server, page);
        console.log(`Fetching users from server ${server} with school ${school}.`); return await Promise.all(playerRows.map((row) => this.fetchUser(row)));
        const res: PlayerRow[][] = []; }
        for (let page = 1; page <= 25; ++page) { async fetchListFromServer(server: string): Promise<PlayerRow[]> {
        const list = await this.fetchList(school, server, page); console.log(`Fetching user list from server ${server}.`);
        if (!list.length) { const resPromises: Promise<PlayerRow[]>[] = [];
        break; for (let school = 1; school < 12; ++school) {
        } resPromises.push(this.fetchListFromSchoolAndServer(school, server));
        res.push(list); }
        } const result = _.flatten(await Promise.all(resPromises));
        const ret = _.flatten(res); console.log(
        console.log(`Fetched users from server ${server} with school ${school}.`); `Fetched user list from server ${server}. ${result.length} users found.`
        return ret; );
        } return result;
        async checkAddRecord(row: PlayerRow) { }
        const latestRows = await this.db.query("select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by date desc limit 1", row.url); async fetchListFromSchoolAndServer(
        if (latestRows.length) { school: number,
        const lrow = latestRows[0]; server: string
        if (_.every(["name", "category", "serverArea", "server", "region"], field => lrow[field] === row[field])) { ): Promise<PlayerRow[]> {
        return; console.log(`Fetching users from server ${server} with school ${school}.`);
        } const res: PlayerRow[][] = [];
        } for (let page = 1; page <= 25; ++page) {
        const sql = "insert into userdata set ?"; const list = await this.fetchList(school, server, page);
        const valueObj = { if (!list.length) {
        date: this.curDate, break;
        ...row }
        }; res.push(list);
        console.log(`Player ${row.name} from ${row.server} has changes. Writing record to database: ${sql} ${JSON.stringify(valueObj)} ${JSON.stringify(await this.db.query(sql, valueObj))}`); }
        } const ret = _.flatten(res);
        async fetchList(school: number, server: string, page: number): Promise<PlayerRow[]> { console.log(`Fetched users from server ${server} with school ${school}.`);
        console.log(`Fetching user list from server ${server} with school ${school} page ${page}.`); return ret;
        try { }
        const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com/bang/ranks`, { async checkAddRecord(row: PlayerRow) {
        responseType: "document", const latestRows = await this.db.query(
        params: { "select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by id desc limit 1",
        order_key: "equ_xiuwei", row.url
        count: 20, );
        school, if (latestRows.length) {
        server, const lrow = latestRows[0];
        page if (
        }, _.every(
        paramsSerializer: (params) => { ["name", "category", "serverArea", "server", "region"],
        return qs.stringify(params); (field) => lrow[field] === row[field]
        } )
        }); ) {
        const playerRows = parsePlayerRows(content); return;
        console.log(`Fetched user list from server ${server} with school ${school} page ${page}. ${playerRows.length} users found.`); }
        if (this.db) { }
        //await Promise.all(playerRows.map(m => this.db.query("delete from userdata where url = ? and date = ?", [m.url, this.curDate]))); const sql = "insert into userdata set ?";
        await Promise.all(playerRows.map(m => this.checkAddRecord(m))); const valueObj = {
        } date: this.curDate,
        return playerRows; ...row,
        } catch(e) { };
        console.error(`Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`); console.log(
        return []; `Player ${row.name} from ${
        } row.server
        } } has changes. Writing record to database: ${sql} ${JSON.stringify(
        async fetchUser(playerRow: PlayerRow): Promise<User> { valueObj
        const id = playerRow.url.split("/").pop(); )} ${JSON.stringify(await this.db.query(sql, valueObj))}`
        try { );
        console.log(`Fetching user ${playerRow.name} from ${playerRow.server}.`); }
        const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com${playerRow.url}`, { async fetchList(
        responseType: "document" school: number,
        }); server: string,
        const user = new User(id, content, playerRow.region); page: number
        console.log(`Fetched user ${playerRow.name} from ${playerRow.server}.`); ): Promise<PlayerRow[]> {
        return user; console.log(
        } catch(e) { `Fetching user list from server ${server} with school ${school} page ${page}.`
        console.error(`Errored fetching role data from ${id}: ${e.toString()}`); );
        return null; try {
        } const content: string = await this.proxyFetcher.getWithProxy(
        } `http://bang.tx3.163.com/bang/ranks`,
        {
        responseType: "document",
        params: {
        order_key: "equ_xiuwei",
        count: 20,
        school,
        server,
        page,
        },
        paramsSerializer: (params) => {
        return qs.stringify(params);
        },
        }
        );
        const playerRows = parsePlayerRows(content);
        console.log(
        `Fetched user list from server ${server} with school ${school} page ${page}. ${playerRows.length} users found.`
        );
        if (this.db) {
        //await Promise.all(playerRows.map(m => this.db.query("delete from userdata where url = ? and date = ?", [m.url, this.curDate])));
        await Promise.all(playerRows.map((m) => this.checkAddRecord(m)));
        }
        return playerRows;
        } catch (e) {
        console.error(
        `Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`
        );
        return [];
        }
        }
        async fetchUser(playerRow: PlayerRow): Promise<User> {
        const id = playerRow.url.split("/").pop();
        try {
        console.log(`Fetching user ${playerRow.name} from ${playerRow.server}.`);
        const content: string = await this.proxyFetcher.getWithProxy(
        `http://bang.tx3.163.com${playerRow.url}`,
        {
        responseType: "document",
        }
        );
        const user = new User(id, content, playerRow.region);
        console.log(`Fetched user ${playerRow.name} from ${playerRow.server}.`);
        return user;
        } catch (e) {
        console.error(`Errored fetching role data from ${id}: ${e.toString()}`);
        return null;
        }
        }
        } }
        src/migrate.ts
        View file @ 593d03fb
        ...@@ -2,100 +2,119 @@ import mysql from "promise-mysql"; ...@@ -2,100 +2,119 @@ import mysql from "promise-mysql";
        import moment from "moment"; import moment from "moment";
        import fs from "fs"; import fs from "fs";
        import _csv_parse from "csv-parse"; import _csv_parse from "csv-parse";
        import util from 'util'; import util from "util";
        import { Config } from "./fetcher"; import { Config } from "./fetcher";
        import { PlayerRow, PlayerRowDated } from "./playerlist"; import { PlayerRow, PlayerRowDated } from "./playerlist";
        import _ from "underscore"; import _ from "underscore";
        import yaml from "yaml"; import yaml from "yaml";
        const parse_csv: (input: Buffer | string, options?: _csv_parse.Options) => Promise<any[]> = util.promisify(_csv_parse); const parse_csv: (
        input: Buffer | string,
        options?: _csv_parse.Options
        ) => Promise<any[]> = util.promisify(_csv_parse);
        let config: Config; let config: Config;
        let db: mysql.Pool; let db: mysql.Pool;
        const serverAreaCache = new Map<string, string>(); const serverAreaCache = new Map<string, string>();
        async function getServerAreaFromServer(server: string) { async function getServerAreaFromServer(server: string) {
        if (serverAreaCache.has(server)) { if (serverAreaCache.has(server)) {
        return serverAreaCache.get(server); return serverAreaCache.get(server);
        } else { } else {
        const [res] = await db.query("select serverArea from userdata where server = ? limit 1", server); const [res] = await db.query(
        if (!res) { "select serverArea from userdata where server = ? limit 1",
        return "none"; server
        } );
        const serverArea = res.serverArea; if (!res) {
        serverAreaCache.set(server, serverArea); return "none";
        return serverArea; }
        } const serverArea = res.serverArea;
        serverAreaCache.set(server, serverArea);
        return serverArea;
        }
        } }
        async function readSingleRecord(col: string[], offset: number, base: PlayerRow): Promise<PlayerRowDated> { async function readSingleRecord(
        let pointer = offset; col: string[],
        const newRecord: PlayerRowDated = { offset: number,
        date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"), base: PlayerRow
        ...(_.clone(base)) ): Promise<PlayerRowDated> {
        }; let pointer = offset;
        newRecord.name = col[pointer++]; const newRecord: PlayerRowDated = {
        newRecord.server = col[pointer++]; date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"),
        newRecord.serverArea = await getServerAreaFromServer(newRecord.server); ..._.clone(base),
        const _region = col[pointer++]; };
        newRecord.region = _region.length ? _region : "none"; newRecord.name = col[pointer++];
        newRecord.category = col[pointer++]; newRecord.server = col[pointer++];
        newRecord.equip = parseInt(col[pointer++]); newRecord.serverArea = await getServerAreaFromServer(newRecord.server);
        newRecord.totalScore = newRecord.equip + newRecord.score; const _region = col[pointer++];
        return newRecord; newRecord.region = _region.length ? _region : "none";
        newRecord.category = col[pointer++];
        newRecord.equip = parseInt(col[pointer++]);
        newRecord.totalScore = newRecord.equip + newRecord.score;
        return newRecord;
        } }
        let leftCount: number; let leftCount: number;
        async function readColumn(col: string[]): Promise<void> { async function readColumn(col: string[]): Promise<void> {
        const recordCount = parseInt(col[0]); const recordCount = parseInt(col[0]);
        const url = `/bang/role/${col[3]}`; const url = `/bang/role/${col[3]}`;
        console.error(`Reading column ${url}.`); console.error(`Reading column ${url}.`);
        let [base] = await db.query("select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by date asc limit 1", url) as PlayerRow[]; let [base] = (await db.query(
        if (!base) { "select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by id asc limit 1",
        console.error(`Base record of ${url} not found. Using default values.`); url
        base = { )) as PlayerRow[];
        url, if (!base) {
        rank: 500, console.error(`Base record of ${url} not found. Using default values.`);
        name: null, base = {
        server: null, url,
        serverArea: null, rank: 500,
        category: null, name: null,
        level: 80, server: null,
        region: "none", serverArea: null,
        score: 0, category: null,
        equip: 0, level: 80,
        totalScore: 0 region: "none",
        } score: 0,
        } equip: 0,
        for (let i = 0; i < recordCount; ++i) { totalScore: 0,
        const offset = 4 + (i * 6); };
        if (!col[offset].length) { }
        continue; for (let i = 0; i < recordCount; ++i) {
        } const offset = 4 + i * 6;
        const record = await readSingleRecord(col, offset, base); if (!col[offset].length) {
        const sql = "insert into userdata set ?"; continue;
        console.log(sql, JSON.stringify(record), JSON.stringify(await db.query(sql, record))); }
        } const record = await readSingleRecord(col, offset, base);
        console.error(`Read column ${url}. ${--leftCount} columns left.`); const sql = "insert into userdata set ?";
        console.log(
        sql,
        JSON.stringify(record),
        JSON.stringify(await db.query(sql, record))
        );
        }
        console.error(`Read column ${url}. ${--leftCount} columns left.`);
        } }
        async function loadCsv(path: string): Promise<string[][]> { async function loadCsv(path: string): Promise<string[][]> {
        const data = await fs.promises.readFile(path); const data = await fs.promises.readFile(path);
        return await parse_csv(data, { return await parse_csv(data, {
        trim: true trim: true,
        }); });
        } }
        async function main() { async function main() {
        console.error("Started."); console.error("Started.");
        const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8")); const config: Config = yaml.parse(
        db = await mysql.createPool(config.MySQLConfig); await fs.promises.readFile("./config.yaml", "utf8")
        const data = await loadCsv(process.argv[2]); );
        leftCount = data.length; db = await mysql.createPool(config.MySQLConfig);
        //await Promise.all(data.map(col => readColumn(col))); const data = await loadCsv(process.argv[2]);
        for (let col of data) { leftCount = data.length;
        await readColumn(col); //await Promise.all(data.map(col => readColumn(col)));
        } for (const col of data) {
        console.error("Finished."); await readColumn(col);
        process.exit(); }
        console.error("Finished.");
        process.exit();
        } }
        main(); main();
        src/playerlist.ts
        View file @ 593d03fb
        import HTML from "posthtml-parser"; import HTML from "posthtml-parser";
        import _, { first } from "underscore"; import {
        import {getDepthOfTree, getNumber, findNodeIndex, findNodeIndexByAttribute, findNodeIndexByContent, findNodeIndexByTag, findAllNodeIndex, getContinuousData, getContinuousNodes, getString} from "./utility"; findAllNodeIndex,
        findNodeIndexByTag,
        getContinuousNodes,
        getDepthOfTree,
        getNumber,
        getString,
        } from "./utility";
        export interface PlayerRow { export interface PlayerRow {
        url: string; url: string;
        rank: number; rank: number;
        name: string; name: string;
        category: string; category: string;
        serverArea: string; serverArea: string;
        server: string; server: string;
        level: number; level: number;
        region: string; region: string;
        score: number; score: number;
        equip: number; equip: number;
        totalScore: number; totalScore: number;
        } }
        export interface PlayerRowDated extends PlayerRow { export interface PlayerRowDated extends PlayerRow {
        date: string; date: string;
        } }
        export interface PlayerRowFull extends PlayerRowDated { export interface PlayerRowFull extends PlayerRowDated {
        id: number; id: number;
        } }
        function getPlayerRowFromTree(tree: HTML.Tree): PlayerRow { function getPlayerRowFromTree(tree: HTML.Tree): PlayerRow {
        const nodes = getContinuousNodes(tree, [1], 0, 2, 10); const nodes = getContinuousNodes(tree, [1], 0, 2, 10);
        return { return {
        url: (nodes[1] as HTML.NodeTag).attrs.href as string, url: (nodes[1] as HTML.NodeTag).attrs.href as string,
        rank: getNumber(nodes[0]), rank: getNumber(nodes[0]),
        name: getString(nodes[1], 7), name: getString(nodes[1], 7),
        serverArea: getString(nodes[2]), serverArea: getString(nodes[2]),
        server: getString(nodes[3]), server: getString(nodes[3]),
        level: getNumber(nodes[4]), level: getNumber(nodes[4]),
        category: getString(nodes[5]), category: getString(nodes[5]),
        region: getString(nodes[6], 7) || "none", region: getString(nodes[6], 7) || "none",
        score: getNumber(nodes[7]), score: getNumber(nodes[7]),
        equip: getNumber(nodes[8]), equip: getNumber(nodes[8]),
        totalScore: getNumber(nodes[9]) totalScore: getNumber(nodes[9]),
        } };
        } }
        export function parsePlayerRows(content: string) { export function parsePlayerRows(content: string) {
        const parsedContent = HTML(content); const parsedContent = HTML(content);
        const tablePos = findNodeIndexByTag(parsedContent, "table", []); const tablePos = findNodeIndexByTag(parsedContent, "table", []);
        const tableTree = getDepthOfTree(parsedContent, tablePos); const tableTree = getDepthOfTree(parsedContent, tablePos);
        const playerPoses = findAllNodeIndex(tableTree, (node) => { const playerPoses = findAllNodeIndex(
        return typeof (node) !== "string" && node.tag === "tr" && node.attrs.class !== "trTop2"; tableTree,
        }, []); (node) => {
        return playerPoses.map(pos => { return (
        const tree = getDepthOfTree(tableTree, pos); typeof node !== "string" &&
        return getPlayerRowFromTree(tree); node.tag === "tr" &&
        }); node.attrs.class !== "trTop2"
        );
        },
        []
        );
        return playerPoses.map((pos) => {
        const tree = getDepthOfTree(tableTree, pos);
        return getPlayerRowFromTree(tree);
        });
        } }
        src/proxy.ts
        View file @ 593d03fb
        ...@@ -9,59 +9,59 @@ import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios"; ...@@ -9,59 +9,59 @@ import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios";
        //} //}
        export interface ProxyConfig { export interface ProxyConfig {
        useProxy: boolean, useProxy: boolean;
        proxySource: string[], proxySource: string[];
        timeout: number timeout: number;
        } }
        const agentList = [ const agentList = [
        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0', "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0",
        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)', "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11', "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11', "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)' "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
        ] ];
        async function testProxy(proxy: AxiosProxyConfig) { async function testProxy(proxy: AxiosProxyConfig) {
        await axios.get("http://mirrors.aliyun.com/debian/pool", { await axios.get("http://mirrors.aliyun.com/debian/pool", {
        proxy, proxy,
        headers: { headers: {
        "User-Agent": agentList[4] "User-Agent": agentList[4],
        }, },
        timeout: this.config.timeout, timeout: this.config.timeout,
        }); });
        return proxy; return proxy;
        } }
        async function checkProxy(proxy: AxiosProxyConfig) { async function checkProxy(proxy: AxiosProxyConfig) {
        let isProxyUsable = false; let isProxyUsable = false;
        try { try {
        await testProxy(proxy); await testProxy(proxy);
        //console.log(`Proxy ${proxy.host} is ok.`); //console.log(`Proxy ${proxy.host} is ok.`);
        isProxyUsable = true; isProxyUsable = true;
        } catch (e) { } catch (e) {
        //console.error(`Proxy ${proxy.host} is broken: ${e.toString()}`); //console.error(`Proxy ${proxy.host} is broken: ${e.toString()}`);
        } }
        return isProxyUsable; return isProxyUsable;
        } }
        async function filterProxies(proxies: AxiosProxyConfig[]) { async function filterProxies(proxies: AxiosProxyConfig[]) {
        const proxiesUsableList = await Promise.all(proxies.map(checkProxy)); const proxiesUsableList = await Promise.all(proxies.map(checkProxy));
        return proxies.filter((proxy, index) => { return proxies.filter((proxy, index) => {
        return proxiesUsableList[index]; return proxiesUsableList[index];
        }); });
        } }
        //async function findFirstUsableProxy(proxies: AxiosProxyConfig[]) { //async function findFirstUsableProxy(proxies: AxiosProxyConfig[]) {
        ...@@ -69,70 +69,86 @@ async function filterProxies(proxies: AxiosProxyConfig[]) { ...@@ -69,70 +69,86 @@ async function filterProxies(proxies: AxiosProxyConfig[]) {
        //} //}
        export class ProxyFetcher { export class ProxyFetcher {
        proxies: AxiosProxyConfig[]; proxies: AxiosProxyConfig[];
        counter: number; counter: number;
        config: ProxyConfig; config: ProxyConfig;
        constructor(config: ProxyConfig) { constructor(config: ProxyConfig) {
        this.config = config; this.config = config;
        this.proxies = []; this.proxies = [];
        this.counter = 0; this.counter = 0;
        } }
        async initProxiesFrom(url: string) { async initProxiesFrom(url: string) {
        if (!this.config.useProxy) { if (!this.config.useProxy) {
        return; return;
        } }
        console.log(`Fetching proxies from ${url}.`) console.log(`Fetching proxies from ${url}.`);
        while (true) { while (true) {
        try { try {
        const proxyPage: string = (await axios.get(url, { const proxyPage: string = (
        responseType: "document", await axios.get(url, {
        })).data; responseType: "document",
        const proxies: AxiosProxyConfig[] = proxyPage.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g).map(proxyString => { })
        const [host, _port] = proxyString.split(":"); ).data;
        const port = parseInt(_port); const proxies: AxiosProxyConfig[] = proxyPage
        const proxy = { host, port }; .match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g)
        return proxy; .map((proxyString) => {
        }); const [host, _port] = proxyString.split(":");
        //const usableProxies = await filterProxies(proxies); const port = parseInt(_port);
        for (let proxy of proxies) { const proxy = { host, port };
        this.proxies.push(proxy); return proxy;
        } });
        console.error(`Got ${proxies.length} proxies from ${url}.`); //const usableProxies = await filterProxies(proxies);
        return; for (const proxy of proxies) {
        } catch (e) { this.proxies.push(proxy);
        console.error(`Failed fetching proxy list from ${url}: ${e.toString()}`) }
        } console.error(`Got ${proxies.length} proxies from ${url}.`);
        } return;
        } } catch (e) {
        async initProxies() { console.error(
        await Promise.all(this.config.proxySource.map((m) => { `Failed fetching proxy list from ${url}: ${e.toString()}`
        return this.initProxiesFrom(m); );
        })); }
        } }
        async getWithProxy(url: string, options: AxiosRequestConfig) { }
        while (true) { async initProxies() {
        if (this.config.useProxy && !this.proxies.length) { await Promise.all(
        await this.initProxies(); this.config.proxySource.map((m) => {
        } return this.initProxiesFrom(m);
        const proxyIndex = !this.config.useProxy ? null : (++this.counter) % this.proxies.length; })
        //const proxyIndex = 0; );
        const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex]; }
        try { async getWithProxy(url: string, options: AxiosRequestConfig) {
        const data = (await axios.get(url, { while (true) {
        proxy, if (this.config.useProxy && !this.proxies.length) {
        headers: { await this.initProxies();
        "User-Agent": agentList[this.counter % agentList.length] }
        }, const proxyIndex = !this.config.useProxy
        timeout: this.config.timeout, ? null
        ...options : ++this.counter % this.proxies.length;
        })).data; //const proxyIndex = 0;
        return data; const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex];
        } catch (e) { try {
        if (this.config.useProxy) { const data = (
        this.proxies.splice(proxyIndex, 1); await axios.get(url, {
        } proxy,
        console.error(`Failed fetching data from ${url}: ${e.toString()} ${this.proxies.length} proxies left.`) headers: {
        } "User-Agent": agentList[this.counter % agentList.length],
        } },
        } timeout: this.config.timeout,
        ...options,
        })
        ).data;
        return data;
        } catch (e) {
        if (this.config.useProxy) {
        this.proxies.splice(proxyIndex, 1);
        }
        console.error(
        `Failed fetching data from ${url}: ${e.toString()} ${
        this.proxies.length
        } proxies left.`
        );
        }
        }
        }
        } }
        src/remove-dup.ts
        View file @ 593d03fb
        ...@@ -6,31 +6,40 @@ import _ from "underscore"; ...@@ -6,31 +6,40 @@ import _ from "underscore";
        import yaml from "yaml"; import yaml from "yaml";
        function checkSameRow(row: PlayerRowFull, lrow: PlayerRowFull) { function checkSameRow(row: PlayerRowFull, lrow: PlayerRowFull) {
        return _.every(["name", "category", "serverArea", "server", "region"], field => lrow[field] === row[field]); return _.every(
        ["name", "category", "serverArea", "server", "region"],
        (field) => lrow[field] === row[field]
        );
        } }
        async function main() { async function main() {
        console.error("Started."); console.error("Started.");
        const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8")); const config: Config = yaml.parse(
        const db = await mysql.createPool(config.MySQLConfig); await fs.promises.readFile("./config.yaml", "utf8")
        const urlDataCache = new Map<string, PlayerRowFull>(); );
        const deleteList: number[] = []; const db = await mysql.createPool(config.MySQLConfig);
        const datas: PlayerRowFull[] = await db.query(`select * from userdata order by date asc`); const urlDataCache = new Map<string, PlayerRowFull>();
        for (let row of datas) { const deleteList: number[] = [];
        if (urlDataCache.has(row.url)) { const datas: PlayerRowFull[] = await db.query(
        const oldRow = urlDataCache.get(row.url); `select * from userdata order by id asc`
        if (checkSameRow(row, oldRow)) { );
        deleteList.push(row.id); for (const row of datas) {
        } if (urlDataCache.has(row.url)) {
        } const oldRow = urlDataCache.get(row.url);
        urlDataCache.set(row.url, row); if (checkSameRow(row, oldRow)) {
        } deleteList.push(row.id);
        console.error(`Deletes: ${deleteList.length}`); }
        for (let id of deleteList) { }
        const sql = `delete from userdata where id = ?`; urlDataCache.set(row.url, row);
        console.error(`Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`); }
        } console.error(`Deletes: ${deleteList.length}`);
        console.error("Finished."); for (const id of deleteList) {
        process.exit(); const sql = `delete from userdata where id = ?`;
        console.error(
        `Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`
        );
        }
        console.error("Finished.");
        process.exit();
        } }
        main(); main();
        src/run.ts
        View file @ 593d03fb
        import {Tx3Fetcher, servers, Config} from "./fetcher"; import { Config, servers, Tx3Fetcher } from "./fetcher";
        import fs from "fs"; import fs from "fs";
        import _ from "underscore";
        import yaml from "yaml"; import yaml from "yaml";
        import { CronJob } from "cron"; import { CronJob } from "cron";
        let config: Config; let config: Config;
        async function loadConfig() { async function loadConfig() {
        config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8")); config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
        } }
        async function runServer(fetcher: Tx3Fetcher, server: string) { async function runServer(fetcher: Tx3Fetcher, server: string) {
        const users = await fetcher.fetchListFromServer(server); const users = await fetcher.fetchListFromServer(server);
        await fs.promises.writeFile(`./output/servers/${server}.json`, JSON.stringify({ await fs.promises.writeFile(
        date: fetcher.curDate, `./output/servers/${server}.json`,
        data: users JSON.stringify(
        }, null, 2)); {
        return users; date: fetcher.curDate,
        data: users,
        },
        null,
        2
        )
        );
        return users;
        } }
        async function run() { async function run() {
        console.log(`Fetch started.`); console.log(`Fetch started.`);
        try { try {
        await fs.promises.access("./output/servers"); await fs.promises.access("./output/servers");
        } catch (e) { } catch (e) {
        await fs.promises.mkdir("./output/servers", { await fs.promises.mkdir("./output/servers", {
        recursive: true recursive: true,
        }); });
        } }
        const fetcher = new Tx3Fetcher(config); const fetcher = new Tx3Fetcher(config);
        await fetcher.init(); await fetcher.init();
        if (config.server) { if (config.server) {
        await Promise.all(config.server.map(server => { await Promise.all(
        return runServer(fetcher, server) config.server.map((server) => {
        })); return runServer(fetcher, server);
        } else { })
        const userListWithServer = await Promise.all(servers.map(server => { );
        return runServer(fetcher, server) } else {
        })); const userListWithServer = await Promise.all(
        const allServersList: any = {}; servers.map((server) => {
        for (let i = 0; i < servers.length;++i) { return runServer(fetcher, server);
        allServersList[servers[i]] = userListWithServer[i]; })
        } );
        await fs.promises.writeFile(`./output/all.json`, JSON.stringify({ const allServersList: any = {};
        date: fetcher.curDate, for (let i = 0; i < servers.length; ++i) {
        data: allServersList allServersList[servers[i]] = userListWithServer[i];
        }, null, 2)); }
        } await fs.promises.writeFile(
        console.log("Finished."); `./output/all.json`,
        JSON.stringify(
        {
        date: fetcher.curDate,
        data: allServersList,
        },
        null,
        2
        )
        );
        }
        console.log("Finished.");
        } }
        async function main() { async function main() {
        await loadConfig(); await loadConfig();
        if (process.argv[2] === "cron") { if (process.argv[2] === "cron") {
        const job = new CronJob(config.cronString, run, null, true, "Asia/Shanghai", null, true); const job = new CronJob(
        job.start(); config.cronString,
        } else { run,
        await run(); null,
        process.exit(); true,
        } "Asia/Shanghai",
        null,
        true
        );
        job.start();
        } else {
        await run();
        process.exit();
        }
        } }
        main(); main();
        src/user.ts
        View file @ 593d03fb
        import HTML from "posthtml-parser"; import HTML from "posthtml-parser";
        import _, { first } from "underscore"; import _ from "underscore";
        import {getDepthOfTree, getNumber, findNodeIndex, findNodeIndexByAttribute, findNodeIndexByContent, findNodeIndexByTag, getContinuousNumber} from "./utility"; import {
        findNodeIndexByAttribute,
        findNodeIndexByContent,
        getContinuousNumber,
        getDepthOfTree,
        getNumber,
        } from "./utility";
        export interface AttackAttribute { export interface AttackAttribute {
        '攻力': number, //大攻取前2字节int16，小攻取后2字节int16，法力也是这样 攻力: number; //大攻取前2字节int16，小攻取后2字节int16，法力也是这样
        '命中': number, 命中: number;
        '法力': number, 法力: number;
        '重击': number, 重击: number;
        '会心一击': number, 会心一击: number;
        '附加伤害': number 附加伤害: number;
        } }
        export interface DefenseAttribute { export interface DefenseAttribute {
        '防御': number, 防御: number;
        '回避': number, 回避: number;
        '法防': number, 法防: number;
        '神明': number, 神明: number;
        '化解': number, 化解: number;
        '知彼': number 知彼: number;
        } }
        export interface SpecialAttribute { export interface SpecialAttribute {
        '身法': number, 身法: number;
        '坚韧': number, 坚韧: number;
        '定力': number, 定力: number;
        '诛心': number, 诛心: number;
        '御心': number, 御心: number;
        '万钧': number, 万钧: number;
        '铁壁': number 铁壁: number;
        } }
        export interface AdvancedAttribute { '追电': number, '骤雨': number, '疾语': number, '明思': number, '扰心': number, '人祸': number } export interface AdvancedAttribute {
        追电: number;
        骤雨: number;
        疾语: number;
        明思: number;
        扰心: number;
        人祸: number;
        }
        export class User { export class User {
        id: string; id: string;
        content: HTML.Tree; content: HTML.Tree;
        name: string; name: string;
        region: string; region: string;
        category: string; category: string;
        serverArea: string; serverArea: string;
        server: string; server: string;
        level: number; level: number;
        equipValue: number; equipValue: number;
        equipRank: number; equipRank: number;
        equipLocalRank: number; equipLocalRank: number;
        equipCategoryRank: number; equipCategoryRank: number;
        scoreValue: number; scoreValue: number;
        scoreRank: number scoreRank: number;
        scoreLocalRank: number; scoreLocalRank: number;
        scoreCategoryRank: number; scoreCategoryRank: number;
        sqStage: number; // 天魂：2，地魂：1，没有神启：null sqStage: number; // 天魂：2，地魂：1，没有神启：null
        sqLevel: number; // 前4位是几境界，后四位是几天 sqLevel: number; // 前4位是几境界，后四位是几天
        qhLevel: number; qhLevel: number;
        tlPoints: number; tlPoints: number;
        hp: number; hp: number;
        mp: number; mp: number;
        li: number; li: number;
        ti: number; ti: number;
        min: number; min: number;
        ji: number; ji: number;
        hun: number; hun: number;
        nian: number; nian: number;
        attackAttributes: AttackAttribute; attackAttributes: AttackAttribute;
        defenseAttributes: DefenseAttribute; defenseAttributes: DefenseAttribute;
        specialAttributes: SpecialAttribute; specialAttributes: SpecialAttribute;
        advancedAttributes: AdvancedAttribute; advancedAttributes: AdvancedAttribute;
        yhz: string[]; yhz: string[];
        private parseMetadata() { private parseMetadata() {
        let namePos = findNodeIndexByAttribute(this.content, "class", "sTitle", []); const namePos = findNodeIndexByAttribute(
        this.name = getDepthOfTree(this.content, namePos.concat([0]))[0] as string; this.content,
        namePos[namePos.length - 1] += 2; "class",
        this.category = getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string; "sTitle",
        namePos[namePos.length - 1] += 2; []
        [this.serverArea, this.server] = (getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string).split("&nbsp;"); );
        this.name = getDepthOfTree(this.content, namePos.concat([0]))[0] as string;
        namePos[namePos.length - 1] += 2;
        this.category = getDepthOfTree(
        this.content,
        namePos.concat([0, 0])
        )[0] as string;
        namePos[namePos.length - 1] += 2;
        [this.serverArea, this.server] = (
        getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string
        ).split("&nbsp;");
        let levelPos = findNodeIndexByContent(this.content, "等级", []); const levelPos = findNodeIndexByContent(this.content, "等级", []);
        levelPos.pop(); levelPos.pop();
        levelPos[levelPos.length - 1]++; levelPos[levelPos.length - 1]++;
        this.level = getNumber(getDepthOfTree(this.content, levelPos)[0]); this.level = getNumber(getDepthOfTree(this.content, levelPos)[0]);
        } }
        private parseEquipmentData() { private parseEquipmentData() {
        let ValuePos = findNodeIndexByContent(this.content, "装备评价:", []); let ValuePos = findNodeIndexByContent(this.content, "装备评价:", []);
        ValuePos.pop(); ValuePos.pop();
        ValuePos[ValuePos.length - 1]++; ValuePos[ValuePos.length - 1]++;
        let datas = getContinuousNumber(this.content, ValuePos, 1, 2, 4); let datas = getContinuousNumber(this.content, ValuePos, 1, 2, 4);
        this.equipValue = datas[0]; this.equipValue = datas[0];
        this.equipRank = datas[1]; this.equipRank = datas[1];
        this.equipLocalRank = datas[2]; this.equipLocalRank = datas[2];
        this.equipCategoryRank = datas[3]; this.equipCategoryRank = datas[3];
        ValuePos = findNodeIndexByContent(this.content, "人物修为:", []); ValuePos = findNodeIndexByContent(this.content, "人物修为:", []);
        ValuePos.pop(); ValuePos.pop();
        ValuePos[ValuePos.length - 1]++; ValuePos[ValuePos.length - 1]++;
        datas = getContinuousNumber(this.content, ValuePos, 1, 2, 8); datas = getContinuousNumber(this.content, ValuePos, 1, 2, 8);
        this.scoreValue = datas[0]; this.scoreValue = datas[0];
        this.scoreRank = datas[1]; this.scoreRank = datas[1];
        this.scoreLocalRank = datas[2]; this.scoreLocalRank = datas[2];
        this.scoreCategoryRank = datas[3]; this.scoreCategoryRank = datas[3];
        this.sqStage = datas[4]; this.sqStage = datas[4];
        this.sqLevel = datas[5]; this.sqLevel = datas[5];
        this.qhLevel = datas[6]; this.qhLevel = datas[6];
        this.tlPoints = datas[7]; this.tlPoints = datas[7];
        } }
        private parseAttributeTable(_pos: number[]): any { private parseAttributeTable(_pos: number[]): any {
        const ret = {}; const ret = {};
        const pos = _.clone(_pos); const pos = _.clone(_pos);
        const tree = getDepthOfTree(this.content, pos); const tree = getDepthOfTree(this.content, pos);
        for (let i = 3; i < tree.length; i += 2) { for (let i = 3; i < tree.length; i += 2) {
        const [_keyNode, valueNode] = getDepthOfTree(tree, [i]); const [_keyNode, valueNode] = getDepthOfTree(tree, [i]);
        const keyNode = (_keyNode as HTML.NodeTag); const keyNode = _keyNode as HTML.NodeTag;
        const key = keyNode.content[0] as string; const key = keyNode.content[0] as string;
        const value = getNumber(valueNode); const value = getNumber(valueNode);
        ret[key] = value; ret[key] = value;
        } }
        return ret; return ret;
        } }
        private parseBasicAttributes() { private parseBasicAttributes() {
        let ValuePos = findNodeIndexByContent(this.content, "命", []); const ValuePos = findNodeIndexByContent(this.content, "命", []);
        ValuePos.pop(); ValuePos.pop();
        ValuePos[ValuePos.length - 1] += 2; ValuePos[ValuePos.length - 1] += 2;
        const datas = getContinuousNumber(this.content, ValuePos, 0, 4, 8); const datas = getContinuousNumber(this.content, ValuePos, 0, 4, 8);
        this.hp = datas[0]; this.hp = datas[0];
        this.mp = datas[1]; this.mp = datas[1];
        this.li = datas[2]; this.li = datas[2];
        this.ti = datas[3]; this.ti = datas[3];
        this.min = datas[4]; this.min = datas[4];
        this.ji = datas[5]; this.ji = datas[5];
        this.hun = datas[6]; this.hun = datas[6];
        this.nian = datas[7]; this.nian = datas[7];
        ValuePos.pop(); ValuePos.pop();
        ValuePos[ValuePos.length - 1] += 2; ValuePos[ValuePos.length - 1] += 2;
        this.attackAttributes = this.parseAttributeTable(ValuePos) as AttackAttribute; this.attackAttributes = this.parseAttributeTable(
        ValuePos[ValuePos.length - 1] += 2; ValuePos
        this.defenseAttributes = this.parseAttributeTable(ValuePos) as DefenseAttribute; ) as AttackAttribute;
        ValuePos[ValuePos.length - 1] += 2; ValuePos[ValuePos.length - 1] += 2;
        ValuePos.push(1); this.defenseAttributes = this.parseAttributeTable(
        this.specialAttributes = this.parseAttributeTable(ValuePos) as SpecialAttribute; ValuePos
        ValuePos[ValuePos.length - 1] += 2; ) as DefenseAttribute;
        this.advancedAttributes = this.parseAttributeTable(ValuePos) as AdvancedAttribute; ValuePos[ValuePos.length - 1] += 2;
        } ValuePos.push(1);
        private parseYHZ() { this.specialAttributes = this.parseAttributeTable(
        let ValuePos = findNodeIndexByAttribute(this.content, "id", "tableYHZ", []); ValuePos
        ValuePos.push(1); ) as SpecialAttribute;
        const tree = getDepthOfTree(this.content, ValuePos); ValuePos[ValuePos.length - 1] += 2;
        this.yhz = []; this.advancedAttributes = this.parseAttributeTable(
        for (let i = 1; i < tree.length; i += 2){ ValuePos
        const node = (tree[i] as HTML.NodeTag); ) as AdvancedAttribute;
        this.yhz.push(node.content[0] as string); }
        } private parseYHZ() {
        } const ValuePos = findNodeIndexByAttribute(
        private parse() { this.content,
        this.parseMetadata(); "id",
        this.parseEquipmentData(); "tableYHZ",
        this.parseBasicAttributes(); []
        this.parseYHZ(); );
        } ValuePos.push(1);
        constructor(id: string, content: string, region: string) { const tree = getDepthOfTree(this.content, ValuePos);
        this.id = id; this.yhz = [];
        this.region = region; for (let i = 1; i < tree.length; i += 2) {
        this.content = HTML(content); const node = tree[i] as HTML.NodeTag;
        this.parse(); this.yhz.push(node.content[0] as string);
        this.content = null; }
        } }
        private parse() {
        this.parseMetadata();
        this.parseEquipmentData();
        this.parseBasicAttributes();
        this.parseYHZ();
        }
        constructor(id: string, content: string, region: string) {
        this.id = id;
        this.region = region;
        this.content = HTML(content);
        this.parse();
        this.content = null;
        }
        } }
        src/utility.ts
        View file @ 593d03fb
        import HTML from "posthtml-parser"; import HTML from "posthtml-parser";
        import _ from "underscore"; import _ from "underscore";
        export function getDepthOfTree(tree: HTML.Tree, indexList: number[]): HTML.Tree {
        if (indexList.length) { export function getDepthOfTree(
        const _indexList = _.clone(indexList); tree: HTML.Tree,
        const index = _indexList.splice(0, 1)[0]; indexList: number[]
        const node = tree[index]; ): HTML.Tree {
        if (typeof (node) === "string" || !node.content) { if (indexList.length) {
        return [node]; const _indexList = _.clone(indexList);
        } const index = _indexList.splice(0, 1)[0];
        return getDepthOfTree(node.content, _indexList); const node = tree[index];
        } else { if (typeof node === "string" || !node.content) {
        return tree; return [node];
        } }
        return getDepthOfTree(node.content, _indexList);
        } else {
        return tree;
        }
        } }
        export function findNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node) => boolean, offset: number[]): number[] { export function findNodeIndex(
        const queue = [offset]; baseTree: HTML.Tree,
        while (queue.length) { condition: (node: HTML.Node) => boolean,
        const indexList = queue.splice(0, 1)[0]; offset: number[]
        const tree = getDepthOfTree(baseTree, indexList); ): number[] {
        for (let i = 0; i < tree.length; ++i) { const queue = [offset];
        const node = tree[i]; while (queue.length) {
        const newList = indexList.concat([i]); const indexList = queue.splice(0, 1)[0];
        if (condition(node)) { const tree = getDepthOfTree(baseTree, indexList);
        return newList; for (let i = 0; i < tree.length; ++i) {
        } else if (typeof (node) !== "string") { const node = tree[i];
        queue.push(newList); const newList = indexList.concat([i]);
        } if (condition(node)) {
        } return newList;
        } } else if (typeof node !== "string") {
        return null; queue.push(newList);
        }
        }
        }
        return null;
        } }
        export function findAllNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node) => boolean, offset: number[]): number[][] { export function findAllNodeIndex(
        const queue = [offset]; baseTree: HTML.Tree,
        const res: number[][] = []; condition: (node: HTML.Node) => boolean,
        while (queue.length) { offset: number[]
        const indexList = queue.splice(0, 1)[0]; ): number[][] {
        const tree = getDepthOfTree(baseTree, indexList); const queue = [offset];
        for (let i = 0; i < tree.length; ++i) { const res: number[][] = [];
        const node = tree[i]; while (queue.length) {
        const newList = indexList.concat([i]); const indexList = queue.splice(0, 1)[0];
        if (condition(node)) { const tree = getDepthOfTree(baseTree, indexList);
        //console.log(newList); for (let i = 0; i < tree.length; ++i) {
        res.push(newList); const node = tree[i];
        } else if (typeof (node) !== "string") { const newList = indexList.concat([i]);
        queue.push(newList); if (condition(node)) {
        } //console.log(newList);
        } res.push(newList);
        } } else if (typeof node !== "string") {
        return res; queue.push(newList);
        }
        }
        }
        return res;
        } }
        export function findNodeIndexByContent(baseTree: HTML.Tree, label: string, offset: number[]): number[] { export function findNodeIndexByContent(
        return findNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        return node === label; label: string,
        }, offset); offset: number[]
        ): number[] {
        return findNodeIndex(
        baseTree,
        (node) => {
        return node === label;
        },
        offset
        );
        } }
        export function findNodeIndexByAttribute(baseTree: HTML.Tree, key: string, value: string, offset: number[]): number[] { export function findNodeIndexByAttribute(
        return findNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        return typeof (node) !== "string" && node.attrs && node.attrs[key] === value; key: string,
        }, offset); value: string,
        offset: number[]
        ): number[] {
        return findNodeIndex(
        baseTree,
        (node) => {
        return (
        typeof node !== "string" && node.attrs && node.attrs[key] === value
        );
        },
        offset
        );
        } }
        export function findNodeIndexByTag(baseTree: HTML.Tree, tag: string, offset: number[]): number[] { export function findNodeIndexByTag(
        return findNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        return typeof (node) !== "string" && node.tag === tag; tag: string,
        }, offset); offset: number[]
        ): number[] {
        return findNodeIndex(
        baseTree,
        (node) => {
        return typeof node !== "string" && node.tag === tag;
        },
        offset
        );
        } }
        export function findAllNodeIndexByContent(baseTree: HTML.Tree, label: string, offset: number[]): number[][] { export function findAllNodeIndexByContent(
        return findAllNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        return node === label; label: string,
        }, offset); offset: number[]
        ): number[][] {
        return findAllNodeIndex(
        baseTree,
        (node) => {
        return node === label;
        },
        offset
        );
        } }
        export function findAllNodeIndexByAttribute(baseTree: HTML.Tree, key: string, value: string, offset: number[]): number[][] { export function findAllNodeIndexByAttribute(
        return findAllNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        return typeof (node) !== "string" && node.attrs && node.attrs[key] === value; key: string,
        }, offset); value: string,
        offset: number[]
        ): number[][] {
        return findAllNodeIndex(
        baseTree,
        (node) => {
        return (
        typeof node !== "string" && node.attrs && node.attrs[key] === value
        );
        },
        offset
        );
        } }
        export function findAllNodeIndexByTag(baseTree: HTML.Tree, tag: string, offset: number[]): number[][] { export function findAllNodeIndexByTag(
        return findAllNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        return typeof (node) !== "string" && node.tag === tag; tag: string,
        }, offset); offset: number[]
        ): number[][] {
        return findAllNodeIndex(
        baseTree,
        (node) => {
        return typeof node !== "string" && node.tag === tag;
        },
        offset
        );
        } }
        const chineseCapitalNumbers = ["零", "壹", "贰", "叁", "肆", "伍", "陆", "柒", "捌", "玖"] const chineseCapitalNumbers = [
        "零",
        "壹",
        "贰",
        "叁",
        "肆",
        "伍",
        "陆",
        "柒",
        "捌",
        "玖",
        ];
        export function getString(node: HTML.Node, lengthLimit?: number) { export function getString(node: HTML.Node, lengthLimit?: number) {
        let resultStr: string; let resultStr: string;
        if (typeof (node) === "string") { if (typeof node === "string") {
        resultStr = node; resultStr = node;
        } else { } else {
        const subTree = node.content; const subTree = node.content;
        if (!subTree) { if (!subTree) {
        return null; return null;
        } }
        const subNode = subTree[0]; const subNode = subTree[0];
        if (typeof (subNode) === "string") { if (typeof subNode === "string") {
        resultStr = subTree[0] as string; resultStr = subTree[0] as string;
        } else { } else {
        resultStr = getString(subTree[0]) as string; resultStr = getString(subTree[0]) as string;
        } }
        } }
        resultStr = resultStr.trim(); resultStr = resultStr.trim();
        if (lengthLimit && resultStr.length > lengthLimit) { if (lengthLimit && resultStr.length > lengthLimit) {
        resultStr = resultStr.slice(0, lengthLimit); resultStr = resultStr.slice(0, lengthLimit);
        } }
        return resultStr; return resultStr;
        } }
        export function getNumber(node: HTML.Node) { export function getNumber(node: HTML.Node) {
        const numberStr = getString(node); const numberStr = getString(node);
        let stringMatch: RegExpMatchArray; let stringMatch: RegExpMatchArray;
        if (numberStr === "没有上榜" || !numberStr) { if (numberStr === "没有上榜" || !numberStr) {
        return null; return null;
        } else if (stringMatch = numberStr.match(/^([天地])魂$/)) { } else if ((stringMatch = numberStr.match(/^([天地])魂$/))) {
        return stringMatch[1] === "天" ? 2 : 1; return stringMatch[1] === "天" ? 2 : 1;
        } else if (stringMatch = numberStr.match(/^(.+)天(.+)境界$/)) { } else if ((stringMatch = numberStr.match(/^(.+)天(.+)境界$/))) {
        return (_.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[1]) << 8) | _.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[2]); return (
        } else if (stringMatch = numberStr.match(/^(\d+)-(\d+)$/)) { (_.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[1]) << 8) |
        const minValue = parseInt(stringMatch[1]); _.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[2])
        const maxValue = parseInt(stringMatch[2]); );
        return (minValue << 16) | maxValue; } else if ((stringMatch = numberStr.match(/^(\d+)-(\d+)$/))) {
        } else { const minValue = parseInt(stringMatch[1]);
        return parseInt(numberStr); const maxValue = parseInt(stringMatch[2]);
        } return (minValue << 16) | maxValue;
        } else {
        return parseInt(numberStr);
        }
        } }
        export function getContinuousNodes(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): HTML.Node[] { export function getContinuousNodes(
        const pos = _.clone(_pos); tree: HTML.Tree,
        const datas: HTML.Node[] = []; _pos: number[],
        for (let i = 0; i < dataCount; ++i) { moveOffset: number,
        const node = getDepthOfTree(tree, pos)[0]; step: number,
        datas.push(node); dataCount: number
        pos[pos.length - (moveOffset + 1)] += step; ): HTML.Node[] {
        } const pos = _.clone(_pos);
        return datas; const datas: HTML.Node[] = [];
        for (let i = 0; i < dataCount; ++i) {
        const node = getDepthOfTree(tree, pos)[0];
        datas.push(node);
        pos[pos.length - (moveOffset + 1)] += step;
        }
        return datas;
        } }
        export function getContinuousData(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): string[] { export function getContinuousData(
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(getString); tree: HTML.Tree,
        _pos: number[],
        moveOffset: number,
        step: number,
        dataCount: number
        ): string[] {
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(
        getString
        );
        } }
        export function getContinuousNumber(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): number[] { export function getContinuousNumber(
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(getNumber); tree: HTML.Tree,
        _pos: number[],
        moveOffset: number,
        step: number,
        dataCount: number
        ): number[] {
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(
        getNumber
        );
        } }
        tests/test-list.ts
        View file @ 593d03fb
        ...@@ -3,9 +3,9 @@ import fs from "fs"; ...@@ -3,9 +3,9 @@ import fs from "fs";
        import { parsePlayerRows } from "../src/playerlist"; import { parsePlayerRows } from "../src/playerlist";
        async function main() { async function main() {
        let html = await fs.promises.readFile("./tests/playerlist.html", "utf-8"); let html = await fs.promises.readFile("./tests/playerlist.html", "utf-8");
        console.log(parsePlayerRows(html)); console.log(parsePlayerRows(html));
        html = await fs.promises.readFile("./tests/playerlist-null.html", "utf-8"); html = await fs.promises.readFile("./tests/playerlist-null.html", "utf-8");
        console.log(parsePlayerRows(html)); console.log(parsePlayerRows(html));
        } }
        main(); main();
        tests/test-proxy.ts
        View file @ 593d03fb
        import { ProxyFetcher } from "../src/proxy"; import { ProxyFetcher } from "../src/proxy";
        async function main() { async function main() {
        const fetcher = new ProxyFetcher({ const fetcher = new ProxyFetcher({
        useProxy: true, useProxy: true,
        proxySource: [ proxySource: [
        "http://www.89ip.cn/tqdl.html?api=1&num=9999", "http://www.89ip.cn/tqdl.html?api=1&num=9999",
        "http://www.66ip.cn/mo.php?tqsl=9999" "http://www.66ip.cn/mo.php?tqsl=9999",
        ], ],
        timeout: 10000 timeout: 10000,
        }); });
        await fetcher.initProxies(); await fetcher.initProxies();
        console.log(await fetcher.getWithProxy("https://mycard.moe", {})); console.log(await fetcher.getWithProxy("https://mycard.moe", {}));
        } }
        main(); main();
        tests/test-user.ts
        View file @ 593d03fb
        ...@@ -3,13 +3,19 @@ import fs from "fs"; ...@@ -3,13 +3,19 @@ import fs from "fs";
        import HTML from "posthtml-parser"; import HTML from "posthtml-parser";
        async function main() { async function main() {
        let id = "28_20588"; let id = "28_20588";
        let html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8"); let html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2)); await fs.promises.writeFile(
        console.log(new User(id, html, null)); `./tests/${id}.json`,
        id = "6_18804822"; JSON.stringify(HTML(html), null, 2)
        html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8"); );
        await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2)); console.log(new User(id, html, null));
        console.log(new User(id, html, null)); id = "6_18804822";
        html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(
        `./tests/${id}.json`,
        JSON.stringify(HTML(html), null, 2)
        );
        console.log(new User(id, html, null));
        } }
        main(); main();
        tsconfig.json
        View file @ 593d03fb
        { {
        "compilerOptions": { "compilerOptions": {
        "outDir": "build", "outDir": "dist",
        "module": "commonjs", "module": "commonjs",
        "target": "esnext", "target": "es2021",
        "esModuleInterop": true, "esModuleInterop": true,
        "emitDecoratorMetadata": true,
        "experimentalDecorators": true,
        "declaration": true, "declaration": true,
        "sourceMap": true "sourceMap": true
        }, },
        "compileOnSave": true, "compileOnSave": true,
        "allowJs": true, "allowJs": true,
        "include": [ "include": [
        "src/*.ts", "*.ts",
        "tests/*.ts" "src/**/*.ts",
        "test/**/*.ts",
        "tests/**/*.ts"
        ] ]
        } }
        Markdown is supported
        0% or
        You are about to add 0 people to the discussion. Proceed with caution.
        Finish editing this message first!
        Please register or to comment