Skip to content

T
tx3-bang-reader
Commit 593d03fb authored by nanahira's avatar nanahira
Browse files
Options

rework

parent 85d2aa62
Pipeline #9850 failed with stages
        in 1 minute and 33 seconds
        Hide whitespace changes
        Inline Side-by-side
        Showing with 12316 additions and 1266 deletions
        .eslintignore 0 → 100644
        View file @ 593d03fb
        webpack.config.js
        dist/*
        build/*
        *.js
        .eslintrc.js 0 → 100644
        View file @ 593d03fb
        module.exports = {
        parser: '@typescript-eslint/parser',
        parserOptions: {
        project: 'tsconfig.json',
        sourceType: 'module',
        },
        plugins: ['@typescript-eslint/eslint-plugin'],
        extends: [
        'plugin:@typescript-eslint/recommended',
        'plugin:prettier/recommended',
        ],
        root: true,
        env: {
        node: true,
        jest: true,
        },
        ignorePatterns: ['.eslintrc.js'],
        rules: {
        '@typescript-eslint/interface-name-prefix': 'off',
        '@typescript-eslint/explicit-function-return-type': 'off',
        '@typescript-eslint/explicit-module-boundary-types': 'off',
        '@typescript-eslint/no-explicit-any': 'off',
        },
        };
        .gitlab-ci.yml
        View file @ 593d03fb
        stages:
        - install
        - build
        - deploy
        variables:
        GIT_DEPTH: "1"
        CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
        CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest
        docker:
        npm_ci:
        stage: install
        tags:
        - linux
        script:
        - npm ci
        artifacts:
        paths:
        - node_modules
        .build_base:
        stage: build
        tags:
        - docker
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        tags:
        - linux
        dependencies:
        - npm_ci
        build:
        extends:
        - .build_base
        script:
        - docker build --pull -t $CONTAINER_TEST_IMAGE .
        - docker push $CONTAINER_TEST_IMAGE
        - npm run build
        artifacts:
        paths:
        - dist/
        deploy_latest:
        stage: deploy
        tags:
        - docker
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        unit-test:
        extends:
        - .build_base
        script:
        - docker pull $CONTAINER_TEST_IMAGE
        - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
        - docker push $CONTAINER_RELEASE_IMAGE
        only:
        - master
        - npm run test
        deploy_tag:
        deploy_npm:
        stage: deploy
        tags:
        - docker
        variables:
        CONTAINER_TAG_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        dependencies:
        - build
        tags:
        - linux
        script:
        - docker pull $CONTAINER_TEST_IMAGE
        - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_TAG_IMAGE
        - docker push $CONTAINER_TAG_IMAGE
        - apt update;apt -y install coreutils
        - echo $NPMRC | base64 --decode > ~/.npmrc
        - npm publish . || true
        only:
        - tags
        - master
        .idea/inspectionProfiles/Project_Default.xml 0 → 100644
        View file @ 593d03fb
        <component name="InspectionProjectProfileManager">
        <profile version="1.0">
        <option name="myName" value="Project Default" />
        <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
        </profile>
        </component>
        \ No newline at end of file
        .idea/jsLinters/eslint.xml 0 → 100644
        View file @ 593d03fb
        <?xml version="1.0" encoding="UTF-8"?>
        <project version="4">
        <component name="EslintConfiguration">
        <option name="fix-on-save" value="true" />
        </component>
        </project>
        \ No newline at end of file
        .npmignore 0 → 100644
        View file @ 593d03fb
        /install-npm.sh
        .git*
        /data
        /output
        /config.yaml
        .idea
        .dockerignore
        Dockerfile
        /src
        /coverage
        /tests
        /dist/tests
        Dockerfile
        View file @ 593d03fb
        FROM node:bullseye-slim
        RUN apt update && apt -y install python3 && rm -rf /var/lib/apt/lists/*
        FROM node:bullseye-slim as base
        LABEL Author="Nanahira <nanahira@momobako.com>"
        RUN apt update && apt -y install python3 build-essential && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/log/*
        WORKDIR /usr/src/app
        COPY ./package*.json ./
        RUN npm ci
        FROM base as builder
        RUN npm ci && npm cache clean --force
        COPY . ./
        RUN npm run build
        CMD ["npm", "run", "start"]
        FROM base
        ENV NODE_ENV production
        RUN npm ci && npm cache clean --force
        COPY --from=builder /usr/src/app/dist ./dist
        CMD [ "npm", "start" ]
        package-lock.json
        View file @ 593d03fb
        This source diff could not be displayed because it is too large. You can view the blob instead.
        package.json
        View file @ 593d03fb
        ......@@ -2,12 +2,11 @@
        "name": "tx3-bang-reader",
        "version": "1.0.0",
        "description": "Read TX3 bang and parse it",
        "main": "build/src/run.js",
        "main": "dist/run.js",
        "scripts": {
        "build": "./node_modules/.bin/tsc",
        "pack": "mkdir dist ; ./node_modules/.bin/pkg --out-path dist .",
        "fetch": "node build/src/run.js",
        "start": "node build/src/run.js cron"
        "build": "tsc",
        "fetch": "node dist/src/run.js",
        "start": "node dist/src/run.js cron"
        },
        "repository": {
        "type": "git",
        ......@@ -22,17 +21,10 @@
        "url": "https://github.com/purerosefallen/tx3-bang-reader/issues"
        },
        "homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme",
        "bin": "build/src/run.js",
        "pkg": {
        "scripts": [
        "build/src/*.js"
        ],
        "assets": []
        },
        "bin": "dist/src/run.js",
        "dependencies": {
        "@types/cron": "^1.7.2",
        "@types/csv-parse": "^1.2.2",
        "@types/node": "^14.0.14",
        "@types/underscore": "^1.10.2",
        "@types/yaml": "^1.9.7",
        "axios": "^0.19.2",
        ......@@ -44,8 +36,21 @@
        "posthtml-parser": "^0.4.2",
        "promise-mysql": "^4.1.3",
        "querystring": "^0.2.0",
        "typescript": "^3.9.5",
        "underscore": "^1.10.2",
        "yaml": "^1.10.0"
        },
        "devDependencies": {
        "@types/jest": "^27.4.0",
        "@types/node": "^17.0.18",
        "@typescript-eslint/eslint-plugin": "^4.33.0",
        "@typescript-eslint/parser": "^4.33.0",
        "eslint": "^7.32.0",
        "eslint-config-prettier": "^8.3.0",
        "eslint-plugin-prettier": "^3.4.1",
        "jest": "^27.5.1",
        "prettier": "^2.5.1",
        "rimraf": "^3.0.2",
        "ts-jest": "^27.1.3",
        "typescript": "^4.5.5"
        }
        }
        src/fetcher.ts
        View file @ 593d03fb
        import axios from "axios";
        import _ from "underscore";
        import { User } from "./user";
        import {ProxyConfig, ProxyFetcher} from "./proxy";
        import { PlayerRow, parsePlayerRows } from "./playerlist";
        import { ProxyConfig, ProxyFetcher } from "./proxy";
        import { parsePlayerRows, PlayerRow } from "./playerlist";
        import qs from "querystring";
        import mysql from "promise-mysql";
        import moment from "moment";
        export const servers = [
        "东方明珠",
        "紫禁之巅",
        "一纪山海",
        "剑心问道",
        "天与秋光",
        "梦回山海",
        "绝代风华",
        "鼎立山河",
        "天府之国",
        "天下无双",
        "情动大荒",
        "琉璃月",
        "齐鲁天下",
        "剑舞香江",
        "白云山",
        "瘦西湖",
        "逐鹿中原",
        "黄鹤楼",
        "武夷九曲",
        "上善若水",
        "君临天下",
        "气壮山河",
        "飞龙在天",
        "烽火关东",
        "盛世长安",
        ]
        "东方明珠",
        "紫禁之巅",
        "一纪山海",
        "剑心问道",
        "天与秋光",
        "梦回山海",
        "绝代风华",
        "鼎立山河",
        "天府之国",
        "天下无双",
        "情动大荒",
        "琉璃月",
        "齐鲁天下",
        "剑舞香江",
        "白云山",
        "瘦西湖",
        "逐鹿中原",
        "黄鹤楼",
        "武夷九曲",
        "上善若水",
        "君临天下",
        "气壮山河",
        "飞龙在天",
        "烽火关东",
        "盛世长安",
        ];
        export interface Config {
        outDir: string;
        server: string[];
        useMySQL: boolean,
        MySQLConfig: mysql.PoolConfig,
        proxy: ProxyConfig,
        cronString: string;
        outDir: string;
        server: string[];
        useMySQL: boolean;
        MySQLConfig: mysql.PoolConfig;
        proxy: ProxyConfig;
        cronString: string;
        }
        export class Tx3Fetcher {
        config: Config;
        proxyFetcher: ProxyFetcher;
        db: mysql.Pool;
        curDate: string;
        constructor(config: Config) {
        this.config = config;
        this.proxyFetcher = new ProxyFetcher(config.proxy);
        }
        async init() {
        this.curDate = moment().format("YYYY-MM-DD HH:mm:ss");
        if(this.config.useMySQL) {
        this.db = await mysql.createPool(this.config.MySQLConfig);
        await this.db.query("CREATE TABLE IF NOT EXISTS `userdata` (\n" +
        " `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" +
        " `date` datetime NOT NULL DEFAULT current_timestamp(),\n" +
        " `url` varchar(50) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `rank` int(11) UNSIGNED NOT NULL,\n" +
        " `name` varchar(7) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `category` varchar(5) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `serverArea` varchar(4) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `server` varchar(4) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `level` tinyint(4) UNSIGNED NOT NULL,\n" +
        " `region` varchar(7) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `score` int(11) UNSIGNED NOT NULL,\n" +
        " `equip` int(11) UNSIGNED NOT NULL,\n" +
        " `totalScore` int(11) UNSIGNED NOT NULL,\n" +
        " PRIMARY KEY (`id`),\n" +
        " INDEX (date),\n" +
        " INDEX (name(7)),\n" +
        " INDEX (url(50)),\n" +
        " INDEX (category(5)),\n" +
        " INDEX (serverArea(4)),\n" +
        " INDEX (server(4)),\n" +
        " INDEX (region(7)),\n" +
        " INDEX (equip)\n" +
        ") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci");
        console.log(`Removing existing records of ${this.curDate}.`);
        //await this.db.query("delete from userdata where date = ?", this.curDate);
        }
        if(this.config.proxy.useProxy) {
        await this.proxyFetcher.initProxies();
        }
        }
        async fetchAll(): Promise<Map<string, User[]>> {
        const res = new Map<string, User[]>();
        const userLists = await Promise.all(servers.map(server => this.fetchUsersFromServer(server)));
        for (let i = 1; i < userLists.length; ++i){
        res.set(servers[i], userLists[i])
        }
        return res;
        }
        async fetchUsersFromServer(server: string): Promise<User[]> {
        console.log(`Fetching user list from server ${server}.`);
        const resPromises: Promise<User[]>[] = [];
        for (let school = 1; school < 12; ++school) {
        resPromises.push(this.fetchUsersFromSchoolAndServer(school, server));
        }
        const result = _.flatten(await Promise.all(resPromises));
        console.log(`Fetched user list with ${result.length} users from server ${server}.`);
        return result;
        }
        async fetchUsersFromSchoolAndServer(school: number, server: string): Promise<User[]> {
        console.log(`Fetching users from server ${server} with school ${school}.`);
        const res: User[][] = [];
        for (let page = 1; page <= 25; ++page) {
        const list = await this.fetchUsers(school, server, page);
        if (!list.length) {
        break;
        }
        res.push(list);
        }
        return _.flatten(res);
        }
        async fetchUsers(school: number, server: string, page: number): Promise<User[]> {
        const playerRows = await this.fetchList(school, server, page);
        return await Promise.all(playerRows.map(row => this.fetchUser(row)));
        }
        async fetchListFromServer(server: string): Promise<PlayerRow[]> {
        console.log(`Fetching user list from server ${server}.`);
        const resPromises: Promise<PlayerRow[]>[] = [];
        for (let school = 1; school < 12; ++school) {
        resPromises.push(this.fetchListFromSchoolAndServer(school, server));
        }
        const result = _.flatten(await Promise.all(resPromises));
        console.log(`Fetched user list from server ${server}. ${result.length} users found.`);
        return result;
        }
        async fetchListFromSchoolAndServer(school: number, server: string): Promise<PlayerRow[]> {
        console.log(`Fetching users from server ${server} with school ${school}.`);
        const res: PlayerRow[][] = [];
        for (let page = 1; page <= 25; ++page) {
        const list = await this.fetchList(school, server, page);
        if (!list.length) {
        break;
        }
        res.push(list);
        }
        const ret = _.flatten(res);
        console.log(`Fetched users from server ${server} with school ${school}.`);
        return ret;
        }
        async checkAddRecord(row: PlayerRow) {
        const latestRows = await this.db.query("select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by date desc limit 1", row.url);
        if (latestRows.length) {
        const lrow = latestRows[0];
        if (_.every(["name", "category", "serverArea", "server", "region"], field => lrow[field] === row[field])) {
        return;
        }
        }
        const sql = "insert into userdata set ?";
        const valueObj = {
        date: this.curDate,
        ...row
        };
        console.log(`Player ${row.name} from ${row.server} has changes. Writing record to database: ${sql} ${JSON.stringify(valueObj)} ${JSON.stringify(await this.db.query(sql, valueObj))}`);
        }
        async fetchList(school: number, server: string, page: number): Promise<PlayerRow[]> {
        console.log(`Fetching user list from server ${server} with school ${school} page ${page}.`);
        try {
        const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com/bang/ranks`, {
        responseType: "document",
        params: {
        order_key: "equ_xiuwei",
        count: 20,
        school,
        server,
        page
        },
        paramsSerializer: (params) => {
        return qs.stringify(params);
        }
        });
        const playerRows = parsePlayerRows(content);
        console.log(`Fetched user list from server ${server} with school ${school} page ${page}. ${playerRows.length} users found.`);
        if (this.db) {
        //await Promise.all(playerRows.map(m => this.db.query("delete from userdata where url = ? and date = ?", [m.url, this.curDate])));
        await Promise.all(playerRows.map(m => this.checkAddRecord(m)));
        }
        return playerRows;
        } catch(e) {
        console.error(`Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`);
        return [];
        }
        }
        async fetchUser(playerRow: PlayerRow): Promise<User> {
        const id = playerRow.url.split("/").pop();
        try {
        console.log(`Fetching user ${playerRow.name} from ${playerRow.server}.`);
        const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com${playerRow.url}`, {
        responseType: "document"
        });
        const user = new User(id, content, playerRow.region);
        console.log(`Fetched user ${playerRow.name} from ${playerRow.server}.`);
        return user;
        } catch(e) {
        console.error(`Errored fetching role data from ${id}: ${e.toString()}`);
        return null;
        }
        }
        config: Config;
        proxyFetcher: ProxyFetcher;
        db: mysql.Pool;
        curDate: string;
        constructor(config: Config) {
        this.config = config;
        this.proxyFetcher = new ProxyFetcher(config.proxy);
        }
        async init() {
        this.curDate = moment().format("YYYY-MM-DD HH:mm:ss");
        if (this.config.useMySQL) {
        this.db = await mysql.createPool(this.config.MySQLConfig);
        await this.db.query(
        "CREATE TABLE IF NOT EXISTS `userdata` (\n" +
        " `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" +
        " `date` datetime NOT NULL DEFAULT current_timestamp(),\n" +
        " `url` varchar(50) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `rank` int(11) UNSIGNED NOT NULL,\n" +
        " `name` varchar(7) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `category` varchar(5) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `serverArea` varchar(4) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `server` varchar(4) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `level` tinyint(4) UNSIGNED NOT NULL,\n" +
        " `region` varchar(7) COLLATE utf8_unicode_ci NOT NULL,\n" +
        " `score` int(11) UNSIGNED NOT NULL,\n" +
        " `equip` int(11) UNSIGNED NOT NULL,\n" +
        " `totalScore` int(11) UNSIGNED NOT NULL,\n" +
        " PRIMARY KEY (`id`),\n" +
        " INDEX (date),\n" +
        " INDEX (name(7)),\n" +
        " INDEX (url(50)),\n" +
        " INDEX (category(5)),\n" +
        " INDEX (serverArea(4)),\n" +
        " INDEX (server(4)),\n" +
        " INDEX (region(7)),\n" +
        " INDEX (equip)\n" +
        ") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci"
        );
        console.log(`Removing existing records of ${this.curDate}.`);
        //await this.db.query("delete from userdata where date = ?", this.curDate);
        }
        if (this.config.proxy.useProxy) {
        await this.proxyFetcher.initProxies();
        }
        }
        async fetchAll(): Promise<Map<string, User[]>> {
        const res = new Map<string, User[]>();
        const userLists = await Promise.all(
        servers.map((server) => this.fetchUsersFromServer(server))
        );
        for (let i = 1; i < userLists.length; ++i) {
        res.set(servers[i], userLists[i]);
        }
        return res;
        }
        async fetchUsersFromServer(server: string): Promise<User[]> {
        console.log(`Fetching user list from server ${server}.`);
        const resPromises: Promise<User[]>[] = [];
        for (let school = 1; school < 12; ++school) {
        resPromises.push(this.fetchUsersFromSchoolAndServer(school, server));
        }
        const result = _.flatten(await Promise.all(resPromises));
        console.log(
        `Fetched user list with ${result.length} users from server ${server}.`
        );
        return result;
        }
        async fetchUsersFromSchoolAndServer(
        school: number,
        server: string
        ): Promise<User[]> {
        console.log(`Fetching users from server ${server} with school ${school}.`);
        const res: User[][] = [];
        for (let page = 1; page <= 25; ++page) {
        const list = await this.fetchUsers(school, server, page);
        if (!list.length) {
        break;
        }
        res.push(list);
        }
        return _.flatten(res);
        }
        async fetchUsers(
        school: number,
        server: string,
        page: number
        ): Promise<User[]> {
        const playerRows = await this.fetchList(school, server, page);
        return await Promise.all(playerRows.map((row) => this.fetchUser(row)));
        }
        async fetchListFromServer(server: string): Promise<PlayerRow[]> {
        console.log(`Fetching user list from server ${server}.`);
        const resPromises: Promise<PlayerRow[]>[] = [];
        for (let school = 1; school < 12; ++school) {
        resPromises.push(this.fetchListFromSchoolAndServer(school, server));
        }
        const result = _.flatten(await Promise.all(resPromises));
        console.log(
        `Fetched user list from server ${server}. ${result.length} users found.`
        );
        return result;
        }
        async fetchListFromSchoolAndServer(
        school: number,
        server: string
        ): Promise<PlayerRow[]> {
        console.log(`Fetching users from server ${server} with school ${school}.`);
        const res: PlayerRow[][] = [];
        for (let page = 1; page <= 25; ++page) {
        const list = await this.fetchList(school, server, page);
        if (!list.length) {
        break;
        }
        res.push(list);
        }
        const ret = _.flatten(res);
        console.log(`Fetched users from server ${server} with school ${school}.`);
        return ret;
        }
        async checkAddRecord(row: PlayerRow) {
        const latestRows = await this.db.query(
        "select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by id desc limit 1",
        row.url
        );
        if (latestRows.length) {
        const lrow = latestRows[0];
        if (
        _.every(
        ["name", "category", "serverArea", "server", "region"],
        (field) => lrow[field] === row[field]
        )
        ) {
        return;
        }
        }
        const sql = "insert into userdata set ?";
        const valueObj = {
        date: this.curDate,
        ...row,
        };
        console.log(
        `Player ${row.name} from ${
        row.server
        } has changes. Writing record to database: ${sql} ${JSON.stringify(
        valueObj
        )} ${JSON.stringify(await this.db.query(sql, valueObj))}`
        );
        }
        async fetchList(
        school: number,
        server: string,
        page: number
        ): Promise<PlayerRow[]> {
        console.log(
        `Fetching user list from server ${server} with school ${school} page ${page}.`
        );
        try {
        const content: string = await this.proxyFetcher.getWithProxy(
        `http://bang.tx3.163.com/bang/ranks`,
        {
        responseType: "document",
        params: {
        order_key: "equ_xiuwei",
        count: 20,
        school,
        server,
        page,
        },
        paramsSerializer: (params) => {
        return qs.stringify(params);
        },
        }
        );
        const playerRows = parsePlayerRows(content);
        console.log(
        `Fetched user list from server ${server} with school ${school} page ${page}. ${playerRows.length} users found.`
        );
        if (this.db) {
        //await Promise.all(playerRows.map(m => this.db.query("delete from userdata where url = ? and date = ?", [m.url, this.curDate])));
        await Promise.all(playerRows.map((m) => this.checkAddRecord(m)));
        }
        return playerRows;
        } catch (e) {
        console.error(
        `Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`
        );
        return [];
        }
        }
        async fetchUser(playerRow: PlayerRow): Promise<User> {
        const id = playerRow.url.split("/").pop();
        try {
        console.log(`Fetching user ${playerRow.name} from ${playerRow.server}.`);
        const content: string = await this.proxyFetcher.getWithProxy(
        `http://bang.tx3.163.com${playerRow.url}`,
        {
        responseType: "document",
        }
        );
        const user = new User(id, content, playerRow.region);
        console.log(`Fetched user ${playerRow.name} from ${playerRow.server}.`);
        return user;
        } catch (e) {
        console.error(`Errored fetching role data from ${id}: ${e.toString()}`);
        return null;
        }
        }
        }
        src/migrate.ts
        View file @ 593d03fb
        ......@@ -2,100 +2,119 @@ import mysql from "promise-mysql";
        import moment from "moment";
        import fs from "fs";
        import _csv_parse from "csv-parse";
        import util from 'util';
        import util from "util";
        import { Config } from "./fetcher";
        import { PlayerRow, PlayerRowDated } from "./playerlist";
        import _ from "underscore";
        import yaml from "yaml";
        const parse_csv: (input: Buffer | string, options?: _csv_parse.Options) => Promise<any[]> = util.promisify(_csv_parse);
        const parse_csv: (
        input: Buffer | string,
        options?: _csv_parse.Options
        ) => Promise<any[]> = util.promisify(_csv_parse);
        let config: Config;
        let db: mysql.Pool;
        const serverAreaCache = new Map<string, string>();
        async function getServerAreaFromServer(server: string) {
        if (serverAreaCache.has(server)) {
        return serverAreaCache.get(server);
        } else {
        const [res] = await db.query("select serverArea from userdata where server = ? limit 1", server);
        if (!res) {
        return "none";
        }
        const serverArea = res.serverArea;
        serverAreaCache.set(server, serverArea);
        return serverArea;
        }
        if (serverAreaCache.has(server)) {
        return serverAreaCache.get(server);
        } else {
        const [res] = await db.query(
        "select serverArea from userdata where server = ? limit 1",
        server
        );
        if (!res) {
        return "none";
        }
        const serverArea = res.serverArea;
        serverAreaCache.set(server, serverArea);
        return serverArea;
        }
        }
        async function readSingleRecord(col: string[], offset: number, base: PlayerRow): Promise<PlayerRowDated> {
        let pointer = offset;
        const newRecord: PlayerRowDated = {
        date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"),
        ...(_.clone(base))
        };
        newRecord.name = col[pointer++];
        newRecord.server = col[pointer++];
        newRecord.serverArea = await getServerAreaFromServer(newRecord.server);
        const _region = col[pointer++];
        newRecord.region = _region.length ? _region : "none";
        newRecord.category = col[pointer++];
        newRecord.equip = parseInt(col[pointer++]);
        newRecord.totalScore = newRecord.equip + newRecord.score;
        return newRecord;
        async function readSingleRecord(
        col: string[],
        offset: number,
        base: PlayerRow
        ): Promise<PlayerRowDated> {
        let pointer = offset;
        const newRecord: PlayerRowDated = {
        date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"),
        ..._.clone(base),
        };
        newRecord.name = col[pointer++];
        newRecord.server = col[pointer++];
        newRecord.serverArea = await getServerAreaFromServer(newRecord.server);
        const _region = col[pointer++];
        newRecord.region = _region.length ? _region : "none";
        newRecord.category = col[pointer++];
        newRecord.equip = parseInt(col[pointer++]);
        newRecord.totalScore = newRecord.equip + newRecord.score;
        return newRecord;
        }
        let leftCount: number;
        async function readColumn(col: string[]): Promise<void> {
        const recordCount = parseInt(col[0]);
        const url = `/bang/role/${col[3]}`;
        console.error(`Reading column ${url}.`);
        let [base] = await db.query("select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by date asc limit 1", url) as PlayerRow[];
        if (!base) {
        console.error(`Base record of ${url} not found. Using default values.`);
        base = {
        url,
        rank: 500,
        name: null,
        server: null,
        serverArea: null,
        category: null,
        level: 80,
        region: "none",
        score: 0,
        equip: 0,
        totalScore: 0
        }
        }
        for (let i = 0; i < recordCount; ++i) {
        const offset = 4 + (i * 6);
        if (!col[offset].length) {
        continue;
        }
        const record = await readSingleRecord(col, offset, base);
        const sql = "insert into userdata set ?";
        console.log(sql, JSON.stringify(record), JSON.stringify(await db.query(sql, record)));
        }
        console.error(`Read column ${url}. ${--leftCount} columns left.`);
        const recordCount = parseInt(col[0]);
        const url = `/bang/role/${col[3]}`;
        console.error(`Reading column ${url}.`);
        let [base] = (await db.query(
        "select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by id asc limit 1",
        url
        )) as PlayerRow[];
        if (!base) {
        console.error(`Base record of ${url} not found. Using default values.`);
        base = {
        url,
        rank: 500,
        name: null,
        server: null,
        serverArea: null,
        category: null,
        level: 80,
        region: "none",
        score: 0,
        equip: 0,
        totalScore: 0,
        };
        }
        for (let i = 0; i < recordCount; ++i) {
        const offset = 4 + i * 6;
        if (!col[offset].length) {
        continue;
        }
        const record = await readSingleRecord(col, offset, base);
        const sql = "insert into userdata set ?";
        console.log(
        sql,
        JSON.stringify(record),
        JSON.stringify(await db.query(sql, record))
        );
        }
        console.error(`Read column ${url}. ${--leftCount} columns left.`);
        }
        async function loadCsv(path: string): Promise<string[][]> {
        const data = await fs.promises.readFile(path);
        return await parse_csv(data, {
        trim: true
        });
        const data = await fs.promises.readFile(path);
        return await parse_csv(data, {
        trim: true,
        });
        }
        async function main() {
        console.error("Started.");
        const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
        db = await mysql.createPool(config.MySQLConfig);
        const data = await loadCsv(process.argv[2]);
        leftCount = data.length;
        //await Promise.all(data.map(col => readColumn(col)));
        for (let col of data) {
        await readColumn(col);
        }
        console.error("Finished.");
        process.exit();
        console.error("Started.");
        const config: Config = yaml.parse(
        await fs.promises.readFile("./config.yaml", "utf8")
        );
        db = await mysql.createPool(config.MySQLConfig);
        const data = await loadCsv(process.argv[2]);
        leftCount = data.length;
        //await Promise.all(data.map(col => readColumn(col)));
        for (const col of data) {
        await readColumn(col);
        }
        console.error("Finished.");
        process.exit();
        }
        main();
        src/playerlist.ts
        View file @ 593d03fb
        import HTML from "posthtml-parser";
        import _, { first } from "underscore";
        import {getDepthOfTree, getNumber, findNodeIndex, findNodeIndexByAttribute, findNodeIndexByContent, findNodeIndexByTag, findAllNodeIndex, getContinuousData, getContinuousNodes, getString} from "./utility";
        import {
        findAllNodeIndex,
        findNodeIndexByTag,
        getContinuousNodes,
        getDepthOfTree,
        getNumber,
        getString,
        } from "./utility";
        export interface PlayerRow {
        url: string;
        rank: number;
        name: string;
        category: string;
        serverArea: string;
        server: string;
        level: number;
        region: string;
        score: number;
        equip: number;
        totalScore: number;
        url: string;
        rank: number;
        name: string;
        category: string;
        serverArea: string;
        server: string;
        level: number;
        region: string;
        score: number;
        equip: number;
        totalScore: number;
        }
        export interface PlayerRowDated extends PlayerRow {
        date: string;
        date: string;
        }
        export interface PlayerRowFull extends PlayerRowDated {
        id: number;
        id: number;
        }
        function getPlayerRowFromTree(tree: HTML.Tree): PlayerRow {
        const nodes = getContinuousNodes(tree, [1], 0, 2, 10);
        return {
        url: (nodes[1] as HTML.NodeTag).attrs.href as string,
        rank: getNumber(nodes[0]),
        name: getString(nodes[1], 7),
        serverArea: getString(nodes[2]),
        server: getString(nodes[3]),
        level: getNumber(nodes[4]),
        category: getString(nodes[5]),
        region: getString(nodes[6], 7) || "none",
        score: getNumber(nodes[7]),
        equip: getNumber(nodes[8]),
        totalScore: getNumber(nodes[9])
        }
        const nodes = getContinuousNodes(tree, [1], 0, 2, 10);
        return {
        url: (nodes[1] as HTML.NodeTag).attrs.href as string,
        rank: getNumber(nodes[0]),
        name: getString(nodes[1], 7),
        serverArea: getString(nodes[2]),
        server: getString(nodes[3]),
        level: getNumber(nodes[4]),
        category: getString(nodes[5]),
        region: getString(nodes[6], 7) || "none",
        score: getNumber(nodes[7]),
        equip: getNumber(nodes[8]),
        totalScore: getNumber(nodes[9]),
        };
        }
        export function parsePlayerRows(content: string) {
        const parsedContent = HTML(content);
        const tablePos = findNodeIndexByTag(parsedContent, "table", []);
        const tableTree = getDepthOfTree(parsedContent, tablePos);
        const playerPoses = findAllNodeIndex(tableTree, (node) => {
        return typeof (node) !== "string" && node.tag === "tr" && node.attrs.class !== "trTop2";
        }, []);
        return playerPoses.map(pos => {
        const tree = getDepthOfTree(tableTree, pos);
        return getPlayerRowFromTree(tree);
        });
        const parsedContent = HTML(content);
        const tablePos = findNodeIndexByTag(parsedContent, "table", []);
        const tableTree = getDepthOfTree(parsedContent, tablePos);
        const playerPoses = findAllNodeIndex(
        tableTree,
        (node) => {
        return (
        typeof node !== "string" &&
        node.tag === "tr" &&
        node.attrs.class !== "trTop2"
        );
        },
        []
        );
        return playerPoses.map((pos) => {
        const tree = getDepthOfTree(tableTree, pos);
        return getPlayerRowFromTree(tree);
        });
        }
        src/proxy.ts
        View file @ 593d03fb
        ......@@ -9,59 +9,59 @@ import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios";
        //}
        export interface ProxyConfig {
        useProxy: boolean,
        proxySource: string[],
        timeout: number
        useProxy: boolean;
        proxySource: string[];
        timeout: number;
        }
        const agentList = [
        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',
        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',
        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
        'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)',
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
        ]
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
        "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
        "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0",
        "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
        "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
        "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
        "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
        ];
        async function testProxy(proxy: AxiosProxyConfig) {
        await axios.get("http://mirrors.aliyun.com/debian/pool", {
        proxy,
        headers: {
        "User-Agent": agentList[4]
        },
        timeout: this.config.timeout,
        });
        return proxy;
        await axios.get("http://mirrors.aliyun.com/debian/pool", {
        proxy,
        headers: {
        "User-Agent": agentList[4],
        },
        timeout: this.config.timeout,
        });
        return proxy;
        }
        async function checkProxy(proxy: AxiosProxyConfig) {
        let isProxyUsable = false;
        try {
        await testProxy(proxy);
        //console.log(`Proxy ${proxy.host} is ok.`);
        isProxyUsable = true;
        } catch (e) {
        //console.error(`Proxy ${proxy.host} is broken: ${e.toString()}`);
        }
        return isProxyUsable;
        let isProxyUsable = false;
        try {
        await testProxy(proxy);
        //console.log(`Proxy ${proxy.host} is ok.`);
        isProxyUsable = true;
        } catch (e) {
        //console.error(`Proxy ${proxy.host} is broken: ${e.toString()}`);
        }
        return isProxyUsable;
        }
        async function filterProxies(proxies: AxiosProxyConfig[]) {
        const proxiesUsableList = await Promise.all(proxies.map(checkProxy));
        return proxies.filter((proxy, index) => {
        return proxiesUsableList[index];
        });
        const proxiesUsableList = await Promise.all(proxies.map(checkProxy));
        return proxies.filter((proxy, index) => {
        return proxiesUsableList[index];
        });
        }
        //async function findFirstUsableProxy(proxies: AxiosProxyConfig[]) {
        ......@@ -69,70 +69,86 @@ async function filterProxies(proxies: AxiosProxyConfig[]) {
        //}
        export class ProxyFetcher {
        proxies: AxiosProxyConfig[];
        counter: number;
        config: ProxyConfig;
        constructor(config: ProxyConfig) {
        this.config = config;
        this.proxies = [];
        this.counter = 0;
        }
        async initProxiesFrom(url: string) {
        if (!this.config.useProxy) {
        return;
        }
        console.log(`Fetching proxies from ${url}.`)
        while (true) {
        try {
        const proxyPage: string = (await axios.get(url, {
        responseType: "document",
        })).data;
        const proxies: AxiosProxyConfig[] = proxyPage.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g).map(proxyString => {
        const [host, _port] = proxyString.split(":");
        const port = parseInt(_port);
        const proxy = { host, port };
        return proxy;
        });
        //const usableProxies = await filterProxies(proxies);
        for (let proxy of proxies) {
        this.proxies.push(proxy);
        }
        console.error(`Got ${proxies.length} proxies from ${url}.`);
        return;
        } catch (e) {
        console.error(`Failed fetching proxy list from ${url}: ${e.toString()}`)
        }
        }
        }
        async initProxies() {
        await Promise.all(this.config.proxySource.map((m) => {
        return this.initProxiesFrom(m);
        }));
        }
        async getWithProxy(url: string, options: AxiosRequestConfig) {
        while (true) {
        if (this.config.useProxy && !this.proxies.length) {
        await this.initProxies();
        }
        const proxyIndex = !this.config.useProxy ? null : (++this.counter) % this.proxies.length;
        //const proxyIndex = 0;
        const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex];
        try {
        const data = (await axios.get(url, {
        proxy,
        headers: {
        "User-Agent": agentList[this.counter % agentList.length]
        },
        timeout: this.config.timeout,
        ...options
        })).data;
        return data;
        } catch (e) {
        if (this.config.useProxy) {
        this.proxies.splice(proxyIndex, 1);
        }
        console.error(`Failed fetching data from ${url}: ${e.toString()} ${this.proxies.length} proxies left.`)
        }
        }
        }
        proxies: AxiosProxyConfig[];
        counter: number;
        config: ProxyConfig;
        constructor(config: ProxyConfig) {
        this.config = config;
        this.proxies = [];
        this.counter = 0;
        }
        async initProxiesFrom(url: string) {
        if (!this.config.useProxy) {
        return;
        }
        console.log(`Fetching proxies from ${url}.`);
        while (true) {
        try {
        const proxyPage: string = (
        await axios.get(url, {
        responseType: "document",
        })
        ).data;
        const proxies: AxiosProxyConfig[] = proxyPage
        .match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g)
        .map((proxyString) => {
        const [host, _port] = proxyString.split(":");
        const port = parseInt(_port);
        const proxy = { host, port };
        return proxy;
        });
        //const usableProxies = await filterProxies(proxies);
        for (const proxy of proxies) {
        this.proxies.push(proxy);
        }
        console.error(`Got ${proxies.length} proxies from ${url}.`);
        return;
        } catch (e) {
        console.error(
        `Failed fetching proxy list from ${url}: ${e.toString()}`
        );
        }
        }
        }
        async initProxies() {
        await Promise.all(
        this.config.proxySource.map((m) => {
        return this.initProxiesFrom(m);
        })
        );
        }
        async getWithProxy(url: string, options: AxiosRequestConfig) {
        while (true) {
        if (this.config.useProxy && !this.proxies.length) {
        await this.initProxies();
        }
        const proxyIndex = !this.config.useProxy
        ? null
        : ++this.counter % this.proxies.length;
        //const proxyIndex = 0;
        const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex];
        try {
        const data = (
        await axios.get(url, {
        proxy,
        headers: {
        "User-Agent": agentList[this.counter % agentList.length],
        },
        timeout: this.config.timeout,
        ...options,
        })
        ).data;
        return data;
        } catch (e) {
        if (this.config.useProxy) {
        this.proxies.splice(proxyIndex, 1);
        }
        console.error(
        `Failed fetching data from ${url}: ${e.toString()} ${
        this.proxies.length
        } proxies left.`
        );
        }
        }
        }
        }
        src/remove-dup.ts
        View file @ 593d03fb
        ......@@ -6,31 +6,40 @@ import _ from "underscore";
        import yaml from "yaml";
        function checkSameRow(row: PlayerRowFull, lrow: PlayerRowFull) {
        return _.every(["name", "category", "serverArea", "server", "region"], field => lrow[field] === row[field]);
        return _.every(
        ["name", "category", "serverArea", "server", "region"],
        (field) => lrow[field] === row[field]
        );
        }
        async function main() {
        console.error("Started.");
        const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
        const db = await mysql.createPool(config.MySQLConfig);
        const urlDataCache = new Map<string, PlayerRowFull>();
        const deleteList: number[] = [];
        const datas: PlayerRowFull[] = await db.query(`select * from userdata order by date asc`);
        for (let row of datas) {
        if (urlDataCache.has(row.url)) {
        const oldRow = urlDataCache.get(row.url);
        if (checkSameRow(row, oldRow)) {
        deleteList.push(row.id);
        }
        }
        urlDataCache.set(row.url, row);
        }
        console.error(`Deletes: ${deleteList.length}`);
        for (let id of deleteList) {
        const sql = `delete from userdata where id = ?`;
        console.error(`Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`);
        }
        console.error("Finished.");
        process.exit();
        console.error("Started.");
        const config: Config = yaml.parse(
        await fs.promises.readFile("./config.yaml", "utf8")
        );
        const db = await mysql.createPool(config.MySQLConfig);
        const urlDataCache = new Map<string, PlayerRowFull>();
        const deleteList: number[] = [];
        const datas: PlayerRowFull[] = await db.query(
        `select * from userdata order by id asc`
        );
        for (const row of datas) {
        if (urlDataCache.has(row.url)) {
        const oldRow = urlDataCache.get(row.url);
        if (checkSameRow(row, oldRow)) {
        deleteList.push(row.id);
        }
        }
        urlDataCache.set(row.url, row);
        }
        console.error(`Deletes: ${deleteList.length}`);
        for (const id of deleteList) {
        const sql = `delete from userdata where id = ?`;
        console.error(
        `Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`
        );
        }
        console.error("Finished.");
        process.exit();
        }
        main();
        src/run.ts
        View file @ 593d03fb
        import {Tx3Fetcher, servers, Config} from "./fetcher";
        import { Config, servers, Tx3Fetcher } from "./fetcher";
        import fs from "fs";
        import _ from "underscore";
        import yaml from "yaml";
        import { CronJob } from "cron";
        let config: Config;
        async function loadConfig() {
        config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
        config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
        }
        async function runServer(fetcher: Tx3Fetcher, server: string) {
        const users = await fetcher.fetchListFromServer(server);
        await fs.promises.writeFile(`./output/servers/${server}.json`, JSON.stringify({
        date: fetcher.curDate,
        data: users
        }, null, 2));
        return users;
        const users = await fetcher.fetchListFromServer(server);
        await fs.promises.writeFile(
        `./output/servers/${server}.json`,
        JSON.stringify(
        {
        date: fetcher.curDate,
        data: users,
        },
        null,
        2
        )
        );
        return users;
        }
        async function run() {
        console.log(`Fetch started.`);
        try {
        await fs.promises.access("./output/servers");
        } catch (e) {
        await fs.promises.mkdir("./output/servers", {
        recursive: true
        });
        }
        const fetcher = new Tx3Fetcher(config);
        await fetcher.init();
        if (config.server) {
        await Promise.all(config.server.map(server => {
        return runServer(fetcher, server)
        }));
        } else {
        const userListWithServer = await Promise.all(servers.map(server => {
        return runServer(fetcher, server)
        }));
        const allServersList: any = {};
        for (let i = 0; i < servers.length;++i) {
        allServersList[servers[i]] = userListWithServer[i];
        }
        await fs.promises.writeFile(`./output/all.json`, JSON.stringify({
        date: fetcher.curDate,
        data: allServersList
        }, null, 2));
        }
        console.log("Finished.");
        console.log(`Fetch started.`);
        try {
        await fs.promises.access("./output/servers");
        } catch (e) {
        await fs.promises.mkdir("./output/servers", {
        recursive: true,
        });
        }
        const fetcher = new Tx3Fetcher(config);
        await fetcher.init();
        if (config.server) {
        await Promise.all(
        config.server.map((server) => {
        return runServer(fetcher, server);
        })
        );
        } else {
        const userListWithServer = await Promise.all(
        servers.map((server) => {
        return runServer(fetcher, server);
        })
        );
        const allServersList: any = {};
        for (let i = 0; i < servers.length; ++i) {
        allServersList[servers[i]] = userListWithServer[i];
        }
        await fs.promises.writeFile(
        `./output/all.json`,
        JSON.stringify(
        {
        date: fetcher.curDate,
        data: allServersList,
        },
        null,
        2
        )
        );
        }
        console.log("Finished.");
        }
        async function main() {
        await loadConfig();
        if (process.argv[2] === "cron") {
        const job = new CronJob(config.cronString, run, null, true, "Asia/Shanghai", null, true);
        job.start();
        } else {
        await run();
        process.exit();
        }
        await loadConfig();
        if (process.argv[2] === "cron") {
        const job = new CronJob(
        config.cronString,
        run,
        null,
        true,
        "Asia/Shanghai",
        null,
        true
        );
        job.start();
        } else {
        await run();
        process.exit();
        }
        }
        main();
        src/user.ts
        View file @ 593d03fb
        import HTML from "posthtml-parser";
        import _, { first } from "underscore";
        import {getDepthOfTree, getNumber, findNodeIndex, findNodeIndexByAttribute, findNodeIndexByContent, findNodeIndexByTag, getContinuousNumber} from "./utility";
        import _ from "underscore";
        import {
        findNodeIndexByAttribute,
        findNodeIndexByContent,
        getContinuousNumber,
        getDepthOfTree,
        getNumber,
        } from "./utility";
        export interface AttackAttribute {
        '攻力': number, //大攻取前2字节int16，小攻取后2字节int16，法力也是这样
        '命中': number,
        '法力': number,
        '重击': number,
        '会心一击': number,
        '附加伤害': number
        攻力: number; //大攻取前2字节int16，小攻取后2字节int16，法力也是这样
        命中: number;
        法力: number;
        重击: number;
        会心一击: number;
        附加伤害: number;
        }
        export interface DefenseAttribute {
        '防御': number,
        '回避': number,
        '法防': number,
        '神明': number,
        '化解': number,
        '知彼': number
        防御: number;
        回避: number;
        法防: number;
        神明: number;
        化解: number;
        知彼: number;
        }
        export interface SpecialAttribute {
        '身法': number,
        '坚韧': number,
        '定力': number,
        '诛心': number,
        '御心': number,
        '万钧': number,
        '铁壁': number
        身法: number;
        坚韧: number;
        定力: number;
        诛心: number;
        御心: number;
        万钧: number;
        铁壁: number;
        }
        export interface AdvancedAttribute { '追电': number, '骤雨': number, '疾语': number, '明思': number, '扰心': number, '人祸': number }
        export interface AdvancedAttribute {
        追电: number;
        骤雨: number;
        疾语: number;
        明思: number;
        扰心: number;
        人祸: number;
        }
        export class User {
        id: string;
        content: HTML.Tree;
        name: string;
        region: string;
        category: string;
        serverArea: string;
        server: string;
        level: number;
        equipValue: number;
        equipRank: number;
        equipLocalRank: number;
        equipCategoryRank: number;
        scoreValue: number;
        scoreRank: number
        scoreLocalRank: number;
        scoreCategoryRank: number;
        sqStage: number; // 天魂：2，地魂：1，没有神启：null
        sqLevel: number; // 前4位是几境界，后四位是几天
        qhLevel: number;
        tlPoints: number;
        hp: number;
        mp: number;
        li: number;
        ti: number;
        min: number;
        ji: number;
        hun: number;
        nian: number;
        attackAttributes: AttackAttribute;
        defenseAttributes: DefenseAttribute;
        specialAttributes: SpecialAttribute;
        advancedAttributes: AdvancedAttribute;
        yhz: string[];
        private parseMetadata() {
        let namePos = findNodeIndexByAttribute(this.content, "class", "sTitle", []);
        this.name = getDepthOfTree(this.content, namePos.concat([0]))[0] as string;
        namePos[namePos.length - 1] += 2;
        this.category = getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string;
        namePos[namePos.length - 1] += 2;
        [this.serverArea, this.server] = (getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string).split("&nbsp;");
        id: string;
        content: HTML.Tree;
        name: string;
        region: string;
        category: string;
        serverArea: string;
        server: string;
        level: number;
        equipValue: number;
        equipRank: number;
        equipLocalRank: number;
        equipCategoryRank: number;
        scoreValue: number;
        scoreRank: number;
        scoreLocalRank: number;
        scoreCategoryRank: number;
        sqStage: number; // 天魂：2，地魂：1，没有神启：null
        sqLevel: number; // 前4位是几境界，后四位是几天
        qhLevel: number;
        tlPoints: number;
        hp: number;
        mp: number;
        li: number;
        ti: number;
        min: number;
        ji: number;
        hun: number;
        nian: number;
        attackAttributes: AttackAttribute;
        defenseAttributes: DefenseAttribute;
        specialAttributes: SpecialAttribute;
        advancedAttributes: AdvancedAttribute;
        yhz: string[];
        private parseMetadata() {
        const namePos = findNodeIndexByAttribute(
        this.content,
        "class",
        "sTitle",
        []
        );
        this.name = getDepthOfTree(this.content, namePos.concat([0]))[0] as string;
        namePos[namePos.length - 1] += 2;
        this.category = getDepthOfTree(
        this.content,
        namePos.concat([0, 0])
        )[0] as string;
        namePos[namePos.length - 1] += 2;
        [this.serverArea, this.server] = (
        getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string
        ).split("&nbsp;");
        let levelPos = findNodeIndexByContent(this.content, "等级", []);
        levelPos.pop();
        levelPos[levelPos.length - 1]++;
        this.level = getNumber(getDepthOfTree(this.content, levelPos)[0]);
        }
        private parseEquipmentData() {
        let ValuePos = findNodeIndexByContent(this.content, "装备评价:", []);
        ValuePos.pop();
        ValuePos[ValuePos.length - 1]++;
        let datas = getContinuousNumber(this.content, ValuePos, 1, 2, 4);
        this.equipValue = datas[0];
        this.equipRank = datas[1];
        this.equipLocalRank = datas[2];
        this.equipCategoryRank = datas[3];
        const levelPos = findNodeIndexByContent(this.content, "等级", []);
        levelPos.pop();
        levelPos[levelPos.length - 1]++;
        this.level = getNumber(getDepthOfTree(this.content, levelPos)[0]);
        }
        private parseEquipmentData() {
        let ValuePos = findNodeIndexByContent(this.content, "装备评价:", []);
        ValuePos.pop();
        ValuePos[ValuePos.length - 1]++;
        let datas = getContinuousNumber(this.content, ValuePos, 1, 2, 4);
        this.equipValue = datas[0];
        this.equipRank = datas[1];
        this.equipLocalRank = datas[2];
        this.equipCategoryRank = datas[3];
        ValuePos = findNodeIndexByContent(this.content, "人物修为:", []);
        ValuePos.pop();
        ValuePos[ValuePos.length - 1]++;
        datas = getContinuousNumber(this.content, ValuePos, 1, 2, 8);
        this.scoreValue = datas[0];
        this.scoreRank = datas[1];
        this.scoreLocalRank = datas[2];
        this.scoreCategoryRank = datas[3];
        this.sqStage = datas[4];
        this.sqLevel = datas[5];
        this.qhLevel = datas[6];
        this.tlPoints = datas[7];
        }
        private parseAttributeTable(_pos: number[]): any {
        const ret = {};
        const pos = _.clone(_pos);
        const tree = getDepthOfTree(this.content, pos);
        for (let i = 3; i < tree.length; i += 2) {
        const [_keyNode, valueNode] = getDepthOfTree(tree, [i]);
        const keyNode = (_keyNode as HTML.NodeTag);
        const key = keyNode.content[0] as string;
        const value = getNumber(valueNode);
        ret[key] = value;
        }
        return ret;
        }
        private parseBasicAttributes() {
        let ValuePos = findNodeIndexByContent(this.content, "命", []);
        ValuePos.pop();
        ValuePos[ValuePos.length - 1] += 2;
        const datas = getContinuousNumber(this.content, ValuePos, 0, 4, 8);
        this.hp = datas[0];
        this.mp = datas[1];
        this.li = datas[2];
        this.ti = datas[3];
        this.min = datas[4];
        this.ji = datas[5];
        this.hun = datas[6];
        this.nian = datas[7];
        ValuePos = findNodeIndexByContent(this.content, "人物修为:", []);
        ValuePos.pop();
        ValuePos[ValuePos.length - 1]++;
        datas = getContinuousNumber(this.content, ValuePos, 1, 2, 8);
        this.scoreValue = datas[0];
        this.scoreRank = datas[1];
        this.scoreLocalRank = datas[2];
        this.scoreCategoryRank = datas[3];
        this.sqStage = datas[4];
        this.sqLevel = datas[5];
        this.qhLevel = datas[6];
        this.tlPoints = datas[7];
        }
        private parseAttributeTable(_pos: number[]): any {
        const ret = {};
        const pos = _.clone(_pos);
        const tree = getDepthOfTree(this.content, pos);
        for (let i = 3; i < tree.length; i += 2) {
        const [_keyNode, valueNode] = getDepthOfTree(tree, [i]);
        const keyNode = _keyNode as HTML.NodeTag;
        const key = keyNode.content[0] as string;
        const value = getNumber(valueNode);
        ret[key] = value;
        }
        return ret;
        }
        private parseBasicAttributes() {
        const ValuePos = findNodeIndexByContent(this.content, "命", []);
        ValuePos.pop();
        ValuePos[ValuePos.length - 1] += 2;
        const datas = getContinuousNumber(this.content, ValuePos, 0, 4, 8);
        this.hp = datas[0];
        this.mp = datas[1];
        this.li = datas[2];
        this.ti = datas[3];
        this.min = datas[4];
        this.ji = datas[5];
        this.hun = datas[6];
        this.nian = datas[7];
        ValuePos.pop();
        ValuePos[ValuePos.length - 1] += 2;
        this.attackAttributes = this.parseAttributeTable(ValuePos) as AttackAttribute;
        ValuePos[ValuePos.length - 1] += 2;
        this.defenseAttributes = this.parseAttributeTable(ValuePos) as DefenseAttribute;
        ValuePos[ValuePos.length - 1] += 2;
        ValuePos.push(1);
        this.specialAttributes = this.parseAttributeTable(ValuePos) as SpecialAttribute;
        ValuePos[ValuePos.length - 1] += 2;
        this.advancedAttributes = this.parseAttributeTable(ValuePos) as AdvancedAttribute;
        }
        private parseYHZ() {
        let ValuePos = findNodeIndexByAttribute(this.content, "id", "tableYHZ", []);
        ValuePos.push(1);
        const tree = getDepthOfTree(this.content, ValuePos);
        this.yhz = [];
        for (let i = 1; i < tree.length; i += 2){
        const node = (tree[i] as HTML.NodeTag);
        this.yhz.push(node.content[0] as string);
        }
        }
        private parse() {
        this.parseMetadata();
        this.parseEquipmentData();
        this.parseBasicAttributes();
        this.parseYHZ();
        }
        constructor(id: string, content: string, region: string) {
        this.id = id;
        this.region = region;
        this.content = HTML(content);
        this.parse();
        this.content = null;
        }
        ValuePos.pop();
        ValuePos[ValuePos.length - 1] += 2;
        this.attackAttributes = this.parseAttributeTable(
        ValuePos
        ) as AttackAttribute;
        ValuePos[ValuePos.length - 1] += 2;
        this.defenseAttributes = this.parseAttributeTable(
        ValuePos
        ) as DefenseAttribute;
        ValuePos[ValuePos.length - 1] += 2;
        ValuePos.push(1);
        this.specialAttributes = this.parseAttributeTable(
        ValuePos
        ) as SpecialAttribute;
        ValuePos[ValuePos.length - 1] += 2;
        this.advancedAttributes = this.parseAttributeTable(
        ValuePos
        ) as AdvancedAttribute;
        }
        private parseYHZ() {
        const ValuePos = findNodeIndexByAttribute(
        this.content,
        "id",
        "tableYHZ",
        []
        );
        ValuePos.push(1);
        const tree = getDepthOfTree(this.content, ValuePos);
        this.yhz = [];
        for (let i = 1; i < tree.length; i += 2) {
        const node = tree[i] as HTML.NodeTag;
        this.yhz.push(node.content[0] as string);
        }
        }
        private parse() {
        this.parseMetadata();
        this.parseEquipmentData();
        this.parseBasicAttributes();
        this.parseYHZ();
        }
        constructor(id: string, content: string, region: string) {
        this.id = id;
        this.region = region;
        this.content = HTML(content);
        this.parse();
        this.content = null;
        }
        }
        src/utility.ts
        View file @ 593d03fb
        import HTML from "posthtml-parser";
        import _ from "underscore";
        export function getDepthOfTree(tree: HTML.Tree, indexList: number[]): HTML.Tree {
        if (indexList.length) {
        const _indexList = _.clone(indexList);
        const index = _indexList.splice(0, 1)[0];
        const node = tree[index];
        if (typeof (node) === "string" || !node.content) {
        return [node];
        }
        return getDepthOfTree(node.content, _indexList);
        } else {
        return tree;
        }
        export function getDepthOfTree(
        tree: HTML.Tree,
        indexList: number[]
        ): HTML.Tree {
        if (indexList.length) {
        const _indexList = _.clone(indexList);
        const index = _indexList.splice(0, 1)[0];
        const node = tree[index];
        if (typeof node === "string" || !node.content) {
        return [node];
        }
        return getDepthOfTree(node.content, _indexList);
        } else {
        return tree;
        }
        }
        export function findNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node) => boolean, offset: number[]): number[] {
        const queue = [offset];
        while (queue.length) {
        const indexList = queue.splice(0, 1)[0];
        const tree = getDepthOfTree(baseTree, indexList);
        for (let i = 0; i < tree.length; ++i) {
        const node = tree[i];
        const newList = indexList.concat([i]);
        if (condition(node)) {
        return newList;
        } else if (typeof (node) !== "string") {
        queue.push(newList);
        }
        }
        }
        return null;
        export function findNodeIndex(
        baseTree: HTML.Tree,
        condition: (node: HTML.Node) => boolean,
        offset: number[]
        ): number[] {
        const queue = [offset];
        while (queue.length) {
        const indexList = queue.splice(0, 1)[0];
        const tree = getDepthOfTree(baseTree, indexList);
        for (let i = 0; i < tree.length; ++i) {
        const node = tree[i];
        const newList = indexList.concat([i]);
        if (condition(node)) {
        return newList;
        } else if (typeof node !== "string") {
        queue.push(newList);
        }
        }
        }
        return null;
        }
        export function findAllNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node) => boolean, offset: number[]): number[][] {
        const queue = [offset];
        const res: number[][] = [];
        while (queue.length) {
        const indexList = queue.splice(0, 1)[0];
        const tree = getDepthOfTree(baseTree, indexList);
        for (let i = 0; i < tree.length; ++i) {
        const node = tree[i];
        const newList = indexList.concat([i]);
        if (condition(node)) {
        //console.log(newList);
        res.push(newList);
        } else if (typeof (node) !== "string") {
        queue.push(newList);
        }
        }
        }
        return res;
        export function findAllNodeIndex(
        baseTree: HTML.Tree,
        condition: (node: HTML.Node) => boolean,
        offset: number[]
        ): number[][] {
        const queue = [offset];
        const res: number[][] = [];
        while (queue.length) {
        const indexList = queue.splice(0, 1)[0];
        const tree = getDepthOfTree(baseTree, indexList);
        for (let i = 0; i < tree.length; ++i) {
        const node = tree[i];
        const newList = indexList.concat([i]);
        if (condition(node)) {
        //console.log(newList);
        res.push(newList);
        } else if (typeof node !== "string") {
        queue.push(newList);
        }
        }
        }
        return res;
        }
        export function findNodeIndexByContent(baseTree: HTML.Tree, label: string, offset: number[]): number[] {
        return findNodeIndex(baseTree, (node) => {
        return node === label;
        }, offset);
        export function findNodeIndexByContent(
        baseTree: HTML.Tree,
        label: string,
        offset: number[]
        ): number[] {
        return findNodeIndex(
        baseTree,
        (node) => {
        return node === label;
        },
        offset
        );
        }
        export function findNodeIndexByAttribute(baseTree: HTML.Tree, key: string, value: string, offset: number[]): number[] {
        return findNodeIndex(baseTree, (node) => {
        return typeof (node) !== "string" && node.attrs && node.attrs[key] === value;
        }, offset);
        export function findNodeIndexByAttribute(
        baseTree: HTML.Tree,
        key: string,
        value: string,
        offset: number[]
        ): number[] {
        return findNodeIndex(
        baseTree,
        (node) => {
        return (
        typeof node !== "string" && node.attrs && node.attrs[key] === value
        );
        },
        offset
        );
        }
        export function findNodeIndexByTag(baseTree: HTML.Tree, tag: string, offset: number[]): number[] {
        return findNodeIndex(baseTree, (node) => {
        return typeof (node) !== "string" && node.tag === tag;
        }, offset);
        export function findNodeIndexByTag(
        baseTree: HTML.Tree,
        tag: string,
        offset: number[]
        ): number[] {
        return findNodeIndex(
        baseTree,
        (node) => {
        return typeof node !== "string" && node.tag === tag;
        },
        offset
        );
        }
        export function findAllNodeIndexByContent(baseTree: HTML.Tree, label: string, offset: number[]): number[][] {
        return findAllNodeIndex(baseTree, (node) => {
        return node === label;
        }, offset);
        export function findAllNodeIndexByContent(
        baseTree: HTML.Tree,
        label: string,
        offset: number[]
        ): number[][] {
        return findAllNodeIndex(
        baseTree,
        (node) => {
        return node === label;
        },
        offset
        );
        }
        export function findAllNodeIndexByAttribute(baseTree: HTML.Tree, key: string, value: string, offset: number[]): number[][] {
        return findAllNodeIndex(baseTree, (node) => {
        return typeof (node) !== "string" && node.attrs && node.attrs[key] === value;
        }, offset);
        export function findAllNodeIndexByAttribute(
        baseTree: HTML.Tree,
        key: string,
        value: string,
        offset: number[]
        ): number[][] {
        return findAllNodeIndex(
        baseTree,
        (node) => {
        return (
        typeof node !== "string" && node.attrs && node.attrs[key] === value
        );
        },
        offset
        );
        }
        export function findAllNodeIndexByTag(baseTree: HTML.Tree, tag: string, offset: number[]): number[][] {
        return findAllNodeIndex(baseTree, (node) => {
        return typeof (node) !== "string" && node.tag === tag;
        }, offset);
        export function findAllNodeIndexByTag(
        baseTree: HTML.Tree,
        tag: string,
        offset: number[]
        ): number[][] {
        return findAllNodeIndex(
        baseTree,
        (node) => {
        return typeof node !== "string" && node.tag === tag;
        },
        offset
        );
        }
        const chineseCapitalNumbers = ["零", "壹", "贰", "叁", "肆", "伍", "陆", "柒", "捌", "玖"]
        const chineseCapitalNumbers = [
        "零",
        "壹",
        "贰",
        "叁",
        "肆",
        "伍",
        "陆",
        "柒",
        "捌",
        "玖",
        ];
        export function getString(node: HTML.Node, lengthLimit?: number) {
        let resultStr: string;
        if (typeof (node) === "string") {
        resultStr = node;
        } else {
        const subTree = node.content;
        if (!subTree) {
        return null;
        }
        const subNode = subTree[0];
        if (typeof (subNode) === "string") {
        resultStr = subTree[0] as string;
        } else {
        resultStr = getString(subTree[0]) as string;
        }
        }
        resultStr = resultStr.trim();
        if (lengthLimit && resultStr.length > lengthLimit) {
        resultStr = resultStr.slice(0, lengthLimit);
        }
        return resultStr;
        let resultStr: string;
        if (typeof node === "string") {
        resultStr = node;
        } else {
        const subTree = node.content;
        if (!subTree) {
        return null;
        }
        const subNode = subTree[0];
        if (typeof subNode === "string") {
        resultStr = subTree[0] as string;
        } else {
        resultStr = getString(subTree[0]) as string;
        }
        }
        resultStr = resultStr.trim();
        if (lengthLimit && resultStr.length > lengthLimit) {
        resultStr = resultStr.slice(0, lengthLimit);
        }
        return resultStr;
        }
        export function getNumber(node: HTML.Node) {
        const numberStr = getString(node);
        let stringMatch: RegExpMatchArray;
        if (numberStr === "没有上榜" || !numberStr) {
        return null;
        } else if (stringMatch = numberStr.match(/^([天地])魂$/)) {
        return stringMatch[1] === "天" ? 2 : 1;
        } else if (stringMatch = numberStr.match(/^(.+)天(.+)境界$/)) {
        return (_.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[1]) << 8) | _.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[2]);
        } else if (stringMatch = numberStr.match(/^(\d+)-(\d+)$/)) {
        const minValue = parseInt(stringMatch[1]);
        const maxValue = parseInt(stringMatch[2]);
        return (minValue << 16) | maxValue;
        } else {
        return parseInt(numberStr);
        }
        const numberStr = getString(node);
        let stringMatch: RegExpMatchArray;
        if (numberStr === "没有上榜" || !numberStr) {
        return null;
        } else if ((stringMatch = numberStr.match(/^([天地])魂$/))) {
        return stringMatch[1] === "天" ? 2 : 1;
        } else if ((stringMatch = numberStr.match(/^(.+)天(.+)境界$/))) {
        return (
        (_.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[1]) << 8) |
        _.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[2])
        );
        } else if ((stringMatch = numberStr.match(/^(\d+)-(\d+)$/))) {
        const minValue = parseInt(stringMatch[1]);
        const maxValue = parseInt(stringMatch[2]);
        return (minValue << 16) | maxValue;
        } else {
        return parseInt(numberStr);
        }
        }
        export function getContinuousNodes(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): HTML.Node[] {
        const pos = _.clone(_pos);
        const datas: HTML.Node[] = [];
        for (let i = 0; i < dataCount; ++i) {
        const node = getDepthOfTree(tree, pos)[0];
        datas.push(node);
        pos[pos.length - (moveOffset + 1)] += step;
        }
        return datas;
        export function getContinuousNodes(
        tree: HTML.Tree,
        _pos: number[],
        moveOffset: number,
        step: number,
        dataCount: number
        ): HTML.Node[] {
        const pos = _.clone(_pos);
        const datas: HTML.Node[] = [];
        for (let i = 0; i < dataCount; ++i) {
        const node = getDepthOfTree(tree, pos)[0];
        datas.push(node);
        pos[pos.length - (moveOffset + 1)] += step;
        }
        return datas;
        }
        export function getContinuousData(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): string[] {
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(getString);
        export function getContinuousData(
        tree: HTML.Tree,
        _pos: number[],
        moveOffset: number,
        step: number,
        dataCount: number
        ): string[] {
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(
        getString
        );
        }
        export function getContinuousNumber(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): number[] {
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(getNumber);
        export function getContinuousNumber(
        tree: HTML.Tree,
        _pos: number[],
        moveOffset: number,
        step: number,
        dataCount: number
        ): number[] {
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(
        getNumber
        );
        }
        tests/test-list.ts
        View file @ 593d03fb
        ......@@ -3,9 +3,9 @@ import fs from "fs";
        import { parsePlayerRows } from "../src/playerlist";
        async function main() {
        let html = await fs.promises.readFile("./tests/playerlist.html", "utf-8");
        console.log(parsePlayerRows(html));
        html = await fs.promises.readFile("./tests/playerlist-null.html", "utf-8");
        console.log(parsePlayerRows(html));
        let html = await fs.promises.readFile("./tests/playerlist.html", "utf-8");
        console.log(parsePlayerRows(html));
        html = await fs.promises.readFile("./tests/playerlist-null.html", "utf-8");
        console.log(parsePlayerRows(html));
        }
        main();
        tests/test-proxy.ts
        View file @ 593d03fb
        import { ProxyFetcher } from "../src/proxy";
        async function main() {
        const fetcher = new ProxyFetcher({
        useProxy: true,
        proxySource: [
        "http://www.89ip.cn/tqdl.html?api=1&num=9999",
        "http://www.66ip.cn/mo.php?tqsl=9999"
        ],
        timeout: 10000
        });
        await fetcher.initProxies();
        console.log(await fetcher.getWithProxy("https://mycard.moe", {}));
        async function main() {
        const fetcher = new ProxyFetcher({
        useProxy: true,
        proxySource: [
        "http://www.89ip.cn/tqdl.html?api=1&num=9999",
        "http://www.66ip.cn/mo.php?tqsl=9999",
        ],
        timeout: 10000,
        });
        await fetcher.initProxies();
        console.log(await fetcher.getWithProxy("https://mycard.moe", {}));
        }
        main();
        tests/test-user.ts
        View file @ 593d03fb
        ......@@ -3,13 +3,19 @@ import fs from "fs";
        import HTML from "posthtml-parser";
        async function main() {
        let id = "28_20588";
        let html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2));
        console.log(new User(id, html, null));
        id = "6_18804822";
        html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2));
        console.log(new User(id, html, null));
        let id = "28_20588";
        let html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(
        `./tests/${id}.json`,
        JSON.stringify(HTML(html), null, 2)
        );
        console.log(new User(id, html, null));
        id = "6_18804822";
        html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(
        `./tests/${id}.json`,
        JSON.stringify(HTML(html), null, 2)
        );
        console.log(new User(id, html, null));
        }
        main();
        tsconfig.json
        View file @ 593d03fb
        {
        "compilerOptions": {
        "outDir": "build",
        "outDir": "dist",
        "module": "commonjs",
        "target": "esnext",
        "target": "es2021",
        "esModuleInterop": true,
        "emitDecoratorMetadata": true,
        "experimentalDecorators": true,
        "declaration": true,
        "sourceMap": true
        },
        "compileOnSave": true,
        "allowJs": true,
        "include": [
        "src/*.ts",
        "tests/*.ts"
        "*.ts",
        "src/**/*.ts",
        "test/**/*.ts",
        "tests/**/*.ts"
        ]
        }
        Markdown is supported
        0% or
        You are about to add 0 people to the discussion. Proceed with caution.
        Finish editing this message first!
        Please register or to comment