Skip to content

T
tx3-bang-reader
Commit 593d03fb authored by nanahira's avatar nanahira
Browse files
Options

rework

parent 85d2aa62
Pipeline #9850 failed with stages
        in 1 minute and 33 seconds
        Expand all Show whitespace changes
        Inline Side-by-side
        Showing with 12316 additions and 1266 deletions
        .eslintignore 0 → 100644
        View file @ 593d03fb
        webpack.config.js
        dist/*
        build/*
        *.js
        .eslintrc.js 0 → 100644
        View file @ 593d03fb
        module.exports = {
        parser: '@typescript-eslint/parser',
        parserOptions: {
        project: 'tsconfig.json',
        sourceType: 'module',
        },
        plugins: ['@typescript-eslint/eslint-plugin'],
        extends: [
        'plugin:@typescript-eslint/recommended',
        'plugin:prettier/recommended',
        ],
        root: true,
        env: {
        node: true,
        jest: true,
        },
        ignorePatterns: ['.eslintrc.js'],
        rules: {
        '@typescript-eslint/interface-name-prefix': 'off',
        '@typescript-eslint/explicit-function-return-type': 'off',
        '@typescript-eslint/explicit-module-boundary-types': 'off',
        '@typescript-eslint/no-explicit-any': 'off',
        },
        };
        .gitlab-ci.yml
        View file @ 593d03fb
        stages: stages:
        - install
        - build - build
        - deploy - deploy
        variables: variables:
        GIT_DEPTH: "1" GIT_DEPTH: "1"
        CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
        CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest
        docker: npm_ci:
        stage: build stage: install
        tags: tags:
        - docker - linux
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        script: script:
        - docker build --pull -t $CONTAINER_TEST_IMAGE . - npm ci
        - docker push $CONTAINER_TEST_IMAGE artifacts:
        paths:
        - node_modules
        deploy_latest: .build_base:
        stage: deploy stage: build
        tags: tags:
        - docker - linux
        before_script: dependencies:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - npm_ci
        build:
        extends:
        - .build_base
        script: script:
        - docker pull $CONTAINER_TEST_IMAGE - npm run build
        - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE artifacts:
        - docker push $CONTAINER_RELEASE_IMAGE paths:
        only: - dist/
        - master
        unit-test:
        extends:
        - .build_base
        script:
        - npm run test
        deploy_tag: deploy_npm:
        stage: deploy stage: deploy
        dependencies:
        - build
        tags: tags:
        - docker - linux
        variables:
        CONTAINER_TAG_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        script: script:
        - docker pull $CONTAINER_TEST_IMAGE - apt update;apt -y install coreutils
        - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_TAG_IMAGE - echo $NPMRC | base64 --decode > ~/.npmrc
        - docker push $CONTAINER_TAG_IMAGE - npm publish . || true
        only: only:
        - tags - master
        .idea/inspectionProfiles/Project_Default.xml 0 → 100644
        View file @ 593d03fb
        <component name="InspectionProjectProfileManager">
        <profile version="1.0">
        <option name="myName" value="Project Default" />
        <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
        </profile>
        </component>
        \ No newline at end of file
        .idea/jsLinters/eslint.xml 0 → 100644
        View file @ 593d03fb
        <?xml version="1.0" encoding="UTF-8"?>
        <project version="4">
        <component name="EslintConfiguration">
        <option name="fix-on-save" value="true" />
        </component>
        </project>
        \ No newline at end of file
        .npmignore 0 → 100644
        View file @ 593d03fb
        /install-npm.sh
        .git*
        /data
        /output
        /config.yaml
        .idea
        .dockerignore
        Dockerfile
        /src
        /coverage
        /tests
        /dist/tests
        Dockerfile
        View file @ 593d03fb
        FROM node:bullseye-slim FROM node:bullseye-slim as base
        RUN apt update && apt -y install python3 && rm -rf /var/lib/apt/lists/* LABEL Author="Nanahira <nanahira@momobako.com>"
        RUN apt update && apt -y install python3 build-essential && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/log/*
        WORKDIR /usr/src/app WORKDIR /usr/src/app
        COPY ./package*.json ./ COPY ./package*.json ./
        RUN npm ci
        FROM base as builder
        RUN npm ci && npm cache clean --force
        COPY . ./ COPY . ./
        RUN npm run build RUN npm run build
        CMD ["npm", "run", "start"] FROM base
        ENV NODE_ENV production
        RUN npm ci && npm cache clean --force
        COPY --from=builder /usr/src/app/dist ./dist
        CMD [ "npm", "start" ]
        package-lock.json
        View file @ 593d03fb
        This diff is collapsed.
        package.json
        View file @ 593d03fb
        ...@@ -2,12 +2,11 @@ ...@@ -2,12 +2,11 @@
        "name": "tx3-bang-reader", "name": "tx3-bang-reader",
        "version": "1.0.0", "version": "1.0.0",
        "description": "Read TX3 bang and parse it", "description": "Read TX3 bang and parse it",
        "main": "build/src/run.js", "main": "dist/run.js",
        "scripts": { "scripts": {
        "build": "./node_modules/.bin/tsc", "build": "tsc",
        "pack": "mkdir dist ; ./node_modules/.bin/pkg --out-path dist .", "fetch": "node dist/src/run.js",
        "fetch": "node build/src/run.js", "start": "node dist/src/run.js cron"
        "start": "node build/src/run.js cron"
        }, },
        "repository": { "repository": {
        "type": "git", "type": "git",
        ...@@ -22,17 +21,10 @@ ...@@ -22,17 +21,10 @@
        "url": "https://github.com/purerosefallen/tx3-bang-reader/issues" "url": "https://github.com/purerosefallen/tx3-bang-reader/issues"
        }, },
        "homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme", "homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme",
        "bin": "build/src/run.js", "bin": "dist/src/run.js",
        "pkg": {
        "scripts": [
        "build/src/*.js"
        ],
        "assets": []
        },
        "dependencies": { "dependencies": {
        "@types/cron": "^1.7.2", "@types/cron": "^1.7.2",
        "@types/csv-parse": "^1.2.2", "@types/csv-parse": "^1.2.2",
        "@types/node": "^14.0.14",
        "@types/underscore": "^1.10.2", "@types/underscore": "^1.10.2",
        "@types/yaml": "^1.9.7", "@types/yaml": "^1.9.7",
        "axios": "^0.19.2", "axios": "^0.19.2",
        ...@@ -44,8 +36,21 @@ ...@@ -44,8 +36,21 @@
        "posthtml-parser": "^0.4.2", "posthtml-parser": "^0.4.2",
        "promise-mysql": "^4.1.3", "promise-mysql": "^4.1.3",
        "querystring": "^0.2.0", "querystring": "^0.2.0",
        "typescript": "^3.9.5",
        "underscore": "^1.10.2", "underscore": "^1.10.2",
        "yaml": "^1.10.0" "yaml": "^1.10.0"
        },
        "devDependencies": {
        "@types/jest": "^27.4.0",
        "@types/node": "^17.0.18",
        "@typescript-eslint/eslint-plugin": "^4.33.0",
        "@typescript-eslint/parser": "^4.33.0",
        "eslint": "^7.32.0",
        "eslint-config-prettier": "^8.3.0",
        "eslint-plugin-prettier": "^3.4.1",
        "jest": "^27.5.1",
        "prettier": "^2.5.1",
        "rimraf": "^3.0.2",
        "ts-jest": "^27.1.3",
        "typescript": "^4.5.5"
        } }
        } }
        src/fetcher.ts
        View file @ 593d03fb
        import axios from "axios";
        import _ from "underscore"; import _ from "underscore";
        import { User } from "./user"; import { User } from "./user";
        import {ProxyConfig, ProxyFetcher} from "./proxy"; import { ProxyConfig, ProxyFetcher } from "./proxy";
        import { PlayerRow, parsePlayerRows } from "./playerlist"; import { parsePlayerRows, PlayerRow } from "./playerlist";
        import qs from "querystring"; import qs from "querystring";
        import mysql from "promise-mysql"; import mysql from "promise-mysql";
        import moment from "moment"; import moment from "moment";
        ...@@ -33,15 +32,14 @@ export const servers = [ ...@@ -33,15 +32,14 @@ export const servers = [
        "飞龙在天", "飞龙在天",
        "烽火关东", "烽火关东",
        "盛世长安", "盛世长安",
        ];
        ]
        export interface Config { export interface Config {
        outDir: string; outDir: string;
        server: string[]; server: string[];
        useMySQL: boolean, useMySQL: boolean;
        MySQLConfig: mysql.PoolConfig, MySQLConfig: mysql.PoolConfig;
        proxy: ProxyConfig, proxy: ProxyConfig;
        cronString: string; cronString: string;
        } }
        export class Tx3Fetcher { export class Tx3Fetcher {
        ...@@ -55,9 +53,10 @@ export class Tx3Fetcher { ...@@ -55,9 +53,10 @@ export class Tx3Fetcher {
        } }
        async init() { async init() {
        this.curDate = moment().format("YYYY-MM-DD HH:mm:ss"); this.curDate = moment().format("YYYY-MM-DD HH:mm:ss");
        if(this.config.useMySQL) { if (this.config.useMySQL) {
        this.db = await mysql.createPool(this.config.MySQLConfig); this.db = await mysql.createPool(this.config.MySQLConfig);
        await this.db.query("CREATE TABLE IF NOT EXISTS `userdata` (\n" + await this.db.query(
        "CREATE TABLE IF NOT EXISTS `userdata` (\n" +
        " `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" + " `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" +
        " `date` datetime NOT NULL DEFAULT current_timestamp(),\n" + " `date` datetime NOT NULL DEFAULT current_timestamp(),\n" +
        " `url` varchar(50) COLLATE utf8_unicode_ci NOT NULL,\n" + " `url` varchar(50) COLLATE utf8_unicode_ci NOT NULL,\n" +
        ...@@ -80,19 +79,22 @@ export class Tx3Fetcher { ...@@ -80,19 +79,22 @@ export class Tx3Fetcher {
        " INDEX (server(4)),\n" + " INDEX (server(4)),\n" +
        " INDEX (region(7)),\n" + " INDEX (region(7)),\n" +
        " INDEX (equip)\n" + " INDEX (equip)\n" +
        ") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci"); ") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci"
        );
        console.log(`Removing existing records of ${this.curDate}.`); console.log(`Removing existing records of ${this.curDate}.`);
        //await this.db.query("delete from userdata where date = ?", this.curDate); //await this.db.query("delete from userdata where date = ?", this.curDate);
        } }
        if(this.config.proxy.useProxy) { if (this.config.proxy.useProxy) {
        await this.proxyFetcher.initProxies(); await this.proxyFetcher.initProxies();
        } }
        } }
        async fetchAll(): Promise<Map<string, User[]>> { async fetchAll(): Promise<Map<string, User[]>> {
        const res = new Map<string, User[]>(); const res = new Map<string, User[]>();
        const userLists = await Promise.all(servers.map(server => this.fetchUsersFromServer(server))); const userLists = await Promise.all(
        for (let i = 1; i < userLists.length; ++i){ servers.map((server) => this.fetchUsersFromServer(server))
        res.set(servers[i], userLists[i]) );
        for (let i = 1; i < userLists.length; ++i) {
        res.set(servers[i], userLists[i]);
        } }
        return res; return res;
        } }
        ...@@ -103,10 +105,15 @@ export class Tx3Fetcher { ...@@ -103,10 +105,15 @@ export class Tx3Fetcher {
        resPromises.push(this.fetchUsersFromSchoolAndServer(school, server)); resPromises.push(this.fetchUsersFromSchoolAndServer(school, server));
        } }
        const result = _.flatten(await Promise.all(resPromises)); const result = _.flatten(await Promise.all(resPromises));
        console.log(`Fetched user list with ${result.length} users from server ${server}.`); console.log(
        `Fetched user list with ${result.length} users from server ${server}.`
        );
        return result; return result;
        } }
        async fetchUsersFromSchoolAndServer(school: number, server: string): Promise<User[]> { async fetchUsersFromSchoolAndServer(
        school: number,
        server: string
        ): Promise<User[]> {
        console.log(`Fetching users from server ${server} with school ${school}.`); console.log(`Fetching users from server ${server} with school ${school}.`);
        const res: User[][] = []; const res: User[][] = [];
        for (let page = 1; page <= 25; ++page) { for (let page = 1; page <= 25; ++page) {
        ...@@ -118,9 +125,13 @@ export class Tx3Fetcher { ...@@ -118,9 +125,13 @@ export class Tx3Fetcher {
        } }
        return _.flatten(res); return _.flatten(res);
        } }
        async fetchUsers(school: number, server: string, page: number): Promise<User[]> { async fetchUsers(
        school: number,
        server: string,
        page: number
        ): Promise<User[]> {
        const playerRows = await this.fetchList(school, server, page); const playerRows = await this.fetchList(school, server, page);
        return await Promise.all(playerRows.map(row => this.fetchUser(row))); return await Promise.all(playerRows.map((row) => this.fetchUser(row)));
        } }
        async fetchListFromServer(server: string): Promise<PlayerRow[]> { async fetchListFromServer(server: string): Promise<PlayerRow[]> {
        console.log(`Fetching user list from server ${server}.`); console.log(`Fetching user list from server ${server}.`);
        ...@@ -129,10 +140,15 @@ export class Tx3Fetcher { ...@@ -129,10 +140,15 @@ export class Tx3Fetcher {
        resPromises.push(this.fetchListFromSchoolAndServer(school, server)); resPromises.push(this.fetchListFromSchoolAndServer(school, server));
        } }
        const result = _.flatten(await Promise.all(resPromises)); const result = _.flatten(await Promise.all(resPromises));
        console.log(`Fetched user list from server ${server}. ${result.length} users found.`); console.log(
        `Fetched user list from server ${server}. ${result.length} users found.`
        );
        return result; return result;
        } }
        async fetchListFromSchoolAndServer(school: number, server: string): Promise<PlayerRow[]> { async fetchListFromSchoolAndServer(
        school: number,
        server: string
        ): Promise<PlayerRow[]> {
        console.log(`Fetching users from server ${server} with school ${school}.`); console.log(`Fetching users from server ${server} with school ${school}.`);
        const res: PlayerRow[][] = []; const res: PlayerRow[][] = [];
        for (let page = 1; page <= 25; ++page) { for (let page = 1; page <= 25; ++page) {
        ...@@ -147,45 +163,72 @@ export class Tx3Fetcher { ...@@ -147,45 +163,72 @@ export class Tx3Fetcher {
        return ret; return ret;
        } }
        async checkAddRecord(row: PlayerRow) { async checkAddRecord(row: PlayerRow) {
        const latestRows = await this.db.query("select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by date desc limit 1", row.url); const latestRows = await this.db.query(
        "select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by id desc limit 1",
        row.url
        );
        if (latestRows.length) { if (latestRows.length) {
        const lrow = latestRows[0]; const lrow = latestRows[0];
        if (_.every(["name", "category", "serverArea", "server", "region"], field => lrow[field] === row[field])) { if (
        _.every(
        ["name", "category", "serverArea", "server", "region"],
        (field) => lrow[field] === row[field]
        )
        ) {
        return; return;
        } }
        } }
        const sql = "insert into userdata set ?"; const sql = "insert into userdata set ?";
        const valueObj = { const valueObj = {
        date: this.curDate, date: this.curDate,
        ...row ...row,
        }; };
        console.log(`Player ${row.name} from ${row.server} has changes. Writing record to database: ${sql} ${JSON.stringify(valueObj)} ${JSON.stringify(await this.db.query(sql, valueObj))}`); console.log(
        } `Player ${row.name} from ${
        async fetchList(school: number, server: string, page: number): Promise<PlayerRow[]> { row.server
        console.log(`Fetching user list from server ${server} with school ${school} page ${page}.`); } has changes. Writing record to database: ${sql} ${JSON.stringify(
        valueObj
        )} ${JSON.stringify(await this.db.query(sql, valueObj))}`
        );
        }
        async fetchList(
        school: number,
        server: string,
        page: number
        ): Promise<PlayerRow[]> {
        console.log(
        `Fetching user list from server ${server} with school ${school} page ${page}.`
        );
        try { try {
        const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com/bang/ranks`, { const content: string = await this.proxyFetcher.getWithProxy(
        `http://bang.tx3.163.com/bang/ranks`,
        {
        responseType: "document", responseType: "document",
        params: { params: {
        order_key: "equ_xiuwei", order_key: "equ_xiuwei",
        count: 20, count: 20,
        school, school,
        server, server,
        page page,
        }, },
        paramsSerializer: (params) => { paramsSerializer: (params) => {
        return qs.stringify(params); return qs.stringify(params);
        },
        } }
        }); );
        const playerRows = parsePlayerRows(content); const playerRows = parsePlayerRows(content);
        console.log(`Fetched user list from server ${server} with school ${school} page ${page}. ${playerRows.length} users found.`); console.log(
        `Fetched user list from server ${server} with school ${school} page ${page}. ${playerRows.length} users found.`
        );
        if (this.db) { if (this.db) {
        //await Promise.all(playerRows.map(m => this.db.query("delete from userdata where url = ? and date = ?", [m.url, this.curDate]))); //await Promise.all(playerRows.map(m => this.db.query("delete from userdata where url = ? and date = ?", [m.url, this.curDate])));
        await Promise.all(playerRows.map(m => this.checkAddRecord(m))); await Promise.all(playerRows.map((m) => this.checkAddRecord(m)));
        } }
        return playerRows; return playerRows;
        } catch(e) { } catch (e) {
        console.error(`Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`); console.error(
        `Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`
        );
        return []; return [];
        } }
        } }
        ...@@ -193,13 +236,16 @@ export class Tx3Fetcher { ...@@ -193,13 +236,16 @@ export class Tx3Fetcher {
        const id = playerRow.url.split("/").pop(); const id = playerRow.url.split("/").pop();
        try { try {
        console.log(`Fetching user ${playerRow.name} from ${playerRow.server}.`); console.log(`Fetching user ${playerRow.name} from ${playerRow.server}.`);
        const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com${playerRow.url}`, { const content: string = await this.proxyFetcher.getWithProxy(
        responseType: "document" `http://bang.tx3.163.com${playerRow.url}`,
        }); {
        responseType: "document",
        }
        );
        const user = new User(id, content, playerRow.region); const user = new User(id, content, playerRow.region);
        console.log(`Fetched user ${playerRow.name} from ${playerRow.server}.`); console.log(`Fetched user ${playerRow.name} from ${playerRow.server}.`);
        return user; return user;
        } catch(e) { } catch (e) {
        console.error(`Errored fetching role data from ${id}: ${e.toString()}`); console.error(`Errored fetching role data from ${id}: ${e.toString()}`);
        return null; return null;
        } }
        ......
        src/migrate.ts
        View file @ 593d03fb
        ...@@ -2,13 +2,16 @@ import mysql from "promise-mysql"; ...@@ -2,13 +2,16 @@ import mysql from "promise-mysql";
        import moment from "moment"; import moment from "moment";
        import fs from "fs"; import fs from "fs";
        import _csv_parse from "csv-parse"; import _csv_parse from "csv-parse";
        import util from 'util'; import util from "util";
        import { Config } from "./fetcher"; import { Config } from "./fetcher";
        import { PlayerRow, PlayerRowDated } from "./playerlist"; import { PlayerRow, PlayerRowDated } from "./playerlist";
        import _ from "underscore"; import _ from "underscore";
        import yaml from "yaml"; import yaml from "yaml";
        const parse_csv: (input: Buffer | string, options?: _csv_parse.Options) => Promise<any[]> = util.promisify(_csv_parse); const parse_csv: (
        input: Buffer | string,
        options?: _csv_parse.Options
        ) => Promise<any[]> = util.promisify(_csv_parse);
        let config: Config; let config: Config;
        let db: mysql.Pool; let db: mysql.Pool;
        ...@@ -17,7 +20,10 @@ async function getServerAreaFromServer(server: string) { ...@@ -17,7 +20,10 @@ async function getServerAreaFromServer(server: string) {
        if (serverAreaCache.has(server)) { if (serverAreaCache.has(server)) {
        return serverAreaCache.get(server); return serverAreaCache.get(server);
        } else { } else {
        const [res] = await db.query("select serverArea from userdata where server = ? limit 1", server); const [res] = await db.query(
        "select serverArea from userdata where server = ? limit 1",
        server
        );
        if (!res) { if (!res) {
        return "none"; return "none";
        } }
        ...@@ -27,11 +33,15 @@ async function getServerAreaFromServer(server: string) { ...@@ -27,11 +33,15 @@ async function getServerAreaFromServer(server: string) {
        } }
        } }
        async function readSingleRecord(col: string[], offset: number, base: PlayerRow): Promise<PlayerRowDated> { async function readSingleRecord(
        col: string[],
        offset: number,
        base: PlayerRow
        ): Promise<PlayerRowDated> {
        let pointer = offset; let pointer = offset;
        const newRecord: PlayerRowDated = { const newRecord: PlayerRowDated = {
        date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"), date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"),
        ...(_.clone(base)) ..._.clone(base),
        }; };
        newRecord.name = col[pointer++]; newRecord.name = col[pointer++];
        newRecord.server = col[pointer++]; newRecord.server = col[pointer++];
        ...@@ -49,7 +59,10 @@ async function readColumn(col: string[]): Promise<void> { ...@@ -49,7 +59,10 @@ async function readColumn(col: string[]): Promise<void> {
        const recordCount = parseInt(col[0]); const recordCount = parseInt(col[0]);
        const url = `/bang/role/${col[3]}`; const url = `/bang/role/${col[3]}`;
        console.error(`Reading column ${url}.`); console.error(`Reading column ${url}.`);
        let [base] = await db.query("select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by date asc limit 1", url) as PlayerRow[]; let [base] = (await db.query(
        "select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by id asc limit 1",
        url
        )) as PlayerRow[];
        if (!base) { if (!base) {
        console.error(`Base record of ${url} not found. Using default values.`); console.error(`Base record of ${url} not found. Using default values.`);
        base = { base = {
        ...@@ -63,17 +76,21 @@ async function readColumn(col: string[]): Promise<void> { ...@@ -63,17 +76,21 @@ async function readColumn(col: string[]): Promise<void> {
        region: "none", region: "none",
        score: 0, score: 0,
        equip: 0, equip: 0,
        totalScore: 0 totalScore: 0,
        } };
        } }
        for (let i = 0; i < recordCount; ++i) { for (let i = 0; i < recordCount; ++i) {
        const offset = 4 + (i * 6); const offset = 4 + i * 6;
        if (!col[offset].length) { if (!col[offset].length) {
        continue; continue;
        } }
        const record = await readSingleRecord(col, offset, base); const record = await readSingleRecord(col, offset, base);
        const sql = "insert into userdata set ?"; const sql = "insert into userdata set ?";
        console.log(sql, JSON.stringify(record), JSON.stringify(await db.query(sql, record))); console.log(
        sql,
        JSON.stringify(record),
        JSON.stringify(await db.query(sql, record))
        );
        } }
        console.error(`Read column ${url}. ${--leftCount} columns left.`); console.error(`Read column ${url}. ${--leftCount} columns left.`);
        } }
        ...@@ -81,18 +98,20 @@ async function readColumn(col: string[]): Promise<void> { ...@@ -81,18 +98,20 @@ async function readColumn(col: string[]): Promise<void> {
        async function loadCsv(path: string): Promise<string[][]> { async function loadCsv(path: string): Promise<string[][]> {
        const data = await fs.promises.readFile(path); const data = await fs.promises.readFile(path);
        return await parse_csv(data, { return await parse_csv(data, {
        trim: true trim: true,
        }); });
        } }
        async function main() { async function main() {
        console.error("Started."); console.error("Started.");
        const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8")); const config: Config = yaml.parse(
        await fs.promises.readFile("./config.yaml", "utf8")
        );
        db = await mysql.createPool(config.MySQLConfig); db = await mysql.createPool(config.MySQLConfig);
        const data = await loadCsv(process.argv[2]); const data = await loadCsv(process.argv[2]);
        leftCount = data.length; leftCount = data.length;
        //await Promise.all(data.map(col => readColumn(col))); //await Promise.all(data.map(col => readColumn(col)));
        for (let col of data) { for (const col of data) {
        await readColumn(col); await readColumn(col);
        } }
        console.error("Finished."); console.error("Finished.");
        ......
        src/playerlist.ts
        View file @ 593d03fb
        import HTML from "posthtml-parser"; import HTML from "posthtml-parser";
        import _, { first } from "underscore"; import {
        import {getDepthOfTree, getNumber, findNodeIndex, findNodeIndexByAttribute, findNodeIndexByContent, findNodeIndexByTag, findAllNodeIndex, getContinuousData, getContinuousNodes, getString} from "./utility"; findAllNodeIndex,
        findNodeIndexByTag,
        getContinuousNodes,
        getDepthOfTree,
        getNumber,
        getString,
        } from "./utility";
        export interface PlayerRow { export interface PlayerRow {
        url: string; url: string;
        ...@@ -37,20 +43,26 @@ function getPlayerRowFromTree(tree: HTML.Tree): PlayerRow { ...@@ -37,20 +43,26 @@ function getPlayerRowFromTree(tree: HTML.Tree): PlayerRow {
        region: getString(nodes[6], 7) || "none", region: getString(nodes[6], 7) || "none",
        score: getNumber(nodes[7]), score: getNumber(nodes[7]),
        equip: getNumber(nodes[8]), equip: getNumber(nodes[8]),
        totalScore: getNumber(nodes[9]) totalScore: getNumber(nodes[9]),
        } };
        } }
        export function parsePlayerRows(content: string) { export function parsePlayerRows(content: string) {
        const parsedContent = HTML(content); const parsedContent = HTML(content);
        const tablePos = findNodeIndexByTag(parsedContent, "table", []); const tablePos = findNodeIndexByTag(parsedContent, "table", []);
        const tableTree = getDepthOfTree(parsedContent, tablePos); const tableTree = getDepthOfTree(parsedContent, tablePos);
        const playerPoses = findAllNodeIndex(tableTree, (node) => { const playerPoses = findAllNodeIndex(
        return typeof (node) !== "string" && node.tag === "tr" && node.attrs.class !== "trTop2"; tableTree,
        }, []); (node) => {
        return playerPoses.map(pos => { return (
        typeof node !== "string" &&
        node.tag === "tr" &&
        node.attrs.class !== "trTop2"
        );
        },
        []
        );
        return playerPoses.map((pos) => {
        const tree = getDepthOfTree(tableTree, pos); const tree = getDepthOfTree(tableTree, pos);
        return getPlayerRowFromTree(tree); return getPlayerRowFromTree(tree);
        }); });
        ......
        src/proxy.ts
        View file @ 593d03fb
        ...@@ -9,36 +9,36 @@ import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios"; ...@@ -9,36 +9,36 @@ import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios";
        //} //}
        export interface ProxyConfig { export interface ProxyConfig {
        useProxy: boolean, useProxy: boolean;
        proxySource: string[], proxySource: string[];
        timeout: number timeout: number;
        } }
        const agentList = [ const agentList = [
        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0', "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0",
        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)', "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11', "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11', "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)' "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
        ] ];
        async function testProxy(proxy: AxiosProxyConfig) { async function testProxy(proxy: AxiosProxyConfig) {
        await axios.get("http://mirrors.aliyun.com/debian/pool", { await axios.get("http://mirrors.aliyun.com/debian/pool", {
        proxy, proxy,
        headers: { headers: {
        "User-Agent": agentList[4] "User-Agent": agentList[4],
        }, },
        timeout: this.config.timeout, timeout: this.config.timeout,
        }); });
        ...@@ -81,57 +81,73 @@ export class ProxyFetcher { ...@@ -81,57 +81,73 @@ export class ProxyFetcher {
        if (!this.config.useProxy) { if (!this.config.useProxy) {
        return; return;
        } }
        console.log(`Fetching proxies from ${url}.`) console.log(`Fetching proxies from ${url}.`);
        while (true) { while (true) {
        try { try {
        const proxyPage: string = (await axios.get(url, { const proxyPage: string = (
        await axios.get(url, {
        responseType: "document", responseType: "document",
        })).data; })
        const proxies: AxiosProxyConfig[] = proxyPage.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g).map(proxyString => { ).data;
        const proxies: AxiosProxyConfig[] = proxyPage
        .match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g)
        .map((proxyString) => {
        const [host, _port] = proxyString.split(":"); const [host, _port] = proxyString.split(":");
        const port = parseInt(_port); const port = parseInt(_port);
        const proxy = { host, port }; const proxy = { host, port };
        return proxy; return proxy;
        }); });
        //const usableProxies = await filterProxies(proxies); //const usableProxies = await filterProxies(proxies);
        for (let proxy of proxies) { for (const proxy of proxies) {
        this.proxies.push(proxy); this.proxies.push(proxy);
        } }
        console.error(`Got ${proxies.length} proxies from ${url}.`); console.error(`Got ${proxies.length} proxies from ${url}.`);
        return; return;
        } catch (e) { } catch (e) {
        console.error(`Failed fetching proxy list from ${url}: ${e.toString()}`) console.error(
        `Failed fetching proxy list from ${url}: ${e.toString()}`
        );
        } }
        } }
        } }
        async initProxies() { async initProxies() {
        await Promise.all(this.config.proxySource.map((m) => { await Promise.all(
        this.config.proxySource.map((m) => {
        return this.initProxiesFrom(m); return this.initProxiesFrom(m);
        })); })
        );
        } }
        async getWithProxy(url: string, options: AxiosRequestConfig) { async getWithProxy(url: string, options: AxiosRequestConfig) {
        while (true) { while (true) {
        if (this.config.useProxy && !this.proxies.length) { if (this.config.useProxy && !this.proxies.length) {
        await this.initProxies(); await this.initProxies();
        } }
        const proxyIndex = !this.config.useProxy ? null : (++this.counter) % this.proxies.length; const proxyIndex = !this.config.useProxy
        ? null
        : ++this.counter % this.proxies.length;
        //const proxyIndex = 0; //const proxyIndex = 0;
        const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex]; const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex];
        try { try {
        const data = (await axios.get(url, { const data = (
        await axios.get(url, {
        proxy, proxy,
        headers: { headers: {
        "User-Agent": agentList[this.counter % agentList.length] "User-Agent": agentList[this.counter % agentList.length],
        }, },
        timeout: this.config.timeout, timeout: this.config.timeout,
        ...options ...options,
        })).data; })
        ).data;
        return data; return data;
        } catch (e) { } catch (e) {
        if (this.config.useProxy) { if (this.config.useProxy) {
        this.proxies.splice(proxyIndex, 1); this.proxies.splice(proxyIndex, 1);
        } }
        console.error(`Failed fetching data from ${url}: ${e.toString()} ${this.proxies.length} proxies left.`) console.error(
        `Failed fetching data from ${url}: ${e.toString()} ${
        this.proxies.length
        } proxies left.`
        );
        } }
        } }
        } }
        ......
        src/remove-dup.ts
        View file @ 593d03fb
        ...@@ -6,17 +6,24 @@ import _ from "underscore"; ...@@ -6,17 +6,24 @@ import _ from "underscore";
        import yaml from "yaml"; import yaml from "yaml";
        function checkSameRow(row: PlayerRowFull, lrow: PlayerRowFull) { function checkSameRow(row: PlayerRowFull, lrow: PlayerRowFull) {
        return _.every(["name", "category", "serverArea", "server", "region"], field => lrow[field] === row[field]); return _.every(
        ["name", "category", "serverArea", "server", "region"],
        (field) => lrow[field] === row[field]
        );
        } }
        async function main() { async function main() {
        console.error("Started."); console.error("Started.");
        const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8")); const config: Config = yaml.parse(
        await fs.promises.readFile("./config.yaml", "utf8")
        );
        const db = await mysql.createPool(config.MySQLConfig); const db = await mysql.createPool(config.MySQLConfig);
        const urlDataCache = new Map<string, PlayerRowFull>(); const urlDataCache = new Map<string, PlayerRowFull>();
        const deleteList: number[] = []; const deleteList: number[] = [];
        const datas: PlayerRowFull[] = await db.query(`select * from userdata order by date asc`); const datas: PlayerRowFull[] = await db.query(
        for (let row of datas) { `select * from userdata order by id asc`
        );
        for (const row of datas) {
        if (urlDataCache.has(row.url)) { if (urlDataCache.has(row.url)) {
        const oldRow = urlDataCache.get(row.url); const oldRow = urlDataCache.get(row.url);
        if (checkSameRow(row, oldRow)) { if (checkSameRow(row, oldRow)) {
        ...@@ -26,9 +33,11 @@ async function main() { ...@@ -26,9 +33,11 @@ async function main() {
        urlDataCache.set(row.url, row); urlDataCache.set(row.url, row);
        } }
        console.error(`Deletes: ${deleteList.length}`); console.error(`Deletes: ${deleteList.length}`);
        for (let id of deleteList) { for (const id of deleteList) {
        const sql = `delete from userdata where id = ?`; const sql = `delete from userdata where id = ?`;
        console.error(`Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`); console.error(
        `Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`
        );
        } }
        console.error("Finished."); console.error("Finished.");
        process.exit(); process.exit();
        ......
        src/run.ts
        View file @ 593d03fb
        import {Tx3Fetcher, servers, Config} from "./fetcher"; import { Config, servers, Tx3Fetcher } from "./fetcher";
        import fs from "fs"; import fs from "fs";
        import _ from "underscore";
        import yaml from "yaml"; import yaml from "yaml";
        import { CronJob } from "cron"; import { CronJob } from "cron";
        ...@@ -12,10 +11,17 @@ async function loadConfig() { ...@@ -12,10 +11,17 @@ async function loadConfig() {
        async function runServer(fetcher: Tx3Fetcher, server: string) { async function runServer(fetcher: Tx3Fetcher, server: string) {
        const users = await fetcher.fetchListFromServer(server); const users = await fetcher.fetchListFromServer(server);
        await fs.promises.writeFile(`./output/servers/${server}.json`, JSON.stringify({ await fs.promises.writeFile(
        `./output/servers/${server}.json`,
        JSON.stringify(
        {
        date: fetcher.curDate, date: fetcher.curDate,
        data: users data: users,
        }, null, 2)); },
        null,
        2
        )
        );
        return users; return users;
        } }
        ...@@ -25,27 +31,38 @@ async function run() { ...@@ -25,27 +31,38 @@ async function run() {
        await fs.promises.access("./output/servers"); await fs.promises.access("./output/servers");
        } catch (e) { } catch (e) {
        await fs.promises.mkdir("./output/servers", { await fs.promises.mkdir("./output/servers", {
        recursive: true recursive: true,
        }); });
        } }
        const fetcher = new Tx3Fetcher(config); const fetcher = new Tx3Fetcher(config);
        await fetcher.init(); await fetcher.init();
        if (config.server) { if (config.server) {
        await Promise.all(config.server.map(server => { await Promise.all(
        return runServer(fetcher, server) config.server.map((server) => {
        })); return runServer(fetcher, server);
        })
        );
        } else { } else {
        const userListWithServer = await Promise.all(servers.map(server => { const userListWithServer = await Promise.all(
        return runServer(fetcher, server) servers.map((server) => {
        })); return runServer(fetcher, server);
        })
        );
        const allServersList: any = {}; const allServersList: any = {};
        for (let i = 0; i < servers.length;++i) { for (let i = 0; i < servers.length; ++i) {
        allServersList[servers[i]] = userListWithServer[i]; allServersList[servers[i]] = userListWithServer[i];
        } }
        await fs.promises.writeFile(`./output/all.json`, JSON.stringify({ await fs.promises.writeFile(
        `./output/all.json`,
        JSON.stringify(
        {
        date: fetcher.curDate, date: fetcher.curDate,
        data: allServersList data: allServersList,
        }, null, 2)); },
        null,
        2
        )
        );
        } }
        console.log("Finished."); console.log("Finished.");
        } }
        ...@@ -53,12 +70,19 @@ async function run() { ...@@ -53,12 +70,19 @@ async function run() {
        async function main() { async function main() {
        await loadConfig(); await loadConfig();
        if (process.argv[2] === "cron") { if (process.argv[2] === "cron") {
        const job = new CronJob(config.cronString, run, null, true, "Asia/Shanghai", null, true); const job = new CronJob(
        config.cronString,
        run,
        null,
        true,
        "Asia/Shanghai",
        null,
        true
        );
        job.start(); job.start();
        } else { } else {
        await run(); await run();
        process.exit(); process.exit();
        } }
        } }
        main(); main();
        src/user.ts
        View file @ 593d03fb
        import HTML from "posthtml-parser"; import HTML from "posthtml-parser";
        import _, { first } from "underscore"; import _ from "underscore";
        import {getDepthOfTree, getNumber, findNodeIndex, findNodeIndexByAttribute, findNodeIndexByContent, findNodeIndexByTag, getContinuousNumber} from "./utility"; import {
        findNodeIndexByAttribute,
        findNodeIndexByContent,
        getContinuousNumber,
        getDepthOfTree,
        getNumber,
        } from "./utility";
        export interface AttackAttribute { export interface AttackAttribute {
        '攻力': number, //大攻取前2字节int16，小攻取后2字节int16，法力也是这样 攻力: number; //大攻取前2字节int16，小攻取后2字节int16，法力也是这样
        '命中': number, 命中: number;
        '法力': number, 法力: number;
        '重击': number, 重击: number;
        '会心一击': number, 会心一击: number;
        '附加伤害': number 附加伤害: number;
        } }
        export interface DefenseAttribute { export interface DefenseAttribute {
        '防御': number, 防御: number;
        '回避': number, 回避: number;
        '法防': number, 法防: number;
        '神明': number, 神明: number;
        '化解': number, 化解: number;
        '知彼': number 知彼: number;
        } }
        export interface SpecialAttribute { export interface SpecialAttribute {
        '身法': number, 身法: number;
        '坚韧': number, 坚韧: number;
        '定力': number, 定力: number;
        '诛心': number, 诛心: number;
        '御心': number, 御心: number;
        '万钧': number, 万钧: number;
        '铁壁': number 铁壁: number;
        } }
        export interface AdvancedAttribute { '追电': number, '骤雨': number, '疾语': number, '明思': number, '扰心': number, '人祸': number } export interface AdvancedAttribute {
        追电: number;
        骤雨: number;
        疾语: number;
        明思: number;
        扰心: number;
        人祸: number;
        }
        export class User { export class User {
        id: string; id: string;
        ...@@ -47,7 +59,7 @@ export class User { ...@@ -47,7 +59,7 @@ export class User {
        equipLocalRank: number; equipLocalRank: number;
        equipCategoryRank: number; equipCategoryRank: number;
        scoreValue: number; scoreValue: number;
        scoreRank: number scoreRank: number;
        scoreLocalRank: number; scoreLocalRank: number;
        scoreCategoryRank: number; scoreCategoryRank: number;
        sqStage: number; // 天魂：2，地魂：1，没有神启：null sqStage: number; // 天魂：2，地魂：1，没有神启：null
        ...@@ -69,14 +81,24 @@ export class User { ...@@ -69,14 +81,24 @@ export class User {
        yhz: string[]; yhz: string[];
        private parseMetadata() { private parseMetadata() {
        let namePos = findNodeIndexByAttribute(this.content, "class", "sTitle", []); const namePos = findNodeIndexByAttribute(
        this.content,
        "class",
        "sTitle",
        []
        );
        this.name = getDepthOfTree(this.content, namePos.concat([0]))[0] as string; this.name = getDepthOfTree(this.content, namePos.concat([0]))[0] as string;
        namePos[namePos.length - 1] += 2; namePos[namePos.length - 1] += 2;
        this.category = getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string; this.category = getDepthOfTree(
        this.content,
        namePos.concat([0, 0])
        )[0] as string;
        namePos[namePos.length - 1] += 2; namePos[namePos.length - 1] += 2;
        [this.serverArea, this.server] = (getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string).split("&nbsp;"); [this.serverArea, this.server] = (
        getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string
        ).split("&nbsp;");
        let levelPos = findNodeIndexByContent(this.content, "等级", []); const levelPos = findNodeIndexByContent(this.content, "等级", []);
        levelPos.pop(); levelPos.pop();
        levelPos[levelPos.length - 1]++; levelPos[levelPos.length - 1]++;
        this.level = getNumber(getDepthOfTree(this.content, levelPos)[0]); this.level = getNumber(getDepthOfTree(this.content, levelPos)[0]);
        ...@@ -110,7 +132,7 @@ export class User { ...@@ -110,7 +132,7 @@ export class User {
        const tree = getDepthOfTree(this.content, pos); const tree = getDepthOfTree(this.content, pos);
        for (let i = 3; i < tree.length; i += 2) { for (let i = 3; i < tree.length; i += 2) {
        const [_keyNode, valueNode] = getDepthOfTree(tree, [i]); const [_keyNode, valueNode] = getDepthOfTree(tree, [i]);
        const keyNode = (_keyNode as HTML.NodeTag); const keyNode = _keyNode as HTML.NodeTag;
        const key = keyNode.content[0] as string; const key = keyNode.content[0] as string;
        const value = getNumber(valueNode); const value = getNumber(valueNode);
        ret[key] = value; ret[key] = value;
        ...@@ -118,7 +140,7 @@ export class User { ...@@ -118,7 +140,7 @@ export class User {
        return ret; return ret;
        } }
        private parseBasicAttributes() { private parseBasicAttributes() {
        let ValuePos = findNodeIndexByContent(this.content, "命", []); const ValuePos = findNodeIndexByContent(this.content, "命", []);
        ValuePos.pop(); ValuePos.pop();
        ValuePos[ValuePos.length - 1] += 2; ValuePos[ValuePos.length - 1] += 2;
        const datas = getContinuousNumber(this.content, ValuePos, 0, 4, 8); const datas = getContinuousNumber(this.content, ValuePos, 0, 4, 8);
        ...@@ -133,22 +155,35 @@ export class User { ...@@ -133,22 +155,35 @@ export class User {
        ValuePos.pop(); ValuePos.pop();
        ValuePos[ValuePos.length - 1] += 2; ValuePos[ValuePos.length - 1] += 2;
        this.attackAttributes = this.parseAttributeTable(ValuePos) as AttackAttribute; this.attackAttributes = this.parseAttributeTable(
        ValuePos
        ) as AttackAttribute;
        ValuePos[ValuePos.length - 1] += 2; ValuePos[ValuePos.length - 1] += 2;
        this.defenseAttributes = this.parseAttributeTable(ValuePos) as DefenseAttribute; this.defenseAttributes = this.parseAttributeTable(
        ValuePos
        ) as DefenseAttribute;
        ValuePos[ValuePos.length - 1] += 2; ValuePos[ValuePos.length - 1] += 2;
        ValuePos.push(1); ValuePos.push(1);
        this.specialAttributes = this.parseAttributeTable(ValuePos) as SpecialAttribute; this.specialAttributes = this.parseAttributeTable(
        ValuePos
        ) as SpecialAttribute;
        ValuePos[ValuePos.length - 1] += 2; ValuePos[ValuePos.length - 1] += 2;
        this.advancedAttributes = this.parseAttributeTable(ValuePos) as AdvancedAttribute; this.advancedAttributes = this.parseAttributeTable(
        ValuePos
        ) as AdvancedAttribute;
        } }
        private parseYHZ() { private parseYHZ() {
        let ValuePos = findNodeIndexByAttribute(this.content, "id", "tableYHZ", []); const ValuePos = findNodeIndexByAttribute(
        this.content,
        "id",
        "tableYHZ",
        []
        );
        ValuePos.push(1); ValuePos.push(1);
        const tree = getDepthOfTree(this.content, ValuePos); const tree = getDepthOfTree(this.content, ValuePos);
        this.yhz = []; this.yhz = [];
        for (let i = 1; i < tree.length; i += 2){ for (let i = 1; i < tree.length; i += 2) {
        const node = (tree[i] as HTML.NodeTag); const node = tree[i] as HTML.NodeTag;
        this.yhz.push(node.content[0] as string); this.yhz.push(node.content[0] as string);
        } }
        } }
        ......
        src/utility.ts
        View file @ 593d03fb
        import HTML from "posthtml-parser"; import HTML from "posthtml-parser";
        import _ from "underscore"; import _ from "underscore";
        export function getDepthOfTree(tree: HTML.Tree, indexList: number[]): HTML.Tree {
        export function getDepthOfTree(
        tree: HTML.Tree,
        indexList: number[]
        ): HTML.Tree {
        if (indexList.length) { if (indexList.length) {
        const _indexList = _.clone(indexList); const _indexList = _.clone(indexList);
        const index = _indexList.splice(0, 1)[0]; const index = _indexList.splice(0, 1)[0];
        const node = tree[index]; const node = tree[index];
        if (typeof (node) === "string" || !node.content) { if (typeof node === "string" || !node.content) {
        return [node]; return [node];
        } }
        return getDepthOfTree(node.content, _indexList); return getDepthOfTree(node.content, _indexList);
        ...@@ -14,7 +18,11 @@ export function getDepthOfTree(tree: HTML.Tree, indexList: number[]): HTML.Tree ...@@ -14,7 +18,11 @@ export function getDepthOfTree(tree: HTML.Tree, indexList: number[]): HTML.Tree
        } }
        } }
        export function findNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node) => boolean, offset: number[]): number[] { export function findNodeIndex(
        baseTree: HTML.Tree,
        condition: (node: HTML.Node) => boolean,
        offset: number[]
        ): number[] {
        const queue = [offset]; const queue = [offset];
        while (queue.length) { while (queue.length) {
        const indexList = queue.splice(0, 1)[0]; const indexList = queue.splice(0, 1)[0];
        ...@@ -24,7 +32,7 @@ export function findNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node) ...@@ -24,7 +32,7 @@ export function findNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node)
        const newList = indexList.concat([i]); const newList = indexList.concat([i]);
        if (condition(node)) { if (condition(node)) {
        return newList; return newList;
        } else if (typeof (node) !== "string") { } else if (typeof node !== "string") {
        queue.push(newList); queue.push(newList);
        } }
        } }
        ...@@ -32,7 +40,11 @@ export function findNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node) ...@@ -32,7 +40,11 @@ export function findNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node)
        return null; return null;
        } }
        export function findAllNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node) => boolean, offset: number[]): number[][] { export function findAllNodeIndex(
        baseTree: HTML.Tree,
        condition: (node: HTML.Node) => boolean,
        offset: number[]
        ): number[][] {
        const queue = [offset]; const queue = [offset];
        const res: number[][] = []; const res: number[][] = [];
        while (queue.length) { while (queue.length) {
        ...@@ -44,7 +56,7 @@ export function findAllNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Nod ...@@ -44,7 +56,7 @@ export function findAllNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Nod
        if (condition(node)) { if (condition(node)) {
        //console.log(newList); //console.log(newList);
        res.push(newList); res.push(newList);
        } else if (typeof (node) !== "string") { } else if (typeof node !== "string") {
        queue.push(newList); queue.push(newList);
        } }
        } }
        ...@@ -52,47 +64,112 @@ export function findAllNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Nod ...@@ -52,47 +64,112 @@ export function findAllNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Nod
        return res; return res;
        } }
        export function findNodeIndexByContent(baseTree: HTML.Tree, label: string, offset: number[]): number[] { export function findNodeIndexByContent(
        return findNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        label: string,
        offset: number[]
        ): number[] {
        return findNodeIndex(
        baseTree,
        (node) => {
        return node === label; return node === label;
        }, offset); },
        offset
        );
        } }
        export function findNodeIndexByAttribute(baseTree: HTML.Tree, key: string, value: string, offset: number[]): number[] { export function findNodeIndexByAttribute(
        return findNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        return typeof (node) !== "string" && node.attrs && node.attrs[key] === value; key: string,
        }, offset); value: string,
        offset: number[]
        ): number[] {
        return findNodeIndex(
        baseTree,
        (node) => {
        return (
        typeof node !== "string" && node.attrs && node.attrs[key] === value
        );
        },
        offset
        );
        } }
        export function findNodeIndexByTag(baseTree: HTML.Tree, tag: string, offset: number[]): number[] { export function findNodeIndexByTag(
        return findNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        return typeof (node) !== "string" && node.tag === tag; tag: string,
        }, offset); offset: number[]
        ): number[] {
        return findNodeIndex(
        baseTree,
        (node) => {
        return typeof node !== "string" && node.tag === tag;
        },
        offset
        );
        } }
        export function findAllNodeIndexByContent(baseTree: HTML.Tree, label: string, offset: number[]): number[][] { export function findAllNodeIndexByContent(
        return findAllNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        label: string,
        offset: number[]
        ): number[][] {
        return findAllNodeIndex(
        baseTree,
        (node) => {
        return node === label; return node === label;
        }, offset); },
        offset
        );
        } }
        export function findAllNodeIndexByAttribute(baseTree: HTML.Tree, key: string, value: string, offset: number[]): number[][] { export function findAllNodeIndexByAttribute(
        return findAllNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        return typeof (node) !== "string" && node.attrs && node.attrs[key] === value; key: string,
        }, offset); value: string,
        offset: number[]
        ): number[][] {
        return findAllNodeIndex(
        baseTree,
        (node) => {
        return (
        typeof node !== "string" && node.attrs && node.attrs[key] === value
        );
        },
        offset
        );
        } }
        export function findAllNodeIndexByTag(baseTree: HTML.Tree, tag: string, offset: number[]): number[][] { export function findAllNodeIndexByTag(
        return findAllNodeIndex(baseTree, (node) => { baseTree: HTML.Tree,
        return typeof (node) !== "string" && node.tag === tag; tag: string,
        }, offset); offset: number[]
        ): number[][] {
        return findAllNodeIndex(
        baseTree,
        (node) => {
        return typeof node !== "string" && node.tag === tag;
        },
        offset
        );
        } }
        const chineseCapitalNumbers = ["零", "壹", "贰", "叁", "肆", "伍", "陆", "柒", "捌", "玖"] const chineseCapitalNumbers = [
        "零",
        "壹",
        "贰",
        "叁",
        "肆",
        "伍",
        "陆",
        "柒",
        "捌",
        "玖",
        ];
        export function getString(node: HTML.Node, lengthLimit?: number) { export function getString(node: HTML.Node, lengthLimit?: number) {
        let resultStr: string; let resultStr: string;
        if (typeof (node) === "string") { if (typeof node === "string") {
        resultStr = node; resultStr = node;
        } else { } else {
        const subTree = node.content; const subTree = node.content;
        ...@@ -100,7 +177,7 @@ export function getString(node: HTML.Node, lengthLimit?: number) { ...@@ -100,7 +177,7 @@ export function getString(node: HTML.Node, lengthLimit?: number) {
        return null; return null;
        } }
        const subNode = subTree[0]; const subNode = subTree[0];
        if (typeof (subNode) === "string") { if (typeof subNode === "string") {
        resultStr = subTree[0] as string; resultStr = subTree[0] as string;
        } else { } else {
        resultStr = getString(subTree[0]) as string; resultStr = getString(subTree[0]) as string;
        ...@@ -118,11 +195,14 @@ export function getNumber(node: HTML.Node) { ...@@ -118,11 +195,14 @@ export function getNumber(node: HTML.Node) {
        let stringMatch: RegExpMatchArray; let stringMatch: RegExpMatchArray;
        if (numberStr === "没有上榜" || !numberStr) { if (numberStr === "没有上榜" || !numberStr) {
        return null; return null;
        } else if (stringMatch = numberStr.match(/^([天地])魂$/)) { } else if ((stringMatch = numberStr.match(/^([天地])魂$/))) {
        return stringMatch[1] === "天" ? 2 : 1; return stringMatch[1] === "天" ? 2 : 1;
        } else if (stringMatch = numberStr.match(/^(.+)天(.+)境界$/)) { } else if ((stringMatch = numberStr.match(/^(.+)天(.+)境界$/))) {
        return (_.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[1]) << 8) | _.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[2]); return (
        } else if (stringMatch = numberStr.match(/^(\d+)-(\d+)$/)) { (_.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[1]) << 8) |
        _.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[2])
        );
        } else if ((stringMatch = numberStr.match(/^(\d+)-(\d+)$/))) {
        const minValue = parseInt(stringMatch[1]); const minValue = parseInt(stringMatch[1]);
        const maxValue = parseInt(stringMatch[2]); const maxValue = parseInt(stringMatch[2]);
        return (minValue << 16) | maxValue; return (minValue << 16) | maxValue;
        ...@@ -131,7 +211,13 @@ export function getNumber(node: HTML.Node) { ...@@ -131,7 +211,13 @@ export function getNumber(node: HTML.Node) {
        } }
        } }
        export function getContinuousNodes(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): HTML.Node[] { export function getContinuousNodes(
        tree: HTML.Tree,
        _pos: number[],
        moveOffset: number,
        step: number,
        dataCount: number
        ): HTML.Node[] {
        const pos = _.clone(_pos); const pos = _.clone(_pos);
        const datas: HTML.Node[] = []; const datas: HTML.Node[] = [];
        for (let i = 0; i < dataCount; ++i) { for (let i = 0; i < dataCount; ++i) {
        ...@@ -142,10 +228,26 @@ export function getContinuousNodes(tree: HTML.Tree, _pos: number[], moveOffset: ...@@ -142,10 +228,26 @@ export function getContinuousNodes(tree: HTML.Tree, _pos: number[], moveOffset:
        return datas; return datas;
        } }
        export function getContinuousData(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): string[] { export function getContinuousData(
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(getString); tree: HTML.Tree,
        _pos: number[],
        moveOffset: number,
        step: number,
        dataCount: number
        ): string[] {
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(
        getString
        );
        } }
        export function getContinuousNumber(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): number[] { export function getContinuousNumber(
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(getNumber); tree: HTML.Tree,
        _pos: number[],
        moveOffset: number,
        step: number,
        dataCount: number
        ): number[] {
        return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(
        getNumber
        );
        } }
        tests/test-proxy.ts
        View file @ 593d03fb
        ...@@ -4,9 +4,9 @@ async function main() { ...@@ -4,9 +4,9 @@ async function main() {
        useProxy: true, useProxy: true,
        proxySource: [ proxySource: [
        "http://www.89ip.cn/tqdl.html?api=1&num=9999", "http://www.89ip.cn/tqdl.html?api=1&num=9999",
        "http://www.66ip.cn/mo.php?tqsl=9999" "http://www.66ip.cn/mo.php?tqsl=9999",
        ], ],
        timeout: 10000 timeout: 10000,
        }); });
        await fetcher.initProxies(); await fetcher.initProxies();
        console.log(await fetcher.getWithProxy("https://mycard.moe", {})); console.log(await fetcher.getWithProxy("https://mycard.moe", {}));
        ......
        tests/test-user.ts
        View file @ 593d03fb
        ...@@ -5,11 +5,17 @@ import HTML from "posthtml-parser"; ...@@ -5,11 +5,17 @@ import HTML from "posthtml-parser";
        async function main() { async function main() {
        let id = "28_20588"; let id = "28_20588";
        let html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8"); let html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2)); await fs.promises.writeFile(
        `./tests/${id}.json`,
        JSON.stringify(HTML(html), null, 2)
        );
        console.log(new User(id, html, null)); console.log(new User(id, html, null));
        id = "6_18804822"; id = "6_18804822";
        html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8"); html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2)); await fs.promises.writeFile(
        `./tests/${id}.json`,
        JSON.stringify(HTML(html), null, 2)
        );
        console.log(new User(id, html, null)); console.log(new User(id, html, null));
        } }
        main(); main();
        tsconfig.json
        View file @ 593d03fb
        { {
        "compilerOptions": { "compilerOptions": {
        "outDir": "build", "outDir": "dist",
        "module": "commonjs", "module": "commonjs",
        "target": "esnext", "target": "es2021",
        "esModuleInterop": true, "esModuleInterop": true,
        "emitDecoratorMetadata": true,
        "experimentalDecorators": true,
        "declaration": true, "declaration": true,
        "sourceMap": true "sourceMap": true
        }, },
        "compileOnSave": true, "compileOnSave": true,
        "allowJs": true, "allowJs": true,
        "include": [ "include": [
        "src/*.ts", "*.ts",
        "tests/*.ts" "src/**/*.ts",
        "test/**/*.ts",
        "tests/**/*.ts"
        ] ]
        } }
        Markdown is supported
        0% or
        You are about to add 0 people to the discussion. Proceed with caution.
        Finish editing this message first!
        Please register or to comment