Skip to content

T
tx3-bang-reader
Commit 593d03fb authored by nanahira's avatar nanahira
Browse files
Options

rework

parent 85d2aa62
Pipeline #9850 failed with stages
        in 1 minute and 33 seconds
        Expand all Hide whitespace changes
        Inline Side-by-side
        Showing with 12316 additions and 1266 deletions
        .eslintignore 0 → 100644
        View file @ 593d03fb
        webpack.config.js
        dist/*
        build/*
        *.js
        .eslintrc.js 0 → 100644
        View file @ 593d03fb
        module.exports = {
        parser: '@typescript-eslint/parser',
        parserOptions: {
        project: 'tsconfig.json',
        sourceType: 'module',
        },
        plugins: ['@typescript-eslint/eslint-plugin'],
        extends: [
        'plugin:@typescript-eslint/recommended',
        'plugin:prettier/recommended',
        ],
        root: true,
        env: {
        node: true,
        jest: true,
        },
        ignorePatterns: ['.eslintrc.js'],
        rules: {
        '@typescript-eslint/interface-name-prefix': 'off',
        '@typescript-eslint/explicit-function-return-type': 'off',
        '@typescript-eslint/explicit-module-boundary-types': 'off',
        '@typescript-eslint/no-explicit-any': 'off',
        },
        };
        .gitlab-ci.yml
        View file @ 593d03fb
        stages: stages:
        - install
        - build - build
        - deploy - deploy
        variables: variables:
        GIT_DEPTH: "1" GIT_DEPTH: "1"
        CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
        CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest
        docker: npm_ci:
        stage: install
        tags:
        - linux
        script:
        - npm ci
        artifacts:
        paths:
        - node_modules
        .build_base:
        stage: build stage: build
        tags: tags:
        - docker - linux
        before_script: dependencies:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - npm_ci
        build:
        extends:
        - .build_base
        script: script:
        - docker build --pull -t $CONTAINER_TEST_IMAGE . - npm run build
        - docker push $CONTAINER_TEST_IMAGE artifacts:
        paths:
        - dist/
        deploy_latest: unit-test:
        stage: deploy extends:
        tags: - .build_base
        - docker
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        script: script:
        - docker pull $CONTAINER_TEST_IMAGE - npm run test
        - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
        - docker push $CONTAINER_RELEASE_IMAGE
        only:
        - master
        deploy_tag: deploy_npm:
        stage: deploy stage: deploy
        tags: dependencies:
        - docker - build
        variables: tags:
        CONTAINER_TAG_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG - linux
        before_script:
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        script: script:
        - docker pull $CONTAINER_TEST_IMAGE - apt update;apt -y install coreutils
        - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_TAG_IMAGE - echo $NPMRC | base64 --decode > ~/.npmrc
        - docker push $CONTAINER_TAG_IMAGE - npm publish . || true
        only: only:
        - tags - master
        .idea/inspectionProfiles/Project_Default.xml 0 → 100644
        View file @ 593d03fb
        <component name="InspectionProjectProfileManager">
        <profile version="1.0">
        <option name="myName" value="Project Default" />
        <inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
        </profile>
        </component>
        \ No newline at end of file
        .idea/jsLinters/eslint.xml 0 → 100644
        View file @ 593d03fb
        <?xml version="1.0" encoding="UTF-8"?>
        <project version="4">
        <component name="EslintConfiguration">
        <option name="fix-on-save" value="true" />
        </component>
        </project>
        \ No newline at end of file
        .npmignore 0 → 100644
        View file @ 593d03fb
        /install-npm.sh
        .git*
        /data
        /output
        /config.yaml
        .idea
        .dockerignore
        Dockerfile
        /src
        /coverage
        /tests
        /dist/tests
        Dockerfile
        View file @ 593d03fb
        FROM node:bullseye-slim FROM node:bullseye-slim as base
        RUN apt update && apt -y install python3 && rm -rf /var/lib/apt/lists/* LABEL Author="Nanahira <nanahira@momobako.com>"
        RUN apt update && apt -y install python3 build-essential && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/log/*
        WORKDIR /usr/src/app WORKDIR /usr/src/app
        COPY ./package*.json ./ COPY ./package*.json ./
        RUN npm ci
        FROM base as builder
        RUN npm ci && npm cache clean --force
        COPY . ./ COPY . ./
        RUN npm run build RUN npm run build
        CMD ["npm", "run", "start"] FROM base
        ENV NODE_ENV production
        RUN npm ci && npm cache clean --force
        COPY --from=builder /usr/src/app/dist ./dist
        CMD [ "npm", "start" ]
        package-lock.json
        View file @ 593d03fb
        This diff is collapsed.
        package.json
        View file @ 593d03fb
        ...@@ -2,12 +2,11 @@ ...@@ -2,12 +2,11 @@
        "name": "tx3-bang-reader", "name": "tx3-bang-reader",
        "version": "1.0.0", "version": "1.0.0",
        "description": "Read TX3 bang and parse it", "description": "Read TX3 bang and parse it",
        "main": "build/src/run.js", "main": "dist/run.js",
        "scripts": { "scripts": {
        "build": "./node_modules/.bin/tsc", "build": "tsc",
        "pack": "mkdir dist ; ./node_modules/.bin/pkg --out-path dist .", "fetch": "node dist/src/run.js",
        "fetch": "node build/src/run.js", "start": "node dist/src/run.js cron"
        "start": "node build/src/run.js cron"
        }, },
        "repository": { "repository": {
        "type": "git", "type": "git",
        ...@@ -22,17 +21,10 @@ ...@@ -22,17 +21,10 @@
        "url": "https://github.com/purerosefallen/tx3-bang-reader/issues" "url": "https://github.com/purerosefallen/tx3-bang-reader/issues"
        }, },
        "homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme", "homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme",
        "bin": "build/src/run.js", "bin": "dist/src/run.js",
        "pkg": {
        "scripts": [
        "build/src/*.js"
        ],
        "assets": []
        },
        "dependencies": { "dependencies": {
        "@types/cron": "^1.7.2", "@types/cron": "^1.7.2",
        "@types/csv-parse": "^1.2.2", "@types/csv-parse": "^1.2.2",
        "@types/node": "^14.0.14",
        "@types/underscore": "^1.10.2", "@types/underscore": "^1.10.2",
        "@types/yaml": "^1.9.7", "@types/yaml": "^1.9.7",
        "axios": "^0.19.2", "axios": "^0.19.2",
        ...@@ -44,8 +36,21 @@ ...@@ -44,8 +36,21 @@
        "posthtml-parser": "^0.4.2", "posthtml-parser": "^0.4.2",
        "promise-mysql": "^4.1.3", "promise-mysql": "^4.1.3",
        "querystring": "^0.2.0", "querystring": "^0.2.0",
        "typescript": "^3.9.5",
        "underscore": "^1.10.2", "underscore": "^1.10.2",
        "yaml": "^1.10.0" "yaml": "^1.10.0"
        },
        "devDependencies": {
        "@types/jest": "^27.4.0",
        "@types/node": "^17.0.18",
        "@typescript-eslint/eslint-plugin": "^4.33.0",
        "@typescript-eslint/parser": "^4.33.0",
        "eslint": "^7.32.0",
        "eslint-config-prettier": "^8.3.0",
        "eslint-plugin-prettier": "^3.4.1",
        "jest": "^27.5.1",
        "prettier": "^2.5.1",
        "rimraf": "^3.0.2",
        "ts-jest": "^27.1.3",
        "typescript": "^4.5.5"
        } }
        } }
        src/fetcher.ts
        View file @ 593d03fb
        This diff is collapsed.
        src/migrate.ts
        View file @ 593d03fb
        ...@@ -2,100 +2,119 @@ import mysql from "promise-mysql"; ...@@ -2,100 +2,119 @@ import mysql from "promise-mysql";
        import moment from "moment"; import moment from "moment";
        import fs from "fs"; import fs from "fs";
        import _csv_parse from "csv-parse"; import _csv_parse from "csv-parse";
        import util from 'util'; import util from "util";
        import { Config } from "./fetcher"; import { Config } from "./fetcher";
        import { PlayerRow, PlayerRowDated } from "./playerlist"; import { PlayerRow, PlayerRowDated } from "./playerlist";
        import _ from "underscore"; import _ from "underscore";
        import yaml from "yaml"; import yaml from "yaml";
        const parse_csv: (input: Buffer | string, options?: _csv_parse.Options) => Promise<any[]> = util.promisify(_csv_parse); const parse_csv: (
        input: Buffer | string,
        options?: _csv_parse.Options
        ) => Promise<any[]> = util.promisify(_csv_parse);
        let config: Config; let config: Config;
        let db: mysql.Pool; let db: mysql.Pool;
        const serverAreaCache = new Map<string, string>(); const serverAreaCache = new Map<string, string>();
        async function getServerAreaFromServer(server: string) { async function getServerAreaFromServer(server: string) {
        if (serverAreaCache.has(server)) { if (serverAreaCache.has(server)) {
        return serverAreaCache.get(server); return serverAreaCache.get(server);
        } else { } else {
        const [res] = await db.query("select serverArea from userdata where server = ? limit 1", server); const [res] = await db.query(
        if (!res) { "select serverArea from userdata where server = ? limit 1",
        return "none"; server
        } );
        const serverArea = res.serverArea; if (!res) {
        serverAreaCache.set(server, serverArea); return "none";
        return serverArea; }
        } const serverArea = res.serverArea;
        serverAreaCache.set(server, serverArea);
        return serverArea;
        }
        } }
        async function readSingleRecord(col: string[], offset: number, base: PlayerRow): Promise<PlayerRowDated> { async function readSingleRecord(
        let pointer = offset; col: string[],
        const newRecord: PlayerRowDated = { offset: number,
        date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"), base: PlayerRow
        ...(_.clone(base)) ): Promise<PlayerRowDated> {
        }; let pointer = offset;
        newRecord.name = col[pointer++]; const newRecord: PlayerRowDated = {
        newRecord.server = col[pointer++]; date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"),
        newRecord.serverArea = await getServerAreaFromServer(newRecord.server); ..._.clone(base),
        const _region = col[pointer++]; };
        newRecord.region = _region.length ? _region : "none"; newRecord.name = col[pointer++];
        newRecord.category = col[pointer++]; newRecord.server = col[pointer++];
        newRecord.equip = parseInt(col[pointer++]); newRecord.serverArea = await getServerAreaFromServer(newRecord.server);
        newRecord.totalScore = newRecord.equip + newRecord.score; const _region = col[pointer++];
        return newRecord; newRecord.region = _region.length ? _region : "none";
        newRecord.category = col[pointer++];
        newRecord.equip = parseInt(col[pointer++]);
        newRecord.totalScore = newRecord.equip + newRecord.score;
        return newRecord;
        } }
        let leftCount: number; let leftCount: number;
        async function readColumn(col: string[]): Promise<void> { async function readColumn(col: string[]): Promise<void> {
        const recordCount = parseInt(col[0]); const recordCount = parseInt(col[0]);
        const url = `/bang/role/${col[3]}`; const url = `/bang/role/${col[3]}`;
        console.error(`Reading column ${url}.`); console.error(`Reading column ${url}.`);
        let [base] = await db.query("select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by date asc limit 1", url) as PlayerRow[]; let [base] = (await db.query(
        if (!base) { "select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by id asc limit 1",
        console.error(`Base record of ${url} not found. Using default values.`); url
        base = { )) as PlayerRow[];
        url, if (!base) {
        rank: 500, console.error(`Base record of ${url} not found. Using default values.`);
        name: null, base = {
        server: null, url,
        serverArea: null, rank: 500,
        category: null, name: null,
        level: 80, server: null,
        region: "none", serverArea: null,
        score: 0, category: null,
        equip: 0, level: 80,
        totalScore: 0 region: "none",
        } score: 0,
        } equip: 0,
        for (let i = 0; i < recordCount; ++i) { totalScore: 0,
        const offset = 4 + (i * 6); };
        if (!col[offset].length) { }
        continue; for (let i = 0; i < recordCount; ++i) {
        } const offset = 4 + i * 6;
        const record = await readSingleRecord(col, offset, base); if (!col[offset].length) {
        const sql = "insert into userdata set ?"; continue;
        console.log(sql, JSON.stringify(record), JSON.stringify(await db.query(sql, record))); }
        } const record = await readSingleRecord(col, offset, base);
        console.error(`Read column ${url}. ${--leftCount} columns left.`); const sql = "insert into userdata set ?";
        console.log(
        sql,
        JSON.stringify(record),
        JSON.stringify(await db.query(sql, record))
        );
        }
        console.error(`Read column ${url}. ${--leftCount} columns left.`);
        } }
        async function loadCsv(path: string): Promise<string[][]> { async function loadCsv(path: string): Promise<string[][]> {
        const data = await fs.promises.readFile(path); const data = await fs.promises.readFile(path);
        return await parse_csv(data, { return await parse_csv(data, {
        trim: true trim: true,
        }); });
        } }
        async function main() { async function main() {
        console.error("Started."); console.error("Started.");
        const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8")); const config: Config = yaml.parse(
        db = await mysql.createPool(config.MySQLConfig); await fs.promises.readFile("./config.yaml", "utf8")
        const data = await loadCsv(process.argv[2]); );
        leftCount = data.length; db = await mysql.createPool(config.MySQLConfig);
        //await Promise.all(data.map(col => readColumn(col))); const data = await loadCsv(process.argv[2]);
        for (let col of data) { leftCount = data.length;
        await readColumn(col); //await Promise.all(data.map(col => readColumn(col)));
        } for (const col of data) {
        console.error("Finished."); await readColumn(col);
        process.exit(); }
        console.error("Finished.");
        process.exit();
        } }
        main(); main();
        src/playerlist.ts
        View file @ 593d03fb
        import HTML from "posthtml-parser"; import HTML from "posthtml-parser";
        import _, { first } from "underscore"; import {
        import {getDepthOfTree, getNumber, findNodeIndex, findNodeIndexByAttribute, findNodeIndexByContent, findNodeIndexByTag, findAllNodeIndex, getContinuousData, getContinuousNodes, getString} from "./utility"; findAllNodeIndex,
        findNodeIndexByTag,
        getContinuousNodes,
        getDepthOfTree,
        getNumber,
        getString,
        } from "./utility";
        export interface PlayerRow { export interface PlayerRow {
        url: string; url: string;
        rank: number; rank: number;
        name: string; name: string;
        category: string; category: string;
        serverArea: string; serverArea: string;
        server: string; server: string;
        level: number; level: number;
        region: string; region: string;
        score: number; score: number;
        equip: number; equip: number;
        totalScore: number; totalScore: number;
        } }
        export interface PlayerRowDated extends PlayerRow { export interface PlayerRowDated extends PlayerRow {
        date: string; date: string;
        } }
        export interface PlayerRowFull extends PlayerRowDated { export interface PlayerRowFull extends PlayerRowDated {
        id: number; id: number;
        } }
        function getPlayerRowFromTree(tree: HTML.Tree): PlayerRow { function getPlayerRowFromTree(tree: HTML.Tree): PlayerRow {
        const nodes = getContinuousNodes(tree, [1], 0, 2, 10); const nodes = getContinuousNodes(tree, [1], 0, 2, 10);
        return { return {
        url: (nodes[1] as HTML.NodeTag).attrs.href as string, url: (nodes[1] as HTML.NodeTag).attrs.href as string,
        rank: getNumber(nodes[0]), rank: getNumber(nodes[0]),
        name: getString(nodes[1], 7), name: getString(nodes[1], 7),
        serverArea: getString(nodes[2]), serverArea: getString(nodes[2]),
        server: getString(nodes[3]), server: getString(nodes[3]),
        level: getNumber(nodes[4]), level: getNumber(nodes[4]),
        category: getString(nodes[5]), category: getString(nodes[5]),
        region: getString(nodes[6], 7) || "none", region: getString(nodes[6], 7) || "none",
        score: getNumber(nodes[7]), score: getNumber(nodes[7]),
        equip: getNumber(nodes[8]), equip: getNumber(nodes[8]),
        totalScore: getNumber(nodes[9]) totalScore: getNumber(nodes[9]),
        } };
        } }
        export function parsePlayerRows(content: string) { export function parsePlayerRows(content: string) {
        const parsedContent = HTML(content); const parsedContent = HTML(content);
        const tablePos = findNodeIndexByTag(parsedContent, "table", []); const tablePos = findNodeIndexByTag(parsedContent, "table", []);
        const tableTree = getDepthOfTree(parsedContent, tablePos); const tableTree = getDepthOfTree(parsedContent, tablePos);
        const playerPoses = findAllNodeIndex(tableTree, (node) => { const playerPoses = findAllNodeIndex(
        return typeof (node) !== "string" && node.tag === "tr" && node.attrs.class !== "trTop2"; tableTree,
        }, []); (node) => {
        return playerPoses.map(pos => { return (
        const tree = getDepthOfTree(tableTree, pos); typeof node !== "string" &&
        return getPlayerRowFromTree(tree); node.tag === "tr" &&
        }); node.attrs.class !== "trTop2"
        );
        },
        []
        );
        return playerPoses.map((pos) => {
        const tree = getDepthOfTree(tableTree, pos);
        return getPlayerRowFromTree(tree);
        });
        } }
        src/proxy.ts
        View file @ 593d03fb
        ...@@ -9,59 +9,59 @@ import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios"; ...@@ -9,59 +9,59 @@ import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios";
        //} //}
        export interface ProxyConfig { export interface ProxyConfig {
        useProxy: boolean, useProxy: boolean;
        proxySource: string[], proxySource: string[];
        timeout: number timeout: number;
        } }
        const agentList = [ const agentList = [
        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
        'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
        'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0', "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0",
        'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)', "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
        'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
        'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11', "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
        'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11', "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11', "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)', "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
        'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)' "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
        ] ];
        async function testProxy(proxy: AxiosProxyConfig) { async function testProxy(proxy: AxiosProxyConfig) {
        await axios.get("http://mirrors.aliyun.com/debian/pool", { await axios.get("http://mirrors.aliyun.com/debian/pool", {
        proxy, proxy,
        headers: { headers: {
        "User-Agent": agentList[4] "User-Agent": agentList[4],
        }, },
        timeout: this.config.timeout, timeout: this.config.timeout,
        }); });
        return proxy; return proxy;
        } }
        async function checkProxy(proxy: AxiosProxyConfig) { async function checkProxy(proxy: AxiosProxyConfig) {
        let isProxyUsable = false; let isProxyUsable = false;
        try { try {
        await testProxy(proxy); await testProxy(proxy);
        //console.log(`Proxy ${proxy.host} is ok.`); //console.log(`Proxy ${proxy.host} is ok.`);
        isProxyUsable = true; isProxyUsable = true;
        } catch (e) { } catch (e) {
        //console.error(`Proxy ${proxy.host} is broken: ${e.toString()}`); //console.error(`Proxy ${proxy.host} is broken: ${e.toString()}`);
        } }
        return isProxyUsable; return isProxyUsable;
        } }
        async function filterProxies(proxies: AxiosProxyConfig[]) { async function filterProxies(proxies: AxiosProxyConfig[]) {
        const proxiesUsableList = await Promise.all(proxies.map(checkProxy)); const proxiesUsableList = await Promise.all(proxies.map(checkProxy));
        return proxies.filter((proxy, index) => { return proxies.filter((proxy, index) => {
        return proxiesUsableList[index]; return proxiesUsableList[index];
        }); });
        } }
        //async function findFirstUsableProxy(proxies: AxiosProxyConfig[]) { //async function findFirstUsableProxy(proxies: AxiosProxyConfig[]) {
        ...@@ -69,70 +69,86 @@ async function filterProxies(proxies: AxiosProxyConfig[]) { ...@@ -69,70 +69,86 @@ async function filterProxies(proxies: AxiosProxyConfig[]) {
        //} //}
        export class ProxyFetcher { export class ProxyFetcher {
        proxies: AxiosProxyConfig[]; proxies: AxiosProxyConfig[];
        counter: number; counter: number;
        config: ProxyConfig; config: ProxyConfig;
        constructor(config: ProxyConfig) { constructor(config: ProxyConfig) {
        this.config = config; this.config = config;
        this.proxies = []; this.proxies = [];
        this.counter = 0; this.counter = 0;
        } }
        async initProxiesFrom(url: string) { async initProxiesFrom(url: string) {
        if (!this.config.useProxy) { if (!this.config.useProxy) {
        return; return;
        } }
        console.log(`Fetching proxies from ${url}.`) console.log(`Fetching proxies from ${url}.`);
        while (true) { while (true) {
        try { try {
        const proxyPage: string = (await axios.get(url, { const proxyPage: string = (
        responseType: "document", await axios.get(url, {
        })).data; responseType: "document",
        const proxies: AxiosProxyConfig[] = proxyPage.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g).map(proxyString => { })
        const [host, _port] = proxyString.split(":"); ).data;
        const port = parseInt(_port); const proxies: AxiosProxyConfig[] = proxyPage
        const proxy = { host, port }; .match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g)
        return proxy; .map((proxyString) => {
        }); const [host, _port] = proxyString.split(":");
        //const usableProxies = await filterProxies(proxies); const port = parseInt(_port);
        for (let proxy of proxies) { const proxy = { host, port };
        this.proxies.push(proxy); return proxy;
        } });
        console.error(`Got ${proxies.length} proxies from ${url}.`); //const usableProxies = await filterProxies(proxies);
        return; for (const proxy of proxies) {
        } catch (e) { this.proxies.push(proxy);
        console.error(`Failed fetching proxy list from ${url}: ${e.toString()}`) }
        } console.error(`Got ${proxies.length} proxies from ${url}.`);
        } return;
        } } catch (e) {
        async initProxies() { console.error(
        await Promise.all(this.config.proxySource.map((m) => { `Failed fetching proxy list from ${url}: ${e.toString()}`
        return this.initProxiesFrom(m); );
        })); }
        } }
        async getWithProxy(url: string, options: AxiosRequestConfig) { }
        while (true) { async initProxies() {
        if (this.config.useProxy && !this.proxies.length) { await Promise.all(
        await this.initProxies(); this.config.proxySource.map((m) => {
        } return this.initProxiesFrom(m);
        const proxyIndex = !this.config.useProxy ? null : (++this.counter) % this.proxies.length; })
        //const proxyIndex = 0; );
        const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex]; }
        try { async getWithProxy(url: string, options: AxiosRequestConfig) {
        const data = (await axios.get(url, { while (true) {
        proxy, if (this.config.useProxy && !this.proxies.length) {
        headers: { await this.initProxies();
        "User-Agent": agentList[this.counter % agentList.length] }
        }, const proxyIndex = !this.config.useProxy
        timeout: this.config.timeout, ? null
        ...options : ++this.counter % this.proxies.length;
        })).data; //const proxyIndex = 0;
        return data; const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex];
        } catch (e) { try {
        if (this.config.useProxy) { const data = (
        this.proxies.splice(proxyIndex, 1); await axios.get(url, {
        } proxy,
        console.error(`Failed fetching data from ${url}: ${e.toString()} ${this.proxies.length} proxies left.`) headers: {
        } "User-Agent": agentList[this.counter % agentList.length],
        } },
        } timeout: this.config.timeout,
        ...options,
        })
        ).data;
        return data;
        } catch (e) {
        if (this.config.useProxy) {
        this.proxies.splice(proxyIndex, 1);
        }
        console.error(
        `Failed fetching data from ${url}: ${e.toString()} ${
        this.proxies.length
        } proxies left.`
        );
        }
        }
        }
        } }
        src/remove-dup.ts
        View file @ 593d03fb
        ...@@ -6,31 +6,40 @@ import _ from "underscore"; ...@@ -6,31 +6,40 @@ import _ from "underscore";
        import yaml from "yaml"; import yaml from "yaml";
        function checkSameRow(row: PlayerRowFull, lrow: PlayerRowFull) { function checkSameRow(row: PlayerRowFull, lrow: PlayerRowFull) {
        return _.every(["name", "category", "serverArea", "server", "region"], field => lrow[field] === row[field]); return _.every(
        ["name", "category", "serverArea", "server", "region"],
        (field) => lrow[field] === row[field]
        );
        } }
        async function main() { async function main() {
        console.error("Started."); console.error("Started.");
        const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8")); const config: Config = yaml.parse(
        const db = await mysql.createPool(config.MySQLConfig); await fs.promises.readFile("./config.yaml", "utf8")
        const urlDataCache = new Map<string, PlayerRowFull>(); );
        const deleteList: number[] = []; const db = await mysql.createPool(config.MySQLConfig);
        const datas: PlayerRowFull[] = await db.query(`select * from userdata order by date asc`); const urlDataCache = new Map<string, PlayerRowFull>();
        for (let row of datas) { const deleteList: number[] = [];
        if (urlDataCache.has(row.url)) { const datas: PlayerRowFull[] = await db.query(
        const oldRow = urlDataCache.get(row.url); `select * from userdata order by id asc`
        if (checkSameRow(row, oldRow)) { );
        deleteList.push(row.id); for (const row of datas) {
        } if (urlDataCache.has(row.url)) {
        } const oldRow = urlDataCache.get(row.url);
        urlDataCache.set(row.url, row); if (checkSameRow(row, oldRow)) {
        } deleteList.push(row.id);
        console.error(`Deletes: ${deleteList.length}`); }
        for (let id of deleteList) { }
        const sql = `delete from userdata where id = ?`; urlDataCache.set(row.url, row);
        console.error(`Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`); }
        } console.error(`Deletes: ${deleteList.length}`);
        console.error("Finished."); for (const id of deleteList) {
        process.exit(); const sql = `delete from userdata where id = ?`;
        console.error(
        `Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`
        );
        }
        console.error("Finished.");
        process.exit();
        } }
        main(); main();
        src/run.ts
        View file @ 593d03fb
        import {Tx3Fetcher, servers, Config} from "./fetcher"; import { Config, servers, Tx3Fetcher } from "./fetcher";
        import fs from "fs"; import fs from "fs";
        import _ from "underscore";
        import yaml from "yaml"; import yaml from "yaml";
        import { CronJob } from "cron"; import { CronJob } from "cron";
        let config: Config; let config: Config;
        async function loadConfig() { async function loadConfig() {
        config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8")); config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
        } }
        async function runServer(fetcher: Tx3Fetcher, server: string) { async function runServer(fetcher: Tx3Fetcher, server: string) {
        const users = await fetcher.fetchListFromServer(server); const users = await fetcher.fetchListFromServer(server);
        await fs.promises.writeFile(`./output/servers/${server}.json`, JSON.stringify({ await fs.promises.writeFile(
        date: fetcher.curDate, `./output/servers/${server}.json`,
        data: users JSON.stringify(
        }, null, 2)); {
        return users; date: fetcher.curDate,
        data: users,
        },
        null,
        2
        )
        );
        return users;
        } }
        async function run() { async function run() {
        console.log(`Fetch started.`); console.log(`Fetch started.`);
        try { try {
        await fs.promises.access("./output/servers"); await fs.promises.access("./output/servers");
        } catch (e) { } catch (e) {
        await fs.promises.mkdir("./output/servers", { await fs.promises.mkdir("./output/servers", {
        recursive: true recursive: true,
        }); });
        } }
        const fetcher = new Tx3Fetcher(config); const fetcher = new Tx3Fetcher(config);
        await fetcher.init(); await fetcher.init();
        if (config.server) { if (config.server) {
        await Promise.all(config.server.map(server => { await Promise.all(
        return runServer(fetcher, server) config.server.map((server) => {
        })); return runServer(fetcher, server);
        } else { })
        const userListWithServer = await Promise.all(servers.map(server => { );
        return runServer(fetcher, server) } else {
        })); const userListWithServer = await Promise.all(
        const allServersList: any = {}; servers.map((server) => {
        for (let i = 0; i < servers.length;++i) { return runServer(fetcher, server);
        allServersList[servers[i]] = userListWithServer[i]; })
        } );
        await fs.promises.writeFile(`./output/all.json`, JSON.stringify({ const allServersList: any = {};
        date: fetcher.curDate, for (let i = 0; i < servers.length; ++i) {
        data: allServersList allServersList[servers[i]] = userListWithServer[i];
        }, null, 2)); }
        } await fs.promises.writeFile(
        console.log("Finished."); `./output/all.json`,
        JSON.stringify(
        {
        date: fetcher.curDate,
        data: allServersList,
        },
        null,
        2
        )
        );
        }
        console.log("Finished.");
        } }
        async function main() { async function main() {
        await loadConfig(); await loadConfig();
        if (process.argv[2] === "cron") { if (process.argv[2] === "cron") {
        const job = new CronJob(config.cronString, run, null, true, "Asia/Shanghai", null, true); const job = new CronJob(
        job.start(); config.cronString,
        } else { run,
        await run(); null,
        process.exit(); true,
        } "Asia/Shanghai",
        null,
        true
        );
        job.start();
        } else {
        await run();
        process.exit();
        }
        } }
        main(); main();
        src/user.ts
        View file @ 593d03fb
        This diff is collapsed.
        src/utility.ts
        View file @ 593d03fb
        This diff is collapsed.
        tests/test-list.ts
        View file @ 593d03fb
        ...@@ -3,9 +3,9 @@ import fs from "fs"; ...@@ -3,9 +3,9 @@ import fs from "fs";
        import { parsePlayerRows } from "../src/playerlist"; import { parsePlayerRows } from "../src/playerlist";
        async function main() { async function main() {
        let html = await fs.promises.readFile("./tests/playerlist.html", "utf-8"); let html = await fs.promises.readFile("./tests/playerlist.html", "utf-8");
        console.log(parsePlayerRows(html)); console.log(parsePlayerRows(html));
        html = await fs.promises.readFile("./tests/playerlist-null.html", "utf-8"); html = await fs.promises.readFile("./tests/playerlist-null.html", "utf-8");
        console.log(parsePlayerRows(html)); console.log(parsePlayerRows(html));
        } }
        main(); main();
        tests/test-proxy.ts
        View file @ 593d03fb
        import { ProxyFetcher } from "../src/proxy"; import { ProxyFetcher } from "../src/proxy";
        async function main() { async function main() {
        const fetcher = new ProxyFetcher({ const fetcher = new ProxyFetcher({
        useProxy: true, useProxy: true,
        proxySource: [ proxySource: [
        "http://www.89ip.cn/tqdl.html?api=1&num=9999", "http://www.89ip.cn/tqdl.html?api=1&num=9999",
        "http://www.66ip.cn/mo.php?tqsl=9999" "http://www.66ip.cn/mo.php?tqsl=9999",
        ], ],
        timeout: 10000 timeout: 10000,
        }); });
        await fetcher.initProxies(); await fetcher.initProxies();
        console.log(await fetcher.getWithProxy("https://mycard.moe", {})); console.log(await fetcher.getWithProxy("https://mycard.moe", {}));
        } }
        main(); main();
        tests/test-user.ts
        View file @ 593d03fb
        ...@@ -3,13 +3,19 @@ import fs from "fs"; ...@@ -3,13 +3,19 @@ import fs from "fs";
        import HTML from "posthtml-parser"; import HTML from "posthtml-parser";
        async function main() { async function main() {
        let id = "28_20588"; let id = "28_20588";
        let html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8"); let html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2)); await fs.promises.writeFile(
        console.log(new User(id, html, null)); `./tests/${id}.json`,
        id = "6_18804822"; JSON.stringify(HTML(html), null, 2)
        html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8"); );
        await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2)); console.log(new User(id, html, null));
        console.log(new User(id, html, null)); id = "6_18804822";
        html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
        await fs.promises.writeFile(
        `./tests/${id}.json`,
        JSON.stringify(HTML(html), null, 2)
        );
        console.log(new User(id, html, null));
        } }
        main(); main();
        tsconfig.json
        View file @ 593d03fb
        { {
        "compilerOptions": { "compilerOptions": {
        "outDir": "build", "outDir": "dist",
        "module": "commonjs", "module": "commonjs",
        "target": "esnext", "target": "es2021",
        "esModuleInterop": true, "esModuleInterop": true,
        "emitDecoratorMetadata": true,
        "experimentalDecorators": true,
        "declaration": true, "declaration": true,
        "sourceMap": true "sourceMap": true
        }, },
        "compileOnSave": true, "compileOnSave": true,
        "allowJs": true, "allowJs": true,
        "include": [ "include": [
        "src/*.ts", "*.ts",
        "tests/*.ts" "src/**/*.ts",
        "test/**/*.ts",
        "tests/**/*.ts"
        ] ]
        } }
        Markdown is supported
        0% or
        You are about to add 0 people to the discussion. Proceed with caution.
        Finish editing this message first!
        Please register or to comment