Commit 593d03fb authored by nanahira's avatar nanahira

rework

parent 85d2aa62
Pipeline #9850 failed with stages
in 1 minute and 33 seconds
webpack.config.js
dist/*
build/*
*.js
module.exports = {
parser: '@typescript-eslint/parser',
parserOptions: {
project: 'tsconfig.json',
sourceType: 'module',
},
plugins: ['@typescript-eslint/eslint-plugin'],
extends: [
'plugin:@typescript-eslint/recommended',
'plugin:prettier/recommended',
],
root: true,
env: {
node: true,
jest: true,
},
ignorePatterns: ['.eslintrc.js'],
rules: {
'@typescript-eslint/interface-name-prefix': 'off',
'@typescript-eslint/explicit-function-return-type': 'off',
'@typescript-eslint/explicit-module-boundary-types': 'off',
'@typescript-eslint/no-explicit-any': 'off',
},
};
stages:
- install
- build
- deploy
variables:
GIT_DEPTH: "1"
CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest
docker:
stage: build
npm_ci:
stage: install
tags:
- docker
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- linux
script:
- docker build --pull -t $CONTAINER_TEST_IMAGE .
- docker push $CONTAINER_TEST_IMAGE
- npm ci
artifacts:
paths:
- node_modules
deploy_latest:
stage: deploy
.build_base:
stage: build
tags:
- docker
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- linux
dependencies:
- npm_ci
build:
extends:
- .build_base
script:
- docker pull $CONTAINER_TEST_IMAGE
- docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
- docker push $CONTAINER_RELEASE_IMAGE
only:
- master
- npm run build
artifacts:
paths:
- dist/
unit-test:
extends:
- .build_base
script:
- npm run test
deploy_tag:
deploy_npm:
stage: deploy
dependencies:
- build
tags:
- docker
variables:
CONTAINER_TAG_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
before_script:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- linux
script:
- docker pull $CONTAINER_TEST_IMAGE
- docker tag $CONTAINER_TEST_IMAGE $CONTAINER_TAG_IMAGE
- docker push $CONTAINER_TAG_IMAGE
- apt update;apt -y install coreutils
- echo $NPMRC | base64 --decode > ~/.npmrc
- npm publish . || true
only:
- tags
- master
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="Eslint" enabled="true" level="WARNING" enabled_by_default="true" />
</profile>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="EslintConfiguration">
<option name="fix-on-save" value="true" />
</component>
</project>
\ No newline at end of file
/install-npm.sh
.git*
/data
/output
/config.yaml
.idea
.dockerignore
Dockerfile
/src
/coverage
/tests
/dist/tests
FROM node:bullseye-slim
RUN apt update && apt -y install python3 && rm -rf /var/lib/apt/lists/*
FROM node:bullseye-slim as base
LABEL Author="Nanahira <nanahira@momobako.com>"
RUN apt update && apt -y install python3 build-essential && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/log/*
WORKDIR /usr/src/app
COPY ./package*.json ./
RUN npm ci
FROM base as builder
RUN npm ci && npm cache clean --force
COPY . ./
RUN npm run build
CMD ["npm", "run", "start"]
FROM base
ENV NODE_ENV production
RUN npm ci && npm cache clean --force
COPY --from=builder /usr/src/app/dist ./dist
CMD [ "npm", "start" ]
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -2,12 +2,11 @@
"name": "tx3-bang-reader",
"version": "1.0.0",
"description": "Read TX3 bang and parse it",
"main": "build/src/run.js",
"main": "dist/run.js",
"scripts": {
"build": "./node_modules/.bin/tsc",
"pack": "mkdir dist ; ./node_modules/.bin/pkg --out-path dist .",
"fetch": "node build/src/run.js",
"start": "node build/src/run.js cron"
"build": "tsc",
"fetch": "node dist/src/run.js",
"start": "node dist/src/run.js cron"
},
"repository": {
"type": "git",
......@@ -22,17 +21,10 @@
"url": "https://github.com/purerosefallen/tx3-bang-reader/issues"
},
"homepage": "https://github.com/purerosefallen/tx3-bang-reader#readme",
"bin": "build/src/run.js",
"pkg": {
"scripts": [
"build/src/*.js"
],
"assets": []
},
"bin": "dist/src/run.js",
"dependencies": {
"@types/cron": "^1.7.2",
"@types/csv-parse": "^1.2.2",
"@types/node": "^14.0.14",
"@types/underscore": "^1.10.2",
"@types/yaml": "^1.9.7",
"axios": "^0.19.2",
......@@ -44,8 +36,21 @@
"posthtml-parser": "^0.4.2",
"promise-mysql": "^4.1.3",
"querystring": "^0.2.0",
"typescript": "^3.9.5",
"underscore": "^1.10.2",
"yaml": "^1.10.0"
},
"devDependencies": {
"@types/jest": "^27.4.0",
"@types/node": "^17.0.18",
"@typescript-eslint/eslint-plugin": "^4.33.0",
"@typescript-eslint/parser": "^4.33.0",
"eslint": "^7.32.0",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-prettier": "^3.4.1",
"jest": "^27.5.1",
"prettier": "^2.5.1",
"rimraf": "^3.0.2",
"ts-jest": "^27.1.3",
"typescript": "^4.5.5"
}
}
import axios from "axios";
import _ from "underscore";
import { User } from "./user";
import {ProxyConfig, ProxyFetcher} from "./proxy";
import { PlayerRow, parsePlayerRows } from "./playerlist";
import { ProxyConfig, ProxyFetcher } from "./proxy";
import { parsePlayerRows, PlayerRow } from "./playerlist";
import qs from "querystring";
import mysql from "promise-mysql";
import moment from "moment";
......@@ -33,15 +32,14 @@ export const servers = [
"飞龙在天",
"烽火关东",
"盛世长安",
]
];
export interface Config {
outDir: string;
server: string[];
useMySQL: boolean,
MySQLConfig: mysql.PoolConfig,
proxy: ProxyConfig,
useMySQL: boolean;
MySQLConfig: mysql.PoolConfig;
proxy: ProxyConfig;
cronString: string;
}
export class Tx3Fetcher {
......@@ -55,9 +53,10 @@ export class Tx3Fetcher {
}
async init() {
this.curDate = moment().format("YYYY-MM-DD HH:mm:ss");
if(this.config.useMySQL) {
if (this.config.useMySQL) {
this.db = await mysql.createPool(this.config.MySQLConfig);
await this.db.query("CREATE TABLE IF NOT EXISTS `userdata` (\n" +
await this.db.query(
"CREATE TABLE IF NOT EXISTS `userdata` (\n" +
" `id` bigint(20) NOT NULL AUTO_INCREMENT,\n" +
" `date` datetime NOT NULL DEFAULT current_timestamp(),\n" +
" `url` varchar(50) COLLATE utf8_unicode_ci NOT NULL,\n" +
......@@ -80,19 +79,22 @@ export class Tx3Fetcher {
" INDEX (server(4)),\n" +
" INDEX (region(7)),\n" +
" INDEX (equip)\n" +
") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci");
") ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci"
);
console.log(`Removing existing records of ${this.curDate}.`);
//await this.db.query("delete from userdata where date = ?", this.curDate);
}
if(this.config.proxy.useProxy) {
if (this.config.proxy.useProxy) {
await this.proxyFetcher.initProxies();
}
}
async fetchAll(): Promise<Map<string, User[]>> {
const res = new Map<string, User[]>();
const userLists = await Promise.all(servers.map(server => this.fetchUsersFromServer(server)));
for (let i = 1; i < userLists.length; ++i){
res.set(servers[i], userLists[i])
const userLists = await Promise.all(
servers.map((server) => this.fetchUsersFromServer(server))
);
for (let i = 1; i < userLists.length; ++i) {
res.set(servers[i], userLists[i]);
}
return res;
}
......@@ -103,10 +105,15 @@ export class Tx3Fetcher {
resPromises.push(this.fetchUsersFromSchoolAndServer(school, server));
}
const result = _.flatten(await Promise.all(resPromises));
console.log(`Fetched user list with ${result.length} users from server ${server}.`);
console.log(
`Fetched user list with ${result.length} users from server ${server}.`
);
return result;
}
async fetchUsersFromSchoolAndServer(school: number, server: string): Promise<User[]> {
async fetchUsersFromSchoolAndServer(
school: number,
server: string
): Promise<User[]> {
console.log(`Fetching users from server ${server} with school ${school}.`);
const res: User[][] = [];
for (let page = 1; page <= 25; ++page) {
......@@ -118,9 +125,13 @@ export class Tx3Fetcher {
}
return _.flatten(res);
}
async fetchUsers(school: number, server: string, page: number): Promise<User[]> {
async fetchUsers(
school: number,
server: string,
page: number
): Promise<User[]> {
const playerRows = await this.fetchList(school, server, page);
return await Promise.all(playerRows.map(row => this.fetchUser(row)));
return await Promise.all(playerRows.map((row) => this.fetchUser(row)));
}
async fetchListFromServer(server: string): Promise<PlayerRow[]> {
console.log(`Fetching user list from server ${server}.`);
......@@ -129,10 +140,15 @@ export class Tx3Fetcher {
resPromises.push(this.fetchListFromSchoolAndServer(school, server));
}
const result = _.flatten(await Promise.all(resPromises));
console.log(`Fetched user list from server ${server}. ${result.length} users found.`);
console.log(
`Fetched user list from server ${server}. ${result.length} users found.`
);
return result;
}
async fetchListFromSchoolAndServer(school: number, server: string): Promise<PlayerRow[]> {
async fetchListFromSchoolAndServer(
school: number,
server: string
): Promise<PlayerRow[]> {
console.log(`Fetching users from server ${server} with school ${school}.`);
const res: PlayerRow[][] = [];
for (let page = 1; page <= 25; ++page) {
......@@ -147,45 +163,72 @@ export class Tx3Fetcher {
return ret;
}
async checkAddRecord(row: PlayerRow) {
const latestRows = await this.db.query("select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by date desc limit 1", row.url);
const latestRows = await this.db.query(
"select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by id desc limit 1",
row.url
);
if (latestRows.length) {
const lrow = latestRows[0];
if (_.every(["name", "category", "serverArea", "server", "region"], field => lrow[field] === row[field])) {
if (
_.every(
["name", "category", "serverArea", "server", "region"],
(field) => lrow[field] === row[field]
)
) {
return;
}
}
const sql = "insert into userdata set ?";
const valueObj = {
date: this.curDate,
...row
...row,
};
console.log(`Player ${row.name} from ${row.server} has changes. Writing record to database: ${sql} ${JSON.stringify(valueObj)} ${JSON.stringify(await this.db.query(sql, valueObj))}`);
}
async fetchList(school: number, server: string, page: number): Promise<PlayerRow[]> {
console.log(`Fetching user list from server ${server} with school ${school} page ${page}.`);
console.log(
`Player ${row.name} from ${
row.server
} has changes. Writing record to database: ${sql} ${JSON.stringify(
valueObj
)} ${JSON.stringify(await this.db.query(sql, valueObj))}`
);
}
async fetchList(
school: number,
server: string,
page: number
): Promise<PlayerRow[]> {
console.log(
`Fetching user list from server ${server} with school ${school} page ${page}.`
);
try {
const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com/bang/ranks`, {
const content: string = await this.proxyFetcher.getWithProxy(
`http://bang.tx3.163.com/bang/ranks`,
{
responseType: "document",
params: {
order_key: "equ_xiuwei",
count: 20,
school,
server,
page
page,
},
paramsSerializer: (params) => {
return qs.stringify(params);
},
}
});
);
const playerRows = parsePlayerRows(content);
console.log(`Fetched user list from server ${server} with school ${school} page ${page}. ${playerRows.length} users found.`);
console.log(
`Fetched user list from server ${server} with school ${school} page ${page}. ${playerRows.length} users found.`
);
if (this.db) {
//await Promise.all(playerRows.map(m => this.db.query("delete from userdata where url = ? and date = ?", [m.url, this.curDate])));
await Promise.all(playerRows.map(m => this.checkAddRecord(m)));
await Promise.all(playerRows.map((m) => this.checkAddRecord(m)));
}
return playerRows;
} catch(e) {
console.error(`Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`);
} catch (e) {
console.error(
`Errored fetching user list with params ${school} ${server} ${page}}: ${e.toString()}`
);
return [];
}
}
......@@ -193,13 +236,16 @@ export class Tx3Fetcher {
const id = playerRow.url.split("/").pop();
try {
console.log(`Fetching user ${playerRow.name} from ${playerRow.server}.`);
const content: string = await this.proxyFetcher.getWithProxy(`http://bang.tx3.163.com${playerRow.url}`, {
responseType: "document"
});
const content: string = await this.proxyFetcher.getWithProxy(
`http://bang.tx3.163.com${playerRow.url}`,
{
responseType: "document",
}
);
const user = new User(id, content, playerRow.region);
console.log(`Fetched user ${playerRow.name} from ${playerRow.server}.`);
return user;
} catch(e) {
} catch (e) {
console.error(`Errored fetching role data from ${id}: ${e.toString()}`);
return null;
}
......
......@@ -2,13 +2,16 @@ import mysql from "promise-mysql";
import moment from "moment";
import fs from "fs";
import _csv_parse from "csv-parse";
import util from 'util';
import util from "util";
import { Config } from "./fetcher";
import { PlayerRow, PlayerRowDated } from "./playerlist";
import _ from "underscore";
import yaml from "yaml";
const parse_csv: (input: Buffer | string, options?: _csv_parse.Options) => Promise<any[]> = util.promisify(_csv_parse);
const parse_csv: (
input: Buffer | string,
options?: _csv_parse.Options
) => Promise<any[]> = util.promisify(_csv_parse);
let config: Config;
let db: mysql.Pool;
......@@ -17,7 +20,10 @@ async function getServerAreaFromServer(server: string) {
if (serverAreaCache.has(server)) {
return serverAreaCache.get(server);
} else {
const [res] = await db.query("select serverArea from userdata where server = ? limit 1", server);
const [res] = await db.query(
"select serverArea from userdata where server = ? limit 1",
server
);
if (!res) {
return "none";
}
......@@ -27,11 +33,15 @@ async function getServerAreaFromServer(server: string) {
}
}
async function readSingleRecord(col: string[], offset: number, base: PlayerRow): Promise<PlayerRowDated> {
async function readSingleRecord(
col: string[],
offset: number,
base: PlayerRow
): Promise<PlayerRowDated> {
let pointer = offset;
const newRecord: PlayerRowDated = {
date: moment(col[pointer++], "YYYY/MM/DD").format("YYYY-MM-DD HH:mm:ss"),
...(_.clone(base))
..._.clone(base),
};
newRecord.name = col[pointer++];
newRecord.server = col[pointer++];
......@@ -49,7 +59,10 @@ async function readColumn(col: string[]): Promise<void> {
const recordCount = parseInt(col[0]);
const url = `/bang/role/${col[3]}`;
console.error(`Reading column ${url}.`);
let [base] = await db.query("select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by date asc limit 1", url) as PlayerRow[];
let [base] = (await db.query(
"select url,rank,name,category,serverArea,server,level,region,score,equip,totalScore from userdata where url = ? order by id asc limit 1",
url
)) as PlayerRow[];
if (!base) {
console.error(`Base record of ${url} not found. Using default values.`);
base = {
......@@ -63,17 +76,21 @@ async function readColumn(col: string[]): Promise<void> {
region: "none",
score: 0,
equip: 0,
totalScore: 0
}
totalScore: 0,
};
}
for (let i = 0; i < recordCount; ++i) {
const offset = 4 + (i * 6);
const offset = 4 + i * 6;
if (!col[offset].length) {
continue;
}
const record = await readSingleRecord(col, offset, base);
const sql = "insert into userdata set ?";
console.log(sql, JSON.stringify(record), JSON.stringify(await db.query(sql, record)));
console.log(
sql,
JSON.stringify(record),
JSON.stringify(await db.query(sql, record))
);
}
console.error(`Read column ${url}. ${--leftCount} columns left.`);
}
......@@ -81,18 +98,20 @@ async function readColumn(col: string[]): Promise<void> {
async function loadCsv(path: string): Promise<string[][]> {
const data = await fs.promises.readFile(path);
return await parse_csv(data, {
trim: true
trim: true,
});
}
async function main() {
console.error("Started.");
const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
const config: Config = yaml.parse(
await fs.promises.readFile("./config.yaml", "utf8")
);
db = await mysql.createPool(config.MySQLConfig);
const data = await loadCsv(process.argv[2]);
leftCount = data.length;
//await Promise.all(data.map(col => readColumn(col)));
for (let col of data) {
for (const col of data) {
await readColumn(col);
}
console.error("Finished.");
......
import HTML from "posthtml-parser";
import _, { first } from "underscore";
import {getDepthOfTree, getNumber, findNodeIndex, findNodeIndexByAttribute, findNodeIndexByContent, findNodeIndexByTag, findAllNodeIndex, getContinuousData, getContinuousNodes, getString} from "./utility";
import {
findAllNodeIndex,
findNodeIndexByTag,
getContinuousNodes,
getDepthOfTree,
getNumber,
getString,
} from "./utility";
export interface PlayerRow {
url: string;
......@@ -37,20 +43,26 @@ function getPlayerRowFromTree(tree: HTML.Tree): PlayerRow {
region: getString(nodes[6], 7) || "none",
score: getNumber(nodes[7]),
equip: getNumber(nodes[8]),
totalScore: getNumber(nodes[9])
}
totalScore: getNumber(nodes[9]),
};
}
export function parsePlayerRows(content: string) {
const parsedContent = HTML(content);
const tablePos = findNodeIndexByTag(parsedContent, "table", []);
const tableTree = getDepthOfTree(parsedContent, tablePos);
const playerPoses = findAllNodeIndex(tableTree, (node) => {
return typeof (node) !== "string" && node.tag === "tr" && node.attrs.class !== "trTop2";
}, []);
return playerPoses.map(pos => {
const playerPoses = findAllNodeIndex(
tableTree,
(node) => {
return (
typeof node !== "string" &&
node.tag === "tr" &&
node.attrs.class !== "trTop2"
);
},
[]
);
return playerPoses.map((pos) => {
const tree = getDepthOfTree(tableTree, pos);
return getPlayerRowFromTree(tree);
});
......
......@@ -9,36 +9,36 @@ import axios, { AxiosProxyConfig, AxiosRequestConfig } from "axios";
//}
export interface ProxyConfig {
useProxy: boolean,
proxySource: string[],
timeout: number
useProxy: boolean;
proxySource: string[];
timeout: number;
}
const agentList = [
'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',
'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
]
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0",
"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
"Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1",
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11",
"Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)",
];
async function testProxy(proxy: AxiosProxyConfig) {
await axios.get("http://mirrors.aliyun.com/debian/pool", {
proxy,
headers: {
"User-Agent": agentList[4]
"User-Agent": agentList[4],
},
timeout: this.config.timeout,
});
......@@ -81,57 +81,73 @@ export class ProxyFetcher {
if (!this.config.useProxy) {
return;
}
console.log(`Fetching proxies from ${url}.`)
console.log(`Fetching proxies from ${url}.`);
while (true) {
try {
const proxyPage: string = (await axios.get(url, {
const proxyPage: string = (
await axios.get(url, {
responseType: "document",
})).data;
const proxies: AxiosProxyConfig[] = proxyPage.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g).map(proxyString => {
})
).data;
const proxies: AxiosProxyConfig[] = proxyPage
.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g)
.map((proxyString) => {
const [host, _port] = proxyString.split(":");
const port = parseInt(_port);
const proxy = { host, port };
return proxy;
});
//const usableProxies = await filterProxies(proxies);
for (let proxy of proxies) {
for (const proxy of proxies) {
this.proxies.push(proxy);
}
console.error(`Got ${proxies.length} proxies from ${url}.`);
return;
} catch (e) {
console.error(`Failed fetching proxy list from ${url}: ${e.toString()}`)
console.error(
`Failed fetching proxy list from ${url}: ${e.toString()}`
);
}
}
}
async initProxies() {
await Promise.all(this.config.proxySource.map((m) => {
await Promise.all(
this.config.proxySource.map((m) => {
return this.initProxiesFrom(m);
}));
})
);
}
async getWithProxy(url: string, options: AxiosRequestConfig) {
while (true) {
if (this.config.useProxy && !this.proxies.length) {
await this.initProxies();
}
const proxyIndex = !this.config.useProxy ? null : (++this.counter) % this.proxies.length;
const proxyIndex = !this.config.useProxy
? null
: ++this.counter % this.proxies.length;
//const proxyIndex = 0;
const proxy = !this.config.useProxy ? null : this.proxies[proxyIndex];
try {
const data = (await axios.get(url, {
const data = (
await axios.get(url, {
proxy,
headers: {
"User-Agent": agentList[this.counter % agentList.length]
"User-Agent": agentList[this.counter % agentList.length],
},
timeout: this.config.timeout,
...options
})).data;
...options,
})
).data;
return data;
} catch (e) {
if (this.config.useProxy) {
this.proxies.splice(proxyIndex, 1);
}
console.error(`Failed fetching data from ${url}: ${e.toString()} ${this.proxies.length} proxies left.`)
console.error(
`Failed fetching data from ${url}: ${e.toString()} ${
this.proxies.length
} proxies left.`
);
}
}
}
......
......@@ -6,17 +6,24 @@ import _ from "underscore";
import yaml from "yaml";
function checkSameRow(row: PlayerRowFull, lrow: PlayerRowFull) {
return _.every(["name", "category", "serverArea", "server", "region"], field => lrow[field] === row[field]);
return _.every(
["name", "category", "serverArea", "server", "region"],
(field) => lrow[field] === row[field]
);
}
async function main() {
console.error("Started.");
const config: Config = yaml.parse(await fs.promises.readFile("./config.yaml", "utf8"));
const config: Config = yaml.parse(
await fs.promises.readFile("./config.yaml", "utf8")
);
const db = await mysql.createPool(config.MySQLConfig);
const urlDataCache = new Map<string, PlayerRowFull>();
const deleteList: number[] = [];
const datas: PlayerRowFull[] = await db.query(`select * from userdata order by date asc`);
for (let row of datas) {
const datas: PlayerRowFull[] = await db.query(
`select * from userdata order by id asc`
);
for (const row of datas) {
if (urlDataCache.has(row.url)) {
const oldRow = urlDataCache.get(row.url);
if (checkSameRow(row, oldRow)) {
......@@ -26,9 +33,11 @@ async function main() {
urlDataCache.set(row.url, row);
}
console.error(`Deletes: ${deleteList.length}`);
for (let id of deleteList) {
for (const id of deleteList) {
const sql = `delete from userdata where id = ?`;
console.error(`Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`);
console.error(
`Deleted: ${sql} ${id} ${JSON.stringify(await db.query(sql, id))}`
);
}
console.error("Finished.");
process.exit();
......
import {Tx3Fetcher, servers, Config} from "./fetcher";
import { Config, servers, Tx3Fetcher } from "./fetcher";
import fs from "fs";
import _ from "underscore";
import yaml from "yaml";
import { CronJob } from "cron";
......@@ -12,10 +11,17 @@ async function loadConfig() {
async function runServer(fetcher: Tx3Fetcher, server: string) {
const users = await fetcher.fetchListFromServer(server);
await fs.promises.writeFile(`./output/servers/${server}.json`, JSON.stringify({
await fs.promises.writeFile(
`./output/servers/${server}.json`,
JSON.stringify(
{
date: fetcher.curDate,
data: users
}, null, 2));
data: users,
},
null,
2
)
);
return users;
}
......@@ -25,27 +31,38 @@ async function run() {
await fs.promises.access("./output/servers");
} catch (e) {
await fs.promises.mkdir("./output/servers", {
recursive: true
recursive: true,
});
}
const fetcher = new Tx3Fetcher(config);
await fetcher.init();
if (config.server) {
await Promise.all(config.server.map(server => {
return runServer(fetcher, server)
}));
await Promise.all(
config.server.map((server) => {
return runServer(fetcher, server);
})
);
} else {
const userListWithServer = await Promise.all(servers.map(server => {
return runServer(fetcher, server)
}));
const userListWithServer = await Promise.all(
servers.map((server) => {
return runServer(fetcher, server);
})
);
const allServersList: any = {};
for (let i = 0; i < servers.length;++i) {
for (let i = 0; i < servers.length; ++i) {
allServersList[servers[i]] = userListWithServer[i];
}
await fs.promises.writeFile(`./output/all.json`, JSON.stringify({
await fs.promises.writeFile(
`./output/all.json`,
JSON.stringify(
{
date: fetcher.curDate,
data: allServersList
}, null, 2));
data: allServersList,
},
null,
2
)
);
}
console.log("Finished.");
}
......@@ -53,12 +70,19 @@ async function run() {
async function main() {
await loadConfig();
if (process.argv[2] === "cron") {
const job = new CronJob(config.cronString, run, null, true, "Asia/Shanghai", null, true);
const job = new CronJob(
config.cronString,
run,
null,
true,
"Asia/Shanghai",
null,
true
);
job.start();
} else {
await run();
process.exit();
}
}
main();
import HTML from "posthtml-parser";
import _, { first } from "underscore";
import {getDepthOfTree, getNumber, findNodeIndex, findNodeIndexByAttribute, findNodeIndexByContent, findNodeIndexByTag, getContinuousNumber} from "./utility";
import _ from "underscore";
import {
findNodeIndexByAttribute,
findNodeIndexByContent,
getContinuousNumber,
getDepthOfTree,
getNumber,
} from "./utility";
export interface AttackAttribute {
'攻力': number, //大攻取前2字节int16,小攻取后2字节int16,法力也是这样
'命中': number,
'法力': number,
'重击': number,
'会心一击': number,
'附加伤害': number
攻力: number; //大攻取前2字节int16,小攻取后2字节int16,法力也是这样
命中: number;
法力: number;
重击: number;
会心一击: number;
附加伤害: number;
}
export interface DefenseAttribute {
'防御': number,
'回避': number,
'法防': number,
'神明': number,
'化解': number,
'知彼': number
防御: number;
回避: number;
法防: number;
神明: number;
化解: number;
知彼: number;
}
export interface SpecialAttribute {
'身法': number,
'坚韧': number,
'定力': number,
'诛心': number,
'御心': number,
'万钧': number,
'铁壁': number
身法: number;
坚韧: number;
定力: number;
诛心: number;
御心: number;
万钧: number;
铁壁: number;
}
export interface AdvancedAttribute { '追电': number, '骤雨': number, '疾语': number, '明思': number, '扰心': number, '人祸': number }
export interface AdvancedAttribute {
追电: number;
骤雨: number;
疾语: number;
明思: number;
扰心: number;
人祸: number;
}
export class User {
id: string;
......@@ -47,7 +59,7 @@ export class User {
equipLocalRank: number;
equipCategoryRank: number;
scoreValue: number;
scoreRank: number
scoreRank: number;
scoreLocalRank: number;
scoreCategoryRank: number;
sqStage: number; // 天魂:2,地魂:1,没有神启:null
......@@ -69,14 +81,24 @@ export class User {
yhz: string[];
private parseMetadata() {
let namePos = findNodeIndexByAttribute(this.content, "class", "sTitle", []);
const namePos = findNodeIndexByAttribute(
this.content,
"class",
"sTitle",
[]
);
this.name = getDepthOfTree(this.content, namePos.concat([0]))[0] as string;
namePos[namePos.length - 1] += 2;
this.category = getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string;
this.category = getDepthOfTree(
this.content,
namePos.concat([0, 0])
)[0] as string;
namePos[namePos.length - 1] += 2;
[this.serverArea, this.server] = (getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string).split("&nbsp;");
[this.serverArea, this.server] = (
getDepthOfTree(this.content, namePos.concat([0, 0]))[0] as string
).split("&nbsp;");
let levelPos = findNodeIndexByContent(this.content, "等级", []);
const levelPos = findNodeIndexByContent(this.content, "等级", []);
levelPos.pop();
levelPos[levelPos.length - 1]++;
this.level = getNumber(getDepthOfTree(this.content, levelPos)[0]);
......@@ -110,7 +132,7 @@ export class User {
const tree = getDepthOfTree(this.content, pos);
for (let i = 3; i < tree.length; i += 2) {
const [_keyNode, valueNode] = getDepthOfTree(tree, [i]);
const keyNode = (_keyNode as HTML.NodeTag);
const keyNode = _keyNode as HTML.NodeTag;
const key = keyNode.content[0] as string;
const value = getNumber(valueNode);
ret[key] = value;
......@@ -118,7 +140,7 @@ export class User {
return ret;
}
private parseBasicAttributes() {
let ValuePos = findNodeIndexByContent(this.content, "", []);
const ValuePos = findNodeIndexByContent(this.content, "", []);
ValuePos.pop();
ValuePos[ValuePos.length - 1] += 2;
const datas = getContinuousNumber(this.content, ValuePos, 0, 4, 8);
......@@ -133,22 +155,35 @@ export class User {
ValuePos.pop();
ValuePos[ValuePos.length - 1] += 2;
this.attackAttributes = this.parseAttributeTable(ValuePos) as AttackAttribute;
this.attackAttributes = this.parseAttributeTable(
ValuePos
) as AttackAttribute;
ValuePos[ValuePos.length - 1] += 2;
this.defenseAttributes = this.parseAttributeTable(ValuePos) as DefenseAttribute;
this.defenseAttributes = this.parseAttributeTable(
ValuePos
) as DefenseAttribute;
ValuePos[ValuePos.length - 1] += 2;
ValuePos.push(1);
this.specialAttributes = this.parseAttributeTable(ValuePos) as SpecialAttribute;
this.specialAttributes = this.parseAttributeTable(
ValuePos
) as SpecialAttribute;
ValuePos[ValuePos.length - 1] += 2;
this.advancedAttributes = this.parseAttributeTable(ValuePos) as AdvancedAttribute;
this.advancedAttributes = this.parseAttributeTable(
ValuePos
) as AdvancedAttribute;
}
private parseYHZ() {
let ValuePos = findNodeIndexByAttribute(this.content, "id", "tableYHZ", []);
const ValuePos = findNodeIndexByAttribute(
this.content,
"id",
"tableYHZ",
[]
);
ValuePos.push(1);
const tree = getDepthOfTree(this.content, ValuePos);
this.yhz = [];
for (let i = 1; i < tree.length; i += 2){
const node = (tree[i] as HTML.NodeTag);
for (let i = 1; i < tree.length; i += 2) {
const node = tree[i] as HTML.NodeTag;
this.yhz.push(node.content[0] as string);
}
}
......
import HTML from "posthtml-parser";
import _ from "underscore";
export function getDepthOfTree(tree: HTML.Tree, indexList: number[]): HTML.Tree {
export function getDepthOfTree(
tree: HTML.Tree,
indexList: number[]
): HTML.Tree {
if (indexList.length) {
const _indexList = _.clone(indexList);
const index = _indexList.splice(0, 1)[0];
const node = tree[index];
if (typeof (node) === "string" || !node.content) {
if (typeof node === "string" || !node.content) {
return [node];
}
return getDepthOfTree(node.content, _indexList);
......@@ -14,7 +18,11 @@ export function getDepthOfTree(tree: HTML.Tree, indexList: number[]): HTML.Tree
}
}
export function findNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node) => boolean, offset: number[]): number[] {
export function findNodeIndex(
baseTree: HTML.Tree,
condition: (node: HTML.Node) => boolean,
offset: number[]
): number[] {
const queue = [offset];
while (queue.length) {
const indexList = queue.splice(0, 1)[0];
......@@ -24,7 +32,7 @@ export function findNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node)
const newList = indexList.concat([i]);
if (condition(node)) {
return newList;
} else if (typeof (node) !== "string") {
} else if (typeof node !== "string") {
queue.push(newList);
}
}
......@@ -32,7 +40,11 @@ export function findNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node)
return null;
}
export function findAllNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Node) => boolean, offset: number[]): number[][] {
export function findAllNodeIndex(
baseTree: HTML.Tree,
condition: (node: HTML.Node) => boolean,
offset: number[]
): number[][] {
const queue = [offset];
const res: number[][] = [];
while (queue.length) {
......@@ -44,7 +56,7 @@ export function findAllNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Nod
if (condition(node)) {
//console.log(newList);
res.push(newList);
} else if (typeof (node) !== "string") {
} else if (typeof node !== "string") {
queue.push(newList);
}
}
......@@ -52,47 +64,112 @@ export function findAllNodeIndex(baseTree: HTML.Tree, condition: (node: HTML.Nod
return res;
}
export function findNodeIndexByContent(baseTree: HTML.Tree, label: string, offset: number[]): number[] {
return findNodeIndex(baseTree, (node) => {
export function findNodeIndexByContent(
baseTree: HTML.Tree,
label: string,
offset: number[]
): number[] {
return findNodeIndex(
baseTree,
(node) => {
return node === label;
}, offset);
},
offset
);
}
export function findNodeIndexByAttribute(baseTree: HTML.Tree, key: string, value: string, offset: number[]): number[] {
return findNodeIndex(baseTree, (node) => {
return typeof (node) !== "string" && node.attrs && node.attrs[key] === value;
}, offset);
export function findNodeIndexByAttribute(
baseTree: HTML.Tree,
key: string,
value: string,
offset: number[]
): number[] {
return findNodeIndex(
baseTree,
(node) => {
return (
typeof node !== "string" && node.attrs && node.attrs[key] === value
);
},
offset
);
}
export function findNodeIndexByTag(baseTree: HTML.Tree, tag: string, offset: number[]): number[] {
return findNodeIndex(baseTree, (node) => {
return typeof (node) !== "string" && node.tag === tag;
}, offset);
export function findNodeIndexByTag(
baseTree: HTML.Tree,
tag: string,
offset: number[]
): number[] {
return findNodeIndex(
baseTree,
(node) => {
return typeof node !== "string" && node.tag === tag;
},
offset
);
}
export function findAllNodeIndexByContent(baseTree: HTML.Tree, label: string, offset: number[]): number[][] {
return findAllNodeIndex(baseTree, (node) => {
export function findAllNodeIndexByContent(
baseTree: HTML.Tree,
label: string,
offset: number[]
): number[][] {
return findAllNodeIndex(
baseTree,
(node) => {
return node === label;
}, offset);
},
offset
);
}
export function findAllNodeIndexByAttribute(baseTree: HTML.Tree, key: string, value: string, offset: number[]): number[][] {
return findAllNodeIndex(baseTree, (node) => {
return typeof (node) !== "string" && node.attrs && node.attrs[key] === value;
}, offset);
export function findAllNodeIndexByAttribute(
baseTree: HTML.Tree,
key: string,
value: string,
offset: number[]
): number[][] {
return findAllNodeIndex(
baseTree,
(node) => {
return (
typeof node !== "string" && node.attrs && node.attrs[key] === value
);
},
offset
);
}
export function findAllNodeIndexByTag(baseTree: HTML.Tree, tag: string, offset: number[]): number[][] {
return findAllNodeIndex(baseTree, (node) => {
return typeof (node) !== "string" && node.tag === tag;
}, offset);
export function findAllNodeIndexByTag(
baseTree: HTML.Tree,
tag: string,
offset: number[]
): number[][] {
return findAllNodeIndex(
baseTree,
(node) => {
return typeof node !== "string" && node.tag === tag;
},
offset
);
}
const chineseCapitalNumbers = ["", "", "", "", "", "", "", "", "", ""]
const chineseCapitalNumbers = [
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
];
export function getString(node: HTML.Node, lengthLimit?: number) {
let resultStr: string;
if (typeof (node) === "string") {
if (typeof node === "string") {
resultStr = node;
} else {
const subTree = node.content;
......@@ -100,7 +177,7 @@ export function getString(node: HTML.Node, lengthLimit?: number) {
return null;
}
const subNode = subTree[0];
if (typeof (subNode) === "string") {
if (typeof subNode === "string") {
resultStr = subTree[0] as string;
} else {
resultStr = getString(subTree[0]) as string;
......@@ -118,11 +195,14 @@ export function getNumber(node: HTML.Node) {
let stringMatch: RegExpMatchArray;
if (numberStr === "没有上榜" || !numberStr) {
return null;
} else if (stringMatch = numberStr.match(/^([天地])魂$/)) {
} else if ((stringMatch = numberStr.match(/^([天地])魂$/))) {
return stringMatch[1] === "" ? 2 : 1;
} else if (stringMatch = numberStr.match(/^(.+)(.+)境界$/)) {
return (_.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[1]) << 8) | _.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[2]);
} else if (stringMatch = numberStr.match(/^(\d+)-(\d+)$/)) {
} else if ((stringMatch = numberStr.match(/^(.+)(.+)境界$/))) {
return (
(_.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[1]) << 8) |
_.findIndex(chineseCapitalNumbers, (m) => m === stringMatch[2])
);
} else if ((stringMatch = numberStr.match(/^(\d+)-(\d+)$/))) {
const minValue = parseInt(stringMatch[1]);
const maxValue = parseInt(stringMatch[2]);
return (minValue << 16) | maxValue;
......@@ -131,7 +211,13 @@ export function getNumber(node: HTML.Node) {
}
}
export function getContinuousNodes(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): HTML.Node[] {
export function getContinuousNodes(
tree: HTML.Tree,
_pos: number[],
moveOffset: number,
step: number,
dataCount: number
): HTML.Node[] {
const pos = _.clone(_pos);
const datas: HTML.Node[] = [];
for (let i = 0; i < dataCount; ++i) {
......@@ -142,10 +228,26 @@ export function getContinuousNodes(tree: HTML.Tree, _pos: number[], moveOffset:
return datas;
}
export function getContinuousData(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): string[] {
return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(getString);
export function getContinuousData(
tree: HTML.Tree,
_pos: number[],
moveOffset: number,
step: number,
dataCount: number
): string[] {
return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(
getString
);
}
export function getContinuousNumber(tree: HTML.Tree, _pos: number[], moveOffset: number, step: number, dataCount: number): number[] {
return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(getNumber);
export function getContinuousNumber(
tree: HTML.Tree,
_pos: number[],
moveOffset: number,
step: number,
dataCount: number
): number[] {
return getContinuousNodes(tree, _pos, moveOffset, step, dataCount).map(
getNumber
);
}
......@@ -4,9 +4,9 @@ async function main() {
useProxy: true,
proxySource: [
"http://www.89ip.cn/tqdl.html?api=1&num=9999",
"http://www.66ip.cn/mo.php?tqsl=9999"
"http://www.66ip.cn/mo.php?tqsl=9999",
],
timeout: 10000
timeout: 10000,
});
await fetcher.initProxies();
console.log(await fetcher.getWithProxy("https://mycard.moe", {}));
......
......@@ -5,11 +5,17 @@ import HTML from "posthtml-parser";
async function main() {
let id = "28_20588";
let html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2));
await fs.promises.writeFile(
`./tests/${id}.json`,
JSON.stringify(HTML(html), null, 2)
);
console.log(new User(id, html, null));
id = "6_18804822";
html = await fs.promises.readFile(`./tests/${id}.html`, "utf-8");
await fs.promises.writeFile(`./tests/${id}.json`, JSON.stringify(HTML(html), null, 2));
await fs.promises.writeFile(
`./tests/${id}.json`,
JSON.stringify(HTML(html), null, 2)
);
console.log(new User(id, html, null));
}
main();
{
"compilerOptions": {
"outDir": "build",
"outDir": "dist",
"module": "commonjs",
"target": "esnext",
"target": "es2021",
"esModuleInterop": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"declaration": true,
"sourceMap": true
},
"compileOnSave": true,
"allowJs": true,
"include": [
"src/*.ts",
"tests/*.ts"
"*.ts",
"src/**/*.ts",
"test/**/*.ts",
"tests/**/*.ts"
]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment