subenv

.dockerignore

@@ -112,3 +112,6 @@ dist
 Dockerfile
 .gitlab-ci.yml
 /config.yaml
+
+/ygopro-*
+/pack.db

.gitignore

@@ -107,4 +107,5 @@ dist
 /output
 /config.yaml
 
-/ygopro-database
+/ygopro-*
+/pack.db

.gitlab-ci.yml

@@ -6,53 +6,47 @@ variables:
   GIT_DEPTH: "1"
   POST_DEPTH: 10
 
-zh-CN:
+.task:
   stage: generate
   tags: 
     - linux
-  image: node:buster
   variables:
     LOCALE: zh-CN
+    NPM_SCRIPT: start
+    CARD_SYMBOL: '简体中文卡'
+    FILE_SYMBOL: 'cn'
   script: ./ci-scripts/generate.sh
   artifacts:
     paths:
       - output
 
+zh-CN:
+  extends: .task
+
 en-US:
-  stage: generate
-  tags: 
-    - linux
-  image: node:buster
+  extends: .task
   variables:
     LOCALE: en-US
-  script: ./ci-scripts/generate.sh
-  artifacts:
-    paths:
-      - output
+
 
 ja-JP:
-  stage: generate
-  tags: 
-    - linux
-  image: node:buster
+  extends: .task
   variables:
     LOCALE: ja-JP
-  script: ./ci-scripts/generate.sh
-  artifacts:
-    paths:
-      - output
 
 ko-KR:
-  stage: generate
-  tags: 
-    - linux
-  image: node:buster
+  extends: .task
   variables:
     LOCALE: ko-KR
-  script: ./ci-scripts/generate.sh
-  artifacts:
-    paths:
-      - output
+
+env408:
+  extends: .task
+  variables:
+    LOCALE: zh-CN
+    NPM_SCRIPT: start:subenv
+    DATE: '2013-01-01'
+    CARD_SYMBOL: '408环境'
+    FILE_SYMBOL: 'env408'
 
 upload_to_minio:
   stage: deploy
@@ -61,6 +55,7 @@ upload_to_minio:
     - en-US
     - ja-JP
     - ko-KR
+    - env408
   tags: 
     - linux
   script:

ci-scripts/generate.sh

@@ -8,13 +8,14 @@ export OUTPUT_PATH="./output/$LOCALE"
 
 apt update ; apt -y install build-essential python3 sqlite3 git tar
 git clone --depth=1 https://code.mycard.moe/mycard/ygopro-database
+wget -O pack.db https://cdn02.moecube.com:444/ygopro-card-list/pack.db
 
 cp -rf ./ygopro-database/locales/ja-JP/cards.cdb "$JP_DATABASE_PATH"
 
 npm ci
 npm run build
-npm start
+npm run "$NPM_SCRIPT"
 
 cd "$OUTPUT_PATH"
-tar zcvf "../cn-data-$LOCALE.tar.gz" deck expansions
+tar zcvf "../$FILE_SYMBOL-data-$LOCALE.tar.gz" deck expansions
 cd ..

package.json

@@ -5,7 +5,8 @@
   "main": "run.ts",
   "scripts": {
     "build": "tsc",
-    "start": "node build/run.js | bunyan"
+    "start": "node build/run.js | bunyan",
+    "start:subenv": "node build/subenv.js | bunyan"
   },
   "repository": {
     "type": "git",
@@ -21,14 +22,17 @@
     "@types/bunyan": "^1.8.6",
     "@types/node": "^14.11.5",
     "@types/sqlite3": "^3.1.6",
-    "@types/underscore": "^1.10.24",
     "axios": "^0.20.0",
     "bunyan": "^1.8.14",
     "jinja-js": "^0.1.8",
+    "lodash": "^4.17.21",
+    "moment": "^2.30.1",
     "sqlite": "^4.0.15",
     "sqlite3": "^5.0.0",
-    "typescript": "^4.0.3",
-    "underscore": "^1.11.0",
     "yaml": "^1.10.0"
+  },
+  "devDependencies": {
+    "@types/lodash": "^4.17.7",
+    "typescript": "^5.5.4"
   }
 }

run.ts

 import { DBReader } from "./src/dbreader";
 import { NWFetcher } from "./src/nwfetcher";
-import _ from "underscore";
+import _ from "lodash";
 import { CNOCGFetcher } from "./src/cnocg";
 import { ExtraCards } from "./src/ExtraCards";
 import { DirectFetcher } from "./src/direct";
@@ -16,7 +16,6 @@ async function main() {
 	//const missingExtraCards = ExtraCards.filter(c => !cards.some(cc => c.code === cc.code));
 	//await dbreader.run(cards.concat(missingExtraCards));
 	const cards = await DirectFetcher.fetchOnce();
-	//console.log(cards);
 	await dbreader.run(cards);
 	process.exit();
 }

src/base.ts

 import { Config, loadConfig } from "./config";
 import bunyan from "bunyan";
-import _ from "underscore";
+import _ from "lodash";
 
 export default abstract class Base {
 	config: Config;

src/config.ts

@@ -5,7 +5,11 @@ export interface Config {
 	postDepth: number;
 	jpDatabasePath: string;
 	cnDatabasePath: string;
+	cardListDatabasePath: string;
 	outputPath: string;
+	descSymbol: string;
+	fileSymbol: string;
+	// cardListCondition?: string;
 }
 
 export async function loadConfig(): Promise<Config> {
@@ -14,6 +18,10 @@ export async function loadConfig(): Promise<Config> {
 		postDepth: process.env.POST_DEPTH ? parseInt(process.env.POST_DEPTH) : 5,
 		jpDatabasePath: process.env.JP_DATABASE_PATH || "./ygopro-database/locales/ja-JP/cards.cdb",
 		cnDatabasePath: process.env.SOURCE_TARGET_PATH || "./ygopro-database/locales/zh-CN/cards.cdb",
-		outputPath: process.env.OUTPUT_PATH || "./output"
+		cardListDatabasePath: process.env.CARD_LIST_DATABASE_PATH || "./pack.db",
+		outputPath: process.env.OUTPUT_PATH || "./output",
+		descSymbol: process.env.CARD_SYMBOL || "简体中文卡",
+		fileSymbol: process.env.FILE_SYMBOL || "cn",
+		// cardListCondition: process.env.CARD_LIST_CONDITION,
 	}
 }

src/dbreader.ts

 import { open, Database } from "sqlite";
 import sqlite3 from "sqlite3";
-import _ from "underscore";
+import _ from "lodash";
 import Base from "./base";
 import { promises as fs } from "fs";
-import { generateBanlistFromCode } from "./utility";
+import { Banlist, generateBanlistFromCode, mergeBanlists } from "./utility";
 import { CardPool, DeckGenerator } from "./deck";
+import moment from "moment";
 
 const textsFields = ["id", "name", "desc"]
 for (let i = 1; i <= 16; ++i) {
@@ -66,7 +67,7 @@ export class Card {
 	}
 	async getRelatedCards(db: Database) {
 		const code = this.code;
-		const moreCodes: number[] = (await db.all('SELECT id FROM datas WHERE id > ? AND id <= ?', [code, code + 10])).map(m => m.id);
+		const moreCodes: number[] = (await db.all('SELECT id FROM datas WHERE id > ? AND id <= ?', [code, code + 20])).map(m => m.id);
 		const cards = await Promise.all(moreCodes.map(code => this.createCardFromRelatedCode(code, db)));
 		return cards;
 	}
@@ -114,7 +115,7 @@ export class DBReader extends Base {
 	jpdb: Database;
 	cndb: Database;
 	outputdb: Database;
-	private async openDatabase(path: string) {
+	async openDatabase(path: string) {
 		return await open({
 			filename: path,
 			driver: sqlite3.Database
@@ -134,9 +135,9 @@ export class DBReader extends Base {
 		}
 	}
 	private async openOutputDatabase() {
-		const fullPath = `${this.config.outputPath}/expansions/cn.cdb`;
+		const fullPath = `${this.config.outputPath}/expansions/${this.config.fileSymbol}.cdb`;
 		const createDirectoryPaths = [
-			`${this.config.outputPath}/deck/cn`,
+			`${this.config.outputPath}/deck/${this.config.fileSymbol}`,
 			`${this.config.outputPath}/expansions`
 		];
 		for (let createDirectoryPath of createDirectoryPaths) {
@@ -191,15 +192,18 @@ export class DBReader extends Base {
 		const otherCodes: number[] = (await this.cndb.all(sql)).map(m => m.id);
 		return otherCodes;
 	}
-	async generateBanlist(codes: number[]) {
+	async generateBanlist(codes: number[], extraBanlists: Banlist[] = []) {
 		const otherCodes = await this.getOtherCardCodes(codes);
 		const banlistString = await generateBanlistFromCode([
+			mergeBanlists([
+				...extraBanlists,
 				{
-				name: "cn",
+					name: moment().format('YYYY.MM') + ' ' + this.config.descSymbol,
 					list: [
 						otherCodes
 					]
 				}
+			]),
 		]);
 		await fs.writeFile(`${this.config.outputPath}/expansions/lflist.conf`, banlistString);
 	}
@@ -219,13 +223,15 @@ export class DBReader extends Base {
 		const cardPool = await this.categorizeCards(cards);
 		const deckGenerator = new DeckGenerator(cardPool);
 		const deckTexts = deckGenerator.getDeckTexts();
-		await Promise.all(_.range(deckTexts.length).map(i => fs.writeFile(`${this.config.outputPath}/deck/cn/cn_${i}.ydk`, deckTexts[i])));
+		await Promise.all(_.range(deckTexts.length).map(i => fs.writeFile(`${this.config.outputPath}/deck/${this.config.fileSymbol}/${this.config.fileSymbol}_${i}.ydk`, deckTexts[i])));
 	}
-	async run(cards: Card[]) {
+	async run(cards: Card[], options: {
+		extraBanlists?: Banlist[]
+	} = {}) {
 		await Promise.all(cards.map(card => card.loadData(this.cndb)));
-		const extendedCards = _.flatten(await Promise.all(cards.map(card => card.getRelatedCards(this.cndb))), true);
-		const allCards = cards.concat(extendedCards);
-		const queries = _.flatten(allCards.map(card => card.getSQLQueries()), true);
+		const extendedCards = (await Promise.all(cards.map(card => card.getRelatedCards(this.cndb)))).flat();
+		const allCards = _.uniqBy(cards.concat(extendedCards), (s) => s.code);
+		const queries = allCards.flatMap(card => card.getSQLQueries());
 		await this.openOutputDatabase();
 		await this.outputdb.run("BEGIN TRANSACTION;");
 		for (let query of queries) {
@@ -235,7 +241,7 @@ export class DBReader extends Base {
 		await this.outputdb.run("COMMIT;");
 		const allCodes = allCards.map(card => card.code);
 		this.log.info(`Database created.`);
-		await this.generateBanlist(allCodes);
+		await this.generateBanlist(allCodes, options.extraBanlists || []);
 		this.log.info(`LFList created.`);
 		await this.generatePatch(allCodes);
 		this.log.info(`Patch created.`);

src/deck.ts

-import _ from "underscore";
+import _ from "lodash";
 import { promises as fs } from "fs";
 
 export interface CardPool {

src/direct.ts

@@ -22,13 +22,13 @@ export class DirectFetcher extends Base {
 			const codeDifference = code - c;
 
 			// 新加入的卡号比某一卡号大，并且 10 以内，则新的卡是那张卡的关联卡，不入。
-			if (codeDifference > 0 && codeDifference <= 10) {
+			if (codeDifference > 0 && codeDifference <= 20) {
 				this.log.debug(`Skipped forward-related dup: ${code} ${c}`);
 				return;
 			}
 
 			// 新加入的卡号比某一卡号小，并且 10 以内，则把旧的卡删掉。-
-			if (codeDifference < 0 && codeDifference >= -10) {
+			if (codeDifference < 0 && codeDifference >= -20) {
 				this.log.debug(`Would remove backword-related dup: ${code} ${c}`);
 				posToRemove.push(i);
 			}

src/nwfetcher.ts

 
 
 import axios from "axios";
-import _ from "underscore";
+import _ from "lodash";
 import Base from "./base";
 import { DBReader } from "./dbreader";
 
@@ -60,7 +60,7 @@ export class NWFetcher extends Base {
 	async fetch(): Promise<string[]> {
 		this.log.debug(`Started fetching...`);
 		const posts = await this.fetchPosts();
-		const strings = _.flatten(await Promise.all(posts.map(m => this.getAllStrings(m))), true);
+		const strings = (await Promise.all(posts.map(m => this.getAllStrings(m)))).flat();
 		this.log.debug(`Done.`);
 		return _.uniq(strings);
 	}

src/packdbfetcher.ts

+import _ from "lodash";
+import Base from "./base";
+import { Card, DBReader } from "./dbreader";
+
+export class PackDbFetcher extends Base { 
+  static async fetchOnce(dbreader: DBReader, cardListCondition?: string) {
+		const fetcher = new PackDbFetcher({ name: "Pack DB Fetcher", level: 'info' });
+		await fetcher.init();
+		return await fetcher.fetch(dbreader, cardListCondition);
+  }
+  
+  async fetch(dbreader: DBReader, cardListCondition?: string) {
+    const packDb = await dbreader.openDatabase(this.config.cardListDatabasePath);
+    const packDbData = await packDb.all<{ id: number, pack_id: string, date: string }[]>("SELECT id FROM pack" + (cardListCondition ? ` WHERE ${cardListCondition}` : ""));
+    return packDbData.map(m => new Card(m.id));
+  }
+}

src/utility.ts

-import _ from "underscore";
+import _ from "lodash";
 import jinja from "jinja-js";
 import { promises as fs } from "fs";
 
-interface Banlist {
+export interface Banlist {
 	name: string
 	list: number[][]
 }
@@ -11,3 +11,55 @@ export async function generateBanlistFromCode(banlists: Banlist[]) {
 	const template = await fs.readFile("./templates/lflist.conf.j2", "utf-8");
 	return jinja.render(template, { banlists });
 }
+
+export async function readLFList(content: string): Promise<Banlist[]> {
+	const lines = content.split("\n");
+	const ret: Banlist[] = [];
+	let name = "";
+	let list: number[][] = [];
+	const nextList = (nextName: string) => { 
+		if (list.length > 0) { 
+			ret.push({ name, list });
+		}
+		list = [];
+		name = nextName;
+	}
+	for (let line of lines) {
+		if (line.startsWith("#")) {
+			continue;
+		}
+		if(line.startsWith("!")) {
+			nextList(line.slice(1));
+			continue;
+		}
+		// xxxxxxxx 0
+		// use regex to match
+		const match = line.match(/^(\d{4,})\s(\d)/);
+		if(!match) {
+			continue;
+		}
+		const code = parseInt(match[1]);
+		const status = parseInt(match[2]);
+		list[status] ??= [];
+		list[status].push(code);
+	}
+	nextList('');
+	return ret;
+}
+
+export function mergeBanlists(lists: Banlist[]): Banlist {
+	const ret: Banlist = { name: lists[0].name, list: [] };
+	const existingCodes = new Set<number>();
+	for (let status = 0; status < 3; ++status) {
+		for (let list of lists) {
+			ret.list[status] ??= [];
+			for(let code of list.list[status] ?? []) {
+				if (!existingCodes.has(code)) {
+					ret.list[status].push(code);
+					existingCodes.add(code);
+				}
+			}
+		}
+	}
+	return ret;
+}

subenv.ts

+import { DBReader } from "./src/dbreader";
+import _ from "lodash";
+import { PackDbFetcher } from "./src/packdbfetcher";
+import axios from "axios";
+import { readLFList } from "./src/utility";
+import moment from "moment";
+
+const DATE = process.env.DATE;
+
+async function getBanlist() {
+	const lflistReq = await axios.get<string>("https://cdn02.moecube.com:444/ygopro-database/zh-CN/lflist.conf", {
+		responseType: 'document'
+	});
+	const lflists = (await readLFList(lflistReq.data)).filter(l => !l.name.includes('TCG')).map(l => ({
+		...l,
+		date: moment(l.name.split(' ')[0], 'YYYY.MM')
+	})).filter(l => l.date.isBefore(DATE));
+	return _.maxBy(lflists, l => l.date.unix());
+}
+
+async function main() {
+	const dbreader = new DBReader({
+		name: "Database",
+		//level: Logger.DEBUG
+	});
+	await dbreader.init();
+	const cards = await PackDbFetcher.fetchOnce(dbreader, `date <= '${DATE}'`);
+	console.log(cards.length);
+	
+	await dbreader.run(cards, {
+		extraBanlists: [await getBanlist()],
+	});
+	process.exit();
+}
+main();

test/deckgen.ts

@@ -2,7 +2,7 @@ import { open, Database } from "sqlite";
 import sqlite3 from "sqlite3";
 import { promises as fs } from "fs";
 import { DeckGenerator } from "../src/deck";
-import _ from "underscore";
+import _ from "lodash";
 
 async function openDatabase(path: string) {
 	return await open({