add deck and lflist

.gitlab-ci.yml

@@ -12,11 +12,14 @@ generate:
     - linux
   image: node
   script:
-    - apt update ; apt -y install build-essential python3 sqlite3 git
+    - apt update ; apt -y install build-essential python3 sqlite3 git tar
     - git clone --depth=1 https://code.mycard.moe/mycard/ygopro-database
     - npm ci
     - npm run build
     - npm start
+    - cd output
+    - tar zcvf cn-data.tar.gz deck cn.cdb lflist.conf
+    - cd ..
   artifacts:
     paths:
       - output
@@ -47,6 +50,6 @@ deploy_to_ygopro_server:
     - ssh-keyscan koishi.momobako.com >> ~/.ssh/known_hosts
     - echo $NANAHIRA_SSH_KEY | base64 --decode > ~/.ssh/id_rsa
     - chmod 600 ~/.ssh/*
-    - scp -r output/cards.cdb nanahira@koishi.momobako.com:~/ygopro-cn/cards.cdb
+    - scp -r output/cn.cdb nanahira@koishi.momobako.com:~/ygopro-cn/cards.cdb
   only:
     - master

package-lock.json

@@ -459,6 +459,11 @@
       "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
       "optional": true
     },
+    "jinja-js": {
+      "version": "0.1.8",
+      "resolved": "https://registry.npmjs.org/jinja-js/-/jinja-js-0.1.8.tgz",
+      "integrity": "sha1-CxPuuW6QmT8EBRTXzmT1N4b1hRE="
+    },
     "jsbn": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",

package.json

@@ -24,6 +24,7 @@
     "@types/underscore": "^1.10.24",
     "axios": "^0.20.0",
     "bunyan": "^1.8.14",
+    "jinja-js": "^0.1.8",
     "sqlite": "^4.0.15",
     "sqlite3": "^5.0.0",
     "typescript": "^4.0.3",

src/dbreader.ts

@@ -3,7 +3,8 @@ import sqlite3 from "sqlite3";
 import _ from "underscore";
 import Base from "./base";
 import { promises as fs } from "fs";
-import SQL from "sql-template-strings";
+import { generateBanlistFromCode } from "./utility";
+import { Cards, DeckGenerator } from "./deck";
 
 const textsFields = ["id", "name", "desc"]
 for (let i = 1; i <= 16; ++i) {
@@ -47,16 +48,17 @@ export class DBReader extends Base {
 		}
 	}
 	private async openOutputDatabase() {
-		const fullPath = `${this.config.outputPath}/cards.cdb`;
+		const fullPath = `${this.config.outputPath}/cn.cdb`;
+		const createDirectoryPath = `${this.config.outputPath}/deck/cn`;
 		try {
-			await fs.access(this.config.outputPath);
+			await fs.access(createDirectoryPath);
 		} catch (e) {
-			this.log.debug(`Creating directory ${this.config.outputPath} ...`);
-			await fs.mkdir(this.config.outputPath, { recursive: true });
+			this.log.debug(`Creating directory ${createDirectoryPath} ...`);
+			await fs.mkdir(createDirectoryPath, { recursive: true });
 		}
 		try {
 			await fs.unlink(fullPath);
-		} catch(e) { }
+		} catch (e) { }
 		this.log.debug(`Creating database ${fullPath} ...`);
 		this.outputdb = await this.openDatabase(fullPath);
 		const initSQLs = [
@@ -127,8 +129,54 @@ export class DBReader extends Base {
 		const queries = _.flatten(await Promise.all(codes.map(s => this.getQueriesFromCode(s))), true);
 		return queries;
 	}
+	async getOtherCardCodes(cnCodes: number[]) {
+		const sql = `SELECT id FROM datas WHERE 1 AND ${cnCodes.map(m => "id != ?").join(" AND ")}`;
+		const otherCodes: number[] = (await this.cndb.all(sql, cnCodes)).map(m => m.id);
+		return otherCodes;
+	}
+	async generateBanlist(codes: number[]) {
+		const otherCodes = await this.getOtherCardCodes(codes);
+		const banlistString = await generateBanlistFromCode([
+			{
+				name: "cn",
+				list: [
+					otherCodes
+				]
+			}
+		]);
+		await fs.writeFile(`${this.config.outputPath}/lflist.conf`, banlistString);
+	}
+	private async checkExtraDeckCards(code: number) {
+		const sql = `select id from datas where type & (0x4000000 | 0x800000 | 0x2000 | 0x40) > 0 AND id = ?`;
+		const result = (await this.cndb.all(sql, [code]));
+		return result.length > 0;
+	}
+	private async checkMainDeckCards(code: number) {
+		const sql = `select id from datas where type & (0x4000000 | 0x800000 | 0x4000 | 0x2000 | 0x40) == 0 AND id = ?`;
+		const result = (await this.cndb.all(sql, [code]));
+		return result.length > 0;
+	}
+	private async categorizeCards(codes: number[]): Promise<Cards> {
+		const [mainResults, extraResults] = await Promise.all([
+			Promise.all(codes.map(code => this.checkMainDeckCards(code))),
+			Promise.all(codes.map(code => this.checkExtraDeckCards(code)))
+		]);
+		return {
+			main: codes.filter((code, index) => mainResults[index]),
+			extra: codes.filter((code, index) => extraResults[index])
+		}
+	}
+	private async generateDecks(codes: number[]) {
+		const cards = await this.categorizeCards(codes);
+		const deckGenerator = new DeckGenerator(cards);
+		const deckTexts = deckGenerator.getDeckTexts();
+		await Promise.all(_.range(deckTexts.length).map(i => fs.writeFile(`${this.config.outputPath}/deck/cn/cn_${i}.ydk`, deckTexts[i])));
+	}
 	async run(strings: string[]) {
-		const codes = await this.getAllCodesFromJapaneseNames(strings);
+		const [codes, pureCodes] = await Promise.all([
+			this.getAllCodesFromJapaneseNames(strings),
+			this.getAllPureCodesFromJapaneseNames(strings)
+		]);
 		const queries = await this.getAllQueries(codes);
 		await this.openOutputDatabase();
 		await this.outputdb.run("BEGIN TRANSACTION;");
@@ -138,5 +186,9 @@ export class DBReader extends Base {
 		}
 		await this.outputdb.run("COMMIT;");
 		this.log.debug(`Database created.`);
+		await this.generateBanlist(codes);
+		this.log.debug(`LFList created.`);
+		await this.generateDecks(pureCodes);
+		this.log.debug(`Decks generated.`);
 	}
 }

src/deck.ts

+import _ from "underscore";
+import { promises as fs } from "fs";
+
+export interface Cards {
+	main: number[],
+	extra: number[]
+}
+export interface Deck extends Cards {
+	side: number[]
+}
+
+
+export class DeckGenerator {
+	cards: Cards;
+	constructor(cards: Cards) {
+		this.cards = cards;
+	}
+	private getExtraCardCountInSide() {
+		if (!this.cards.main.length) {
+			return 15;
+		}
+		if (!this.cards.extra.length) {
+			return 0;
+		}
+		const mainExtraRatio = this.cards.main.length / this.cards.extra.length;
+		const maxRatio = (60 + 15) / 15;
+		const minRatio = 60 / (15 + 15);
+		if (mainExtraRatio >= maxRatio) {
+			return 0;
+		}
+		if (mainExtraRatio <= minRatio) {
+			return 15;
+		}
+		const simplifiedRatio = (mainExtraRatio - minRatio) / (maxRatio - minRatio);
+		return Math.ceil(15 * simplifiedRatio);
+	}
+	private splitCards(codes: number[], unit: number) {
+		const count = Math.ceil(codes.length / unit);
+		return _.range(count).map(i => {
+			return codes.slice(i * unit, (i + 1) * unit);
+		});
+	}
+	private getDeckString(deck: Deck) {
+		const deckText = '#generated by ygopro-cn-database-generator\n#main\n' + deck.main.join('\n') + '\n#extra\n' + deck.extra.join('\n') + '\n!side\n' + deck.side.join('\n') + '\n';
+		return deckText;
+	}
+	private getDeckFromPart(main: number[], extra: number[]) {
+		const targetMain = main.slice(0, 60);
+		const targetExtra = extra.slice(0, 15);
+		const targetSide = main.slice(60).concat(extra.slice(15));
+		return {
+			main: targetMain,
+			extra: targetExtra,
+			side: targetSide
+		}
+	}
+private getDecks() {
+		const extraInSide = this.getExtraCardCountInSide();
+		const mainInSide = 15 - extraInSide;
+		const extraCount = 15 + extraInSide;
+		const mainCount = 60 + mainInSide;
+		const mainParts = this.splitCards(this.cards.main, mainCount);
+		const extraParts = this.splitCards(this.cards.extra, extraCount);
+		return _.range(Math.max(mainParts.length, extraParts.length)).map(i => this.getDeckFromPart(mainParts[i] || [], extraParts[i] || []));
+	}
+	getDeckTexts() {
+		const decks = this.getDecks();
+		return decks.map(d => this.getDeckString(d));
+	}
+}

src/utility.ts

+import _ from "underscore";
+import jinja from "jinja-js";
+import { promises as fs } from "fs";
+
+interface Banlist {
+	name: string
+	list: number[][]
+}
+
+export async function generateBanlistFromCode(banlists: Banlist[]) {
+	const template = await fs.readFile("./templates/lflist.conf.j2", "utf-8");
+	return jinja.render(template, { banlists });
+}

templates/lflist.conf.j2

+#{% for banlist in banlists %}[{{banlist.name}}]{% endfor %}
+
+{% for banlist in banlists %}
+!{{banlist.name}}
+
+#forbidden
+{% for code in banlist.list[0] %}
+{{code}} 0
+{%- endfor %}
+
+#limit
+{% for code in banlist.list[1] %}
+{{code}} 1
+{%- endfor %}
+
+#semi limit
+{% for code in banlist.list[2] %}
+{{code}} 2
+{%- endfor %}
+
+{% endfor %}

test/fetch.ts

@@ -9,6 +9,7 @@ async function main() {
 	await dbreader.init();
 	const strings = await fetcher.fetch();
 	await dbreader.run(strings);
+	await dbreader.finalize();
 	process.exit();
 }
 main();

test/lflist.ts

+import { generateBanlistFromCode } from "../src/utility";
+
+const banlists = [
+	{
+		name: "test1",
+		list: [
+			[111, 222, 333],
+			[444, 555, 666],
+			[777, 888, 999]
+		]
+	},
+	{
+		name: "test2",
+		list: [
+			[111, 222, 333],
+			[444, 555, 666],
+			[777, 888, 9999]
+		]
+	}
+]
+
+async function main() {
+	const listText = await generateBanlistFromCode(banlists);
+	console.log(listText);
+}
+main();