rework dbreader

run.ts

@@ -8,7 +8,8 @@ async function main() {
 	const dbreader = new DBReader({ name: "Database", level: "debug" });
 	await dbreader.init();
 	const strings = await fetcher.fetch();
-	await dbreader.run(strings);
+	const cards = await dbreader.getAllCardsFromJapaneseNames(strings);
+	await dbreader.run(cards);
 	process.exit();
 }
 main();

src/dbreader.ts

@@ -4,7 +4,7 @@ import _ from "underscore";
 import Base from "./base";
 import { promises as fs } from "fs";
 import { generateBanlistFromCode } from "./utility";
-import { Cards, DeckGenerator } from "./deck";
+import { CardPool, DeckGenerator } from "./deck";
 
 const textsFields = ["id", "name", "desc"]
 for (let i = 1; i <= 16; ++i) {
@@ -12,7 +12,7 @@ for (let i = 1; i <= 16; ++i) {
 }
 const datasFields = ["id", "ot", "alias", "setcode", "type", "atk", "def", "level", "race", "attribute", "category"];
 
-class SQLQuery {
+export class SQLQuery {
 	sql: string;
 	values: any[];
 	constructor(sql: string, values: any[]) {
@@ -24,6 +24,73 @@ class SQLQuery {
 	}
 }
 
+export class Card {
+	code: number;
+	datas: any;
+	texts: any;
+	constructor(code: number) {
+		this.code = code;
+		this.datas = {};
+		this.texts = {};
+	}
+	private getDatasArray(): any[] {
+		const ret = [];
+		for (let field of datasFields) {
+			ret.push(this.datas[field]);
+		}
+		return ret;
+	}
+	private getTextsArray(): any[] {
+		const ret = [];
+		for (let field of textsFields) {
+			ret.push(this.texts[field]);
+		}
+		return ret;
+	}
+	async getRelatedCards(db: Database) {
+		const code = this.code;
+		const moreCodes: number[] = (await db.all('SELECT id FROM datas WHERE id > ? AND id <= ?', [code, code + 10])).map(m => m.id);
+		const cards = moreCodes.map(code => new Card(code));
+		await Promise.all(cards.map(card => card.loadData(db)));
+		return cards;
+	}
+	isInMainDeck() {
+		const cardType: number = this.datas.type;
+		if (cardType === undefined) {
+			throw `Card data is not loaded`;
+		}
+		return (cardType & (0x4000000 | 0x800000 | 0x4000 | 0x2000 | 0x40)) == 0;
+	}
+	isInExtraDeck() {
+		const cardType: number = this.datas.type;
+		if (cardType === undefined) {
+			throw `Card data is not loaded`;
+		}
+		return (cardType & (0x4000000 | 0x800000 | 0x2000 | 0x40)) > 0;
+	}
+	async loadData(db: Database) {
+		const datas = await db.get("select * from datas where id = ?", [this.code]);
+		const texts = await db.get("select * from texts where id = ?", [this.code]);
+		texts.desc += '\r\n\r\n\u2605简体中文卡';
+		this.datas = {
+			...this.datas,
+			...datas
+		}
+		this.texts = {
+			...this.texts,
+			...texts
+		}
+	}
+	getSQLQueries() {
+		const datasArray = this.getDatasArray();
+		const textsArray = this.getTextsArray();
+		return [
+			new SQLQuery(`INSERT INTO texts VALUES(${_.range(textsArray.length).map(m => "?")});`, textsArray),
+			new SQLQuery(`INSERT INTO datas VALUES(${_.range(datasArray.length).map(m => "?")});`, datasArray)
+		]
+	}
+}
+
 export class DBReader extends Base {
 	jpdb: Database;
 	cndb: Database;
@@ -77,63 +144,26 @@ export class DBReader extends Base {
 			await this.outputdb.run(sql);
 		}
 	}
-	async getCodeFromJapaneseName(name: string): Promise<number> {
+	async getCardFromJapaneseName(name: string) {
 		this.log.debug(`Reading JP database for code of name ${name}.`);
 		const output = await this.jpdb.get('SELECT id FROM texts WHERE name = ?', name);
 		if (!output) {
 			this.log.debug(`Code of ${name} not found.`);
-			return 0;
+			return null;
 		}
 		const code: number = output.id;
-		return code;
-	}
-	async getExtendedCodeFromJapaneseName(name: string): Promise<number[]> {
-		const code: number = await this.getCodeFromJapaneseName(name);
-		if (!code) {
-			return [];
-		}
-		this.log.debug(`Reading CN database for more codes of id ${code}.`);
-		const moreCodes: number[] = (await this.cndb.all('SELECT id FROM datas WHERE id >= ? AND id <= ?', [code, code + 10])).map(m => m.id);
-		this.log.debug(`${name} => ${moreCodes.join(",")}`);
-		return moreCodes;
-	}
-	async getAllPureCodesFromJapaneseNames(names: string[]): Promise<number[]> {
-		const codes = await Promise.all(names.map(s => this.getCodeFromJapaneseName(s)));
-		return _.uniq(codes);
-	}
-	async getAllCodesFromJapaneseNames(names: string[]): Promise<number[]> {
-		const codes = _.flatten(await Promise.all(names.map(s => this.getExtendedCodeFromJapaneseName(s))), true);
-		return _.uniq(codes);
-	}
-	private getDatasArray(datas: any): any[] {
-		const ret = [];
-		for (let field of datasFields) {
-			ret.push(datas[field]);
+		this.log.debug(`${name} => ${code}`);
+		return new Card(code);
+	}
+	async getAllCardsFromJapaneseNames(names: string[]) {
+		const cards: Card[] = [];
+		for (let name of names) {
+			const card = await this.getCardFromJapaneseName(name);
+			if (card && _.every(cards, c => c.code !== card.code)) {
+				cards.push(card);
+			}
 		}
-		return ret;
-	}
-	private getTextsArray(texts: any): any[] {
-		const ret = [];
-		for (let field of textsFields) {
-			ret.push(texts[field]);
-		}
-		return ret;
-	}
-	async getQueriesFromCode(code: number): Promise<SQLQuery[]> {
-		this.log.debug(`Reading card ${code}.`);
-		const datas = await this.cndb.get("select * from datas where id = ?", [code]);
-		const texts = await this.cndb.get("select * from texts where id = ?", [code]);
-		texts.desc += '\r\n\r\n\u2605简体中文卡';
-		const datasArray = this.getDatasArray(datas);
-		const textsArray = this.getTextsArray(texts);
-		return [
-			new SQLQuery(`INSERT INTO texts VALUES(${_.range(textsArray.length).map(m => "?")});`, textsArray),
-			new SQLQuery(`INSERT INTO datas VALUES(${_.range(datasArray.length).map(m => "?")});`, datasArray)
-		]
-	}
-	async getAllQueries(codes: number[]): Promise<SQLQuery[]> {
-		const queries = _.flatten(await Promise.all(codes.map(s => this.getQueriesFromCode(s))), true);
-		return queries;
+		return cards;
 	}
 	async getOtherCardCodes(cnCodes: number[]) {
 		const sql = `SELECT id FROM datas WHERE 1 AND ${cnCodes.map(m => "id != ?").join(" AND ")}`;
@@ -152,38 +182,25 @@ export class DBReader extends Base {
 		]);
 		await fs.writeFile(`${this.config.outputPath}/expansions/lflist.conf`, banlistString);
 	}
-	private async checkExtraDeckCards(code: number) {
-		const sql = `select id from datas where type & (0x4000000 | 0x800000 | 0x2000 | 0x40) > 0 AND id = ?`;
-		const result = (await this.cndb.all(sql, [code]));
-		return result.length > 0;
-	}
-	private async checkMainDeckCards(code: number) {
-		const sql = `select id from datas where type & (0x4000000 | 0x800000 | 0x4000 | 0x2000 | 0x40) == 0 AND id = ?`;
-		const result = (await this.cndb.all(sql, [code]));
-		return result.length > 0;
-	}
-	private async categorizeCards(codes: number[]): Promise<Cards> {
-		const [mainResults, extraResults] = await Promise.all([
-			Promise.all(codes.map(code => this.checkMainDeckCards(code))),
-			Promise.all(codes.map(code => this.checkExtraDeckCards(code)))
-		]);
+	private async categorizeCards(cards: Card[]): Promise<CardPool> {
+		const main = cards.filter(card => card.isInMainDeck()).map(card => card.code);
+		const extra = cards.filter(card => card.isInExtraDeck()).map(card => card.code);
 		return {
-			main: codes.filter((code, index) => mainResults[index]),
-			extra: codes.filter((code, index) => extraResults[index])
+			main,
+			extra
 		}
 	}
-	private async generateDecks(codes: number[]) {
-		const cards = await this.categorizeCards(codes);
-		const deckGenerator = new DeckGenerator(cards);
+	private async generateDecks(cards: Card[]) {
+		const cardPool = await this.categorizeCards(cards);
+		const deckGenerator = new DeckGenerator(cardPool);
 		const deckTexts = deckGenerator.getDeckTexts();
 		await Promise.all(_.range(deckTexts.length).map(i => fs.writeFile(`${this.config.outputPath}/deck/cn/cn_${i}.ydk`, deckTexts[i])));
 	}
-	async run(strings: string[]) {
-		const [codes, pureCodes] = await Promise.all([
-			this.getAllCodesFromJapaneseNames(strings),
-			this.getAllPureCodesFromJapaneseNames(strings)
-		]);
-		const queries = await this.getAllQueries(codes);
+	async run(cards: Card[]) {
+		await Promise.all(cards.map(card => card.loadData(this.cndb)));
+		const extendedCards = _.flatten(await Promise.all(cards.map(card => card.getRelatedCards(this.cndb))), true);
+		const allCards = cards.concat(extendedCards);
+		const queries = _.flatten(allCards.map(card => card.getSQLQueries()), true);
 		await this.openOutputDatabase();
 		await this.outputdb.run("BEGIN TRANSACTION;");
 		for (let query of queries) {
@@ -192,9 +209,9 @@ export class DBReader extends Base {
 		}
 		await this.outputdb.run("COMMIT;");
 		this.log.debug(`Database created.`);
-		await this.generateBanlist(codes);
+		await this.generateBanlist(extendedCards.map(card => card.code));
 		this.log.debug(`LFList created.`);
-		await this.generateDecks(pureCodes);
+		await this.generateDecks(cards);
 		this.log.debug(`Decks generated.`);
 	}
 }

src/deck.ts

 import _ from "underscore";
 import { promises as fs } from "fs";
 
-export interface Cards {
+export interface CardPool {
 	main: number[],
 	extra: number[]
 }
-export interface Deck extends Cards {
+export interface Deck extends CardPool {
 	side: number[]
 }
 
 
 export class DeckGenerator {
-	cards: Cards;
-	constructor(cards: Cards) {
+	cards: CardPool;
+	constructor(cards: CardPool) {
 		this.cards = cards;
 	}
 	private getExtraCardCountInSide() {

test/fetch.ts

-import { DBReader } from "../src/dbreader";
-import { CNFetcher } from "../src/fetcher";
-import _ from "underscore";
-
-async function main() {
-	const fetcher = new CNFetcher({ name: "Test fetch", level: "debug" });
-	await fetcher.init();
-	const dbreader = new DBReader({ name: "Test database", level: "debug" });
-	await dbreader.init();
-	const strings = await fetcher.fetch();
-	await dbreader.run(strings);
-	await dbreader.finalize();
-	process.exit();
-}
-main();