migrate to ygopro-cdb-encode

package.json

@@ -20,20 +20,19 @@
   "license": "AGPL-3.0",
   "dependencies": {
     "@types/bunyan": "^1.8.6",
-    "@types/node": "^14.11.5",
-    "@types/sqlite3": "^3.1.6",
-    "axios": "^0.20.0",
+    "axios": "^1.13.5",
     "bunyan": "^1.8.14",
-    "jinja-js": "^0.1.8",
     "lodash": "^4.17.21",
     "moment": "^2.30.1",
-    "sqlite": "^4.0.15",
-    "sqlite3": "^5.0.0",
+    "sql.js": "^1.13.0",
     "yaml": "^1.10.0",
+    "ygopro-cdb-encode": "^1.0.1",
     "ygopro-lflist-encode": "^1.0.3"
   },
   "devDependencies": {
     "@types/lodash": "^4.17.7",
-    "typescript": "^5.5.4"
+    "@types/node": "^25.2.2",
+    "@types/sql.js": "^1.4.9",
+    "typescript": "^5.9.3"
   }
 }

src/dbreader.ts

-import { open, Database } from "sqlite";
-import sqlite3 from "sqlite3";
 import _ from "lodash";
 import Base from "./base";
 import { promises as fs } from "fs";
@@ -7,24 +5,25 @@ import { generateBanlistFromCode } from "./utility";
 import { CardPool, DeckGenerator } from "./deck";
 import moment from "moment";
 import { YGOProLFListItem } from "ygopro-lflist-encode";
+import initSqlJs, { SqlJsStatic } from "sql.js";
+import { CardDataEntry, YGOProCdb } from "ygopro-cdb-encode";
 
-const textsFields = ["id", "name", "desc"]
-for (let i = 1; i <= 16; ++i) {
-	textsFields.push(`str${i}`);
-}
-const datasFields = ["id", "ot", "alias", "setcode", "type", "atk", "def", "level", "race", "attribute", "category"];
-
-export class SQLQuery {
-	sql: string;
-	values: any[];
-	constructor(sql: string, values: any[]) {
-		this.sql = sql;
-		this.values = values;
+export function queryAll<T = any>(db: YGOProCdb, sql: string, params?: any[] | Record<string, any>) {
+	const stmt = db.database.prepare(sql);
+	if (params !== undefined) {
+		stmt.bind(params as any);
 	}
-	async perform(db: Database) {
-		//console.log(db.config.filename, this.sql, this.values);
-		await db.run(this.sql, this.values);
+	const rows: T[] = [];
+	while (stmt.step()) {
+		rows.push(stmt.getAsObject() as T);
 	}
+	stmt.free();
+	return rows;
+}
+
+export function queryOne<T = any>(db: YGOProCdb, sql: string, params?: any[] | Record<string, any>) {
+	const rows = queryAll<T>(db, sql, params);
+	return rows[0];
 }
 
 export class Card {
@@ -40,21 +39,7 @@ export class Card {
 		this.datas = {};
 		this.texts = {};
 	}
-	private getDatasArray(): any[] {
-		const ret = [];
-		for (let field of datasFields) {
-			ret.push(this.datas[field]);
-		}
-		return ret;
-	}
-	private getTextsArray(): any[] {
-		const ret = [];
-		for (let field of textsFields) {
-			ret.push(this.texts[field]);
-		}
-		return ret;
-	}
-	private async createCardFromRelatedCode(code: number, db: Database) {
+	private async createCardFromRelatedCode(code: number, db: YGOProCdb) {
 		const card = new Card(code);
 		await card.loadData(db);
 		if (this.loadedTexts.name === card.loadedTexts.name) { // is alias
@@ -66,10 +51,10 @@ export class Card {
 		}
 		return card;
 	}
-	async getRelatedCards(db: Database) {
+	async getRelatedCards(db: YGOProCdb) {
 		const code = this.code;
 		if (code === 5405694) return [];
-		const moreCodes: number[] = (await db.all('SELECT id FROM datas WHERE id > ? AND id <= ? AND (alias = ? OR (type & 0x4000) > 0)', [code, code + 20, code])).map(m => m.id);
+		const moreCodes: number[] = queryAll<{ id: number }>(db, 'SELECT id FROM datas WHERE id > ? AND id <= ? AND (alias = ? OR (type & 0x4000) > 0)', [code, code + 20, code]).map(m => m.id);
 		const cards = await Promise.all(moreCodes.map(code => this.createCardFromRelatedCode(code, db)));
 		return cards;
 	}
@@ -87,11 +72,18 @@ export class Card {
 		}
 		return (cardType & (0x4000000 | 0x800000 | 0x2000 | 0x40)) > 0;
 	}
-	async loadData(db: Database) {
+	async loadData(db: YGOProCdb) {
 		this.preDatas = this.datas;
 		this.preTexts = this.texts;
-		this.loadedDatas = await db.get("select * from datas where id = ?", [this.code]);
-		this.loadedTexts = await db.get("select * from texts where id = ?", [this.code]);
+		const entry = db.findOne("datas.id = :id", { id: this.code });
+		if (!entry) {
+			this.loadedDatas = {};
+			this.loadedTexts = {};
+		} else {
+			const row = entry.toSqljsRow();
+			this.loadedDatas = row.datas;
+			this.loadedTexts = row.texts;
+		}
 		this.datas = {
 			...this.loadedDatas,
 			...this.preDatas
@@ -101,45 +93,55 @@ export class Card {
 			...this.preTexts
 		}
 	}
-	getSQLQueries() {
-		const datasArray = this.getDatasArray();
-		const textsArray = this.getTextsArray();
-		return [
-			new SQLQuery(`INSERT INTO texts VALUES(${_.range(textsArray.length).map(m => "?")});`, textsArray),
-			new SQLQuery(`INSERT INTO datas VALUES(${_.range(datasArray.length).map(m => "?")});`, datasArray)
-		]
+	toCardDataEntry() {
+		const row = {
+			...this.datas,
+			...this.texts
+		};
+		return new CardDataEntry().fromSqljsRow(row as any);
 	}
 }
 
 export class DBReader extends Base {
-	jpdb: Database;
-	cndb: Database;
-	outputdb: Database;
-	overrideTextDb: Database;
+	jpdb: YGOProCdb;
+	cndb: YGOProCdb;
+	outputdb?: YGOProCdb;
+	overrideTextDb?: YGOProCdb;
+	private sqljs?: SqlJsStatic;
+	private outputDbPath?: string;
+	private async getSqlJs() {
+		if (!this.sqljs) {
+			this.sqljs = await initSqlJs();
+		}
+		return this.sqljs;
+	}
 	async openDatabase(path: string, mustExist: boolean = false) {
 		if (mustExist) { 
 			await fs.access(path);
 		}
-		return await open({
-			filename: path,
-			driver: sqlite3.Database
-		});
+		const SQL = await this.getSqlJs();
+		const data = new Uint8Array(await fs.readFile(path));
+		return new YGOProCdb(SQL).from(data);
 	}
 	async init() {
 		await super.init();
 		this.log.debug(`Opening databases...`);
 		this.cndb = await this.openDatabase(this.config.cnDatabasePath, true);
 		this.jpdb = await this.openDatabase(this.config.jpDatabasePath, true);
-		this.overrideTextDb = this.config.overrideTextDbPath ? await this.openDatabase(this.config.overrideTextDbPath) : null;
+		this.overrideTextDb = this.config.overrideTextDbPath ? await this.openDatabase(this.config.overrideTextDbPath) : undefined;
 	}
 	async finalize() {
-		await this.cndb.close();
-		await this.jpdb.close();
+		if (this.cndb) {
+			this.cndb.finalize();
+		}
+		if (this.jpdb) {
+			this.jpdb.finalize();
+		}
 		if (this.outputdb) {
-			await this.outputdb.close();
+			this.outputdb.finalize();
 		}
 		if (this.overrideTextDb) {
-			await this.overrideTextDb.close();
+			this.overrideTextDb.finalize();
 		}
 	}
 	private async openOutputDatabase() {
@@ -156,25 +158,14 @@ export class DBReader extends Base {
 				await fs.mkdir(createDirectoryPath, { recursive: true });
 			}
 		}
-		try {
-			await fs.unlink(fullPath);
-		} catch (e) { }
+		this.outputDbPath = fullPath;
 		this.log.debug(`Creating database ${fullPath} ...`);
-		this.outputdb = await this.openDatabase(fullPath);
-		const initSQLs = [
-			"PRAGMA foreign_keys=OFF;",
-			"BEGIN TRANSACTION;",
-			"CREATE TABLE texts(id integer primary key,name text,desc text,str1 text,str2 text,str3 text,str4 text,str5 text,str6 text,str7 text,str8 text,str9 text,str10 text,str11 text,str12 text,str13 text,str14 text,str15 text,str16 text);",
-			"CREATE TABLE datas(id integer primary key,ot integer,alias integer,setcode integer,type integer,atk integer,def integer,level integer,race integer,attribute integer,category integer);",
-			"COMMIT;"
-		];
-		for (let sql of initSQLs) {
-			await this.outputdb.run(sql);
-		}
+		const SQL = await this.getSqlJs();
+		this.outputdb = new YGOProCdb(SQL);
 	}
 	async getCardFromJapaneseName(name: string) {
 		this.log.debug(`Reading JP database for code of name ${name}.`);
-		const output = await this.jpdb.get('SELECT id FROM texts WHERE name = ?', name);
+		const output = queryOne<{ id: number }>(this.jpdb, 'SELECT id FROM texts WHERE name = ?', [name]);
 		if (!output) {
 			this.log.debug(`Code of ${name} not found.`);
 			return null;
@@ -197,7 +188,7 @@ export class DBReader extends Base {
 	}
 	async getOtherCardCodes(cnCodes: number[]) {
 		const sql = `SELECT id FROM datas WHERE id not IN (${cnCodes.join(",")})`;
-		const otherCodes: number[] = (await this.cndb.all(sql)).map(m => m.id);
+		const otherCodes: number[] = queryAll<{ id: number }>(this.cndb, sql).map(m => m.id);
 		return otherCodes;
 	}
 	async generateBanlist(codes: number[], extraBanlists: YGOProLFListItem[] = []) {
@@ -242,21 +233,17 @@ export class DBReader extends Base {
 				card.datas.ot |= 0x8;
 			}
 			if (this.overrideTextDb) {
-				const overrideTextEntry = await this.overrideTextDb.get('SELECT desc FROM texts WHERE id = ?', [card.code]);
-				if (overrideTextEntry) {
-					card.texts.desc = overrideTextEntry.desc;
+				const overrideEntry = this.overrideTextDb.findOne('texts.id = :id', { id: card.code });
+				if (overrideEntry) {
+					card.texts.desc = overrideEntry.desc;
 				}
 			}
 			card.texts.desc += '\r\n\r\n\u2605' + this.config.descSymbol;
 		}));
-		const queries = allCards.flatMap(card => card.getSQLQueries());
+		const entries = allCards.map(card => card.toCardDataEntry());
 		await this.openOutputDatabase();
-		await this.outputdb.run("BEGIN TRANSACTION;");
-		for (let query of queries) {
-			this.log.debug(`Writing database: ${query.sql} ${query.values.join(",")}`);
-			await query.perform(this.outputdb);
-		}
-		await this.outputdb.run("COMMIT;");
+		this.outputdb.addCard(entries);
+		await fs.writeFile(this.outputDbPath, Buffer.from(this.outputdb.export()));
 		const allCodes = allCards.map(card => card.code);
 		this.log.info(`Database created.`);
 		await this.generateBanlist(allCodes, options.extraBanlists || []);
@@ -272,8 +259,8 @@ export class DBReader extends Base {
 	async getAllCardsFromDatabase(dbPath: string) {
 		try {
 			const db = await this.openDatabase(dbPath, true);
-			const codes: number[] = (await db.all('SELECT id FROM datas where type & 0x4000 = 0 and not exists (select 1 from datas orig where orig.id = datas.alias and orig.id < datas.id and orig.id >= (datas.id - 20))')).map(m => m.id);
-			await db.close();
+			const codes: number[] = queryAll<{ id: number }>(db, 'SELECT id FROM datas where type & 0x4000 = 0 and not exists (select 1 from datas orig where orig.id = datas.alias and orig.id < datas.id and orig.id >= (datas.id - 20))').map(m => m.id);
+			db.finalize();
 			return codes;
 		} catch (e) {
 			this.log.error(`Error getting all cards from database: ${e}`);

src/direct.ts

 import Base from "./base";
 import { promises as fs } from "fs";
-import { Card, DBReader } from "./dbreader";
+import { Card, DBReader, queryAll } from "./dbreader";
 import { CNOCGFetcher } from "./cnocg";
 
 export class DirectFetcher extends Base {
@@ -57,11 +57,12 @@ export class DirectFetcher extends Base {
 	private async workWithExisting() {
 		const cnDb = await new DBReader({ name: 'Existing Reader', level: 'info' }).openDatabase(this.config.cnDatabasePath);
 		this.log.info(`Reading existing YGOPro database.`);
-		const cnDbData = await cnDb.all<{ id: number }[]>("SELECT id FROM datas where (ot & 0x8) > 0 and (type & 0x4000) = 0");
+		const cnDbData = queryAll<{ id: number }>(cnDb, "SELECT id FROM datas where (ot & 0x8) > 0 and (type & 0x4000) = 0");
 		this.log.info(`${cnDbData.length} existing cards found.`);
 		for (const row of cnDbData) {
 			this.addCode(row.id);
 		}
+		cnDb.finalize();
 	}
 
 	async fetch(): Promise<Card[]> {

src/packdbfetcher.ts

 import _ from "lodash";
 import Base from "./base";
-import { Card, DBReader } from "./dbreader";
+import { Card, DBReader, queryAll } from "./dbreader";
 
 export class PackDbFetcher extends Base { 
   static async fetchOnce(dbreader: DBReader, cardListCondition?: string) {
@@ -11,7 +11,8 @@ export class PackDbFetcher extends Base {
   
   async fetch(dbreader: DBReader, cardListCondition?: string) {
     const packDb = await dbreader.openDatabase(this.config.cardListDatabasePath);
-    const packDbData = await packDb.all<{ id: number, pack_id: string, date: string }[]>("SELECT id FROM pack" + (cardListCondition ? ` WHERE ${cardListCondition}` : ""));
+    const packDbData = queryAll<{ id: number, pack_id: string, date: string }>(packDb, "SELECT id FROM pack" + (cardListCondition ? ` WHERE ${cardListCondition}` : ""));
+    packDb.finalize();
     return packDbData.map(m => new Card(m.id));
   }
 }

test/deckgen.ts

-import { open, Database } from "sqlite";
-import sqlite3 from "sqlite3";
 import { promises as fs } from "fs";
 import { DeckGenerator } from "../src/deck";
 import _ from "lodash";
+import initSqlJs from "sql.js";
+import { YGOProCdb } from "ygopro-cdb-encode";
 
 async function openDatabase(path: string) {
-	return await open({
-		filename: path,
-		driver: sqlite3.Database
-	});
+	const SQL = await initSqlJs();
+	const data = new Uint8Array(await fs.readFile(path));
+	return new YGOProCdb(SQL).from(data);
 }
 
 const targetDirectory = process.argv[3];
@@ -38,10 +37,26 @@ async function main() {
 	console.log(`Opening database ${process.argv[2]}.`);
 	const db = await openDatabase(process.argv[2]);
 	console.log(`Reading database ${process.argv[2]}.`);
-	const main = (await db.all('select id from datas where (type & 0x4000) = 0 and (type & (0x4000000 | 0x800000 | 0x4000 | 0x2000 | 0x40)) = 0')).map(m => m.id);
-	const extra = (await db.all('select id from datas where (type & 0x4000) = 0 and (type & (0x4000000 | 0x800000 | 0x4000 | 0x2000 | 0x40)) > 0')).map(m => m.id);
+	const mainStmt = db.database.prepare('select id from datas where (type & 0x4000) = 0 and (type & (0x4000000 | 0x800000 | 0x4000 | 0x2000 | 0x40)) = 0');
+	const main: number[] = [];
+	while (mainStmt.step()) {
+		const row = mainStmt.getAsObject() as { id: number };
+		if (row.id != null) {
+			main.push(row.id);
+		}
+	}
+	mainStmt.free();
+	const extraStmt = db.database.prepare('select id from datas where (type & 0x4000) = 0 and (type & (0x4000000 | 0x800000 | 0x4000 | 0x2000 | 0x40)) > 0');
+	const extra: number[] = [];
+	while (extraStmt.step()) {
+		const row = extraStmt.getAsObject() as { id: number };
+		if (row.id != null) {
+			extra.push(row.id);
+		}
+	}
+	extraStmt.free();
 	console.log(`${main.length} mains and ${extra.length} extras.`);
-	await db.close();
+	db.finalize();
 	console.log(`Generating decks to ${targetDirectory}.`);
 	await generateDecks(main, extra);
 }