Skip to content

Y
ygopro-cn-database-generator
Commit ebbdbf7e authored by nanahira's avatar nanahira
Browse files
Options

migrate to ygopro-cdb-encode

parent e568febb
Pipeline #42959 passed with stages
          in 3 minutes and 15 seconds
          Expand all Show whitespace changes
          Inline Side-by-side
          Showing with 402 additions and 1132 deletions
          package-lock.json
          View file @ ebbdbf7e
          This diff is collapsed.
          src/dbreader.ts
          View file @ ebbdbf7e
          import { open, Database } from "sqlite";
          import sqlite3 from "sqlite3";
          import _ from "lodash";
          import Base from "./base";
          import { promises as fs } from "fs";
          ......@@ -7,24 +5,25 @@ import { generateBanlistFromCode } from "./utility";
          import { CardPool, DeckGenerator } from "./deck";
          import moment from "moment";
          import { YGOProLFListItem } from "ygopro-lflist-encode";
          import initSqlJs, { SqlJsStatic } from "sql.js";
          import { CardDataEntry, YGOProCdb } from "ygopro-cdb-encode";
          const textsFields = ["id", "name", "desc"]
          for (let i = 1; i <= 16; ++i) {
          textsFields.push(`str${i}`);
          }
          const datasFields = ["id", "ot", "alias", "setcode", "type", "atk", "def", "level", "race", "attribute", "category"];
          export class SQLQuery {
          sql: string;
          values: any[];
          constructor(sql: string, values: any[]) {
          this.sql = sql;
          this.values = values;
          export function queryAll<T = any>(db: YGOProCdb, sql: string, params?: any[] | Record<string, any>) {
          const stmt = db.database.prepare(sql);
          if (params !== undefined) {
          stmt.bind(params as any);
          }
          async perform(db: Database) {
          //console.log(db.config.filename, this.sql, this.values);
          await db.run(this.sql, this.values);
          const rows: T[] = [];
          while (stmt.step()) {
          rows.push(stmt.getAsObject() as T);
          }
          stmt.free();
          return rows;
          }
          export function queryOne<T = any>(db: YGOProCdb, sql: string, params?: any[] | Record<string, any>) {
          const rows = queryAll<T>(db, sql, params);
          return rows[0];
          }
          export class Card {
          ......@@ -40,21 +39,7 @@ export class Card {
          this.datas = {};
          this.texts = {};
          }
          private getDatasArray(): any[] {
          const ret = [];
          for (let field of datasFields) {
          ret.push(this.datas[field]);
          }
          return ret;
          }
          private getTextsArray(): any[] {
          const ret = [];
          for (let field of textsFields) {
          ret.push(this.texts[field]);
          }
          return ret;
          }
          private async createCardFromRelatedCode(code: number, db: Database) {
          private async createCardFromRelatedCode(code: number, db: YGOProCdb) {
          const card = new Card(code);
          await card.loadData(db);
          if (this.loadedTexts.name === card.loadedTexts.name) { // is alias
          ......@@ -66,10 +51,10 @@ export class Card {
          }
          return card;
          }
          async getRelatedCards(db: Database) {
          async getRelatedCards(db: YGOProCdb) {
          const code = this.code;
          if (code === 5405694) return [];
          const moreCodes: number[] = (await db.all('SELECT id FROM datas WHERE id > ? AND id <= ? AND (alias = ? OR (type & 0x4000) > 0)', [code, code + 20, code])).map(m => m.id);
          const moreCodes: number[] = queryAll<{ id: number }>(db, 'SELECT id FROM datas WHERE id > ? AND id <= ? AND (alias = ? OR (type & 0x4000) > 0)', [code, code + 20, code]).map(m => m.id);
          const cards = await Promise.all(moreCodes.map(code => this.createCardFromRelatedCode(code, db)));
          return cards;
          }
          ......@@ -87,11 +72,18 @@ export class Card {
          }
          return (cardType & (0x4000000 | 0x800000 | 0x2000 | 0x40)) > 0;
          }
          async loadData(db: Database) {
          async loadData(db: YGOProCdb) {
          this.preDatas = this.datas;
          this.preTexts = this.texts;
          this.loadedDatas = await db.get("select * from datas where id = ?", [this.code]);
          this.loadedTexts = await db.get("select * from texts where id = ?", [this.code]);
          const entry = db.findOne("datas.id = :id", { id: this.code });
          if (!entry) {
          this.loadedDatas = {};
          this.loadedTexts = {};
          } else {
          const row = entry.toSqljsRow();
          this.loadedDatas = row.datas;
          this.loadedTexts = row.texts;
          }
          this.datas = {
          ...this.loadedDatas,
          ...this.preDatas
          ......@@ -101,45 +93,55 @@ export class Card {
          ...this.preTexts
          }
          }
          getSQLQueries() {
          const datasArray = this.getDatasArray();
          const textsArray = this.getTextsArray();
          return [
          new SQLQuery(`INSERT INTO texts VALUES(${_.range(textsArray.length).map(m => "?")});`, textsArray),
          new SQLQuery(`INSERT INTO datas VALUES(${_.range(datasArray.length).map(m => "?")});`, datasArray)
          ]
          toCardDataEntry() {
          const row = {
          ...this.datas,
          ...this.texts
          };
          return new CardDataEntry().fromSqljsRow(row as any);
          }
          }
          export class DBReader extends Base {
          jpdb: Database;
          cndb: Database;
          outputdb: Database;
          overrideTextDb: Database;
          jpdb: YGOProCdb;
          cndb: YGOProCdb;
          outputdb?: YGOProCdb;
          overrideTextDb?: YGOProCdb;
          private sqljs?: SqlJsStatic;
          private outputDbPath?: string;
          private async getSqlJs() {
          if (!this.sqljs) {
          this.sqljs = await initSqlJs();
          }
          return this.sqljs;
          }
          async openDatabase(path: string, mustExist: boolean = false) {
          if (mustExist) {
          await fs.access(path);
          }
          return await open({
          filename: path,
          driver: sqlite3.Database
          });
          const SQL = await this.getSqlJs();
          const data = new Uint8Array(await fs.readFile(path));
          return new YGOProCdb(SQL).from(data);
          }
          async init() {
          await super.init();
          this.log.debug(`Opening databases...`);
          this.cndb = await this.openDatabase(this.config.cnDatabasePath, true);
          this.jpdb = await this.openDatabase(this.config.jpDatabasePath, true);
          this.overrideTextDb = this.config.overrideTextDbPath ? await this.openDatabase(this.config.overrideTextDbPath) : null;
          this.overrideTextDb = this.config.overrideTextDbPath ? await this.openDatabase(this.config.overrideTextDbPath) : undefined;
          }
          async finalize() {
          await this.cndb.close();
          await this.jpdb.close();
          if (this.cndb) {
          this.cndb.finalize();
          }
          if (this.jpdb) {
          this.jpdb.finalize();
          }
          if (this.outputdb) {
          await this.outputdb.close();
          this.outputdb.finalize();
          }
          if (this.overrideTextDb) {
          await this.overrideTextDb.close();
          this.overrideTextDb.finalize();
          }
          }
          private async openOutputDatabase() {
          ......@@ -156,25 +158,14 @@ export class DBReader extends Base {
          await fs.mkdir(createDirectoryPath, { recursive: true });
          }
          }
          try {
          await fs.unlink(fullPath);
          } catch (e) { }
          this.outputDbPath = fullPath;
          this.log.debug(`Creating database ${fullPath} ...`);
          this.outputdb = await this.openDatabase(fullPath);
          const initSQLs = [
          "PRAGMA foreign_keys=OFF;",
          "BEGIN TRANSACTION;",
          "CREATE TABLE texts(id integer primary key,name text,desc text,str1 text,str2 text,str3 text,str4 text,str5 text,str6 text,str7 text,str8 text,str9 text,str10 text,str11 text,str12 text,str13 text,str14 text,str15 text,str16 text);",
          "CREATE TABLE datas(id integer primary key,ot integer,alias integer,setcode integer,type integer,atk integer,def integer,level integer,race integer,attribute integer,category integer);",
          "COMMIT;"
          ];
          for (let sql of initSQLs) {
          await this.outputdb.run(sql);
          }
          const SQL = await this.getSqlJs();
          this.outputdb = new YGOProCdb(SQL);
          }
          async getCardFromJapaneseName(name: string) {
          this.log.debug(`Reading JP database for code of name ${name}.`);
          const output = await this.jpdb.get('SELECT id FROM texts WHERE name = ?', name);
          const output = queryOne<{ id: number }>(this.jpdb, 'SELECT id FROM texts WHERE name = ?', [name]);
          if (!output) {
          this.log.debug(`Code of ${name} not found.`);
          return null;
          ......@@ -197,7 +188,7 @@ export class DBReader extends Base {
          }
          async getOtherCardCodes(cnCodes: number[]) {
          const sql = `SELECT id FROM datas WHERE id not IN (${cnCodes.join(",")})`;
          const otherCodes: number[] = (await this.cndb.all(sql)).map(m => m.id);
          const otherCodes: number[] = queryAll<{ id: number }>(this.cndb, sql).map(m => m.id);
          return otherCodes;
          }
          async generateBanlist(codes: number[], extraBanlists: YGOProLFListItem[] = []) {
          ......@@ -242,21 +233,17 @@ export class DBReader extends Base {
          card.datas.ot |= 0x8;
          }
          if (this.overrideTextDb) {
          const overrideTextEntry = await this.overrideTextDb.get('SELECT desc FROM texts WHERE id = ?', [card.code]);
          if (overrideTextEntry) {
          card.texts.desc = overrideTextEntry.desc;
          const overrideEntry = this.overrideTextDb.findOne('texts.id = :id', { id: card.code });
          if (overrideEntry) {
          card.texts.desc = overrideEntry.desc;
          }
          }
          card.texts.desc += '\r\n\r\n\u2605' + this.config.descSymbol;
          }));
          const queries = allCards.flatMap(card => card.getSQLQueries());
          const entries = allCards.map(card => card.toCardDataEntry());
          await this.openOutputDatabase();
          await this.outputdb.run("BEGIN TRANSACTION;");
          for (let query of queries) {
          this.log.debug(`Writing database: ${query.sql} ${query.values.join(",")}`);
          await query.perform(this.outputdb);
          }
          await this.outputdb.run("COMMIT;");
          this.outputdb.addCard(entries);
          await fs.writeFile(this.outputDbPath, Buffer.from(this.outputdb.export()));
          const allCodes = allCards.map(card => card.code);
          this.log.info(`Database created.`);
          await this.generateBanlist(allCodes, options.extraBanlists || []);
          ......@@ -272,8 +259,8 @@ export class DBReader extends Base {
          async getAllCardsFromDatabase(dbPath: string) {
          try {
          const db = await this.openDatabase(dbPath, true);
          const codes: number[] = (await db.all('SELECT id FROM datas where type & 0x4000 = 0 and not exists (select 1 from datas orig where orig.id = datas.alias and orig.id < datas.id and orig.id >= (datas.id - 20))')).map(m => m.id);
          await db.close();
          const codes: number[] = queryAll<{ id: number }>(db, 'SELECT id FROM datas where type & 0x4000 = 0 and not exists (select 1 from datas orig where orig.id = datas.alias and orig.id < datas.id and orig.id >= (datas.id - 20))').map(m => m.id);
          db.finalize();
          return codes;
          } catch (e) {
          this.log.error(`Error getting all cards from database: ${e}`);
          ......
          src/direct.ts
          View file @ ebbdbf7e
          import Base from "./base";
          import { promises as fs } from "fs";
          import { Card, DBReader } from "./dbreader";
          import { Card, DBReader, queryAll } from "./dbreader";
          import { CNOCGFetcher } from "./cnocg";
          export class DirectFetcher extends Base {
          ......@@ -57,11 +57,12 @@ export class DirectFetcher extends Base {
          private async workWithExisting() {
          const cnDb = await new DBReader({ name: 'Existing Reader', level: 'info' }).openDatabase(this.config.cnDatabasePath);
          this.log.info(`Reading existing YGOPro database.`);
          const cnDbData = await cnDb.all<{ id: number }[]>("SELECT id FROM datas where (ot & 0x8) > 0 and (type & 0x4000) = 0");
          const cnDbData = queryAll<{ id: number }>(cnDb, "SELECT id FROM datas where (ot & 0x8) > 0 and (type & 0x4000) = 0");
          this.log.info(`${cnDbData.length} existing cards found.`);
          for (const row of cnDbData) {
          this.addCode(row.id);
          }
          cnDb.finalize();
          }
          async fetch(): Promise<Card[]> {
          ......
          src/packdbfetcher.ts
          View file @ ebbdbf7e
          import _ from "lodash";
          import Base from "./base";
          import { Card, DBReader } from "./dbreader";
          import { Card, DBReader, queryAll } from "./dbreader";
          export class PackDbFetcher extends Base {
          static async fetchOnce(dbreader: DBReader, cardListCondition?: string) {
          ......@@ -11,7 +11,8 @@ export class PackDbFetcher extends Base {
          async fetch(dbreader: DBReader, cardListCondition?: string) {
          const packDb = await dbreader.openDatabase(this.config.cardListDatabasePath);
          const packDbData = await packDb.all<{ id: number, pack_id: string, date: string }[]>("SELECT id FROM pack" + (cardListCondition ? ` WHERE ${cardListCondition}` : ""));
          const packDbData = queryAll<{ id: number, pack_id: string, date: string }>(packDb, "SELECT id FROM pack" + (cardListCondition ? ` WHERE ${cardListCondition}` : ""));
          packDb.finalize();
          return packDbData.map(m => new Card(m.id));
          }
          }
          test/deckgen.ts
          View file @ ebbdbf7e
          import { open, Database } from "sqlite";
          import sqlite3 from "sqlite3";
          import { promises as fs } from "fs";
          import { DeckGenerator } from "../src/deck";
          import _ from "lodash";
          import initSqlJs from "sql.js";
          import { YGOProCdb } from "ygopro-cdb-encode";
          async function openDatabase(path: string) {
          return await open({
          filename: path,
          driver: sqlite3.Database
          });
          const SQL = await initSqlJs();
          const data = new Uint8Array(await fs.readFile(path));
          return new YGOProCdb(SQL).from(data);
          }
          const targetDirectory = process.argv[3];
          ......@@ -38,10 +37,26 @@ async function main() {
          console.log(`Opening database ${process.argv[2]}.`);
          const db = await openDatabase(process.argv[2]);
          console.log(`Reading database ${process.argv[2]}.`);
          const main = (await db.all('select id from datas where (type & 0x4000) = 0 and (type & (0x4000000 | 0x800000 | 0x4000 | 0x2000 | 0x40)) = 0')).map(m => m.id);
          const extra = (await db.all('select id from datas where (type & 0x4000) = 0 and (type & (0x4000000 | 0x800000 | 0x4000 | 0x2000 | 0x40)) > 0')).map(m => m.id);
          const mainStmt = db.database.prepare('select id from datas where (type & 0x4000) = 0 and (type & (0x4000000 | 0x800000 | 0x4000 | 0x2000 | 0x40)) = 0');
          const main: number[] = [];
          while (mainStmt.step()) {
          const row = mainStmt.getAsObject() as { id: number };
          if (row.id != null) {
          main.push(row.id);
          }
          }
          mainStmt.free();
          const extraStmt = db.database.prepare('select id from datas where (type & 0x4000) = 0 and (type & (0x4000000 | 0x800000 | 0x4000 | 0x2000 | 0x40)) > 0');
          const extra: number[] = [];
          while (extraStmt.step()) {
          const row = extraStmt.getAsObject() as { id: number };
          if (row.id != null) {
          extra.push(row.id);
          }
          }
          extraStmt.free();
          console.log(`${main.length} mains and ${extra.length} extras.`);
          await db.close();
          db.finalize();
          console.log(`Generating decks to ${targetDirectory}.`);
          await generateDecks(main, extra);
          }
          ......
          Markdown is supported
          0% or
          You are about to add 0 people to the discussion. Proceed with caution.
          Finish editing this message first!
          Please register or to comment