use SAB for sharing card reader

.dockerignore

@@ -44,3 +44,4 @@ Dockerfile
 /ssl
 /ygopro
 /windbot
+/researches

.eslintignore

@@ -4,3 +4,4 @@ build/*
 *.js
 /ygopro
 /windbot
+/researches

.gitignore

@@ -41,3 +41,4 @@ lerna-debug.log*
 /ygopro
 /windbot
 /plugins
+/researches

src/ocgcore-worker/ocgcore-worker-options.ts

 import YGOProDeck from 'ygopro-deck-encode';
 import { HostInfo } from 'ygopro-msg-encode';
 import { TransportType } from 'yuzuthread';
+import { CardStorage } from '../ygopro';
 
 export class OcgcoreWorkerOptions {
   ygoproPaths: string[];
   extraScriptPaths: string[];
+  @TransportType(() => CardStorage)
+  cardStorage: CardStorage;
   ocgcoreWasmPath?: string;
   seed: number[];
   hostinfo: HostInfo;

src/ocgcore-worker/ocgcore-worker.ts

@@ -4,7 +4,6 @@ import {
   OcgcoreWrapper,
   createOcgcoreWrapper,
   DirScriptReaderEx,
-  DirCardReader,
   _OcgcoreConstants,
   parseCardQuery,
   parseFieldCardQuery,
@@ -31,14 +30,12 @@ import {
 import { OcgcoreWorkerOptions } from './ocgcore-worker-options';
 import { ReplaySubject, Subject } from 'rxjs';
 import { calculateDuelOptions } from '../utility/calculate-duel-options';
-import initSqlJs from 'sql.js';
 import {
   YGOProMessages,
   YGOProMsgResponseBase,
   YGOProMsgRetry,
 } from 'ygopro-msg-encode';
 import * as fs from 'node:fs';
-import { isMainThread } from 'node:worker_threads';
 
 const { OcgcoreScriptConstants } = _OcgcoreConstants;
 const OCGCORE_MESSAGE_REPLAY_BUFFER_SIZE = 128;
@@ -112,9 +109,8 @@ export class OcgcoreWorker {
     });
 
     // Load script reader and card reader
-    const sqlJs = await initSqlJs();
     const scriptReader = await DirScriptReaderEx(...this.options.ygoproPaths);
-    const cardReader = await DirCardReader(sqlJs, ...this.options.ygoproPaths);
+    const cardReader = this.options.cardStorage.toCardReader();
     this.ocgcore.setScriptReader(scriptReader);
     this.ocgcore.setCardReader(cardReader);
 

src/room/room-module.ts

 import { createAppContext } from 'nfkit';
 import { ContextState } from '../app';
-import { YGOProResourceLoader } from './ygopro-resource-loader';
+import { YGOProResourceLoader } from '../ygopro';
 import { DefaultHostInfoProvider } from './default-hostinfo-provder';
 import { RoomManager } from './room-manager';
 import { DefaultDeckChecker } from './default-deck-checker';

src/room/room.ts

@@ -60,7 +60,7 @@ import {
   OcgcoreMessageType,
   _OcgcoreConstants,
 } from 'koishipro-core.js';
-import { YGOProResourceLoader } from './ygopro-resource-loader';
+import { YGOProResourceLoader } from '../ygopro';
 import { blankLFList } from '../utility/blank-lflist';
 import { Client } from '../client';
 import { RoomMethod } from '../utility/decorators';
@@ -1337,6 +1337,7 @@ export class Room {
     const ocgcoreWasmPath = ocgcoreWasmPathConfig
       ? path.resolve(process.cwd(), ocgcoreWasmPathConfig)
       : undefined;
+    const cardStorage = await this.resourceLoader.getCardStorage();
 
     try {
       this.ocgcore = await initWorker(OcgcoreWorker, {
@@ -1344,6 +1345,7 @@ export class Room {
         hostinfo: this.hostinfo,
         ygoproPaths: this.resourceLoader.ygoproPaths,
         extraScriptPaths,
+        cardStorage,
         ocgcoreWasmPath,
         registry,
         decks: duelRecord.toSwappedPlayers().map((p) => p.deck),

src/ygopro/card-data-with-ot.ts

+import { BinaryField } from 'ygopro-msg-encode';
+import { CardDataEntry } from 'ygopro-cdb-encode';
+
+const CARD_DATA_PAYLOAD_SIZE = new CardDataEntry().toPayload().length;
+
+export class CardDataWithOt extends CardDataEntry {
+  @BinaryField('u8', CARD_DATA_PAYLOAD_SIZE)
+  declare ot: number;
+}
+
+export const CARD_DATA_WITH_OT_PAYLOAD_SIZE =
+  new CardDataWithOt().toPayload().length;

src/ygopro/card-storage.ts

+import type { CardReaderFn } from 'koishipro-core.js';
+import { CardDataEntry } from 'ygopro-cdb-encode';
+import { TransportType } from 'yuzuthread';
+import {
+  CARD_DATA_WITH_OT_PAYLOAD_SIZE,
+  CardDataWithOt,
+} from './card-data-with-ot';
+
+const CARD_ID_MIN = 1;
+const CARD_ID_MAX = 0x7fffffff;
+const HASH_LOAD_FACTOR = 0.75;
+const HASH_MIN_CAPACITY = 16;
+const CARD_ENTRY_SIZE = CARD_DATA_WITH_OT_PAYLOAD_SIZE;
+
+const isValidCardId = (cardId: number) =>
+  Number.isInteger(cardId) && cardId >= CARD_ID_MIN && cardId <= CARD_ID_MAX;
+
+export class CardStorage {
+  @TransportType(() => Buffer)
+  private entries: Buffer;
+
+  @TransportType(() => Buffer)
+  private hashKeys: Buffer;
+
+  @TransportType(() => Buffer)
+  private hashValues: Buffer;
+
+  private hashMask: number;
+  size: number;
+
+  constructor(
+    entries: Buffer,
+    hashKeys: Buffer,
+    hashValues: Buffer,
+    hashMask: number,
+    size: number,
+  ) {
+    this.entries = entries;
+    this.hashKeys = hashKeys;
+    this.hashValues = hashValues;
+    this.hashMask = hashMask;
+    this.size = size;
+  }
+
+  static fromCards(cards: Iterable<CardDataEntry>): CardStorage {
+    const uniqueCards: CardDataEntry[] = [];
+    const seen = new Set<number>();
+    for (const card of cards) {
+      const cardId = (card.code ?? 0) >>> 0;
+      if (!isValidCardId(cardId) || seen.has(cardId)) {
+        continue;
+      }
+      seen.add(cardId);
+      uniqueCards.push(card);
+    }
+
+    const hashCapacity = this.computeHashCapacity(uniqueCards.length);
+    const entries = Buffer.alloc(uniqueCards.length * CARD_ENTRY_SIZE);
+    const hashKeys = Buffer.alloc(hashCapacity * 4);
+    const hashValues = Buffer.alloc(hashCapacity * 4);
+
+    const storage = new CardStorage(
+      entries,
+      hashKeys,
+      hashValues,
+      hashCapacity - 1,
+      uniqueCards.length,
+    );
+
+    for (let i = 0; i < uniqueCards.length; i++) {
+      storage.writeEntry(i, uniqueCards[i]);
+      storage.insertHash(uniqueCards[i].code >>> 0, i);
+    }
+
+    return storage;
+  }
+
+  get byteLength() {
+    return this.entries.length + this.hashKeys.length + this.hashValues.length;
+  }
+
+  readCard(cardId: number): CardDataWithOt | undefined {
+    if (!isValidCardId(cardId)) {
+      return undefined;
+    }
+
+    const entryIndex = this.findEntryIndex(cardId >>> 0);
+    if (entryIndex < 0) {
+      return undefined;
+    }
+
+    return this.readEntry(entryIndex);
+  }
+
+  toCardReader(): CardReaderFn {
+    return (cardId: number) => {
+      const data = this.readCard(cardId);
+      if (!data) {
+        return undefined;
+      }
+      return {
+        code: data.code,
+        ot: data.ot,
+        alias: data.alias,
+        setcode: [...(data.setcode ?? [])],
+        type: data.type,
+        level: data.level,
+        attribute: data.attribute,
+        race: data.race,
+        attack: data.attack,
+        defense: data.defense,
+        lscale: data.lscale,
+        rscale: data.rscale,
+        linkMarker: data.linkMarker,
+      };
+    };
+  }
+
+  private static computeHashCapacity(cardCount: number) {
+    const required = Math.max(1, Math.ceil(cardCount / HASH_LOAD_FACTOR));
+    let capacity = HASH_MIN_CAPACITY;
+    while (capacity < required) {
+      capacity <<= 1;
+    }
+    return capacity;
+  }
+
+  private hash(cardId: number) {
+    return (Math.imul(cardId, 0x9e3779b1) >>> 0) & this.hashMask;
+  }
+
+  private getHashKeysView() {
+    return new Uint32Array(
+      this.hashKeys.buffer,
+      this.hashKeys.byteOffset,
+      this.hashKeys.length >>> 2,
+    );
+  }
+
+  private getHashValuesView() {
+    return new Uint32Array(
+      this.hashValues.buffer,
+      this.hashValues.byteOffset,
+      this.hashValues.length >>> 2,
+    );
+  }
+
+  private getEntryPayload(entryIndex: number) {
+    const offset = entryIndex * CARD_ENTRY_SIZE;
+    return this.entries.subarray(offset, offset + CARD_ENTRY_SIZE);
+  }
+
+  private writeEntry(entryIndex: number, card: CardDataEntry) {
+    const payload =
+      card instanceof CardDataWithOt
+        ? card.toPayload()
+        : new CardDataWithOt()
+            .fromPartial({
+              code: card.code,
+              ot: card.ot,
+              alias: card.alias,
+              setcode: card.setcode,
+              type: card.type,
+              level: card.level,
+              attribute: card.attribute,
+              race: card.race,
+              attack: card.attack,
+              defense: card.defense,
+              lscale: card.lscale,
+              rscale: card.rscale,
+              linkMarker: card.linkMarker,
+              category: card.category,
+            })
+            .toPayload();
+    if (payload.length !== CARD_ENTRY_SIZE) {
+      throw new TypeError(
+        `Unexpected card entry payload size: ${payload.length}`,
+      );
+    }
+    this.entries.set(payload, entryIndex * CARD_ENTRY_SIZE);
+  }
+
+  private readEntry(entryIndex: number): CardDataWithOt {
+    return new CardDataWithOt().fromPayload(this.getEntryPayload(entryIndex));
+  }
+
+  private insertHash(cardId: number, entryIndex: number) {
+    const keys = this.getHashKeysView();
+    const values = this.getHashValuesView();
+    let slot = this.hash(cardId);
+
+    while (true) {
+      const key = keys[slot];
+      if (key === 0) {
+        keys[slot] = cardId;
+        values[slot] = entryIndex + 1;
+        return;
+      }
+      if (key === cardId) {
+        return;
+      }
+      slot = (slot + 1) & this.hashMask;
+    }
+  }
+
+  private findEntryIndex(cardId: number) {
+    const keys = this.getHashKeysView();
+    const values = this.getHashValuesView();
+    let slot = this.hash(cardId);
+
+    while (true) {
+      const key = keys[slot];
+      if (key === 0) {
+        return -1;
+      }
+      if (key === cardId) {
+        const value = values[slot];
+        return value > 0 ? value - 1 : -1;
+      }
+      slot = (slot + 1) & this.hashMask;
+    }
+  }
+}

src/ygopro/index.ts

+export * from './card-data-with-ot';
+export * from './card-storage';
+export * from './ygopro-resource-loader';

src/room/ygopro-resource-loader.ts → src/ygopro/ygopro-resource-loader.ts

 import { Context } from '../app';
-import { searchYGOProResource, SqljsCardReader } from 'koishipro-core.js';
+import type { CardReaderFn } from 'koishipro-core.js';
+import { searchYGOProResource } from 'koishipro-core.js';
 import { YGOProLFList } from 'ygopro-lflist-encode';
 import path from 'node:path';
+import type { CardDataEntry } from 'ygopro-cdb-encode';
 import { YGOProCdb } from 'ygopro-cdb-encode';
+import { toShared } from 'yuzuthread';
+import BetterLock from 'better-lock';
+import { CardStorage } from './card-storage';
 
 export class YGOProResourceLoader {
-  constructor(private ctx: Context) {}
+  constructor(private ctx: Context) {
+    void this.loadYGOProCdbs();
+  }
 
   ygoproPaths = this.ctx.config
     .getStringArray('YGOPRO_PATH')
@@ -16,16 +23,56 @@ export class YGOProResourceLoader {
     .map((p) => path.resolve(process.cwd(), p));
 
   private logger = this.ctx.createLogger(this.constructor.name);
+  private loadingLock = new BetterLock();
+  private loadingCardStorage?: Promise<CardStorage>;
+  private currentCardStorage?: CardStorage;
+  private currentCardReader?: CardReaderFn;
+
+  async getCardStorage() {
+    if (this.currentCardStorage) {
+      return this.currentCardStorage;
+    }
+    if (this.loadingCardStorage) {
+      return this.loadingCardStorage;
+    }
+    return this.loadYGOProCdbs();
+  }
 
-  private cardReader = this.mergeDatabase().then((db) => SqljsCardReader(db));
+  async getCardReader(): Promise<CardReaderFn> {
+    if (this.currentCardReader) {
+      return this.currentCardReader;
+    }
+    const storage = await this.getCardStorage();
+    const reader = storage.toCardReader();
+    this.currentCardReader = reader;
+    return reader;
+  }
 
-  async getCardReader() {
-    return this.cardReader;
+  async loadYGOProCdbs() {
+    if (this.loadingCardStorage) {
+      return this.loadingCardStorage;
+    }
+    const loading = this.loadingLock.acquire(async () => {
+      const storage = await this.loadCardStorage();
+      this.currentCardStorage = storage;
+      this.currentCardReader = storage.toCardReader();
+      return storage;
+    });
+    this.loadingCardStorage = loading;
+    try {
+      return await loading;
+    } finally {
+      if (this.loadingCardStorage === loading) {
+        this.loadingCardStorage = undefined;
+      }
+    }
   }
 
-  private async mergeDatabase() {
-    const db = new YGOProCdb(this.ctx.SQL).noTexts();
+  private async loadCardStorage() {
+    const cards: CardDataEntry[] = [];
+    const seen = new Set<number>();
     let dbCount = 0;
+
     for await (const file of searchYGOProResource(...this.ygoproPaths)) {
       const filename = path.basename(file.path);
       if (!filename?.endsWith('.cdb')) {
@@ -35,11 +82,13 @@ export class YGOProResourceLoader {
         const currentDb = new this.ctx.SQL.Database(await file.read());
         try {
           const currentCdb = new YGOProCdb(currentDb).noTexts();
-          const cards = currentCdb.find();
-          for (const card of cards) {
-            if (!db.findById(card.code)) {
-              db.addCard(card);
+          for (const card of currentCdb.find()) {
+            const cardId = card.code >>> 0;
+            if (seen.has(cardId)) {
+              continue;
             }
+            seen.add(cardId);
+            cards.push(card);
           }
           ++dbCount;
         } finally {
@@ -50,10 +99,15 @@ export class YGOProResourceLoader {
         continue;
       }
     }
+
+    const storage = toShared(CardStorage.fromCards(cards));
     this.logger.info(
-      `Merged database from ${dbCount} databases with ${db.find().length} cards`,
+      {
+        size: storage.byteLength,
+      },
+      `Merged database from ${dbCount} databases with ${storage.size} cards`,
     );
-    return db;
+    return storage;
   }
 
   async *getLFLists() {