migrate sqlite3 to sql.js

package.json

@@ -30,7 +30,7 @@
     "querystring": "^0.2.0",
     "reflect-metadata": "^0.1.13",
     "request": "^2.88.2",
-    "sqlite3": "^5.0.2",
+    "sql.js": "^1.13.0",
     "typeorm": "^0.2.29",
     "underscore": "^1.11.0",
     "underscore.string": "^3.3.6",
@@ -53,6 +53,7 @@
     "@types/formidable": "^3.4.6",
     "@types/ip6addr": "^0.2.3",
     "@types/node": "^16.18.126",
+    "@types/sql.js": "^1.4.9",
     "@types/underscore": "^1.13.0",
     "@types/ws": "^8.5.3",
     "coffeescript": "^2.7.0",

ygopro-deck-stats.js

@@ -6,8 +6,8 @@
  
  读取指定文件夹里所有卡组，得出卡片使用量，生成csv
 */
-var sqlite3 = require('sqlite3').verbose();
 var fs = require('fs');
+var initSqlJs = require('sql.js');
 var loadJSON = require('load-json-file').sync;
 var config = loadJSON('./config/deckstats.json'); //{ "deckpath": "../decks", "dbfile": "cards.cdb" }
 var constants = loadJSON('./data/constants.json');
@@ -16,6 +16,23 @@ var ALL_MAIN_CARDS={};
 var ALL_SIDE_CARDS={};
 var ALL_CARD_DATAS={};
 
+var sqlJsPromise;
+var SQL = null;
+function ensureSqlJs() {
+    if (!sqlJsPromise) {
+        var wasmPath = require.resolve('sql.js/dist/sql-wasm.wasm');
+        sqlJsPromise = initSqlJs({
+            locateFile: function() {
+                return wasmPath;
+            }
+        }).then(function(SQLLib) {
+            SQL = SQLLib;
+            return SQLLib;
+        });
+    }
+    return sqlJsPromise;
+}
+
 function add_to_deck(deck,id) {
     if (deck[id]) {
         deck[id]=deck[id]+1;
@@ -70,15 +87,14 @@ function read_deck_file(filename) {
 }
 
 function load_database(callback) {
-    var db=new sqlite3.Database(config.dbfile);
-    db.each("select * from datas,texts where datas.id=texts.id", function (err,result) {
-        if (err) {
-            console.log(config.dbfile + ":" + err);
-            return;
-        }
-        else {
+    ensureSqlJs().then(function() {
+        var dbBuffer = fs.readFileSync(config.dbfile);
+        var db = new SQL.Database(dbBuffer);
+        var stmt = db.prepare("select * from datas,texts where datas.id=texts.id");
+        while (stmt.step()) {
+            var result = stmt.getAsObject();
             if (result.type & constants.TYPES.TYPE_TOKEN) {
-                return;
+                continue;
             }
             
             var card={};
@@ -140,7 +156,17 @@ function load_database(callback) {
             
             ALL_CARD_DATAS[result.id]=card;
         }
-    }, callback);
+        stmt.free();
+        db.close();
+        if (callback) {
+            callback();
+        }
+    }).catch(function(err) {
+        console.log(config.dbfile + ":" + err);
+        if (callback) {
+            callback(err);
+        }
+    });
 }
 
 function read_decks() {
@@ -190,5 +216,8 @@ function output_csv(list,filename) {
     }
 }
 
-load_database(read_decks);
-
+ensureSqlJs().then(function() {
+    load_database(read_decks);
+}).catch(function(err) {
+    console.error("Failed to load sql.js: " + err);
+});

ygopro-pre.js

@@ -9,8 +9,8 @@
 */
 var http = require('http');
 var https = require('https');
-var sqlite3 = require('sqlite3').verbose();
 var fs = require('fs');
+var initSqlJs = require('sql.js');
 var exec = require('child_process').exec;
 var execSync = require('child_process').execSync;
 var spawn = require('child_process').spawn;
@@ -36,6 +36,23 @@ var responder;
 var dataver = moment().format("YYYYMMDDHHmmss");
 const _async = require("async");
 
+var sqlJsPromise;
+var SQL = null;
+function ensureSqlJs() {
+    if (!sqlJsPromise) {
+        var wasmPath = require.resolve('sql.js/dist/sql-wasm.wasm');
+        sqlJsPromise = initSqlJs({
+            locateFile: function() {
+                return wasmPath;
+            }
+        }).then(function(SQLLib) {
+            SQL = SQLLib;
+            return SQLLib;
+        });
+    }
+    return sqlJsPromise;
+}
+
 //输出反馈信息，如有http长连接则输出到http，否则输出到控制台
 var sendResponse = function(text) {
     text=""+text;
@@ -50,18 +67,17 @@ var sendResponse = function(text) {
 
 //读取数据库内内容到cardHTMLs，异步
 var loadDb = function(db_file, callback) {
-    var db = new sqlite3.Database(db_file);
-    
-    db.each("select * from datas,texts where datas.id=texts.id", function (err,result) {
-        if (err) {
-            sendResponse(db_file + ":" + err);
-            return;
-        }
-        else {
+    ensureSqlJs().then(function() {
+        var dbBuffer = fs.readFileSync(db_file);
+        var db = new SQL.Database(dbBuffer);
+        var stmt = db.prepare("select * from datas,texts where datas.id=texts.id");
+        var num = 0;
+        while (stmt.step()) {
+            var result = stmt.getAsObject();
             if (result.type & constants.TYPES.TYPE_TOKEN) {
-                return;
+                continue;
             }
-            
+
             var cardHTML="<tr>";
             
             cardHTML+='<td><a href="'+ config.html_img_rel_path + result.id +'.jpg" target="_blank"><img data-original="'+config.html_img_rel_path+config.html_img_thumbnail+ result.id +'.jpg'+ config.html_img_thumbnail_suffix +'" alt="'+ result.name +'"></a></td>';
@@ -191,16 +207,16 @@ var loadDb = function(db_file, callback) {
             cardHTML+='</tr>';
             
             cardHTMLs.push(cardHTML);
+            num++;
         }
-    }, function(err, num) {
-        if(err) {
-            sendResponse(db_file + ":" + err);
-        }
-        else {
-            dataver = moment().format("YYYYMMDDHHmmss");
-            sendResponse("已加载数据库"+db_file+"，共"+num+"张卡。");
-        }
-            callback(err, num);
+        stmt.free();
+        db.close();
+        dataver = moment().format("YYYYMMDDHHmmss");
+        sendResponse("已加载数据库"+db_file+"，共"+num+"张卡。");
+        callback(null, num);
+    }).catch(function(err) {
+        sendResponse(db_file + ":" + err);
+        callback(err);
     });
 }
 
@@ -576,14 +592,20 @@ async function requestListener(req, res) {
 
 }
 
-if (ssl_config.enabled) {
-    const ssl_cert = fs.readFileSync(ssl_config.cert);
-    const ssl_key = fs.readFileSync(ssl_config.key);
-    const options = {
-        cert: ssl_cert,
-        key: ssl_key
+function startServer() {
+    if (ssl_config.enabled) {
+        const ssl_cert = fs.readFileSync(ssl_config.cert);
+        const ssl_key = fs.readFileSync(ssl_config.key);
+        const options = {
+            cert: ssl_cert,
+            key: ssl_key
+        }
+        https.createServer(options, requestListener).listen(config.port);
+    } else { 
+        http.createServer(requestListener).listen(config.port);
     }
-    https.createServer(options, requestListener).listen(config.port);
-} else { 
-    http.createServer(requestListener).listen(config.port);
 }
+
+ensureSqlJs().then(startServer).catch(function(err) {
+    console.error("Failed to load sql.js: " + err);
+});

ygopro-update.js

@@ -9,8 +9,8 @@
 */
 var http = require('http');
 var https = require('https');
-var sqlite3 = require('sqlite3').verbose();
 var fs = require('fs');
+var initSqlJs = require('sql.js');
 var execSync = require('child_process').execSync;
 var spawn = require('child_process').spawn;
 var spawnSync = require('child_process').spawnSync;
@@ -34,6 +34,23 @@ var changelog=[];
 //http长连接
 var responder;
 
+var sqlJsPromise;
+var SQL = null;
+function ensureSqlJs() {
+    if (!sqlJsPromise) {
+        var wasmPath = require.resolve('sql.js/dist/sql-wasm.wasm');
+        sqlJsPromise = initSqlJs({
+            locateFile: function() {
+                return wasmPath;
+            }
+        }).then(function(SQLLib) {
+            SQL = SQLLib;
+            return SQLLib;
+        });
+    }
+    return sqlJsPromise;
+}
+
 //输出反馈信息，如有http长连接则输出到http，否则输出到控制台
 var sendResponse = function(text) {
     text=""+text;
@@ -48,23 +65,21 @@ var sendResponse = function(text) {
 
 //读取数据库内内容到cardNames，异步
 var loadDb = function(db_file) {
-    var db = new sqlite3.Database(db_file);
-    
-    db.each("select id,name from texts", function (err,result) {
-        if (err) {
-            sendResponse(db_file + ":" + err);
-            return;
-        }
-        else {
+    ensureSqlJs().then(function() {
+        var dbBuffer = fs.readFileSync(db_file);
+        var db = new SQL.Database(dbBuffer);
+        var stmt = db.prepare("select id,name from texts");
+        var num = 0;
+        while (stmt.step()) {
+            var result = stmt.getAsObject();
             cardNames[result.id] = result.name;
+            num++;
         }
-    }, function(err, num) {
-        if(err) {
-            sendResponse(db_file + ":" + err);
-        }
-        else {
-            sendResponse("已加载数据库"+db_file+"，共"+num+"张卡。");
-        }
+        stmt.free();
+        db.close();
+        sendResponse("已加载数据库"+db_file+"，共"+num+"张卡。");
+    }).catch(function(err) {
+        sendResponse(db_file + ":" + err);
     });
 }
 
@@ -294,14 +309,20 @@ async function requestListener(req, res) {
 
 }
 
-if (ssl_config.enabled) {
-    const ssl_cert = fs.readFileSync(ssl_config.cert);
-    const ssl_key = fs.readFileSync(ssl_config.key);
-    const options = {
-        cert: ssl_cert,
-        key: ssl_key
+function startServer() {
+    if (ssl_config.enabled) {
+        const ssl_cert = fs.readFileSync(ssl_config.cert);
+        const ssl_key = fs.readFileSync(ssl_config.key);
+        const options = {
+            cert: ssl_cert,
+            key: ssl_key
+        }
+        https.createServer(options, requestListener).listen(config.port);
+    } else { 
+        http.createServer(requestListener).listen(config.port);
     }
-    https.createServer(options, requestListener).listen(config.port);
-} else { 
-    http.createServer(requestListener).listen(config.port);
 }
+
+ensureSqlJs().then(startServer).catch(function(err) {
+    console.error("Failed to load sql.js: " + err);
+});