fix tiktoken

package-lock.json

@@ -25,6 +25,7 @@
         "pino": "^10.3.1",
         "pino-pretty": "^13.1.3",
         "rxjs": "^7.8.2",
+        "tiktoken": "^1.0.22",
         "typed-reflector": "^1.0.14",
         "typed-struct": "^2.7.1",
         "typeorm": "^0.3.28",
@@ -8226,6 +8227,12 @@
         "node": ">=20"
       }
     },
+    "node_modules/tiktoken": {
+      "version": "1.0.22",
+      "resolved": "https://registry.npmjs.org/tiktoken/-/tiktoken-1.0.22.tgz",
+      "integrity": "sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA==",
+      "license": "MIT"
+    },
     "node_modules/tinyglobby": {
       "version": "0.2.15",
       "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz",

package.json

@@ -78,6 +78,7 @@
     "pino": "^10.3.1",
     "pino-pretty": "^13.1.3",
     "rxjs": "^7.8.2",
+    "tiktoken": "^1.0.22",
     "typed-reflector": "^1.0.14",
     "typed-struct": "^2.7.1",
     "typeorm": "^0.3.28",

src/feats/chatgpt-service.ts

@@ -103,11 +103,12 @@ export class ChatgptService {
     }
 
     const moduleName = 'tiktoken';
-    this.tiktokenModulePromise = import(moduleName).catch(() => {
+    this.tiktokenModulePromise = import(moduleName).catch((e) => {
       this.tiktokenUnavailable = true;
       if (!this.tiktokenUnavailableLogged) {
         this.tiktokenUnavailableLogged = true;
         this.logger.warn(
+          { error: (e as Error).toString() },
           'tiktoken is unavailable, using approximate token counting',
         );
       }
@@ -377,11 +378,12 @@ export class ChatgptService {
       }
       this.tokenizerByModel.set(model, encoder);
       return encoder;
-    } catch {
+    } catch(e) {
       this.tiktokenUnavailable = true;
       if (!this.tiktokenUnavailableLogged) {
         this.tiktokenUnavailableLogged = true;
         this.logger.warn(
+          { error: (e as Error).toString() },
           'tiktoken is unavailable, using approximate token counting',
         );
       }