file datasource & sync accounts

config.example.yaml

@@ -3,9 +3,15 @@ port: 3000
 redisUrl: ''
 token: ''
 accounts:
+  type: 'static',
+  config:
     - email: 'test@example.com'
       password: 'wwww'
       pro: false
       loginType: 'default'
-proxies: []
-apiKeys: []
+proxies:
+  type: 'static'
\ No newline at end of file
+  config: []
+apiKeys:
+  type: 'static'
+  config: []
\ No newline at end of file

package-lock.json

@@ -13,6 +13,7 @@
         "@nestjs/config": "^2.3.1",
         "@nestjs/core": "^9.0.0",
         "@nestjs/platform-express": "^9.0.0",
+        "@nestjs/schedule": "^2.2.0",
         "@nestjs/swagger": "^6.2.1",
         "aragami": "^1.1.3",
         "axios": "^1.3.3",
@@ -1647,6 +1648,20 @@
         "@nestjs/core": "^9.0.0"
       }
     },
+    "node_modules/@nestjs/schedule": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-2.2.0.tgz",
+      "integrity": "sha512-wrDnUONTxBkD6lTWh9ecYk/kvJTbA3PylotjBoRsECmcS1SNvgInFXuL38UnHiFnXM3CHSFnzRLB259Bc1mOdQ==",
+      "dependencies": {
+        "cron": "2.2.0",
+        "uuid": "9.0.0"
+      },
+      "peerDependencies": {
+        "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0",
+        "@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0",
+        "reflect-metadata": "^0.1.12"
+      }
+    },
     "node_modules/@nestjs/schematics": {
       "version": "9.0.4",
       "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-9.0.4.tgz",
@@ -3658,6 +3673,14 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "dev": true
     },
+    "node_modules/cron": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/cron/-/cron-2.2.0.tgz",
+      "integrity": "sha512-GPiI3OgMv83XRtEUc2gUdaLvJhO3XbLN288layOBkDTupg0RK5IECNGpkykIMHg+muVR2bxt29b0xvCAcBrjYQ==",
+      "dependencies": {
+        "luxon": "^3.2.1"
+      }
+    },
     "node_modules/cross-fetch": {
       "version": "3.1.5",
       "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
@@ -6646,6 +6669,14 @@
         "yallist": "^3.0.2"
       }
     },
+    "node_modules/luxon": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.2.1.tgz",
+      "integrity": "sha512-QrwPArQCNLAKGO/C+ZIilgIuDnEnKx5QYODdDtbFaxzsbZcc/a7WFq7MhsVYgRlwawLtvOUESTlfJ+hc/USqPg==",
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/macos-release": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.5.0.tgz",
@@ -11637,6 +11668,15 @@
         "tslib": "2.5.0"
       }
     },
+    "@nestjs/schedule": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-2.2.0.tgz",
+      "integrity": "sha512-wrDnUONTxBkD6lTWh9ecYk/kvJTbA3PylotjBoRsECmcS1SNvgInFXuL38UnHiFnXM3CHSFnzRLB259Bc1mOdQ==",
+      "requires": {
+        "cron": "2.2.0",
+        "uuid": "9.0.0"
+      }
+    },
     "@nestjs/schematics": {
       "version": "9.0.4",
       "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-9.0.4.tgz",
@@ -13184,6 +13224,14 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "dev": true
     },
+    "cron": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/cron/-/cron-2.2.0.tgz",
+      "integrity": "sha512-GPiI3OgMv83XRtEUc2gUdaLvJhO3XbLN288layOBkDTupg0RK5IECNGpkykIMHg+muVR2bxt29b0xvCAcBrjYQ==",
+      "requires": {
+        "luxon": "^3.2.1"
+      }
+    },
     "cross-fetch": {
       "version": "3.1.5",
       "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz",
@@ -15415,6 +15463,11 @@
         "yallist": "^3.0.2"
       }
     },
+    "luxon": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.2.1.tgz",
+      "integrity": "sha512-QrwPArQCNLAKGO/C+ZIilgIuDnEnKx5QYODdDtbFaxzsbZcc/a7WFq7MhsVYgRlwawLtvOUESTlfJ+hc/USqPg=="
+    },
     "macos-release": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/macos-release/-/macos-release-2.5.0.tgz",

package.json

@@ -25,6 +25,7 @@
     "@nestjs/config": "^2.3.1",
     "@nestjs/core": "^9.0.0",
     "@nestjs/platform-express": "^9.0.0",
+    "@nestjs/schedule": "^2.2.0",
     "@nestjs/swagger": "^6.2.1",
     "aragami": "^1.1.3",
     "axios": "^1.3.3",

src/account-pool/account-pool.service.ts

@@ -10,6 +10,7 @@ import { OpenAIAccount } from '../utility/config';
 import { AccountState } from './account-state';
 import { AccountProviderService } from '../account-provider/account-provider.service';
 import { AccountPoolStatusDto } from './account-pool-status.dto';
+import { Interval } from '@nestjs/schedule';
 
 @Injectable()
 export class AccountPoolService
@@ -42,9 +43,11 @@ export class AccountPoolService
         proxyServer: proxy,
       }),
     );
+    this.log(`Initializing account ${account.email}`);
     const success = await state.init();
     if (success) {
       this.accounts.set(account.email, state);
+      this.log(`Initialized account ${account.email}`);
       return true;
     } else if (retry) {
       await Promise.all([
@@ -60,6 +63,25 @@ export class AccountPoolService
 
   private accountInfos: OpenAIAccount[];
 
+  @Interval(1000 * 60 * 5)
+  async syncAccounts() {
+    this.accountInfos = await this.accountProvider.get();
+    const accountEmails = new Set(this.accountInfos.map((a) => a.email));
+    const addedAccounts = this.accountInfos.filter(
+      (a) => !this.accounts.has(a.email),
+    );
+    const removedAccounts = Array.from(this.accounts.keys()).filter(
+      (a) => !accountEmails.has(a),
+    );
+    this.log(
+      `Sync accounts: ${addedAccounts.length} added, ${removedAccounts.length} removed`,
+    );
+    await Promise.all([
+      ...addedAccounts.map((a) => this.addAccount(a)),
+      ...removedAccounts.map((a) => this.removeAccount(a)),
+    ]);
+  }
+
   async onModuleInit() {
     this.ChatGPTAPIBrowserConstructor = (
       await eval("import('chatgpt3')")
@@ -112,26 +134,18 @@ export class AccountPoolService
     if (this.accounts.has(account.email)) {
       return false;
     }
-    const result = await this.initAccount(account, false);
-    if (result) {
-      this.accountInfos.push(account);
-    }
-    await this.accountProvider.writeBack(this.accountInfos, 'add', account);
-    return result;
+    return this.initAccount(account, false);
   }
 
   async removeAccount(email: string) {
-    const index = this.accountInfos.findIndex((a) => a.email === email);
-    if (index === -1) {
+    const state = this.accounts.get(email);
+    if (!state) {
       return false;
     }
-    const [account] = this.accountInfos.splice(index, 1);
-    const state = this.accounts.get(email);
-    if (state) {
+    this.log(`Removing account ${email}`);
     this.accounts.delete(email);
     await state.close();
-    }
-    await this.accountProvider.writeBack(this.accountInfos, 'remove', account);
+    this.log(`Removed account ${email}`);
     return true;
   }
 

src/account-provider/account-provider.service.ts

@@ -8,9 +8,4 @@ export class AccountProviderService extends BaseProvider<OpenAIAccount[]> {
   constructor(config: ConfigService) {
     super(config, 'accounts');
   }
-  async writeBack(
-    accounts: OpenAIAccount[],
-    type: 'add' | 'remove',
-    account: OpenAIAccount,
-  ) {}
 }

src/app.module.ts

@@ -12,6 +12,7 @@ import { AccountProviderService } from './account-provider/account-provider.serv
 import { ProxyProviderService } from './proxy-provider/proxy-provider.service';
 import { KeyProviderService } from './key-provider/key-provider.service';
 import { DavinciService } from './davinci/davinci.service';
+import { ScheduleModule } from '@nestjs/schedule';
 
 @Module({
   imports: [
@@ -35,6 +36,7 @@ import { DavinciService } from './davinci/davinci.service';
         return {};
       },
     }),
+    ScheduleModule.forRoot(),
   ],
   controllers: [AppController],
   providers: [

src/datasource/base.ts

@@ -6,6 +6,4 @@ export class BaseDatasource<T, K extends keyof DatasourceConfigMap<T>> {
   async getData(): Promise<any> {
     throw new Error('Not implemented');
   }
-
-  async writeBack(data: T) {}
 }

src/datasource/datasource.ts

 import { StaticDatasource } from './source/static';
 import { ApiDatasource } from './source/api';
 import { BaseDatasource } from './base';
+import { FileDatasource } from './source/file';
 
 export interface DatasourceConfigMap<T> {
   static: T;
   api: { endpoint: string; headers: Record<string, string | number> };
+  file: { path: string; keys: string[]; separator: string };
 }
 export interface DatasourceConfig<
   T,
@@ -23,3 +25,4 @@ export const Datasources = new Map<
 
 Datasources.set('static', StaticDatasource as any);
 Datasources.set('api', ApiDatasource as any);
+Datasources.set('file', FileDatasource as any);

src/datasource/source/file.ts

+import { BaseDatasource } from '../base';
+import * as fs from 'fs';
+
+export class FileDatasource<T> extends BaseDatasource<T, 'file'> {
+  async getData() {
+    const content = await fs.promises.readFile(this.config.path, 'utf-8');
+    return content
+      .split('\n')
+      .map((_line) => {
+        const line = _line.trim();
+        if (!line) {
+          return '';
+        }
+        if (!this.config.separator) {
+          return line;
+        }
+        const values = line.split(new RegExp(this.config.separator));
+        if (!this.config.keys) {
+          return values;
+        }
+        const obj: Record<string, string> = {};
+        this.config.keys.forEach((key, index) => {
+          obj[key] = values[index];
+        });
+        return obj;
+      })
+      .filter((d) => !!d);
+  }
+}

src/utility/config.ts

 import yaml from 'yaml';
 import * as fs from 'fs';
+import { DatasourceConfig } from '../datasource/datasource';
 
 export interface OpenAIAccount {
   email: string;
@@ -11,11 +12,20 @@ export interface OpenAIAccount {
 const defaultConfig = {
   host: '::',
   port: 3000,
-  accounts: [] as OpenAIAccount[],
+  accounts: {
+    type: 'static',
+    config: [],
+  } as DatasourceConfig<OpenAIAccount[]>,
   redisUrl: '',
   token: '',
-  proxies: [] as string[],
-  apiKeys: [] as string[],
+  proxies: {
+    type: 'static',
+    config: [],
+  } as DatasourceConfig<string[]>,
+  apiKeys: {
+    type: 'static',
+    config: [],
+  } as DatasourceConfig<string[]>,
 };
 
 export type Config = typeof defaultConfig;