datasource

package-lock.json

@@ -15,6 +15,7 @@
         "@nestjs/platform-express": "^9.0.0",
         "@nestjs/swagger": "^6.2.1",
         "aragami": "^1.1.3",
+        "axios": "^1.3.3",
         "better-lock": "^2.0.3",
         "chatgpt3": "npm:chatgpt@^3.5.2",
         "chatgpt4": "npm:chatgpt@^4.3.2",
@@ -2812,8 +2813,17 @@
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
-      "dev": true
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
+    "node_modules/axios": {
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.3.3.tgz",
+      "integrity": "sha512-eYq77dYIFS77AQlhzEL937yUBSepBfPIe8FcgEDN35vMNZKMrs81pgnyrQpwfy4NF4b4XWX1Zgx7yX+25w8QJA==",
+      "dependencies": {
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
+      }
     },
     "node_modules/babel-jest": {
       "version": "28.1.3",
@@ -3510,7 +3520,6 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dev": true,
       "dependencies": {
         "delayed-stream": "~1.0.0"
       },
@@ -3792,7 +3801,6 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
       "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "dev": true,
       "engines": {
         "node": ">=0.4.0"
       }
@@ -4713,6 +4721,25 @@
       "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
       "dev": true
     },
+    "node_modules/follow-redirects": {
+      "version": "1.15.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/for-in": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
@@ -4800,7 +4827,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
       "dependencies": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -8001,8 +8027,7 @@
     "node_modules/proxy-from-env": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
-      "peer": true
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
     },
     "node_modules/pump": {
       "version": "3.0.0",
@@ -12540,8 +12565,17 @@
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
-      "dev": true
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
+    "axios": {
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.3.3.tgz",
+      "integrity": "sha512-eYq77dYIFS77AQlhzEL937yUBSepBfPIe8FcgEDN35vMNZKMrs81pgnyrQpwfy4NF4b4XWX1Zgx7yX+25w8QJA==",
+      "requires": {
+        "follow-redirects": "^1.15.0",
+        "form-data": "^4.0.0",
+        "proxy-from-env": "^1.1.0"
+      }
     },
     "babel-jest": {
       "version": "28.1.3",
@@ -13037,7 +13071,6 @@
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
       "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
-      "dev": true,
       "requires": {
         "delayed-stream": "~1.0.0"
       }
@@ -13247,8 +13280,7 @@
     "delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
-      "dev": true
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="
     },
     "denque": {
       "version": "2.1.0",
@@ -13961,6 +13993,11 @@
       "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==",
       "dev": true
     },
+    "follow-redirects": {
+      "version": "1.15.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA=="
+    },
     "for-in": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
@@ -14021,7 +14058,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
       "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
-      "dev": true,
       "requires": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.8",
@@ -16289,8 +16325,7 @@
     "proxy-from-env": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==",
-      "peer": true
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
     },
     "pump": {
       "version": "3.0.0",

package.json

@@ -27,6 +27,7 @@
     "@nestjs/platform-express": "^9.0.0",
     "@nestjs/swagger": "^6.2.1",
     "aragami": "^1.1.3",
+    "axios": "^1.3.3",
     "better-lock": "^2.0.3",
     "chatgpt3": "npm:chatgpt@^3.5.2",
     "chatgpt4": "npm:chatgpt@^4.3.2",

src/account-provider/account-provider.service.ts

 import { Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
 import { OpenAIAccount } from '../utility/config';
+import { fetchDatasource } from '../datasource/datasource';
 
 @Injectable()
 export class AccountProviderService {
   constructor(private config: ConfigService) {}
   async getAccounts() {
-    return this.config.get<OpenAIAccount[]>('accounts') || [];
+    return (
+      (await fetchDatasource<OpenAIAccount[]>(this.config.get('accounts'))) ||
+      []
+    );
   }
 
   async writeBack(

src/datasource/base.ts

+import { DatasourceConfigMap } from './datasource';
+
+export class BaseDatasource<T, K extends keyof DatasourceConfigMap<T>> {
+  constructor(public config: DatasourceConfigMap<T>[K]) {}
+
+  async getData(): Promise<any> {
+    throw new Error('Not implemented');
+  }
+
+  async writeBack(data: T) {}
+}

src/datasource/datasource.ts

+import { StaticDatasource } from './source/static';
+import { ApiDatasource } from './source/api';
+import { BaseDatasource } from './base';
+
+export interface DatasourceConfigMap<T> {
+  static: T;
+  api: { endpoint: string; headers: Record<string, string | number> };
+}
+export interface DatasourceConfig<T, K extends keyof DatasourceConfigMap<T>> {
+  type: K;
+  config: DatasourceConfigMap<T>[K];
+}
+
+export const Datasources = new Map<
+  string,
+  new <T, K extends keyof DatasourceConfigMap<T>>(
+    config: DatasourceConfigMap<T>[K],
+  ) => BaseDatasource<T, K>
+>();
+
+Datasources.set('static', StaticDatasource as any);
+Datasources.set('api', ApiDatasource as any);
+
+export async function fetchDatasource<T>(
+  config: DatasourceConfig<T, keyof DatasourceConfigMap<any>>,
+): Promise<T> {
+  if (!config) {
+    return;
+  }
+  if (Array.isArray(config)) {
+    // legacy config
+    return config as any;
+  }
+  const Datasource = Datasources.get(config.type);
+  if (!Datasource) {
+    throw new Error(`Unknown datasource type: ${config.type}`);
+  }
+  return new Datasource(config.config).getData();
+}

src/datasource/source/api.ts

+import { BaseDatasource } from '../base';
+import axios from 'axios';
+
+export class ApiDatasource<T> extends BaseDatasource<T, 'api'> {
+  async getData() {
+    const { endpoint, headers } = this.config;
+    const res = await axios.get(endpoint, { headers });
+    return res.data;
+  }
+}

src/datasource/source/static.ts

+import { BaseDatasource } from '../base';
+
+export class StaticDatasource<T> extends BaseDatasource<T, 'static'> {
+  async getData() {
+    return this.config;
+  }
+}

src/key-provider/key-provider.service.ts

 import { Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
+import { fetchDatasource } from '../datasource/datasource';
 
 @Injectable()
 export class KeyProviderService {
   constructor(private config: ConfigService) {}
   async getKeys() {
-    return this.config.get<string[]>('apiKeys') || [];
+    return (await fetchDatasource<string[]>(this.config.get('apiKeys'))) || [];
   }
 }

src/proxy-provider/proxy-provider.service.ts

 import { Injectable } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
+import { fetchDatasource } from '../datasource/datasource';
 
 @Injectable()
 export class ProxyProviderService {
   constructor(private config: ConfigService) {}
   async getProxies() {
-    return this.config.get<string[]>('proxies') || [];
+    return (await fetchDatasource<string[]>(this.config.get('proxies'))) || [];
   }
 }