fix too many files things

console-api/src/packager/packager.service.ts

@@ -23,6 +23,8 @@ interface FileWithHash {
   hash: string;
 }
 
+const ARG_SIZE_LIMIT = 30000;
+
 class ArchiveTask {
   readonly path: string;
   constructor(public readonly role: ArchiveType, public readonly files: FileWithHash[], public readonly altFiles?: string[]) {
@@ -200,11 +202,52 @@ export class PackagerService extends ConsoleLogger {
   }
 
   async checksum(root: string, directories: string[], files: string[]): Promise<Record<string, string>> {
-    const { stdout } = await util.promisify(child_process.execFile)('sha256sum', files, { maxBuffer: 1 * 1024 ** 3, cwd: root });
+    // 通常系统参数限制在 128KB 左右，保守一点设为 32KB
+
+
+    function chunkFiles(files: string[], limit = ARG_SIZE_LIMIT): string[][] {
+      const chunks: string[][] = [];
+      let currentChunk: string[] = [];
+      let currentSize = 0;
+
+      for (const file of files) {
+        const size = Buffer.byteLength(file) + 1; // +1 for space
+        if (currentSize + size > limit && currentChunk.length > 0) {
+          chunks.push(currentChunk);
+          currentChunk = [];
+          currentSize = 0;
+        }
+        currentChunk.push(file);
+        currentSize += size;
+      }
+
+      if (currentChunk.length > 0) {
+        chunks.push(currentChunk);
+      }
+
+      return chunks;
+    }
+
+    const fileChunks = chunkFiles(files);
+
+    const chunkPromises = fileChunks.map((chunk) =>
+      util.promisify(child_process.execFile)('sha256sum', chunk, {
+        cwd: root,
+        maxBuffer: 1 * 1024 ** 3,
+      }).then(({ stdout }) =>
+        stdout
+          .trim()
+          .split('\n')
+          .map((line) => line.split('  ', 2).reverse())
+      )
+    );
+  
+    const results = (await Promise.all(chunkPromises)).flat();
+  
     return Object.fromEntries([
       ['.', ''],
       ...directories.map((d) => [d, '']),
-      ...stdout.split('\n').map((line) => line.split('  ', 2).reverse()),
+      ...results,
     ]);
   }
 
@@ -245,8 +288,23 @@ export class PackagerService extends ConsoleLogger {
     return this.archiveQueue.add(async () => {
       const files = archiveTask.filePaths;
       this.log(`Packaging archive ${archiveName} with ${archiveTask.exactFilePaths.length} files.`);
+      let tmpFilename: string;
+
+      let fileParams = files;
 
-      const child = child_process.spawn('tar', ['--zstd', '-cf', '-'].concat(files), {
+      const shouldUseFileList = (files: string[], limit = ARG_SIZE_LIMIT): boolean => {
+        const totalLength = files.reduce((sum, f) => sum + Buffer.byteLength(f) + 1, 0); // +1 for space/newline
+        return totalLength > limit;
+      };
+
+      if (shouldUseFileList(files)) {
+        this.warn(`Too many files in archive ${archiveName}, using file list.`);
+        tmpFilename = path.join(this.packagerWorkingDirectory, `filelist-${archiveName}.txt`);
+        await fs.promises.writeFile(tmpFilename, files.join('\n'), 'utf8');
+        fileParams = ['--files-from', tmpFilename];
+      }
+
+      const child = child_process.spawn('tar', ['--zstd', '-cf', '-', ...fileParams], {
         cwd: root,
       });
       const childPromise = new Promise<void>((resolve, reject) => {
@@ -270,7 +328,6 @@ export class PackagerService extends ConsoleLogger {
       });
 
       let uploadStream = child.stdout;
-      let tmpFilename: string;
 
       try {
        /* if (files.length > 1000) {