first

.dockerignore

@@ -36,6 +36,8 @@ lerna-debug.log*
 /data
 /output
 /config.yaml
+/repo
+/uploads
 
 .git*
 Dockerfile

.gitignore

@@ -36,3 +36,5 @@ lerna-debug.log*
 /data
 /output
 /config.yaml
+/repo
+/uploads
\ No newline at end of file

.npmignore

@@ -5,4 +5,6 @@
 /config.yaml
 .idea
 .dockerignore
-Dockerfile
\ No newline at end of file
+Dockerfile
+/repo
+/uploads
\ No newline at end of file

package.json

@@ -29,12 +29,17 @@
     "@nestjs/typeorm": "^8.0.2",
     "class-transformer": "^0.4.0",
     "class-validator": "^0.13.1",
+    "delay": "^5.0.0",
+    "hasha": "^5.2.2",
     "lodash": "^4.17.21",
+    "multer": "^1.4.2",
+    "p-queue": "6.6.2",
     "pg": "^8.7.1",
     "pg-native": "^3.0.0",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^7.2.0",
+    "simple-git": "^2.45.0",
     "swagger-ui-express": "^4.1.6",
     "typeorm": "^0.2.37"
   },

src/app.controller.ts

-import { Controller, Get } from '@nestjs/common';
+import {
+  Controller,
+  Post,
+  UploadedFile,
+  UseInterceptors,
+} from '@nestjs/common';
 import { AppService } from './app.service';
+import { FileInterceptor } from '@nestjs/platform-express';
+import * as os from 'os';
+import path from 'path';
+import { FileUploadDto } from './dto/FileUpload.dto';
+import { ApiBody, ApiConsumes, ApiCreatedResponse } from '@nestjs/swagger';
+import { StringReturnMessageDto } from './dto/ReturnMessage.dto';
+import multer from 'multer';
 
 @Controller()
 export class AppController {
   constructor(private readonly appService: AppService) {}
 
-  @Get()
-  getHello(): string {
-    return this.appService.getHello();
+  @Post('upload')
+  @ApiConsumes('multipart/form-data')
+  @ApiBody({
+    description: '上传的文件',
+    type: FileUploadDto,
+  })
+  @UseInterceptors(
+    FileInterceptor('file', {
+      storage: multer.diskStorage({
+        destination: path.join(os.tmpdir(), 'uploads'),
+      }),
+      limits: {
+        fileSize: 50 * 1024 * 1024,
+        fieldNameSize: 128,
+      },
+    }),
+  )
+  @ApiCreatedResponse({ type: StringReturnMessageDto })
+  upload(@UploadedFile() file: Express.Multer.File) {
+    return this.appService.upload(file);
   }
 }

src/app.module.ts

@@ -3,7 +3,7 @@ import { AppController } from './app.controller';
 import { AppService } from './app.service';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { ConfigModule, ConfigService } from '@nestjs/config';
-import { User } from './entities/User.entity';
+import { File } from './entities/File.entity';
 
 const configModule = ConfigModule.forRoot();
 
@@ -17,13 +17,14 @@ const configModule = ConfigModule.forRoot();
       useFactory: async (config: ConfigService) => {
         return {
           type: 'postgres',
-          entities: [User], // entities here
+          entities: [File], // entities here
           synchronize: !config.get('DB_NO_INIT'),
           host: config.get('DB_HOST'),
           port: parseInt(config.get('DB_PORT')) || 5432,
           username: config.get('DB_USER'),
           password: config.get('DB_PASS'),
           database: config.get('DB_NAME'),
+          //logging: true,
         };
       },
     }),

src/app.service.ts

-import { Connection } from 'typeorm';
+import { Connection, Repository } from 'typeorm';
 import { InjectConnection } from '@nestjs/typeorm';
 import { Injectable, ConsoleLogger } from '@nestjs/common';
+import simpleGit, { SimpleGit } from 'simple-git';
+import { ConfigService } from '@nestjs/config';
+import path from 'path';
+import { File } from './entities/File.entity';
+import { promises as fs } from 'fs';
+import * as os from 'os';
+import delay from 'delay';
+import { ReturnMessageDto } from './dto/ReturnMessage.dto';
+import PQueue from 'p-queue';
 
 @Injectable()
 export class AppService extends ConsoleLogger {
+  dbRepo: Repository<File>;
+  git: SimpleGit;
+  githubRepo: string;
+  githubUser: string;
+  githubToken: string;
+  repoPath: string;
+  maxBranchSize: number;
+  commitQueue: File[] = [];
+  inQueueQueue = new PQueue({ concurrency: 1 });
+  queueHashmap = new Map<string, File>();
+
+  repoUrl() {
+    return `https://${this.githubToken}@github.com/${this.githubUser}/${this.githubRepo}.git`;
+  }
+
   constructor(
     @InjectConnection('app')
     private db: Connection,
+    private config: ConfigService,
   ) {
     super('app');
+    this.githubRepo = config.get('GITHUB_REPO');
+    this.githubUser = config.get('GITHUB_USER');
+    this.githubToken = config.get('GITHUB_TOKEN');
+    this.repoPath = config.get('REPO_PATH') || path.join(os.tmpdir(), 'repo');
+    this.dbRepo = this.db.getRepository(File);
+    this.maxBranchSize =
+      parseInt(config.get('BRANCH_SIZE')) || 50 * 1024 * 1024;
+    this.main().then();
+  }
+
+  async main() {
+    /*try {
+      await fs.mkdir(path.join(os.tmpdir(), 'uploads'), { recursive: true });
+    } catch (e) {}*/
+    try {
+      await this.initRepo();
+    } catch (e) {
+      this.error(`Initialize failed: ${e.toString()}`);
+    }
+    while (true) {
+      try {
+        await this.mainLoop();
+      } catch (e) {
+        this.error(`Loop failed: ${e.toString()}`);
+      }
+    }
+  }
+
+  createGitInstance() {
+    this.git = simpleGit({
+      baseDir: this.repoPath,
+      binary: this.config.get('GIT_BINARY') || 'git',
+    });
+  }
+
+  gatherFileTasks(possibleSize: number) {
+    const files: File[] = [];
+    let size = 0;
+    while (this.commitQueue.length > 0 && size <= possibleSize) {
+      const file = this.commitQueue.pop();
+      size += file.size;
+      if (size <= possibleSize) {
+        files.push(file);
+      } else {
+        this.commitQueue.push(file);
+        break;
+      }
+    }
+    return files;
+  }
+
+  async mainLoop() {
+    //this.log('Running loop.');
+    if (!this.commitQueue.length) {
+      return delay(3000);
+    }
+    this.log(`Processing files.`);
+    let checkoutResult = await this.checkoutCorrectBranch();
+    let files = this.gatherFileTasks(this.maxBranchSize - checkoutResult.size);
+    if (!files.length) {
+      checkoutResult = await this.checkoutCorrectBranch(true);
+      files = this.gatherFileTasks(this.maxBranchSize - checkoutResult.size);
+    }
+    if (!files.length) {
+      this.error(`Cannot fetch any files, exiting.`);
+      return;
+    }
+    this.log(`Will process ${files.length} files.`);
+    try {
+      this.log(`Moving files.`);
+      for (const file of files) {
+        file.branchId = checkoutResult.branchId;
+        await fs.rename(file.savePath, path.join(this.repoPath, file.filename));
+      }
+      this.log(`Committing files.`);
+      await this.git
+        .add(files.map((f) => f.filename))
+        .commit('upload')
+        .push('origin', checkoutResult.branchId.toString(36), ['-u', '-f']);
+      this.log(`Saving file entries to database.`);
+      await this.dbRepo.save(files);
+      this.log(`Finished processing files.`);
+      for (const file of files) {
+        file.resolve(this.githubUser, this.githubRepo);
+      }
+    } catch (e) {
+      this.error(`Errored processing files: ${e.toString()}`);
+      for (const file of files) {
+        try {
+          await fs.unlink(file.savePath);
+          await fs.unlink(path.join(this.repoPath, file.filename));
+        } catch (e) {}
+        file.reject(new ReturnMessageDto(500, 'upload failed'));
+      }
+    } finally {
+      for (const file of files) {
+        this.queueHashmap.delete(file.filename);
+      }
+    }
+  }
+
+  async initRepo() {
+    this.log(`Initializing repo at ${this.repoPath}`);
+    try {
+      await fs.access(path.join(this.repoPath, '.git'));
+      this.createGitInstance();
+      this.log(`Repo exists, skipping.`);
+    } catch (e) {
+      try {
+        await fs.access(this.repoPath);
+      } catch (e) {
+        await fs.mkdir(this.repoPath, { recursive: true });
+      }
+      this.createGitInstance();
+      await this.git
+        .init()
+        .addRemote('origin', this.repoUrl())
+        .addConfig('core.autocrlf', 'false', false);
+      this.log(`Initialized repo at ${this.repoPath}`);
+      await this.checkoutCorrectBranch(false, true);
+      this.log(`Initializing finished.`);
+    }
+  }
+
+  async checkoutCorrectBranch(
+    forceNew?: boolean,
+    fetch?: boolean,
+  ): Promise<{ branchId: number; size: number }> {
+    const res = await this.getLatestBranchId(forceNew);
+    const branchName = res.branchId.toString(36);
+    if (!res.size) {
+      this.log(`Checking out to a new branch ${branchName}`);
+      await this.git.checkout(['--orphan', branchName]);
+      try {
+        await this.git.raw(['rm', '-rf', '.']);
+      } catch (e) {}
+      this.log(`Checked out to a new branch ${branchName}`);
+    } else {
+      this.log(`Checking out existing branch ${branchName}`);
+      try {
+        if (fetch) {
+          await this.git.fetch('origin', branchName);
+        }
+        await this.git.checkout(branchName, ['-f']);
+        this.log(`Checked out existing branch ${branchName}`);
+      } catch (e) {
+        this.error(
+          `Checking out to existing branch failed, will checkout to a new branch: ${e.toString()}`,
+        );
+        return this.checkoutCorrectBranch(true, fetch);
+      }
+    }
+    return res;
+  }
+
+  async getLatestBranchId(
+    forceNew?: boolean,
+  ): Promise<{ branchId: number; size: number }> {
+    const [latestFile] = await this.dbRepo.find({
+      select: ['branchId'],
+      order: { id: 'DESC' },
+      take: 1,
+    });
+    if (!latestFile) {
+      return { branchId: 1, size: 0 };
+    }
+    const branchId = parseInt(latestFile.branchIdValue.toString());
+    if (forceNew) {
+      return { branchId: branchId + 1, size: 0 };
+    }
+    const { totalSizeRaw } = await this.dbRepo
+      .createQueryBuilder('file')
+      .select('sum(file.size)', 'totalSizeRaw')
+      .where('file.branchId = :branchId', { branchId })
+      .getRawOne();
+    const size = parseInt(totalSizeRaw);
+    if (size >= this.maxBranchSize) {
+      this.log(`Will switch to branch ${branchId.toString(36)}`);
+      return { branchId: branchId + 1, size: 0 };
+    } else {
+      this.log(`Will remain on branch ${branchId.toString(36)}`);
+      return { branchId, size };
+    }
+  }
+
+  async addFileToQueue(file: File) {
+    return new Promise<File>(async (resolve, reject) => {
+      await this.inQueueQueue.add(() => {
+        const inQueueFile = this.queueHashmap.get(file.filename);
+        if (inQueueFile) {
+          inQueueFile.resolveFunction.push(resolve);
+          inQueueFile.rejectFunction.push(reject);
+        } else {
+          this.log(`Queued file ${file.filename}`);
+          file.resolveFunction.push(resolve);
+          file.rejectFunction.push(reject);
+          this.commitQueue.unshift(file);
+          this.queueHashmap.set(file.filename, file);
+        }
+      });
+    });
   }
 
-  getHello(): string {
-    return 'Hello World!';
+  async upload(mutFile: Express.Multer.File) {
+    const file = new File();
+    await file.fromMulterFile(mutFile);
+    const existingFile = await this.dbRepo.findOne({
+      where: { hash: file.hash, name: file.name },
+      select: ['hash', 'name', 'branchId'],
+    });
+    if (existingFile) {
+      return new ReturnMessageDto(
+        201,
+        'success',
+        existingFile.getJsdelivrUrl(this.githubUser, this.githubRepo),
+      );
+    }
+    const uploadedFile = await this.addFileToQueue(file);
+    return new ReturnMessageDto(201, 'success', uploadedFile.url);
   }
 }

src/dto/FileUpload.dto.ts

+import { ApiProperty } from '@nestjs/swagger';
+
+export class FileUploadDto {
+  @ApiProperty({ type: 'string', format: 'binary' })
+  file: any;
+}

src/dto/ReturnMessage.dto.ts

 import { ApiProperty } from '@nestjs/swagger';
 import { HttpException } from '@nestjs/common';
+import { File } from '../entities/File.entity';
 
 export class BlankReturnMessageDto {
   @ApiProperty({ description: '返回状态' })
@@ -27,3 +28,7 @@ export class ReturnMessageDto<T> extends BlankReturnMessageDto {
     this.data = data;
   }
 }
+
+export class StringReturnMessageDto extends BlankReturnMessageDto {
+  data: string;
+}

src/entities/File.entity.ts

+import { TimeBase } from './TimeBase.entity';
+import { Column, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';
+import path from 'path';
+import hasha from 'hasha';
+import { ReturnMessageDto } from '../dto/ReturnMessage.dto';
+
+@Entity()
+@Index((f) => [f.hash, f.name], { unique: true })
+export class File extends TimeBase {
+  @PrimaryGeneratedColumn({ type: 'int8' })
+  id: number;
+
+  @Index()
+  @Column('varchar', { length: 128 })
+  hash: string;
+
+  @Index()
+  @Column('varchar', { length: 128 })
+  name: string;
+
+  @Index()
+  @Column('int8')
+  branchId: number;
+
+  get branchIdValue() {
+    return parseInt(this.branchId.toString());
+  }
+
+  @Index()
+  @Column('int4')
+  size: number;
+
+  url: string;
+
+  //multerFile: Express.Multer.File;
+  savePath: string;
+
+  resolveFunction: ((_this: File) => void)[] = [];
+  rejectFunction: ((error: any) => void)[] = [];
+
+  resolve(user: string, repo: string) {
+    this.url = this.getJsdelivrUrl(user, repo);
+    this.savePath = undefined;
+    for (const func of this.resolveFunction) {
+      func(this);
+    }
+  }
+
+  reject(error: any) {
+    for (const func of this.rejectFunction) {
+      func(error);
+    }
+  }
+
+  async fromMulterFile(multerFile: Express.Multer.File) {
+    this.savePath = multerFile.path;
+    this.size = multerFile.size;
+    const dotSplit = multerFile.originalname.split('/').pop().split('.');
+    this.name = multerFile.originalname;
+    if (this.name.length > 128) {
+      throw new ReturnMessageDto(400, 'name too long').toException();
+    }
+    this.hash = await hasha.fromFile(multerFile.path);
+  }
+
+  get filename() {
+    return `${this.hash}-${this.name}`;
+  }
+
+  getJsdelivrUrl(user: string, repo: string) {
+    return `https://cdn.jsdelivr.net/gh/${user}/${repo}@${this.branch}/${this.filename}`;
+  }
+
+  get branch() {
+    return this.branchIdValue.toString(36);
+  }
+
+  set branch(branchName: string) {
+    this.branchId = parseInt(branchName, 36);
+  }
+
+  setSuffix(filename: string) {
+    this.name = path.extname(filename);
+    return this.name;
+  }
+}

src/entities/User.entity.ts

-import { TimeBase } from './TimeBase.entity';
-import { Column, Entity, Index, PrimaryColumn } from 'typeorm';
-
-@Entity()
-export class User extends TimeBase {
-  @PrimaryColumn('varchar', { length: 32 })
-  id: string;
-
-  @Index()
-  @Column('varchar', { length: 32 })
-  name: string;
-}

tsconfig.build.json

 {
   "extends": "./tsconfig.json",
-  "exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "repo", "uploads"]
 }

tsconfig.json

@@ -14,5 +14,6 @@
     "esModuleInterop": true
   },
   "compileOnSave": true,
-  "allowJs": true
+  "allowJs": true,
+  "exclude": ["repo", "uploads"]
 }