Skip to content

J
jsdelivr-uploader
Commit eac1b651 authored by nanahira's avatar nanahira
Browse files
Options

first

parent 28223860
Hide whitespace changes
Inline Side-by-side
Showing with 564 additions and 27 deletions
.dockerignore
View file @ eac1b651
......@@ -36,6 +36,8 @@ lerna-debug.log*
/data
/output
/config.yaml
/repo
/uploads
.git*
Dockerfile
......
.gitignore
View file @ eac1b651
......@@ -36,3 +36,5 @@ lerna-debug.log*
/data
/output
/config.yaml
/repo
/uploads
\ No newline at end of file
.npmignore
View file @ eac1b651
......@@ -5,4 +5,6 @@
/config.yaml
.idea
.dockerignore
Dockerfile
\ No newline at end of file
Dockerfile
/repo
/uploads
\ No newline at end of file
package-lock.json
View file @ eac1b651
......@@ -16,12 +16,17 @@
"@nestjs/typeorm": "^8.0.2",
"class-transformer": "^0.4.0",
"class-validator": "^0.13.1",
"delay": "^5.0.0",
"hasha": "^5.2.2",
"lodash": "^4.17.21",
"multer": "^1.4.2",
"p-queue": "6.6.2",
"pg": "^8.7.1",
"pg-native": "^3.0.0",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.2.0",
"simple-git": "^2.45.0",
"swagger-ui-express": "^4.1.6",
"typeorm": "^0.2.37"
},
......@@ -1335,6 +1340,19 @@
"url": "https://github.com/chalk/chalk?sponsor=1"
}
},
"node_modules/@kwsites/file-exists": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz",
"integrity": "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==",
"dependencies": {
"debug": "^4.1.1"
}
},
"node_modules/@kwsites/promise-deferred": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz",
"integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw=="
},
"node_modules/@nestjs/cli": {
"version": "8.1.1",
"resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-8.1.1.tgz",
......@@ -3778,6 +3796,17 @@
"clone": "^1.0.2"
}
},
"node_modules/delay": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
"integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
......@@ -4404,6 +4433,11 @@
"node": ">= 0.6"
}
},
"node_modules/eventemitter3": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
"integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="
},
"node_modules/events": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
......@@ -5087,6 +5121,29 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/hasha": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/hasha/-/hasha-5.2.2.tgz",
"integrity": "sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==",
"dependencies": {
"is-stream": "^2.0.0",
"type-fest": "^0.8.0"
},
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/hasha/node_modules/type-fest": {
"version": "0.8.1",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz",
"integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==",
"engines": {
"node": ">=8"
}
},
"node_modules/highlight.js": {
"version": "10.7.3",
"resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz",
......@@ -5470,7 +5527,6 @@
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
"dev": true,
"engines": {
"node": ">=8"
},
......@@ -7550,6 +7606,14 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/p-finally": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
"integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=",
"engines": {
"node": ">=4"
}
},
"node_modules/p-limit": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
......@@ -7577,6 +7641,32 @@
"node": ">=8"
}
},
"node_modules/p-queue": {
"version": "6.6.2",
"resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",
"integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==",
"dependencies": {
"eventemitter3": "^4.0.4",
"p-timeout": "^3.2.0"
},
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/p-timeout": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
"integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==",
"dependencies": {
"p-finally": "^1.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/p-try": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
......@@ -8533,6 +8623,16 @@
"integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==",
"dev": true
},
"node_modules/simple-git": {
"version": "2.45.0",
"resolved": "https://registry.npmjs.org/simple-git/-/simple-git-2.45.0.tgz",
"integrity": "sha512-wu/Ujs9IXn0HuyYm4HyRvne+EKsjJSWKEMkB3wQa3gNHSMHt7y3oeNX9zRQ3UBPk7bRRMLLHAdIZCZfHT9ehPg==",
"dependencies": {
"@kwsites/file-exists": "^1.1.1",
"@kwsites/promise-deferred": "^1.1.1",
"debug": "^4.3.1"
}
},
"node_modules/sisteransi": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
......@@ -11504,6 +11604,19 @@
}
}
},
"@kwsites/file-exists": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz",
"integrity": "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==",
"requires": {
"debug": "^4.1.1"
}
},
"@kwsites/promise-deferred": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz",
"integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw=="
},
"@nestjs/cli": {
"version": "8.1.1",
"resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-8.1.1.tgz",
......@@ -13375,6 +13488,11 @@
"clone": "^1.0.2"
}
},
"delay": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
"integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw=="
},
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
......@@ -13836,6 +13954,11 @@
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
"integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
},
"eventemitter3": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
"integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="
},
"events": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
......@@ -14367,6 +14490,22 @@
"integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==",
"dev": true
},
"hasha": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/hasha/-/hasha-5.2.2.tgz",
"integrity": "sha512-Hrp5vIK/xr5SkeN2onO32H0MgNZ0f17HRNH39WfL0SYUNOTZ5Lz1TJ8Pajo/87dYGEFlLMm7mIc/k/s6Bvz9HQ==",
"requires": {
"is-stream": "^2.0.0",
"type-fest": "^0.8.0"
},
"dependencies": {
"type-fest": {
"version": "0.8.1",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz",
"integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA=="
}
}
},
"highlight.js": {
"version": "10.7.3",
"resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz",
......@@ -14649,8 +14788,7 @@
"is-stream": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
"dev": true
"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="
},
"is-typedarray": {
"version": "1.0.0",
......@@ -16247,6 +16385,11 @@
"integrity": "sha512-ycIL2+1V32th+8scbpTvyHNaHe02z0sjgh91XXjAk+ZeXoPN4Z46DVUnzdso0aX4KckKw0FNNFHdjZ2UsZvxiA==",
"dev": true
},
"p-finally": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
"integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4="
},
"p-limit": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
......@@ -16265,6 +16408,23 @@
"p-limit": "^2.2.0"
}
},
"p-queue": {
"version": "6.6.2",
"resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",
"integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==",
"requires": {
"eventemitter3": "^4.0.4",
"p-timeout": "^3.2.0"
}
},
"p-timeout": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
"integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==",
"requires": {
"p-finally": "^1.0.0"
}
},
"p-try": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
......@@ -16991,6 +17151,16 @@
"integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==",
"dev": true
},
"simple-git": {
"version": "2.45.0",
"resolved": "https://registry.npmjs.org/simple-git/-/simple-git-2.45.0.tgz",
"integrity": "sha512-wu/Ujs9IXn0HuyYm4HyRvne+EKsjJSWKEMkB3wQa3gNHSMHt7y3oeNX9zRQ3UBPk7bRRMLLHAdIZCZfHT9ehPg==",
"requires": {
"@kwsites/file-exists": "^1.1.1",
"@kwsites/promise-deferred": "^1.1.1",
"debug": "^4.3.1"
}
},
"sisteransi": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
......
src/app.controller.ts
View file @ eac1b651
import { Controller, Get } from '@nestjs/common';
import {
Controller,
Post,
UploadedFile,
UseInterceptors,
} from '@nestjs/common';
import { AppService } from './app.service';
import { FileInterceptor } from '@nestjs/platform-express';
import * as os from 'os';
import path from 'path';
import { FileUploadDto } from './dto/FileUpload.dto';
import { ApiBody, ApiConsumes, ApiCreatedResponse } from '@nestjs/swagger';
import { StringReturnMessageDto } from './dto/ReturnMessage.dto';
import multer from 'multer';
@Controller()
export class AppController {
constructor(private readonly appService: AppService) {}
@Get()
getHello(): string {
return this.appService.getHello();
@Post('upload')
@ApiConsumes('multipart/form-data')
@ApiBody({
description: '上传的文件',
type: FileUploadDto,
})
@UseInterceptors(
FileInterceptor('file', {
storage: multer.diskStorage({
destination: path.join(os.tmpdir(), 'uploads'),
}),
limits: {
fileSize: 50 * 1024 * 1024,
fieldNameSize: 128,
},
}),
)
@ApiCreatedResponse({ type: StringReturnMessageDto })
upload(@UploadedFile() file: Express.Multer.File) {
return this.appService.upload(file);
}
}
src/app.module.ts
View file @ eac1b651
......@@ -3,7 +3,7 @@ import { AppController } from './app.controller';
import { AppService } from './app.service';
import { TypeOrmModule } from '@nestjs/typeorm';
import { ConfigModule, ConfigService } from '@nestjs/config';
import { User } from './entities/User.entity';
import { File } from './entities/File.entity';
const configModule = ConfigModule.forRoot();
......@@ -17,13 +17,14 @@ const configModule = ConfigModule.forRoot();
useFactory: async (config: ConfigService) => {
return {
type: 'postgres',
entities: [User], // entities here
entities: [File], // entities here
synchronize: !config.get('DB_NO_INIT'),
host: config.get('DB_HOST'),
port: parseInt(config.get('DB_PORT')) || 5432,
username: config.get('DB_USER'),
password: config.get('DB_PASS'),
database: config.get('DB_NAME'),
//logging: true,
};
},
}),
......
src/app.service.ts
View file @ eac1b651
import { Connection } from 'typeorm';
import { Connection, Repository } from 'typeorm';
import { InjectConnection } from '@nestjs/typeorm';
import { Injectable, ConsoleLogger } from '@nestjs/common';
import simpleGit, { SimpleGit } from 'simple-git';
import { ConfigService } from '@nestjs/config';
import path from 'path';
import { File } from './entities/File.entity';
import { promises as fs } from 'fs';
import * as os from 'os';
import delay from 'delay';
import { ReturnMessageDto } from './dto/ReturnMessage.dto';
import PQueue from 'p-queue';
@Injectable()
export class AppService extends ConsoleLogger {
dbRepo: Repository<File>;
git: SimpleGit;
githubRepo: string;
githubUser: string;
githubToken: string;
repoPath: string;
maxBranchSize: number;
commitQueue: File[] = [];
inQueueQueue = new PQueue({ concurrency: 1 });
queueHashmap = new Map<string, File>();
repoUrl() {
return `https://${this.githubToken}@github.com/${this.githubUser}/${this.githubRepo}.git`;
}
constructor(
@InjectConnection('app')
private db: Connection,
private config: ConfigService,
) {
super('app');
this.githubRepo = config.get('GITHUB_REPO');
this.githubUser = config.get('GITHUB_USER');
this.githubToken = config.get('GITHUB_TOKEN');
this.repoPath = config.get('REPO_PATH') || path.join(os.tmpdir(), 'repo');
this.dbRepo = this.db.getRepository(File);
this.maxBranchSize =
parseInt(config.get('BRANCH_SIZE')) || 50 * 1024 * 1024;
this.main().then();
}
async main() {
/*try {
await fs.mkdir(path.join(os.tmpdir(), 'uploads'), { recursive: true });
} catch (e) {}*/
try {
await this.initRepo();
} catch (e) {
this.error(`Initialize failed: ${e.toString()}`);
}
while (true) {
try {
await this.mainLoop();
} catch (e) {
this.error(`Loop failed: ${e.toString()}`);
}
}
}
createGitInstance() {
this.git = simpleGit({
baseDir: this.repoPath,
binary: this.config.get('GIT_BINARY') || 'git',
});
}
gatherFileTasks(possibleSize: number) {
const files: File[] = [];
let size = 0;
while (this.commitQueue.length > 0 && size <= possibleSize) {
const file = this.commitQueue.pop();
size += file.size;
if (size <= possibleSize) {
files.push(file);
} else {
this.commitQueue.push(file);
break;
}
}
return files;
}
async mainLoop() {
//this.log('Running loop.');
if (!this.commitQueue.length) {
return delay(3000);
}
this.log(`Processing files.`);
let checkoutResult = await this.checkoutCorrectBranch();
let files = this.gatherFileTasks(this.maxBranchSize - checkoutResult.size);
if (!files.length) {
checkoutResult = await this.checkoutCorrectBranch(true);
files = this.gatherFileTasks(this.maxBranchSize - checkoutResult.size);
}
if (!files.length) {
this.error(`Cannot fetch any files, exiting.`);
return;
}
this.log(`Will process ${files.length} files.`);
try {
this.log(`Moving files.`);
for (const file of files) {
file.branchId = checkoutResult.branchId;
await fs.rename(file.savePath, path.join(this.repoPath, file.filename));
}
this.log(`Committing files.`);
await this.git
.add(files.map((f) => f.filename))
.commit('upload')
.push('origin', checkoutResult.branchId.toString(36), ['-u', '-f']);
this.log(`Saving file entries to database.`);
await this.dbRepo.save(files);
this.log(`Finished processing files.`);
for (const file of files) {
file.resolve(this.githubUser, this.githubRepo);
}
} catch (e) {
this.error(`Errored processing files: ${e.toString()}`);
for (const file of files) {
try {
await fs.unlink(file.savePath);
await fs.unlink(path.join(this.repoPath, file.filename));
} catch (e) {}
file.reject(new ReturnMessageDto(500, 'upload failed'));
}
} finally {
for (const file of files) {
this.queueHashmap.delete(file.filename);
}
}
}
async initRepo() {
this.log(`Initializing repo at ${this.repoPath}`);
try {
await fs.access(path.join(this.repoPath, '.git'));
this.createGitInstance();
this.log(`Repo exists, skipping.`);
} catch (e) {
try {
await fs.access(this.repoPath);
} catch (e) {
await fs.mkdir(this.repoPath, { recursive: true });
}
this.createGitInstance();
await this.git
.init()
.addRemote('origin', this.repoUrl())
.addConfig('core.autocrlf', 'false', false);
this.log(`Initialized repo at ${this.repoPath}`);
await this.checkoutCorrectBranch(false, true);
this.log(`Initializing finished.`);
}
}
async checkoutCorrectBranch(
forceNew?: boolean,
fetch?: boolean,
): Promise<{ branchId: number; size: number }> {
const res = await this.getLatestBranchId(forceNew);
const branchName = res.branchId.toString(36);
if (!res.size) {
this.log(`Checking out to a new branch ${branchName}`);
await this.git.checkout(['--orphan', branchName]);
try {
await this.git.raw(['rm', '-rf', '.']);
} catch (e) {}
this.log(`Checked out to a new branch ${branchName}`);
} else {
this.log(`Checking out existing branch ${branchName}`);
try {
if (fetch) {
await this.git.fetch('origin', branchName);
}
await this.git.checkout(branchName, ['-f']);
this.log(`Checked out existing branch ${branchName}`);
} catch (e) {
this.error(
`Checking out to existing branch failed, will checkout to a new branch: ${e.toString()}`,
);
return this.checkoutCorrectBranch(true, fetch);
}
}
return res;
}
async getLatestBranchId(
forceNew?: boolean,
): Promise<{ branchId: number; size: number }> {
const [latestFile] = await this.dbRepo.find({
select: ['branchId'],
order: { id: 'DESC' },
take: 1,
});
if (!latestFile) {
return { branchId: 1, size: 0 };
}
const branchId = parseInt(latestFile.branchIdValue.toString());
if (forceNew) {
return { branchId: branchId + 1, size: 0 };
}
const { totalSizeRaw } = await this.dbRepo
.createQueryBuilder('file')
.select('sum(file.size)', 'totalSizeRaw')
.where('file.branchId = :branchId', { branchId })
.getRawOne();
const size = parseInt(totalSizeRaw);
if (size >= this.maxBranchSize) {
this.log(`Will switch to branch ${branchId.toString(36)}`);
return { branchId: branchId + 1, size: 0 };
} else {
this.log(`Will remain on branch ${branchId.toString(36)}`);
return { branchId, size };
}
}
async addFileToQueue(file: File) {
return new Promise<File>(async (resolve, reject) => {
await this.inQueueQueue.add(() => {
const inQueueFile = this.queueHashmap.get(file.filename);
if (inQueueFile) {
inQueueFile.resolveFunction.push(resolve);
inQueueFile.rejectFunction.push(reject);
} else {
this.log(`Queued file ${file.filename}`);
file.resolveFunction.push(resolve);
file.rejectFunction.push(reject);
this.commitQueue.unshift(file);
this.queueHashmap.set(file.filename, file);
}
});
});
}
getHello(): string {
return 'Hello World!';
async upload(mutFile: Express.Multer.File) {
const file = new File();
await file.fromMulterFile(mutFile);
const existingFile = await this.dbRepo.findOne({
where: { hash: file.hash, name: file.name },
select: ['hash', 'name', 'branchId'],
});
if (existingFile) {
return new ReturnMessageDto(
201,
'success',
existingFile.getJsdelivrUrl(this.githubUser, this.githubRepo),
);
}
const uploadedFile = await this.addFileToQueue(file);
return new ReturnMessageDto(201, 'success', uploadedFile.url);
}
}
src/dto/FileUpload.dto.ts 0 → 100644
View file @ eac1b651
import { ApiProperty } from '@nestjs/swagger';
export class FileUploadDto {
@ApiProperty({ type: 'string', format: 'binary' })
file: any;
}
src/dto/ReturnMessage.dto.ts
View file @ eac1b651
import { ApiProperty } from '@nestjs/swagger';
import { HttpException } from '@nestjs/common';
import { File } from '../entities/File.entity';
export class BlankReturnMessageDto {
@ApiProperty({ description: '返回状态' })
......@@ -27,3 +28,7 @@ export class ReturnMessageDto<T> extends BlankReturnMessageDto {
this.data = data;
}
}
export class StringReturnMessageDto extends BlankReturnMessageDto {
data: string;
}
src/entities/File.entity.ts 0 → 100644
View file @ eac1b651
import { TimeBase } from './TimeBase.entity';
import { Column, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';
import path from 'path';
import hasha from 'hasha';
import { ReturnMessageDto } from '../dto/ReturnMessage.dto';
@Entity()
@Index((f) => [f.hash, f.name], { unique: true })
export class File extends TimeBase {
@PrimaryGeneratedColumn({ type: 'int8' })
id: number;
@Index()
@Column('varchar', { length: 128 })
hash: string;
@Index()
@Column('varchar', { length: 128 })
name: string;
@Index()
@Column('int8')
branchId: number;
get branchIdValue() {
return parseInt(this.branchId.toString());
}
@Index()
@Column('int4')
size: number;
url: string;
//multerFile: Express.Multer.File;
savePath: string;
resolveFunction: ((_this: File) => void)[] = [];
rejectFunction: ((error: any) => void)[] = [];
resolve(user: string, repo: string) {
this.url = this.getJsdelivrUrl(user, repo);
this.savePath = undefined;
for (const func of this.resolveFunction) {
func(this);
}
}
reject(error: any) {
for (const func of this.rejectFunction) {
func(error);
}
}
async fromMulterFile(multerFile: Express.Multer.File) {
this.savePath = multerFile.path;
this.size = multerFile.size;
const dotSplit = multerFile.originalname.split('/').pop().split('.');
this.name = multerFile.originalname;
if (this.name.length > 128) {
throw new ReturnMessageDto(400, 'name too long').toException();
}
this.hash = await hasha.fromFile(multerFile.path);
}
get filename() {
return `${this.hash}-${this.name}`;
}
getJsdelivrUrl(user: string, repo: string) {
return `https://cdn.jsdelivr.net/gh/${user}/${repo}@${this.branch}/${this.filename}`;
}
get branch() {
return this.branchIdValue.toString(36);
}
set branch(branchName: string) {
this.branchId = parseInt(branchName, 36);
}
setSuffix(filename: string) {
this.name = path.extname(filename);
return this.name;
}
}
src/entities/User.entity.ts deleted 100644 → 0
View file @ 28223860
import { TimeBase } from './TimeBase.entity';
import { Column, Entity, Index, PrimaryColumn } from 'typeorm';
@Entity()
export class User extends TimeBase {
@PrimaryColumn('varchar', { length: 32 })
id: string;
@Index()
@Column('varchar', { length: 32 })
name: string;
}
tsconfig.build.json
View file @ eac1b651
{
"extends": "./tsconfig.json",
"exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
"exclude": ["node_modules", "test", "dist", "**/*spec.ts", "repo", "uploads"]
}
tsconfig.json
View file @ eac1b651
......@@ -14,5 +14,6 @@
"esModuleInterop": true
},
"compileOnSave": true,
"allowJs": true
"allowJs": true,
"exclude": ["repo", "uploads"]
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment