first

.dockerignore

+# compiled output
+/dist
+/node_modules
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# OS
+.DS_Store
+
+# Tests
+/coverage
+/.nyc_output
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+/data
+/output
+/config.yaml
+
+.git*
+Dockerfile
+.dockerignore

.gitignore

+# compiled output
+/dist
+/node_modules
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# OS
+.DS_Store
+
+# Tests
+/coverage
+/.nyc_output
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+/data
+/output
+/config.yaml
\ No newline at end of file

.gitlab-ci.yml

+stages:
+  - build
+  - deploy
+variables:
+  GIT_DEPTH: "1"
+
+before_script:
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+
+.build-image:
+  stage: build
+  script:
+    - docker build --pull -t $TARGET_IMAGE .
+    - docker push $TARGET_IMAGE
+
+build-x86:
+  extends: .build-image
+  tags:
+    - docker
+  variables:
+    TARGET_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-x86
+
+build-arm:
+  extends: .build-image
+  tags:
+    - docker-arm
+  variables:
+    TARGET_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-arm
+
+.deploy:
+  stage: deploy
+  tags:
+    - docker
+  script:
+    - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-x86
+    - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-arm
+    - docker manifest create $TARGET_IMAGE --amend $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-x86 --amend
+      $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-arm
+    - docker manifest push $TARGET_IMAGE
+
+deploy_latest:
+  extends: .deploy
+  variables:
+    TARGET_IMAGE: $CI_REGISTRY_IMAGE:latest
+  only:
+    - master
+
+deploy_branch:
+  extends: .deploy
+  variables:
+    TARGET_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG

.npmignore

+/install-npm.sh
+.git*
+/data
+/output
+/config.yaml
+.idea
+.dockerignore
+Dockerfile
+/src

.prettierrc

+{
+  "singleQuote": true,
+  "trailingComma": "all"
+}
\ No newline at end of file

Dockerfile

+FROM node:lts-bookworm-slim as base
+LABEL Author="Nanahira <nanahira@momobako.com>"
+
+RUN apt update && apt -y install python3 build-essential && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/log/*
+WORKDIR /usr/src/app
+COPY ./package*.json ./
+
+FROM base as builder
+RUN npm ci && npm cache clean --force
+COPY . ./
+RUN npm run build
+
+FROM base
+ENV NODE_ENV production
+RUN npm ci && npm cache clean --force
+COPY --from=builder /usr/src/app/dist ./dist
+COPY ./config.example.yaml ./config.yaml
+
+EXPOSE 3000
+CMD [ "npm", "run", "start:prod" ]

README.md

+# App name
+
+App description.
+
+## Installation
+
+```bash
+$ npm install
+```
+
+## Config
+
+Make a copy of `config.example.yaml` to `config.yaml`.
+
+## Running the app
+
+```bash
+# development
+$ npm run start
+
+# watch mode
+$ npm run start:dev
+
+# production mode
+$ npm run start:prod
+```
+
+## License
+
+AGPLv3

config.example.yaml

+host: '::'
+port: 3000
+S3_ENDPOINT: ''
+S3_BUCKET: ''
+S3_PATH_PREFIX: ''
+S3_ACCESS_KEY_ID: ''
+S3_SECRET_ACCESS_KEY: ''
+S3_PATH_STYLE: ''
+HTTP_TIMEOUT: 30000
\ No newline at end of file

eslint.config.mjs

+// @ts-check
+import eslint from '@eslint/js';
+import eslintPluginPrettierRecommended from 'eslint-plugin-prettier/recommended';
+import globals from 'globals';
+import tseslint from 'typescript-eslint';
+
+export default tseslint.config(
+  {
+    ignores: ['eslint.config.mjs'],
+  },
+  eslint.configs.recommended,
+  ...tseslint.configs.recommendedTypeChecked,
+  eslintPluginPrettierRecommended,
+  {
+    languageOptions: {
+      globals: {
+        ...globals.node,
+        ...globals.jest,
+      },
+      sourceType: 'commonjs',
+      parserOptions: {
+        projectService: true,
+        tsconfigRootDir: import.meta.dirname,
+      },
+    },
+  },
+  {
+    rules: {
+      '@typescript-eslint/no-explicit-any': 'off',
+      '@typescript-eslint/no-floating-promises': 'warn',
+      '@typescript-eslint/no-unsafe-argument': 'warn'
+    },
+  },
+);
\ No newline at end of file

install-npm.sh

+#!/bin/bash
+npm install --save \
+	class-validator \
+	class-transformer \
+	@nestjs/swagger \
+	@nestjs/config \
+	yaml \
+	nesties
+
+npm install --save-dev \
+	@types/express
+
+# npm i --save-exact --save-dev eslint@8.22.0

nest-cli.json

+{
+  "collection": "@nestjs/schematics",
+  "sourceRoot": "src",
+  "compilerOptions": {
+    "plugins": ["@nestjs/swagger"]
+  }
+}

package.json

+{
+  "name": "mat-cacher",
+  "version": "0.0.1",
+  "description": "",
+  "author": "",
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "build": "nest build",
+    "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
+    "start": "nest start",
+    "start:dev": "nest start --watch",
+    "start:debug": "nest start --debug --watch",
+    "start:prod": "node dist/main",
+    "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:cov": "jest --coverage",
+    "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
+    "test:e2e": "jest --config ./test/jest-e2e.json"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.828.0",
+    "@nestjs/axios": "^4.0.0",
+    "@nestjs/common": "^11.0.1",
+    "@nestjs/config": "^4.0.2",
+    "@nestjs/core": "^11.0.1",
+    "@nestjs/platform-express": "^11.0.1",
+    "@nestjs/swagger": "^11.2.0",
+    "class-transformer": "^0.5.1",
+    "class-validator": "^0.14.2",
+    "nesties": "^1.1.2",
+    "reflect-metadata": "^0.2.2",
+    "rxjs": "^7.8.1",
+    "yaml": "^2.8.0"
+  },
+  "devDependencies": {
+    "@eslint/eslintrc": "^3.2.0",
+    "@eslint/js": "^9.18.0",
+    "@nestjs/cli": "^11.0.0",
+    "@nestjs/schematics": "^11.0.0",
+    "@nestjs/testing": "^11.0.1",
+    "@swc/cli": "^0.6.0",
+    "@swc/core": "^1.10.7",
+    "@types/express": "^5.0.3",
+    "@types/jest": "^29.5.14",
+    "@types/node": "^22.10.7",
+    "@types/supertest": "^6.0.2",
+    "eslint": "^9.18.0",
+    "eslint-config-prettier": "^10.0.1",
+    "eslint-plugin-prettier": "^5.2.2",
+    "globals": "^16.0.0",
+    "jest": "^29.7.0",
+    "prettier": "^3.4.2",
+    "source-map-support": "^0.5.21",
+    "supertest": "^7.0.0",
+    "ts-jest": "^29.2.5",
+    "ts-loader": "^9.5.2",
+    "ts-node": "^10.9.2",
+    "tsconfig-paths": "^4.2.0",
+    "typescript": "^5.7.3",
+    "typescript-eslint": "^8.20.0"
+  },
+  "jest": {
+    "moduleFileExtensions": [
+      "js",
+      "json",
+      "ts"
+    ],
+    "rootDir": "src",
+    "testRegex": ".*\\.spec\\.ts$",
+    "transform": {
+      "^.+\\.(t|j)s$": "ts-jest"
+    },
+    "collectCoverageFrom": [
+      "**/*.(t|j)s"
+    ],
+    "coverageDirectory": "../coverage",
+    "testEnvironment": "node"
+  }
+}

src/app.controller.spec.ts

+import { Test, TestingModule } from '@nestjs/testing';
+import { AppController } from './app.controller';
+import { AppService } from './app.service';
+
+describe('AppController', () => {
+  let appController: AppController;
+
+  beforeEach(async () => {
+    const app: TestingModule = await Test.createTestingModule({
+      controllers: [AppController],
+      providers: [AppService],
+    }).compile();
+
+    appController = app.get<AppController>(AppController);
+  });
+
+  describe('root', () => {
+    it('should be defined', () => {
+      expect(appController).toBeDefined();
+    });
+  });
+});

src/app.controller.ts

+import { Controller, Get, Req, Res } from '@nestjs/common';
+import { AppService } from './app.service';
+import { Request, Response } from 'express';
+
+@Controller()
+export class AppController {
+  constructor(private readonly appService: AppService) {}
+
+  @Get('*')
+  async render(@Req() req: Request, @Res({ passthrough: true }) res: Response) {
+    const result = await this.appService.getResult(req, res);
+    res.header('Cache-Control', 'public, max-age=63072000, immutable');
+    return result;
+  }
+}

src/app.module.ts

+import { Module } from '@nestjs/common';
+import { AppController } from './app.controller';
+import { AppService } from './app.service';
+import { ConfigModule } from '@nestjs/config';
+import { loadConfig } from './utility/config';
+import { HttpModule } from '@nestjs/axios';
+
+@Module({
+  imports: [
+    ConfigModule.forRoot({
+      load: [loadConfig],
+      isGlobal: true,
+      ignoreEnvVars: true,
+      ignoreEnvFile: true,
+    }),
+    HttpModule,
+  ],
+  controllers: [AppController],
+  providers: [AppService],
+})
+export class AppModule {}

src/app.service.ts

+import { Injectable, ConsoleLogger, StreamableFile } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { Request, Response } from 'express';
+import { HttpService } from '@nestjs/axios';
+import { Readable } from 'node:stream';
+import {
+  DeleteObjectCommand,
+  GetObjectCommand,
+  CopyObjectCommand,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { BlankReturnMessageDto } from 'nesties';
+import { lastValueFrom } from 'rxjs';
+import { teex } from './utility/tee-stream';
+import { createHash } from 'node:crypto';
+
+@Injectable()
+export class AppService extends ConsoleLogger {
+  constructor(
+    private readonly config: ConfigService,
+    private readonly httpService: HttpService,
+  ) {
+    super('app');
+  }
+
+  bucket = this.config.get<string>('S3_BUCKET');
+  endpoint = this.config.get<string>('S3_ENDPOINT');
+  pathPrefix = this.config.get<string>('S3_PATH_PREFIX') || '';
+
+  s3 = new S3Client({
+    credentials: {
+      accessKeyId: this.config.get<string>('S3_ACCESS_KEY_ID'),
+      secretAccessKey: this.config.get<string>('S3_SECRET_ACCESS_KEY'),
+    },
+    region: 'us-east-1',
+    endpoint: this.endpoint,
+    forcePathStyle: !!this.config.get<boolean>('S3_PATH_STYLE'),
+  });
+  async getResult(req: Request, res: Response) {
+    const url = req.originalUrl.startsWith('/')
+      ? req.originalUrl.slice(1)
+      : req.originalUrl;
+    const urlObj = new URL(url);
+    const filename = urlObj.pathname.split('/').pop();
+    if (!filename) {
+      throw new BlankReturnMessageDto(400, 'Bad filename').toException();
+    }
+
+    this.log(`Client ${req.ip} requested file ${filename} from ${url}`);
+
+    const s3Key = `${this.pathPrefix}${filename}`;
+    const runExisting = async () => {
+      try {
+        const existing = await this.s3.send(
+          new GetObjectCommand({
+            Bucket: this.bucket,
+            Key: s3Key,
+          }),
+        );
+        if (existing) {
+          this.log(`Found existing file ${filename}`);
+          const stream = existing.Body as Readable;
+          return new StreamableFile(stream, {
+            type: existing.ContentType,
+            disposition: `attachment; filename="${filename}"`,
+            length: existing.ContentLength,
+          });
+        }
+      } catch (e) {}
+      return undefined;
+    };
+
+    const existing = await runExisting();
+    if (existing) return existing;
+
+    this.log(`Downloading file ${filename} from upstream ${url}`);
+    const dl = await lastValueFrom(
+      this.httpService.get<Readable>(url, {
+        responseType: 'stream',
+        headers: {
+          ...req.headers,
+          host: undefined,
+          Host: undefined,
+        },
+        timeout: this.config.get<number>('HTTP_TIMEOUT') || 30000,
+        validateStatus: () => true,
+      }),
+    );
+    if (dl.status >= 300) {
+      throw new BlankReturnMessageDto(
+        dl.status,
+        'Upstream error',
+      ).toException();
+    }
+    const st = dl.data;
+    const contentType =
+      (dl.headers['content-type'] as string) || 'application/octet-stream';
+    const contentLength =
+      parseInt(dl.headers['content-length'], 10) || undefined;
+
+    const [st1, st2, st3] = teex(st, 3);
+
+    // st1: the stream to upload to S3
+    // st2: the stream to return to the client
+    // st3: the stream to md5sum
+
+    const streamMd5Promise = new Promise<string>((resolve, reject) => {
+      const hash = createHash('md5');
+      st3.on('data', (chunk) => hash.update(chunk));
+      st3.on('end', () => resolve(hash.digest('hex')));
+      st3.on('error', (err) => reject(err));
+    });
+
+    const tempKey = `${s3Key}-temp-${Date.now()}`;
+
+    this.s3
+      .send(
+        new PutObjectCommand({
+          Bucket: this.bucket,
+          Key: tempKey,
+          Body: st1,
+          ContentType: contentType,
+          ContentLength: contentLength,
+        }),
+      )
+      .then(async (res) => {
+        // successfully uploaded the partial file
+        this.log(`Uploaded partial file ${filename} to S3`);
+        const etag = res.ETag;
+        try {
+          if (etag) {
+            const md5 = await streamMd5Promise;
+            if (etag && etag.replace(/"/g, '') !== md5) {
+              this.warn(`MD5 mismatch for ${filename}: ${etag} != ${md5}`);
+              return;
+            }
+          } else {
+            this.warn(`ETag is missing for ${filename}, skipping MD5 check`);
+          }
+          // now we can rename the file to the final name
+          await this.s3.send(
+            new CopyObjectCommand({
+              Bucket: this.bucket,
+              CopySource: `${this.bucket}/${tempKey}`,
+              Key: s3Key,
+            }),
+          );
+          this.log(`Uploaded file ${filename} to S3 successfully`);
+        } finally {
+          await this.s3.send(
+            new DeleteObjectCommand({
+              Bucket: this.bucket,
+              Key: tempKey,
+            }),
+          );
+        }
+      })
+      .catch((err) => {
+        this.warn(`Failed to upload ${s3Key} to S3: ${err}`);
+      });
+
+    return new StreamableFile(st2, {
+      type: contentType,
+      disposition: `attachment; filename="${filename}"`,
+      length: contentLength,
+    });
+  }
+}

src/main.ts

+import { NestFactory } from '@nestjs/core';
+import { SwaggerModule, DocumentBuilder } from '@nestjs/swagger';
+import { NestExpressApplication } from '@nestjs/platform-express';
+import { AppModule } from './app.module';
+import { ConfigService } from '@nestjs/config';
+
+async function bootstrap() {
+  const app = await NestFactory.create<NestExpressApplication>(AppModule);
+  app.enableCors();
+  app.set('trust proxy', ['172.16.0.0/12', 'loopback']);
+
+  const config = app.get(ConfigService);
+
+  /*
+  if (!config.get('NO_OPENAPI')) {
+    const documentConfig = new DocumentBuilder()
+      .setTitle('app')
+      .setDescription('The app')
+      .setVersion('1.0')
+      .build();
+
+    const document = SwaggerModule.createDocument(app, documentConfig);
+    SwaggerModule.setup('docs', app, document);
+  }
+   */
+
+  await app.listen(
+    config.get<number>('port') || 3000,
+    config.get<string>('host') || '::',
+  );
+}
+bootstrap();

src/utility/config.ts

+import yaml from 'yaml';
+import * as fs from 'fs';
+
+const defaultConfig = {
+  host: '::',
+  port: 3000,
+};
+
+export type Config = typeof defaultConfig;
+
+export async function loadConfig(): Promise<Config> {
+  let readConfig: Partial<Config> = {};
+  try {
+    const configText = await fs.promises.readFile('./config.yaml', 'utf-8');
+    readConfig = yaml.parse(configText);
+  } catch (e) {
+    console.error(`Failed to read config: ${e.toString()}`);
+  }
+  return {
+    ...defaultConfig,
+    ...readConfig,
+    ...process.env,
+  };
+}

test/app.e2e-spec.ts

+import { Test, TestingModule } from '@nestjs/testing';
+import { INestApplication } from '@nestjs/common';
+import * as request from 'supertest';
+import { AppModule } from './../src/app.module';
+
+describe('AppController (e2e)', () => {
+  let app: INestApplication;
+
+  beforeEach(async () => {
+    const moduleFixture: TestingModule = await Test.createTestingModule({
+      imports: [AppModule],
+    }).compile();
+
+    app = moduleFixture.createNestApplication();
+    await app.init();
+  });
+
+  /* it('/ (GET)', () => {
+    return request(app.getHttpServer())
+      .get('/')
+      .expect(200)
+      .expect('Hello World!');
+  }); */
+});

test/jest-e2e.json

+{
+  "moduleFileExtensions": ["js", "json", "ts"],
+  "rootDir": ".",
+  "testEnvironment": "node",
+  "testRegex": ".e2e-spec.ts$",
+  "transform": {
+    "^.+\\.(t|j)s$": "ts-jest"
+  }
+}

tsconfig.build.json

+{
+  "extends": "./tsconfig.json",
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
+}

tsconfig.json

+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "declaration": true,
+    "removeComments": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "allowSyntheticDefaultImports": true,
+    "target": "es2021",
+    "sourceMap": true,
+    "outDir": "./dist",
+    "baseUrl": "./",
+    "incremental": true,
+    "esModuleInterop": true
+  },
+  "compileOnSave": true,
+  "allowJs": true
+}