first

.dockerignore

+# compiled output
+/dist
+/node_modules
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# OS
+.DS_Store
+
+# Tests
+/coverage
+/.nyc_output
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+/data
+/output
+/config.yaml
+
+.git*
+Dockerfile
+.dockerignore

.eslintrc.js

+module.exports = {
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: 'tsconfig.json',
+    sourceType: 'module',
+  },
+  plugins: ['@typescript-eslint/eslint-plugin'],
+  extends: [
+    'plugin:@typescript-eslint/recommended',
+    'prettier/@typescript-eslint',
+    'plugin:prettier/recommended',
+  ],
+  root: true,
+  env: {
+    node: true,
+    jest: true,
+  },
+  rules: {
+    '@typescript-eslint/interface-name-prefix': 'off',
+    '@typescript-eslint/explicit-function-return-type': 'off',
+    '@typescript-eslint/explicit-module-boundary-types': 'off',
+    '@typescript-eslint/no-explicit-any': 'off',
+  },
+};

.gitignore

+# compiled output
+/dist
+/node_modules
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# OS
+.DS_Store
+
+# Tests
+/coverage
+/.nyc_output
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+/data
+/output
+/config.yaml

.gitlab-ci.yml

+stages:
+  - build
+  - combine
+  - deploy
+
+variables:
+  GIT_DEPTH: "1"
+  CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
+  CONTAINER_TEST_ARM_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-arm
+  CONTAINER_TEST_X86_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-x86
+  CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest
+
+before_script:
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+
+build-x86:
+  stage: build
+  tags: 
+    - docker
+  script:
+    - docker build --pull -t $CONTAINER_TEST_X86_IMAGE .
+    - docker push $CONTAINER_TEST_X86_IMAGE
+
+build-arm:
+  stage: build
+  tags: 
+    - docker-arm
+  script:
+    - docker build --pull -t $CONTAINER_TEST_ARM_IMAGE .
+    - docker push $CONTAINER_TEST_ARM_IMAGE
+
+combine:
+  stage: combine
+  tags:
+    - docker
+  script:
+    - docker pull $CONTAINER_TEST_X86_IMAGE
+    - docker pull $CONTAINER_TEST_ARM_IMAGE
+    - docker manifest create $CONTAINER_TEST_IMAGE --amend $CONTAINER_TEST_X86_IMAGE --amend $CONTAINER_TEST_ARM_IMAGE
+    - docker manifest push $CONTAINER_TEST_IMAGE
+
+deploy_latest:
+  stage: deploy
+  tags: 
+    - docker
+  script:
+    - docker pull $CONTAINER_TEST_IMAGE
+    - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
+    - docker push $CONTAINER_RELEASE_IMAGE
+  only:
+    - master
+
+deploy_tag:
+  stage: deploy
+  tags: 
+    - docker
+  variables:
+    CONTAINER_TAG_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
+  script:
+    - docker pull $CONTAINER_TEST_IMAGE
+    - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_TAG_IMAGE
+    - docker push $CONTAINER_TAG_IMAGE
+  only:
+    - tags

.prettierrc

+{
+  "singleQuote": true,
+  "trailingComma": "all"
+}
\ No newline at end of file

Dockerfile

+FROM node:buster-slim
+RUN apt update && apt -y install python3 build-essential && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+WORKDIR /usr/src/app
+COPY ./package*.json ./
+RUN npm ci
+COPY . ./
+RUN npm run build
+
+CMD ["npm", "run", "start:prod"]

README.md

+<p align="center">
+  <a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo_text.svg" width="320" alt="Nest Logo" /></a>
+</p>
+
+[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
+[circleci-url]: https://circleci.com/gh/nestjs/nest
+
+  <p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p>
+    <p align="center">
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a>
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a>
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a>
+<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a>
+<a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a>
+<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a>
+<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a>
+<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a>
+  <a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a>
+    <a href="https://opencollective.com/nest#sponsor"  target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a>
+  <a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a>
+</p>
+  <!--[![Backers on Open Collective](https://opencollective.com/nest/backers/badge.svg)](https://opencollective.com/nest#backer)
+  [![Sponsors on Open Collective](https://opencollective.com/nest/sponsors/badge.svg)](https://opencollective.com/nest#sponsor)-->
+
+## Description
+
+[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
+
+## Installation
+
+```bash
+$ npm install
+```
+
+## Running the app
+
+```bash
+# development
+$ npm run start
+
+# watch mode
+$ npm run start:dev
+
+# production mode
+$ npm run start:prod
+```
+
+## Test
+
+```bash
+# unit tests
+$ npm run test
+
+# e2e tests
+$ npm run test:e2e
+
+# test coverage
+$ npm run test:cov
+```
+
+## Support
+
+Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
+
+## Stay in touch
+
+- Author - [Kamil Myśliwiec](https://kamilmysliwiec.com)
+- Website - [https://nestjs.com](https://nestjs.com/)
+- Twitter - [@nestframework](https://twitter.com/nestframework)
+
+## License
+
+Nest is [MIT licensed](LICENSE).

app.controller.spec.ts

+import { Test, TestingModule } from '@nestjs/testing';
+import { AppController } from './app.controller';
+import { AppService } from './app.service';
+
+describe('AppController', () => {
+  let appController: AppController;
+
+  beforeEach(async () => {
+    const app: TestingModule = await Test.createTestingModule({
+      controllers: [AppController],
+      providers: [AppService],
+    }).compile();
+
+    appController = app.get<AppController>(AppController);
+  });
+
+  describe('root', () => {
+    /* it('should return "Hello World!"', () => {
+      expect(appController.getHello()).toBe('Hello World!');
+    }); */
+  });
+});

nest-cli.json

+{
+  "collection": "@nestjs/schematics",
+  "sourceRoot": "src"
+}

package.json

+{
+  "name": "ygopro-tips-biu-generator",
+  "version": "0.0.1",
+  "description": "",
+  "author": "",
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "prebuild": "rimraf dist",
+    "build": "nest build",
+    "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
+    "start": "nest start",
+    "start:dev": "nest start --watch",
+    "start:debug": "nest start --debug --watch",
+    "start:prod": "node dist/main",
+    "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:cov": "jest --coverage",
+    "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
+    "test:e2e": "jest --config ./test/jest-e2e.json"
+  },
+  "dependencies": {
+    "@nestjs/common": "^7.5.1",
+    "@nestjs/core": "^7.5.1",
+    "@nestjs/platform-express": "^7.5.1",
+    "@types/lodash": "^4.14.168",
+    "axios": "^0.21.1",
+    "lodash": "^4.17.20",
+    "moment": "^2.29.1",
+    "p-queue": "^6.6.2",
+    "reflect-metadata": "^0.1.13",
+    "rimraf": "^3.0.2",
+    "rxjs": "^6.6.3"
+  },
+  "devDependencies": {
+    "@nestjs/cli": "^7.5.1",
+    "@nestjs/schematics": "^7.1.3",
+    "@nestjs/testing": "^7.5.1",
+    "@types/express": "^4.17.8",
+    "@types/jest": "^26.0.15",
+    "@types/node": "^14.14.6",
+    "@types/supertest": "^2.0.10",
+    "@typescript-eslint/eslint-plugin": "^4.6.1",
+    "@typescript-eslint/parser": "^4.6.1",
+    "eslint": "^7.12.1",
+    "eslint-config-prettier": "^6.15.0",
+    "eslint-plugin-prettier": "^3.1.4",
+    "jest": "^26.6.3",
+    "prettier": "^2.1.2",
+    "supertest": "^6.0.0",
+    "ts-jest": "^26.4.3",
+    "ts-loader": "^8.0.8",
+    "ts-node": "^9.0.0",
+    "tsconfig-paths": "^3.9.0",
+    "typescript": "^4.0.5"
+  },
+  "jest": {
+    "moduleFileExtensions": [
+      "js",
+      "json",
+      "ts"
+    ],
+    "rootDir": "src",
+    "testRegex": ".*\\.spec\\.ts$",
+    "transform": {
+      "^.+\\.(t|j)s$": "ts-jest"
+    },
+    "collectCoverageFrom": [
+      "**/*.(t|j)s"
+    ],
+    "coverageDirectory": "../coverage",
+    "testEnvironment": "node"
+  }
+}

src/app.controller.spec.ts

+import { Test, TestingModule } from '@nestjs/testing';
+import { AppController } from './app.controller';
+import { AppService } from './app.service';
+
+describe('AppController', () => {
+  let appController: AppController;
+
+  beforeEach(async () => {
+    const app: TestingModule = await Test.createTestingModule({
+      controllers: [AppController],
+      providers: [AppService],
+    }).compile();
+
+    appController = app.get<AppController>(AppController);
+  });
+
+  describe('root', () => {
+    /* it('should return "Hello World!"', () => {
+      expect(appController.getHello()).toBe('Hello World!');
+    }); */
+  });
+});

src/app.controller.ts

+import { Controller, Get } from '@nestjs/common';
+import { AppService } from './app.service';
+
+@Controller()
+export class AppController {
+  constructor(private readonly appService: AppService) {}
+
+  @Get('tips.json')
+  async getTips(): Promise<string[]> {
+    return await this.appService.getTips();
+  }
+}

src/app.logger.ts

+import { Injectable, Scope, Logger } from '@nestjs/common';
+
+@Injectable()
+export class AppLogger extends Logger {}

src/app.module.ts

+import { Module } from '@nestjs/common';
+import { AppController } from './app.controller';
+import { AppService } from './app.service';
+import { AppLogger } from './app.logger';
+
+@Module({
+  imports: [],
+  controllers: [AppController],
+  providers: [AppService, AppLogger],
+})
+export class AppModule {}

src/app.service.ts

+import { Injectable } from '@nestjs/common';
+import { AppLogger } from './app.logger';
+import moment from 'moment';
+import PQueue from 'p-queue';
+import axios from 'axios';
+import _ from 'lodash';
+
+const blacklistedWords = ['曲', '词', '曲', '詞', '[Vv]ocal', ':', '：'];
+const blacklistedRegex = blacklistedWords.map((word) => new RegExp(word));
+
+@Injectable()
+export class AppService {
+  cache: string[];
+  lastFetchTime: moment.Moment;
+  queue: PQueue;
+  constructor(private log: AppLogger) {
+    this.log.setContext('ygopro-tips-biu-generator');
+    this.queue = new PQueue({ concurrency: 1 });
+  }
+
+  async getTips(): Promise<string[]> {
+    if (
+      this.cache &&
+      this.lastFetchTime &&
+      moment().diff(this.lastFetchTime) <= 3600000
+    ) {
+      return this.cache;
+    }
+    this.log.log(`Queue size: ${this.queue.size}`);
+    if (this.queue.size > 0) {
+      await this.queue.onEmpty();
+      return this.cache;
+    } else {
+      return await this.fetchTips();
+    }
+  }
+
+  async fetchTips(): Promise<string[]> {
+    return await this.queue.add(async () => await this.fetchTipsTask());
+  }
+
+  async getLyricsOfSong(id: number): Promise<string[]> {
+    try {
+      this.log.log(`Fetching lyrics of song ${id}.`);
+      const { data } = await axios.get(`https://web.biu.moe/s${id}`, {
+        responseType: 'document',
+        timeout: 30000,
+      });
+      const lyricMatches = (data as string).match(
+        /\[\d+:\d+[:\.]\d+\]([^<]+)/g,
+      );
+      if (!lyricMatches) {
+        this.log.log(`Song ${id} has no lyrics.`);
+        return [];
+      }
+      const lyrics = lyricMatches
+        .map((m) => m.match(/\[\d+:\d+[:\.]\d+\]([^<]+)/)[1].trim())
+        .filter((lyric) =>
+          blacklistedRegex.every((regex) => !lyric.match(regex)),
+        );
+      this.log.log(`${lyrics.length} lines of lyrics found in song ${id}.`);
+      return lyrics;
+    } catch (e) {
+      this.log.error(`Failed fetching lyrics of ${id}: ${e.toString()}`);
+      return [];
+    }
+  }
+
+  async fetchTipsTask(): Promise<string[]> {
+    try {
+      this.log.log(`Fetching song list.`);
+      const { data } = await axios.get('https://web.biu.moe/Index/home', {
+        responseType: 'document',
+        timeout: 30000,
+      });
+      const urls = (data as string).match(/href="\/s(\d+)"/g);
+      if (!urls) {
+        this.log.error(`No songs found, retrying.`);
+        return await this.fetchTipsTask();
+      }
+      this.log.log(`${urls.length} songs found.`);
+      const songIds = urls.map((m) => parseInt(m.match(/href="\/s(\d+)"/)[1]));
+      const lyrics = _.flatten(
+        await Promise.all(songIds.map((id) => this.getLyricsOfSong(id))),
+      );
+      this.log.log(`${lyrics.length} lines of lyrics got.`);
+      if (lyrics.length) {
+        this.cache = lyrics;
+        this.lastFetchTime = moment();
+        return lyrics;
+      } else {
+        this.log.log(`No lyrics found, retrying.`);
+        return await this.fetchTipsTask();
+      }
+    } catch (e) {
+      this.log.log(`Errored fetching song list, retrying: ${e.toString()}`);
+      return await this.fetchTipsTask();
+    }
+  }
+}

src/main.ts

+import { NestFactory } from '@nestjs/core';
+import { AppModule } from './app.module';
+
+async function bootstrap() {
+  const app = await NestFactory.create(AppModule);
+  app.enableCors();
+  await app.listen(3000);
+}
+bootstrap();

test/app.e2e-spec.ts

+import { Test, TestingModule } from '@nestjs/testing';
+import { INestApplication } from '@nestjs/common';
+import * as request from 'supertest';
+import { AppModule } from './../src/app.module';
+
+describe('AppController (e2e)', () => {
+  let app: INestApplication;
+
+  beforeEach(async () => {
+    const moduleFixture: TestingModule = await Test.createTestingModule({
+      imports: [AppModule],
+    }).compile();
+
+    app = moduleFixture.createNestApplication();
+    await app.init();
+  });
+
+  /* it('/ (GET)', () => {
+    return request(app.getHttpServer())
+      .get('/')
+      .expect(200)
+      .expect('Hello World!');
+  }); */
+});

test/jest-e2e.json

+{
+  "moduleFileExtensions": ["js", "json", "ts"],
+  "rootDir": ".",
+  "testEnvironment": "node",
+  "testRegex": ".e2e-spec.ts$",
+  "transform": {
+    "^.+\\.(t|j)s$": "ts-jest"
+  }
+}

tsconfig.build.json

+{
+  "extends": "./tsconfig.json",
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
+}

tsconfig.json

+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "declaration": true,
+    "removeComments": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "allowSyntheticDefaultImports": true,
+    "target": "es2017",
+    "sourceMap": true,
+    "outDir": "./dist",
+    "baseUrl": "./",
+    "incremental": true,
+    "esModuleInterop": true
+  },
+  "compileOnSave": true,
+  "allowJs": true
+}