Commit 075d0d39 authored by nanahira's avatar nanahira

move to redis

parent d8bb19f4
Pipeline #14454 failed with stages
in 2 minutes and 52 seconds
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
"@aws-sdk/client-s3": "^3.26.0", "@aws-sdk/client-s3": "^3.26.0",
"@aws-sdk/lib-storage": "^3.26.0", "@aws-sdk/lib-storage": "^3.26.0",
"@cityssm/map-expire": "^1.1.1", "@cityssm/map-expire": "^1.1.1",
"@nestjs-modules/ioredis": "^1.0.1",
"@nestjs/axios": "^0.0.1", "@nestjs/axios": "^0.0.1",
"@nestjs/cli": "^8.0.0", "@nestjs/cli": "^8.0.0",
"@nestjs/common": "^8.0.0", "@nestjs/common": "^8.0.0",
...@@ -25,7 +26,7 @@ ...@@ -25,7 +26,7 @@
"busboy": "^0.2.14", "busboy": "^0.2.14",
"class-transformer": "^0.4.0", "class-transformer": "^0.4.0",
"class-validator": "^0.13.1", "class-validator": "^0.13.1",
"delay": "^5.0.0", "ioredis": "^4.28.5",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"moment": "^2.29.1", "moment": "^2.29.1",
"mustache": "^4.2.0", "mustache": "^4.2.0",
...@@ -33,6 +34,7 @@ ...@@ -33,6 +34,7 @@
"p-queue": "6.6.2", "p-queue": "6.6.2",
"pg": "^8.7.1", "pg": "^8.7.1",
"readdirp": "^3.6.0", "readdirp": "^3.6.0",
"redlock": "^5.0.0-beta.2",
"reflect-metadata": "^0.1.13", "reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",
"rxjs": "^7.2.0", "rxjs": "^7.2.0",
...@@ -44,6 +46,7 @@ ...@@ -44,6 +46,7 @@
"@nestjs/testing": "^8.0.0", "@nestjs/testing": "^8.0.0",
"@types/busboy": "^0.2.4", "@types/busboy": "^0.2.4",
"@types/express": "^4.17.13", "@types/express": "^4.17.13",
"@types/ioredis": "^4.28.10",
"@types/jest": "^26.0.24", "@types/jest": "^26.0.24",
"@types/lodash": "^4.14.172", "@types/lodash": "^4.14.172",
"@types/multer": "^1.4.7", "@types/multer": "^1.4.7",
...@@ -2391,6 +2394,16 @@ ...@@ -2391,6 +2394,16 @@
"url": "https://github.com/chalk/chalk?sponsor=1" "url": "https://github.com/chalk/chalk?sponsor=1"
} }
}, },
"node_modules/@nestjs-modules/ioredis": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@nestjs-modules/ioredis/-/ioredis-1.0.1.tgz",
"integrity": "sha512-UhIOqL5pXeJlCucHMkJONwqecvLFySW/95uw34Umh0ld7v+6tvYRK3JHQYcJqzlmNiq52bfNxf7CUpuHCSdA3A==",
"peerDependencies": {
"@nestjs/common": "^6.7.0 || ^7.0.0 || ^8.0.0",
"@nestjs/core": "^6.7.0 || ^7.0.0 || ^8.0.0",
"ioredis": "^4.0.0"
}
},
"node_modules/@nestjs/axios": { "node_modules/@nestjs/axios": {
"version": "0.0.1", "version": "0.0.1",
"resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-0.0.1.tgz", "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-0.0.1.tgz",
...@@ -3167,6 +3180,15 @@ ...@@ -3167,6 +3180,15 @@
"@types/node": "*" "@types/node": "*"
} }
}, },
"node_modules/@types/ioredis": {
"version": "4.28.10",
"resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz",
"integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/istanbul-lib-coverage": { "node_modules/@types/istanbul-lib-coverage": {
"version": "2.0.3", "version": "2.0.3",
"resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz",
...@@ -4490,6 +4512,14 @@ ...@@ -4490,6 +4512,14 @@
"node": ">=0.8" "node": ">=0.8"
} }
}, },
"node_modules/cluster-key-slot": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz",
"integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/co": { "node_modules/co": {
"version": "4.6.0", "version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
...@@ -4799,17 +4829,6 @@ ...@@ -4799,17 +4829,6 @@
"clone": "^1.0.2" "clone": "^1.0.2"
} }
}, },
"node_modules/delay": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
"integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/delayed-stream": { "node_modules/delayed-stream": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
...@@ -4819,6 +4838,14 @@ ...@@ -4819,6 +4838,14 @@
"node": ">=0.4.0" "node": ">=0.4.0"
} }
}, },
"node_modules/denque": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz",
"integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==",
"engines": {
"node": ">=0.10"
}
},
"node_modules/depd": { "node_modules/depd": {
"version": "1.1.2", "version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
...@@ -6362,6 +6389,31 @@ ...@@ -6362,6 +6389,31 @@
"node": ">= 0.10" "node": ">= 0.10"
} }
}, },
"node_modules/ioredis": {
"version": "4.28.5",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
"integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
"dependencies": {
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.1",
"denque": "^1.1.0",
"lodash.defaults": "^4.2.0",
"lodash.flatten": "^4.4.0",
"lodash.isarguments": "^3.1.0",
"p-map": "^2.1.0",
"redis-commands": "1.7.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"engines": {
"node": ">=6"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/ioredis"
}
},
"node_modules/ipaddr.js": { "node_modules/ipaddr.js": {
"version": "1.9.1", "version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
...@@ -7985,6 +8037,16 @@ ...@@ -7985,6 +8037,16 @@
"integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=", "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=",
"dev": true "dev": true
}, },
"node_modules/lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="
},
"node_modules/lodash.flatten": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
"integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g=="
},
"node_modules/lodash.get": { "node_modules/lodash.get": {
"version": "4.4.2", "version": "4.4.2",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
...@@ -7995,6 +8057,11 @@ ...@@ -7995,6 +8057,11 @@
"resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz", "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz",
"integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=" "integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI="
}, },
"node_modules/lodash.isarguments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="
},
"node_modules/lodash.merge": { "node_modules/lodash.merge": {
"version": "4.6.2", "version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
...@@ -8345,6 +8412,11 @@ ...@@ -8345,6 +8412,11 @@
"resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
"integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="
}, },
"node_modules/node-abort-controller": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.0.1.tgz",
"integrity": "sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw=="
},
"node_modules/node-emoji": { "node_modules/node-emoji": {
"version": "1.10.0", "version": "1.10.0",
"resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.10.0.tgz", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.10.0.tgz",
...@@ -8594,6 +8666,14 @@ ...@@ -8594,6 +8666,14 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/p-map": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
"integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==",
"engines": {
"node": ">=6"
}
},
"node_modules/p-queue": { "node_modules/p-queue": {
"version": "6.6.2", "version": "6.6.2",
"resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",
...@@ -9179,6 +9259,41 @@ ...@@ -9179,6 +9259,41 @@
"node": ">= 0.10" "node": ">= 0.10"
} }
}, },
"node_modules/redis-commands": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
"integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
},
"node_modules/redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
"engines": {
"node": ">=4"
}
},
"node_modules/redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
"dependencies": {
"redis-errors": "^1.0.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/redlock": {
"version": "5.0.0-beta.2",
"resolved": "https://registry.npmjs.org/redlock/-/redlock-5.0.0-beta.2.tgz",
"integrity": "sha512-2RDWXg5jgRptDrB1w9O/JgSZC0j7y4SlaXnor93H/UJm/QyDiFgBKNtrh0TI6oCXqYSaSoXxFh6Sd3VtYfhRXw==",
"dependencies": {
"node-abort-controller": "^3.0.1"
},
"engines": {
"node": ">=12"
}
},
"node_modules/reflect-metadata": { "node_modules/reflect-metadata": {
"version": "0.1.13", "version": "0.1.13",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz",
...@@ -9704,6 +9819,11 @@ ...@@ -9704,6 +9819,11 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/standard-as-callback": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
"integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="
},
"node_modules/statuses": { "node_modules/statuses": {
"version": "1.5.0", "version": "1.5.0",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
...@@ -13370,6 +13490,12 @@ ...@@ -13370,6 +13490,12 @@
} }
} }
}, },
"@nestjs-modules/ioredis": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@nestjs-modules/ioredis/-/ioredis-1.0.1.tgz",
"integrity": "sha512-UhIOqL5pXeJlCucHMkJONwqecvLFySW/95uw34Umh0ld7v+6tvYRK3JHQYcJqzlmNiq52bfNxf7CUpuHCSdA3A==",
"requires": {}
},
"@nestjs/axios": { "@nestjs/axios": {
"version": "0.0.1", "version": "0.0.1",
"resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-0.0.1.tgz", "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-0.0.1.tgz",
...@@ -13925,6 +14051,15 @@ ...@@ -13925,6 +14051,15 @@
"@types/node": "*" "@types/node": "*"
} }
}, },
"@types/ioredis": {
"version": "4.28.10",
"resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz",
"integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/istanbul-lib-coverage": { "@types/istanbul-lib-coverage": {
"version": "2.0.3", "version": "2.0.3",
"resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz",
...@@ -14949,6 +15084,11 @@ ...@@ -14949,6 +15084,11 @@
"resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
"integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=" "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4="
}, },
"cluster-key-slot": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz",
"integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw=="
},
"co": { "co": {
"version": "4.6.0", "version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
...@@ -15208,17 +15348,17 @@ ...@@ -15208,17 +15348,17 @@
"clone": "^1.0.2" "clone": "^1.0.2"
} }
}, },
"delay": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
"integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw=="
},
"delayed-stream": { "delayed-stream": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
"dev": true "dev": true
}, },
"denque": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz",
"integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw=="
},
"depd": { "depd": {
"version": "1.1.2", "version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
...@@ -16375,6 +16515,24 @@ ...@@ -16375,6 +16515,24 @@
"resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz",
"integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==" "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA=="
}, },
"ioredis": {
"version": "4.28.5",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz",
"integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==",
"requires": {
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.1",
"denque": "^1.1.0",
"lodash.defaults": "^4.2.0",
"lodash.flatten": "^4.4.0",
"lodash.isarguments": "^3.1.0",
"p-map": "^2.1.0",
"redis-commands": "1.7.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
}
},
"ipaddr.js": { "ipaddr.js": {
"version": "1.9.1", "version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
...@@ -17611,6 +17769,16 @@ ...@@ -17611,6 +17769,16 @@
"integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=", "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=",
"dev": true "dev": true
}, },
"lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="
},
"lodash.flatten": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz",
"integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g=="
},
"lodash.get": { "lodash.get": {
"version": "4.4.2", "version": "4.4.2",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
...@@ -17621,6 +17789,11 @@ ...@@ -17621,6 +17789,11 @@
"resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz", "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz",
"integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI=" "integrity": "sha1-0Z9NwQlQWMzL4rDN9O4P5Ko3yGI="
}, },
"lodash.isarguments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="
},
"lodash.merge": { "lodash.merge": {
"version": "4.6.2", "version": "4.6.2",
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
...@@ -17896,6 +18069,11 @@ ...@@ -17896,6 +18069,11 @@
"resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
"integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw=="
}, },
"node-abort-controller": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.0.1.tgz",
"integrity": "sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw=="
},
"node-emoji": { "node-emoji": {
"version": "1.10.0", "version": "1.10.0",
"resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.10.0.tgz", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.10.0.tgz",
...@@ -18075,6 +18253,11 @@ ...@@ -18075,6 +18253,11 @@
"p-limit": "^2.2.0" "p-limit": "^2.2.0"
} }
}, },
"p-map": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz",
"integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw=="
},
"p-queue": { "p-queue": {
"version": "6.6.2", "version": "6.6.2",
"resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",
...@@ -18511,6 +18694,32 @@ ...@@ -18511,6 +18694,32 @@
"resolve": "^1.1.6" "resolve": "^1.1.6"
} }
}, },
"redis-commands": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
"integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
},
"redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="
},
"redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
"requires": {
"redis-errors": "^1.0.0"
}
},
"redlock": {
"version": "5.0.0-beta.2",
"resolved": "https://registry.npmjs.org/redlock/-/redlock-5.0.0-beta.2.tgz",
"integrity": "sha512-2RDWXg5jgRptDrB1w9O/JgSZC0j7y4SlaXnor93H/UJm/QyDiFgBKNtrh0TI6oCXqYSaSoXxFh6Sd3VtYfhRXw==",
"requires": {
"node-abort-controller": "^3.0.1"
}
},
"reflect-metadata": { "reflect-metadata": {
"version": "0.1.13", "version": "0.1.13",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz",
...@@ -18913,6 +19122,11 @@ ...@@ -18913,6 +19122,11 @@
} }
} }
}, },
"standard-as-callback": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
"integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="
},
"statuses": { "statuses": {
"version": "1.5.0", "version": "1.5.0",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
......
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
"@aws-sdk/client-s3": "^3.26.0", "@aws-sdk/client-s3": "^3.26.0",
"@aws-sdk/lib-storage": "^3.26.0", "@aws-sdk/lib-storage": "^3.26.0",
"@cityssm/map-expire": "^1.1.1", "@cityssm/map-expire": "^1.1.1",
"@nestjs-modules/ioredis": "^1.0.1",
"@nestjs/axios": "^0.0.1", "@nestjs/axios": "^0.0.1",
"@nestjs/cli": "^8.0.0", "@nestjs/cli": "^8.0.0",
"@nestjs/common": "^8.0.0", "@nestjs/common": "^8.0.0",
...@@ -37,7 +38,7 @@ ...@@ -37,7 +38,7 @@
"busboy": "^0.2.14", "busboy": "^0.2.14",
"class-transformer": "^0.4.0", "class-transformer": "^0.4.0",
"class-validator": "^0.13.1", "class-validator": "^0.13.1",
"delay": "^5.0.0", "ioredis": "^4.28.5",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"moment": "^2.29.1", "moment": "^2.29.1",
"mustache": "^4.2.0", "mustache": "^4.2.0",
...@@ -45,6 +46,7 @@ ...@@ -45,6 +46,7 @@
"p-queue": "6.6.2", "p-queue": "6.6.2",
"pg": "^8.7.1", "pg": "^8.7.1",
"readdirp": "^3.6.0", "readdirp": "^3.6.0",
"redlock": "^5.0.0-beta.2",
"reflect-metadata": "^0.1.13", "reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",
"rxjs": "^7.2.0", "rxjs": "^7.2.0",
...@@ -59,6 +61,7 @@ ...@@ -59,6 +61,7 @@
"@nestjs/testing": "^8.0.0", "@nestjs/testing": "^8.0.0",
"@types/busboy": "^0.2.4", "@types/busboy": "^0.2.4",
"@types/express": "^4.17.13", "@types/express": "^4.17.13",
"@types/ioredis": "^4.28.10",
"@types/jest": "^26.0.24", "@types/jest": "^26.0.24",
"@types/lodash": "^4.14.172", "@types/lodash": "^4.14.172",
"@types/multer": "^1.4.7", "@types/multer": "^1.4.7",
......
...@@ -19,19 +19,20 @@ import path from 'path'; ...@@ -19,19 +19,20 @@ import path from 'path';
import { ArchiveMirror } from './entities/ArchiveMirror.dto'; import { ArchiveMirror } from './entities/ArchiveMirror.dto';
import { MirrorService } from './mirror/mirror.service'; import { MirrorService } from './mirror/mirror.service';
import { HttpModule } from '@nestjs/axios'; import { HttpModule } from '@nestjs/axios';
import { RedisModule } from '@nestjs-modules/ioredis';
const configModule = ConfigModule.forRoot(); import { LockService } from './lock/lock.service';
@Module({ @Module({
imports: [ imports: [
ServeStaticModule.forRoot({ ServeStaticModule.forRoot({
rootPath: path.join(__dirname, '..', 'public'), rootPath: path.join(__dirname, '..', 'public'),
}), }),
configModule, ConfigModule.forRoot({
isGlobal: true
}),
HttpModule, HttpModule,
TypeOrmModule.forRootAsync({ TypeOrmModule.forRootAsync({
name: 'app', name: 'app',
imports: [configModule],
inject: [ConfigService], inject: [ConfigService],
useFactory: async (config: ConfigService) => { useFactory: async (config: ConfigService) => {
return { return {
...@@ -49,8 +50,16 @@ const configModule = ConfigModule.forRoot(); ...@@ -49,8 +50,16 @@ const configModule = ConfigModule.forRoot();
}; };
}, },
}), }),
RedisModule.forRootAsync({
inject: [ConfigService],
useFactory: async (config: ConfigService) => ({
config: {
url: config.get('REDIS_URL'),
},
}),
}),
], ],
controllers: [AppController, AdminController, UpdateController], controllers: [AppController, AdminController, UpdateController],
providers: [AppService, PackagerService, AssetsS3Service, PackageS3Service, UpdateService, MirrorService], providers: [AppService, PackagerService, AssetsS3Service, PackageS3Service, UpdateService, MirrorService, LockService],
}) })
export class AppModule {} export class AppModule {}
import { Test, TestingModule } from '@nestjs/testing';
import { LockService } from './lock.service';
describe('LockService', () => {
let service: LockService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [LockService],
}).compile();
service = module.get<LockService>(LockService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});
import { InjectRedis, Redis } from '@nestjs-modules/ioredis';
import { Injectable } from '@nestjs/common';
import Redlock from 'redlock';
@Injectable()
export class LockService extends Redlock {
constructor(@InjectRedis() private readonly redis: Redis) {
super([redis]);
}
}
...@@ -14,9 +14,10 @@ import { ConsoleLogger, forwardRef, Inject, Injectable } from '@nestjs/common'; ...@@ -14,9 +14,10 @@ import { ConsoleLogger, forwardRef, Inject, Injectable } from '@nestjs/common';
import { Archive, ArchiveType } from '../entities/Archive.entity'; import { Archive, ArchiveType } from '../entities/Archive.entity';
import { AppService } from '../app.service'; import { AppService } from '../app.service';
import { createHash } from 'crypto'; import { createHash } from 'crypto';
import delay from 'delay';
import { Cache } from '@cityssm/map-expire'; import { Cache } from '@cityssm/map-expire';
import PQueue from 'p-queue'; import PQueue from 'p-queue';
import { LockService } from 'src/lock/lock.service';
import { InjectRedis, Redis } from '@nestjs-modules/ioredis';
export interface FileWithHash { export interface FileWithHash {
file: readdirp.EntryInfo; file: readdirp.EntryInfo;
...@@ -63,17 +64,19 @@ export class ArchiveTask { ...@@ -63,17 +64,19 @@ export class ArchiveTask {
@Injectable() @Injectable()
export class PackagerService extends ConsoleLogger { export class PackagerService extends ConsoleLogger {
bucket_max = 10 * 1024 ** 2; bucket_max = 10 * 1024 ** 2;
bucket_enter = 1 * 1024 ** 2; bucket_enter = 1024 ** 2;
noGatherExts = new Set<string>(); noGatherExts = new Set<string>();
packagerWorkingDirectory: string; packagerWorkingDirectory: string;
private uploadLock = new Set<string>(); // private uploadLock = new Set<string>();
private hashCache = new Cache<string, string>(); // private hashCache = new Cache<string, string>();
constructor( constructor(
@Inject(forwardRef(() => AppService)) private readonly appService: AppService, @Inject(forwardRef(() => AppService)) private readonly appService: AppService,
private s3: PackageS3Service, private s3: PackageS3Service,
config: ConfigService config: ConfigService,
private redlock: LockService,
@InjectRedis() private readonly redis: Redis
) { ) {
super('packager'); super('packager');
this.bucket_max = (parseInt(config.get('PACKAGE_BUCKET_MAX')) || 10) * 1024 ** 2; this.bucket_max = (parseInt(config.get('PACKAGE_BUCKET_MAX')) || 10) * 1024 ** 2;
...@@ -85,19 +88,6 @@ export class PackagerService extends ConsoleLogger { ...@@ -85,19 +88,6 @@ export class PackagerService extends ConsoleLogger {
} }
} }
private async waitForLock(key: string) {
while (this.uploadLock.has(key)) {
await delay(10);
}
this.uploadLock.add(key);
}
private releaseLock(key: string) {
if (this.uploadLock.has(key)) {
this.uploadLock.delete(key);
}
}
async build( async build(
stream: NodeJS.ReadableStream, stream: NodeJS.ReadableStream,
pathPrefix?: string, pathPrefix?: string,
...@@ -208,21 +198,23 @@ export class PackagerService extends ConsoleLogger { ...@@ -208,21 +198,23 @@ export class PackagerService extends ConsoleLogger {
} }
private async lookForExistingArchiveHash(path: string) { private async lookForExistingArchiveHash(path: string) {
let hash = this.hashCache.get(path); const hashKey = `hash:${path}`;
let hash = await this.redis.get(hashKey);
if (hash) { if (hash) {
return hash; return hash;
} }
hash = await this.appService.lookForExistingArchiveHash(path); hash = await this.appService.lookForExistingArchiveHash(path);
if (hash) { if (hash) {
this.hashCache.set(path, hash, 24 * 60 * 60 * 1000); await this.redis.set(hashKey, hash, 'EX', 60 * 60 * 24);
return hash; return hash;
} }
return null; return null;
} }
async archive(root: string, archiveTask: ArchiveTask): Promise<Archive> { async archive(root: string, archiveTask: ArchiveTask): Promise<Archive> {
await this.waitForLock(archiveTask.path); return this.redlock.using([`archive:${archiveTask.path}`], 1800 * 1000, async () =>
return this.archiveQueue.add(() => this.archiveProcess(root, archiveTask)); this.archiveQueue.add(() => this.archiveProcess(root, archiveTask))
);
} }
private archiveQueue = new PQueue({ concurrency: parseInt(process.env.PACKAGE_COCURRENCY) || os.cpus().length }); private archiveQueue = new PQueue({ concurrency: parseInt(process.env.PACKAGE_COCURRENCY) || os.cpus().length });
...@@ -230,7 +222,6 @@ export class PackagerService extends ConsoleLogger { ...@@ -230,7 +222,6 @@ export class PackagerService extends ConsoleLogger {
private async archiveProcess(root: string, archiveTask: ArchiveTask): Promise<Archive> { private async archiveProcess(root: string, archiveTask: ArchiveTask): Promise<Archive> {
const archive = archiveTask.archive; const archive = archiveTask.archive;
const archiveName = archiveTask.archiveFullPath; const archiveName = archiveTask.archiveFullPath;
try {
const existing = await this.s3.fileExists(archiveName); const existing = await this.s3.fileExists(archiveName);
if (existing) { if (existing) {
const hash = await this.lookForExistingArchiveHash(archiveTask.path); const hash = await this.lookForExistingArchiveHash(archiveTask.path);
...@@ -272,13 +263,9 @@ export class PackagerService extends ConsoleLogger { ...@@ -272,13 +263,9 @@ export class PackagerService extends ConsoleLogger {
}); });
const [, { object }] = await Promise.all([childPromise, uploadPromise]); const [, { object }] = await Promise.all([childPromise, uploadPromise]);
archive.hash = hashObject.digest('hex'); archive.hash = hashObject.digest('hex');
this.hashCache.set(archive.path, archive.hash, 24 * 60 * 60 * 1000); await this.redis.set(`hash:${archive.path}`, archive.hash, 'EX', 60 * 60 * 24);
archive.size = object.Size; archive.size = object.Size;
} catch (e) {
throw e;
} finally {
this.releaseLock(archiveTask.path);
}
return archive; return archive;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment