Commit 05f6259c authored by Him188's avatar Him188 Committed by GitHub

Cui cloud publishing & GitHub republishing (#177)

* Add CuiCloud tasks

* Add CuiCloud workflow

* Fix shadowJar

* Fix `reply` function prohibition in MessageSelectBuilder

* Fix doc

* Update CI

* Update CI

* Test CI

* Fix file

* Increase jvm memory limitation

* Add comments

* Remove unnecessary experimental api use

* Increase upload task timeout

* Fix sha

* Increase timeout

* Fix github

* Trigger publishing on release
parent ffd9a512
# This is a basic workflow to help you get started with Actions
name: CuiCloud Publish
# Controls when the action will run. Triggers the workflow on push or pull request
# events but only for the master branch
on:
release:
types:
- created
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# This workflow contains a single job called "build"
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
java-version: 1.8
- name: Grant execute permission for gradlew
run: chmod +x gradlew
- name: Gradle clean
run: ./gradlew clean
- name: Gradle build
run: ./gradlew build # if test's failed, don't publish
- name: Gradle :mirai-core:cuiCloudUpload
run: ./gradlew :mirai-core:cuiCloudUpload -Dcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Pcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Dcui_cloud_url=${{ secrets.CUI_CLOUD_URL }} -Pcui_cloud_url=${{ secrets.CUI_CLOUD_URL }}
- name: Gradle :mirai-core-qqandroid:cuiCloudUpload
run: ./gradlew :mirai-core-qqandroid:cuiCloudUpload -Dcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Pcui_cloud_key=${{ secrets.CUI_CLOUD_KEY }} -Dcui_cloud_url=${{ secrets.CUI_CLOUD_URL }} -Pcui_cloud_url=${{ secrets.CUI_CLOUD_URL }}
# - name: Upload artifact
# uses: actions/upload-artifact@v1.0.0
# with:
# # Artifact name
# name: mirai-core
# # Directory containing files to upload
# path: "mirai-core/build/libs/mirai-core-*-all.jar"
# - name: Upload artifact
# uses: actions/upload-artifact@v1.0.0
# with:
# # Artifact name
# name: mirai-core-qqandroid-all
# # Directory containing files to upload
# path: "mirai-core-qqandroid/build/libs/mirai-core-qqandroid-*-all.jar"
# This is a basic workflow to help you get started with Actions # This is a basic workflow to help you get started with Actions
name: Shadow Publish name: mirai-repo Publish
# Controls when the action will run. Triggers the workflow on push or pull request # Controls when the action will run. Triggers the workflow on push or pull request
# events but only for the master branch # events but only for the master branch
...@@ -25,7 +25,7 @@ jobs: ...@@ -25,7 +25,7 @@ jobs:
- name: Gradle clean - name: Gradle clean
run: ./gradlew clean run: ./gradlew clean
- name: Gradle build - name: Gradle build
run: ./gradlew build run: ./gradlew build # if test's failed, don't publish
- name: Gradle :mirai-core:githubUpload - name: Gradle :mirai-core:githubUpload
run: ./gradlew :mirai-core:githubUpload -Dgithub_token=${{ secrets.MAMOE_TOKEN }} -Pgithub_token=${{ secrets.MAMOE_TOKEN }} run: ./gradlew :mirai-core:githubUpload -Dgithub_token=${{ secrets.MAMOE_TOKEN }} -Pgithub_token=${{ secrets.MAMOE_TOKEN }}
- name: Gradle :mirai-core-qqandroid:githubUpload - name: Gradle :mirai-core-qqandroid:githubUpload
......
...@@ -84,36 +84,64 @@ subprojects { ...@@ -84,36 +84,64 @@ subprojects {
dependsOn(shadowJvmJar) dependsOn(shadowJvmJar)
doFirst { doFirst {
timeout.set(Duration.ofMinutes(10)) timeout.set(Duration.ofHours(3))
File(projectDir, "build/libs").walk() findLatestFile()?.let { (_, file) ->
.filter { it.isFile } val filename = file.name
.onEach { println("all files=$it") } println("Uploading file $filename")
.filter { it.name.matches(Regex("""${project.name}-([0-9]|\.)*\.jar""")) } runCatching {
.onEach { println("matched file: ${it.name}") } upload.GitHub.upload(
.associateBy { it.nameWithoutExtension.substringAfterLast('-') } file,
.onEach { println("versions: $it") } "https://api.github.com/repos/mamoe/mirai-repo/contents/shadow/${project.name}/$filename",
.maxBy { project
it.key.split('.').foldRightIndexed(0) { index: Int, s: String, acc: Int -> )
acc + 100.0.pow(2 - index).toInt() * (s.toIntOrNull() ?: 0) }.exceptionOrNull()?.let {
} System.err.println("GitHub Upload failed")
}?.let { (_, file) -> it.printStackTrace() // force show stacktrace
val filename = file.name throw it
println("Uploading file $filename")
runCatching {
upload.GitHub.upload(
file,
"https://api.github.com/repos/mamoe/mirai-repo/contents/shadow/${project.name}/$filename",
project
)
}.exceptionOrNull()?.let {
System.err.println("Upload failed")
it.printStackTrace() // force show stacktrace
throw it
}
} }
}
}
}
val cuiCloudUpload by tasks.creating {
group = "mirai"
dependsOn(shadowJvmJar)
doFirst {
timeout.set(Duration.ofHours(3))
findLatestFile()?.let { (_, file) ->
val filename = file.name
println("Uploading file $filename")
runCatching {
upload.CuiCloud.upload(
file,
project
)
}.exceptionOrNull()?.let {
System.err.println("CuiCloud Upload failed")
it.printStackTrace() // force show stacktrace
throw it
}
}
} }
} }
} }
}
fun Project.findLatestFile(): Map.Entry<String, File>? {
return File(projectDir, "build/libs").walk()
.filter { it.isFile }
.onEach { println("all files=$it") }
.filter { it.name.matches(Regex("""${project.name}-([0-9]|\.)*\.jar""")) }
.onEach { println("matched file: ${it.name}") }
.associateBy { it.nameWithoutExtension.substringAfterLast('-') }
.onEach { println("versions: $it") }
.maxBy {
it.key.split('.').foldRightIndexed(0) { index: Int, s: String, acc: Int ->
acc + 100.0.pow(2 - index).toInt() * (s.toIntOrNull() ?: 0)
}
}
} }
\ No newline at end of file
...@@ -6,10 +6,20 @@ repositories { ...@@ -6,10 +6,20 @@ repositories {
jcenter() jcenter()
} }
kotlin {
sourceSets.all {
languageSettings.useExperimentalAnnotation("kotlin.Experimental")
languageSettings.useExperimentalAnnotation("kotlin.RequiresOptIn")
}
}
dependencies { dependencies {
fun kotlinx(id: String, version: String) = "org.jetbrains.kotlinx:kotlinx-$id:$version" fun kotlinx(id: String, version: String) = "org.jetbrains.kotlinx:kotlinx-$id:$version"
fun ktor(id: String, version: String) = "io.ktor:ktor-$id:$version" fun ktor(id: String, version: String) = "io.ktor:ktor-$id:$version"
api("org.jsoup:jsoup:1.12.1")
api("com.google.code.gson:gson:2.8.6")
api(kotlinx("coroutines-core", "1.3.3")) api(kotlinx("coroutines-core", "1.3.3"))
api(ktor("client-core", "1.3.2")) api(ktor("client-core", "1.3.2"))
api(ktor("client-cio", "1.3.2")) api(ktor("client-cio", "1.3.2"))
......
/*
* Copyright 2020 Mamoe Technologies and contributors.
*
* 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证.
* Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link.
*
* https://github.com/mamoe/mirai/blob/master/LICENSE
*/
package upload
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import org.gradle.api.Project
import org.gradle.kotlin.dsl.provideDelegate
import org.jsoup.Connection
import org.jsoup.Jsoup
import java.io.File
import java.util.*
object CuiCloud {
private fun getUrl(project: Project): String {
kotlin.runCatching {
@Suppress("UNUSED_VARIABLE", "LocalVariableName")
val cui_cloud_url: String by project
return cui_cloud_url
}
System.getProperty("cui_cloud_url", null)?.let {
return it.trim()
}
error("cannot find url for CuiCloud")
}
private fun getKey(project: Project): String {
kotlin.runCatching {
@Suppress("UNUSED_VARIABLE", "LocalVariableName")
val cui_cloud_key: String by project
return cui_cloud_key
}
System.getProperty("cui_cloud_key", null)?.let {
return it.trim()
}
error("cannot find key for CuiCloud")
}
fun upload(file: File, project: Project) {
val cuiCloudUrl = getUrl(project)
val key = getKey(project)
runBlocking {
uploadToCuiCloud(
cuiCloudUrl,
key,
"/mirai/${project.name}/${file.nameWithoutExtension}.mp4",
file.readBytes()
)
}
}
private suspend fun uploadToCuiCloud(
cuiCloudUrl: String,
cuiToken: String,
filePath: String,
content: ByteArray
) {
val response = withContext(Dispatchers.IO) {
Jsoup.connect(cuiCloudUrl).method(Connection.Method.POST)
.data("base64", Base64.getEncoder().encodeToString(content))
.data("filePath", filePath)
.data("key", cuiToken)
.timeout(Int.MAX_VALUE)
.execute()
}
if (response.statusCode() != 200) {
println(response.body())
error("Cui Cloud Does Not Return 200")
}
}
}
\ No newline at end of file
...@@ -2,13 +2,19 @@ ...@@ -2,13 +2,19 @@
package upload package upload
import com.google.gson.JsonObject
import com.google.gson.JsonParser
import io.ktor.client.HttpClient import io.ktor.client.HttpClient
import io.ktor.client.engine.cio.CIO import io.ktor.client.engine.cio.CIO
import io.ktor.client.features.HttpTimeout import io.ktor.client.features.HttpTimeout
import io.ktor.client.request.put import io.ktor.client.request.put
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import org.gradle.api.Project import org.gradle.api.Project
import org.gradle.kotlin.dsl.provideDelegate import org.gradle.kotlin.dsl.provideDelegate
import org.jsoup.Connection
import org.jsoup.Jsoup
import java.io.File import java.io.File
import java.util.* import java.util.*
...@@ -57,16 +63,114 @@ object GitHub { ...@@ -57,16 +63,114 @@ object GitHub {
connectTimeoutMillis = 600_000 connectTimeoutMillis = 600_000
} }
}.put<String>("$url?access_token=$token") { }.put<String>("$url?access_token=$token") {
//header("token", token) val sha = getGithubSha("mirai-repo", "shadow/${project.name}/${file.name}", "master", project)
println("sha=$sha")
val content = String(Base64.getEncoder().encode(file.readBytes())) val content = String(Base64.getEncoder().encode(file.readBytes()))
body = """ body = """
{ {
"message": "automatically upload on release", "message": "automatically upload on release",
"content": "$content" "content": "$content"
${if (sha == null) "" else """, "sha": "$sha" """}
} }
""".trimIndent() """.trimIndent()
}.let { }.let {
println("Upload response: $it") println("Upload response: $it")
} }
} }
private suspend fun getGithubSha(
repo: String,
filePath: String,
branch: String,
project: Project
): String? {
fun String.asJson(): JsonObject {
return JsonParser.parseString(this).asJsonObject
}
/*
* 只能获取1M以内/branch为master的sha
* */
class TargetTooLargeException() : Exception("Target TOO Large")
suspend fun getShaSmart(repo: String, filePath: String, project: Project): String? {
return withContext(Dispatchers.IO) {
val response = Jsoup
.connect(
"https://api.github.com/repos/mamoe/$repo/contents/$filePath?access_token=" + getGithubToken(
project
)
)
.ignoreContentType(true)
.ignoreHttpErrors(true)
.method(Connection.Method.GET)
.execute()
if (response.statusCode() == 404) {
null
} else {
val p = response.body().asJson()
if (p.has("message") && p["message"].asString == "This API returns blobs up to 1 MB in size. The requested blob is too large to fetch via the API, but you can use the Git Data API to request blobs up to 100 MB in size.") {
throw TargetTooLargeException()
}
p.get("sha").asString
}
}
}
suspend fun getShaStupid(
repo: String,
filePath: String,
branch: String,
project: Project
): String? {
val resp = withContext(Dispatchers.IO) {
Jsoup
.connect(
"https://api.github.com/repos/mamoe/$repo/git/ref/heads/$branch?access_token=" + getGithubToken(
project
)
)
.ignoreContentType(true)
.ignoreHttpErrors(true)
.method(Connection.Method.GET)
.execute()
}
if (resp.statusCode() == 404) {
println("Branch Not Found")
return null
}
val info = resp.body().asJson().get("object").asJsonObject.get("url").asString
var parentNode = withContext(Dispatchers.IO) {
Jsoup.connect(info + "?access_token=" + getGithubToken(project)).ignoreContentType(true)
.method(Connection.Method.GET)
.execute().body().asJson().get("tree").asJsonObject.get("url").asString
}
filePath.split("/").forEach { subPath ->
withContext(Dispatchers.IO) {
Jsoup.connect(parentNode + "?access_token=" + getGithubToken(project)).ignoreContentType(true)
.method(Connection.Method.GET).execute().body().asJson().get("tree").asJsonArray
}.forEach list@{
with(it.asJsonObject) {
if (this.get("path").asString == subPath) {
parentNode = this.get("url").asString
return@list
}
}
}
}
check(parentNode.contains("/blobs/"))
return parentNode.substringAfterLast("/")
}
return if (branch == "master") {
try {
getShaSmart(repo, filePath, project)
} catch (e: TargetTooLargeException) {
getShaStupid(repo, filePath, branch, project)
}
} else {
getShaStupid(repo, filePath, branch, project)
}
}
} }
\ No newline at end of file
...@@ -2,4 +2,5 @@ ...@@ -2,4 +2,5 @@
kotlin.code.style=official kotlin.code.style=official
# config # config
kotlin.incremental.multiplatform=true kotlin.incremental.multiplatform=true
kotlin.parallel.tasks.in.project=true kotlin.parallel.tasks.in.project=true
\ No newline at end of file org.gradle.jvmargs=-Xmx4096m -XX:MaxPermSize=512m -Dfile.encoding=UTF-8
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment