More QOL improvements

- Rewrite GithubDownload to use ktor and ktx.serialization instead of curl, jq, and grep
- Handle printing http errors from GitHub better
- Remove dependency on JsonPath, instead use ktx.serialization to read the json object
- Format info messages to be more gay
- Add done in x seconds at the end
- Stop SLF4J info message from printing

build.gradle.kts

@@ -24,13 +24,12 @@ dependencies {
     implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1")
     implementation("com.github.ajalt.clikt:clikt:3.5.0")
     implementation("com.lordcodes.turtle:turtle:0.8.0")
-    implementation("com.jayway.jsonpath:json-path:2.7.0")
     implementation("io.ktor:ktor-client-core:2.3.8")
     implementation("io.ktor:ktor-client-cio:2.3.8")
+    implementation("io.ktor:ktor-client-cio-jvm:2.3.8")
     implementation("com.github.ajalt.mordant:mordant:2.3.0")
     implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0-RC2")
-    implementation("com.sealwu:kscript-tools:1.0.22")
-    implementation("io.ktor:ktor-client-cio-jvm:2.3.8")
+    implementation("org.slf4j:slf4j-nop:2.0.12")
     testImplementation(kotlin("test"))
 }
 
@@ -48,6 +47,8 @@ kotlin {
 
 tasks {
     shadowJar {
-        minimize()
+        minimize {
+            exclude { it.moduleGroup == "org.slf4j" }
+        }
     }
 }

gradle.properties

@@ -1,2 +1,2 @@
 kotlin.code.style=official
-version=2.0.0-beta.2
+version=2.0.0-beta.3

src/main/kotlin/Main.kt

@@ -15,7 +15,6 @@ import com.github.ajalt.mordant.animation.progressAnimation
 import com.github.ajalt.mordant.rendering.TextColors.brightGreen
 import com.github.ajalt.mordant.rendering.TextColors.yellow
 import com.github.ajalt.mordant.terminal.Terminal
-import com.jayway.jsonpath.JsonPath
 import config.GithubConfig
 import downloading.DownloadParser
 import downloading.DownloadResult
@@ -28,11 +27,15 @@ import kotlinx.coroutines.channels.Channel
 import kotlinx.coroutines.joinAll
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.jsonObject
 import kotlin.io.path.absolute
 import kotlin.io.path.createDirectories
 import kotlin.io.path.div
 import kotlin.time.Duration
 import kotlin.time.Duration.Companion.minutes
+import kotlin.time.DurationUnit
+import kotlin.time.TimeSource
 
 val keepup by lazy { Keepup() }
 val t by lazy { Terminal() }
@@ -54,7 +57,7 @@ class Keepup : CliktCommand() {
         .path(mustExist = true, canBeFile = false, mustBeWritable = true)
 
     // === Options ===
-    val jsonPath by option(help = "JsonPath to the root value to keep")
+    val jsonPath by option(help = "Path to the root object to download from, uses keys separated by .")
         .default("$")
 
     val fileType by option(help = "Type of file for the input stream")
@@ -92,18 +95,30 @@ class Keepup : CliktCommand() {
         if (overrideGithubRelease != GithubReleaseOverride.NONE)
             t.println("${yellow("[!]")} Overriding GitHub release versions to $overrideGithubRelease")
 
+        val startTime = TimeSource.Monotonic.markNow()
+
         val jsonInput = if (fileType == "hocon") {
-            t.println("Converting HOCON to JSON")
+            t.println("${MSG.info} Converting HOCON to JSON")
             renderHocon(input)
         } else input
 
-        t.println("Parsing input")
-        val parsed = JsonPath.parse(jsonInput)
-        val items = parsed.read<Map<String, Any?>>(jsonPath)
+        t.println("${MSG.info} Parsing input")
+        val parsed = Json.parseToJsonElement(jsonInput.reader().use { it.readText() })
+        val items = jsonPath.removePrefix("$").split(".").fold(parsed.jsonObject) { acc, key ->
+            if (key == "") return@fold acc
+            acc.getValue(key).jsonObject
+        }
         val strings = getLeafStrings(items)
 
-        t.println("Clearing symlinks")
+        t.println("${MSG.info} Clearing symlinks")
         clearSymlinks(dest)
+
+        t.println(
+            "${MSG.info} Running Keepup on ${yellow(strings.size.toString())} items" + if (jsonPath != "$") " from path ${
+                yellow(jsonPath)
+            }" else ""
+        )
+
         val progress = if (hideProgressBar) null else t.progressAnimation {
             text("Keepup!")
             percentage()
@@ -146,7 +161,8 @@ class Keepup : CliktCommand() {
 
             progress?.clear()
             progress?.stop()
-            t.println(brightGreen("Keepup done!"))
+            val elapsed = startTime.elapsedNow().toString(unit = DurationUnit.SECONDS, decimals = 2)
+            t.println("${MSG.info} ${brightGreen("done in $elapsed!")}")
         }
     }
 }

src/main/kotlin/downloading/DownloadParser.kt

@@ -1,7 +1,6 @@
 package downloading
 
 import SimilarFileChecker
-import com.lordcodes.turtle.ShellRunException
 import config.GithubConfig
 import downloading.github.GithubArtifact
 import downloading.github.GithubDownload
@@ -46,11 +45,11 @@ class DownloadParser(
         val results = runCatching {
             downloader.download()
         }.getOrElse { error ->
-            val message = (error.message).takeIf { error is ShellRunException } ?: error.stackTraceToString()
+            val message = error.stackTraceToString()
 
             listOf(
                 DownloadResult.Failure(
-                    message = "Program errored,\n${message.prependIndent("\t")}",
+                    message = "Program errored,\n$message",
                     keyInConfig = source.keyInConfig,
                 )
             )

src/main/kotlin/downloading/github/GithubDownload.kt

@@ -1,19 +1,24 @@
 package downloading.github
 
 import com.github.ajalt.mordant.rendering.TextColors
-import com.github.ajalt.mordant.rendering.TextColors.gray
-import commands.CachedCommand
 import config.GithubConfig
 import downloading.DownloadResult
 import downloading.Downloader
 import downloading.HttpDownload
 import downloading.Source
+import helpers.CachedRequest
 import helpers.GithubReleaseOverride
 import helpers.MSG
 import io.ktor.client.*
+import io.ktor.client.request.*
+import io.ktor.client.statement.*
+import io.ktor.http.*
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.coroutineScope
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.builtins.ListSerializer
+import kotlinx.serialization.json.Json
 import t
 import java.nio.file.Path
 import kotlin.io.path.div
@@ -27,40 +32,78 @@ class GithubDownload(
     val artifact: GithubArtifact,
     val targetDir: Path,
 ) : Downloader {
+    val json = Json { ignoreUnknownKeys = true }
+
+    @Serializable
+    data class GithubRelease(
+        val published_at: String,
+        val assets: List<Asset>
+    )
+
+    @Serializable
+    data class Asset(
+        val browser_download_url: String
+    )
+
+    @Serializable
+    data class GithubErrorMessage(
+        val message: String
+    )
 
     override suspend fun download(): List<DownloadResult> {
         val version = when (config.overrideGithubRelease) {
-            GithubReleaseOverride.LATEST_RELEASE -> "latest-release"
             GithubReleaseOverride.LATEST -> "latest"
             else -> artifact.releaseVersion
         }
-        val releaseURL = if (version == "latest") "latest" else "tags/${artifact.releaseVersion}"
 
-        //TODO convert to ktor
-        val commandResult = CachedCommand(
-            buildString {
-                append("curl -s ")
-                if (config.githubAuthToken != null)
-                    append("-H \"Authorization: token ${config.githubAuthToken}\" ")
-                if (config.overrideGithubRelease == GithubReleaseOverride.LATEST)
-                    append("https://api.github.com/repos/${artifact.repo}/releases | jq 'map(select(.draft == false)) | sort_by(.published_at) | last'")
-                else
-                    append("https://api.github.com/repos/${artifact.repo}/releases/$releaseURL")
-                append(" | grep 'browser_download_url'")
-            },
+        val response = CachedRequest(
             targetDir / "response-${artifact.repo.replace("/", "-")}-$version",
             expiration = config.cacheExpirationTime.takeIf { version == "latest" }
-        ).getFromCacheOrEval()
+        ) {
+            val response = client.get {
+                if (config.overrideGithubRelease == GithubReleaseOverride.LATEST)
+                    url("https://api.github.com/repos/${artifact.repo}/releases")
+                else {
+                    val releaseURL = if (version == "latest") "latest" else "tags/${artifact.releaseVersion}"
+                    url("https://api.github.com/repos/${artifact.repo}/releases/$releaseURL")
+                }
+                headers {
+                    if (config.githubAuthToken != null)
+                        append(HttpHeaders.Authorization, "token ${config.githubAuthToken}")
+                }
+            }
+            if (response.status != HttpStatusCode.OK) {
+                return@CachedRequest Result.failure(RuntimeException("GET responded with error: ${response.status}, ${response.bodyAsText()}"))
+            }
 
-        val downloadURLs = commandResult
-            .result
-            .split("\n")
-            .map { it.trim().removePrefix("\"browser_download_url\": \"").trim('"') }
+            Result.success(response.bodyAsText())
+        }.getFromCacheOrEval().getOrElse {
+            return listOf(DownloadResult.Failure(it.message ?: "", artifact.source.keyInConfig))
+        }
+
+        val body = response.result
+
+        val release: GithubRelease = runCatching {
+            if (config.overrideGithubRelease == GithubReleaseOverride.LATEST) {
+                json.decodeFromString(ListSerializer(GithubRelease.serializer()), body)
+                    .maxBy { it.published_at }
+            } else json.decodeFromString(GithubRelease.serializer(), body)
+        }.getOrElse {
+            return listOf(
+                DownloadResult.Failure(
+                    "Failed to parse GitHub response:\n${it.message}",
+                    artifact.source.keyInConfig
+                )
+            )
+        }
+        val downloadURLs = release.assets
+            .map { it.browser_download_url }
             .filter { it.contains(artifact.calculatedRegex) }
 
-        val fullName = TextColors.yellow("github:${artifact.repo}:$version:${artifact.regex}")
-        if (!commandResult.wasCached) {
-            t.println(gray("${MSG.github} Got artifact URLs for $fullName"))
+        val fullName = TextColors.yellow(artifact.source.keyInConfig)
+
+        if (!response.wasCached) {
+            t.println(TextColors.gray("${MSG.github} $fullName GET artifact URLs"))
         }
 
         return coroutineScope {

src/main/kotlin/commands/CachedCommand.kt → src/main/kotlin/helpers/CachedRequest.kt

@@ -1,40 +1,37 @@
-package commands
+package helpers
 
-import evalBash
 import java.nio.file.Path
 import java.time.LocalDateTime
 import kotlin.io.path.*
 import kotlin.time.Duration
 import kotlin.time.toJavaDuration
 
-class CachedCommand(val command: String, val path: Path, val expiration: Duration? = null) {
-    class Result(
+class CachedRequest(val path: Path, val expiration: Duration? = null, val evaluate: suspend () -> Result<String>) {
+    class Returned(
         val wasCached: Boolean,
         val result: String,
     )
 
-    fun getFromCacheOrEval(): Result {
+    suspend fun getFromCacheOrEval(): Result<Returned> {
         val expirationPath = path.parent / (path.name + ".expiration")
         // get current time
         val time = LocalDateTime.now()
         val expiryDate = expirationPath.takeIf { it.exists() }?.readText()?.let { LocalDateTime.parse(it) }
 
         if (path.exists() && (expiryDate == null || time < expiryDate)) {
-            return Result(true, path.readText())
+            return Result.success(Returned(true, path.readText()))
         }
 
-        val evaluated = command.evalBash(env = mapOf())
-            .onFailure {
-                throw IllegalStateException("Failed to evaluate command $command, result:\n${this.stderr.joinToString("\n")}")
-            }
-            .getOrThrow()
-        path.deleteIfExists()
-        path.createParentDirectories().createFile().writeText(evaluated)
+        val evaluated = evaluate()
+        evaluated.onSuccess {
+            path.deleteIfExists()
+            path.createParentDirectories().createFile().writeText(it)
+        }
         // write expiration date
         if (expiration != null) {
             expirationPath.deleteIfExists()
             expirationPath.createFile().writeText((time + expiration.toJavaDuration()).toString())
         }
-        return Result(false, evaluated)
+        return evaluated.map { Returned(false, it) }
     }
 }

src/main/kotlin/helpers/Helpers.kt

@@ -3,6 +3,8 @@ package helpers
 import com.typesafe.config.ConfigFactory
 import com.typesafe.config.ConfigRenderOptions
 import downloading.DownloadResult
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
 import java.io.ByteArrayInputStream
 import java.io.InputStream
 import java.nio.file.Path
@@ -25,11 +27,12 @@ fun clearSymlinks(path: Path) {
 }
 
 /** Fold leaf Strings of [map] into a list of Strings */
-fun getLeafStrings(map: Map<String, Any?>, acc: MutableMap<String, String> = mutableMapOf()): Map<String, String> {
+fun getLeafStrings(map: JsonObject, acc: MutableMap<String, String> = mutableMapOf()): Map<String, String> {
     map.entries.forEach { (key, value) ->
         when (value) {
-            is String -> acc[key] = value
-            is Map<*, *> -> getLeafStrings(value as Map<String, Any?>, acc)
+            is JsonPrimitive -> acc[key] = value.content
+            is JsonObject -> getLeafStrings(value, acc)
+            else -> return acc
         }
     }
     return acc

src/main/kotlin/helpers/Messages.kt

@@ -3,6 +3,16 @@ package helpers
 import com.github.ajalt.mordant.rendering.TextColors.*
 
 object MSG {
+    val info = buildString {
+        append(brightWhite("["))
+        append(brightRed("K"))
+        append(brightYellow("e"))
+        append(brightGreen("e"))
+        append(brightCyan("p"))
+        append(brightBlue("u"))
+        append(brightMagenta("p"))
+        append(brightWhite("]"))
+    }
     val download = brightBlue("[Downloaded]")
     val failure = brightRed("[Failure]   ")
     val cached = brightGreen("[Use Cached]")

src/main/kotlin/helpers/PrintToConsole.kt

@@ -11,7 +11,7 @@ fun DownloadResult.printToConsole() {
 
     when (this) {
         is DownloadResult.Failure -> {
-            t.println(brightRed("${MSG.failure} for $formattedKey: $message"), stderr = true)
+            t.println(brightRed("${MSG.failure} $formattedKey: $message"), stderr = true)
         }
 
         is DownloadResult.Downloaded -> {