Rewrite scip-java CLI to Kotlin

Dockerfile

@@ -18,7 +18,7 @@ RUN git config --global --add safe.directory *
 COPY . .
 
 RUN sbt publishLocal dumpScipJavaVersion
-RUN mkdir -p /app && coursier bootstrap "com.sourcegraph:scip-java_2.13:$(cat VERSION)" -f -o /app/scip-java -M com.sourcegraph.scip_java.ScipJava
+RUN mkdir -p /app && coursier bootstrap "com.sourcegraph:scip-java:$(cat VERSION)" -f -o /app/scip-java -M com.sourcegraph.scip_java.ScipJava
 
 COPY ./bin/scip-java-docker-script.sh /usr/bin/scip-java
 

build.sbt

@@ -11,13 +11,14 @@ lazy val V =
     val protobuf = "4.34.2"
     val scipBindings = "0.8.0"
     val scalaXml = "2.1.0"
-    val moped = "0.2.0"
     val gradle = "8.10"
     val scala213 = "2.13.13"
     val scalameta = "4.9.3"
     val kotlinVersion = "2.2.0"
     val kotest = "4.6.3"
     val kctfork = "0.7.1"
+    val clikt = "5.0.3"
+    val kotlinxSerialization = "1.9.0"
   }
 
 // sbt-git's bundled JGit can't read linked worktrees; shell out to
@@ -196,28 +197,29 @@ lazy val mavenPlugin = project
 
 lazy val cli = project
   .in(file("scip-java"))
+  .enablePlugins(KotlinPlugin, PackPlugin, DockerPlugin)
   .settings(
     moduleName := "scip-java",
+    crossPaths := false,
+    autoScalaLibrary := false,
+    kotlinVersion := V.kotlinVersion,
+    kotlincJvmTarget := "11",
+    Compile / javacOptions ++= Seq("--release", "11"),
     (Compile / mainClass) := Some("com.sourcegraph.scip_java.ScipJava"),
     (run / baseDirectory) := (ThisBuild / baseDirectory).value,
     // ScipJava.main can call System.exit, so we always fork the JVM when
     // sbt invokes it directly (e.g. from the scip-kotlinc snapshots
     // task) so it cannot kill the surrounding sbt process.
     Compile / run / fork := true,
-    buildInfoKeys :=
-      Seq[BuildInfoKey](
-        version,
-        sbtVersion,
-        scalaVersion,
-        "javacModuleOptions" -> javacModuleOptions,
-        "scalametaVersion" -> V.scalameta,
-        "scala213" -> V.scala213
-      ),
-    buildInfoPackage := "com.sourcegraph.scip_java",
+    // Generate a tiny Java `BuildInfo` class replacing the previous
+    // sbt-buildinfo-generated Scala object. Same shape as the Gradle plugin's
+    // `GradlePluginBuildInfo` (introduced in the Gradle plugin Kotlin port).
+    Compile / sourceGenerators += scipJavaCliBuildInfoGenerator.taskValue,
     libraryDependencies ++=
       List(
-        "org.scala-lang.modules" %% "scala-xml" % V.scalaXml,
-        "org.scalameta" %% "moped" % V.moped,
+        "com.github.ajalt.clikt" % "clikt-jvm" % V.clikt,
+        "org.jetbrains.kotlinx" % "kotlinx-serialization-json-jvm" %
+          V.kotlinxSerialization,
         "org.jetbrains.kotlin" % "kotlin-compiler-embeddable" % V.kotlinVersion,
         "org.jetbrains.kotlin" % "kotlin-scripting-common" % V.kotlinVersion,
         "org.jetbrains.kotlin" % "kotlin-scripting-jvm" % V.kotlinVersion,
@@ -289,10 +291,59 @@ lazy val cli = project
     docker / dockerfile :=
       NativeDockerfile((ThisBuild / baseDirectory).value / "Dockerfile")
   )
-  .enablePlugins(PackPlugin, DockerPlugin, BuildInfoPlugin)
   .dependsOn(scip)
 
-// Task key for regenerating the SCIP/SCIP golden snapshots emitted by
+// Source-generator for the CLI's build-info Java class. Replaces the
+// sbt-buildinfo-generated Scala BuildInfo object so the CLI module stays
+// Kotlin/Java-only (and the generated class is straightforward to consume
+// from Kotlin).
+lazy val scipJavaCliBuildInfoGenerator = Def.task {
+  val out = (Compile / sourceManaged).value / "com" / "sourcegraph" /
+    "scip_java" / "BuildInfo.java"
+  IO.createDirectory(out.getParentFile)
+  val optionsLiteral = javacModuleOptions
+    .map(javaStringLiteral)
+    .mkString("Arrays.asList(", ", ", ")")
+  val versionLiteral = javaStringLiteral(version.value)
+  val contents =
+    s"""package com.sourcegraph.scip_java;
+       |
+       |import java.util.Arrays;
+       |import java.util.Collections;
+       |import java.util.List;
+       |
+       |public final class BuildInfo {
+       |    private BuildInfo() {}
+       |    public static final String version = $versionLiteral;
+       |    public static final List<String> javacModuleOptions =
+       |        Collections.unmodifiableList($optionsLiteral);
+       |}
+       |""".stripMargin
+  IO.write(out, contents)
+  Seq(out)
+}
+
+def javaStringLiteral(value: String): String = {
+  val escaped = value.flatMap {
+    case '\\' =>
+      "\\\\"
+    case '"' =>
+      "\\\""
+    case '\n' =>
+      "\\n"
+    case '\r' =>
+      "\\r"
+    case '\t' =>
+      "\\t"
+    case c if c.isControl =>
+      f"\\u${c.toInt}%04x"
+    case c =>
+      c.toString
+  }
+  "\"" + escaped + "\""
+}
+
+// Task key for regenerating the SCIP golden snapshots emitted by
 // the scip-kotlinc compiler plugin over the Kotlin minimized fixtures.
 // We deliberately do NOT call this `snapshots` to avoid colliding with the
 // existing top-level `snapshots` test project (`lazy val snapshots = project`).
@@ -615,8 +666,8 @@ val testSettings = List(
   libraryDependencies ++=
     List(
       "org.scalameta" %% "munit" % "0.7.29",
-      "org.scalameta" %% "moped-testkit" % V.moped,
       "org.scalameta" %% "scalameta" % V.scalameta,
+      "com.lihaoyi" %% "os-lib" % "0.9.3",
       "com.lihaoyi" %% "pprint" % "0.6.6"
     )
 )

docs/getting-started.md

@@ -194,7 +194,7 @@ Run `scip-java index --help` to learn more about the available command-line
 options.
 
 ```scala mdoc:passthrough
-com.sourcegraph.scip_java.ScipJava.printHelp(Console.out)
+com.sourcegraph.scip_java.ScipJava.INSTANCE.printHelp(Console.out)
 ```
 
 ## Supported programming languages

scip-aggregator/src/main/java/com/sourcegraph/scip_aggregator/ScipAggregator.java

@@ -109,6 +109,9 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) {
     for (Path root : options.targetroots) {
       if (JAR_PATTERN.matches(root)) shards.add(root);
       else if (Files.isDirectory(root)) Files.walkFileTree(root, visitor);
+      else
+        options.reporter.warning(
+            "ignoring target root that does not exist or is not a directory: " + root);
     }
     return shards;
   }

scip-aggregator/src/main/java/com/sourcegraph/scip_aggregator/ScipAggregatorReporter.java

@@ -13,6 +13,8 @@ public void error(String message) {
     error(new MessageOnlyException(message));
   }
 
+  public void warning(String message) {}
+
   public void startProcessing(int taskSize) {}
 
   public void processedOneItem() {}

scip-java/src/main/kotlin/com/sourcegraph/scip_java/CliEnvironment.kt

@@ -0,0 +1,25 @@
+package com.sourcegraph.scip_java
+
+import java.io.PrintStream
+import java.nio.file.Path
+import java.nio.file.Paths
+
+/**
+ * Captures the per-invocation environment of a scip-java CLI run.
+ *
+ * Tests inject a custom environment to redirect stdout/stderr into a
+ * byte buffer, point the working directory at a temporary fixture
+ * directory, and so on.
+ */
+data class CliEnvironment(
+    val workingDirectory: Path = Paths.get("").toAbsolutePath(),
+    val environmentVariables: Map<String, String> = System.getenv(),
+    val standardOutput: PrintStream = System.out,
+    val standardError: PrintStream = System.err,
+) {
+    fun withWorkingDirectory(cwd: Path): CliEnvironment = copy(workingDirectory = cwd)
+
+    fun withStandardOutput(out: PrintStream): CliEnvironment = copy(standardOutput = out)
+
+    fun withStandardError(err: PrintStream): CliEnvironment = copy(standardError = err)
+}

scip-java/src/main/kotlin/com/sourcegraph/scip_java/CliReporter.kt

@@ -0,0 +1,80 @@
+package com.sourcegraph.scip_java
+
+import com.sourcegraph.scip_aggregator.ScipAggregatorReporter
+import java.nio.file.NoSuchFileException
+import java.util.concurrent.atomic.AtomicInteger
+
+/**
+ * Console reporter: `info` goes to stdout, `warning`/`error` to stderr.
+ * Doubles as the [ScipAggregatorReporter] consumed by the aggregator.
+ */
+class CliReporter(private val env: CliEnvironment) : ScipAggregatorReporter() {
+    private val errorCount = AtomicInteger()
+
+    private var totalShards = 0
+    private val processedShards = AtomicInteger()
+    private val lastReportedDecile = AtomicInteger()
+
+    fun info(message: String) {
+        env.standardOutput.println(message)
+    }
+
+    override fun warning(message: String) {
+        env.standardError.println("warning: $message")
+    }
+
+    override fun error(message: String) {
+        errorCount.incrementAndGet()
+        env.standardError.println("error: $message")
+    }
+
+    /** Dropped to avoid leaking noise into snapshot tests. */
+    @Suppress("UNUSED_PARAMETER")
+    fun debug(message: String) {}
+
+    override fun error(e: Throwable) {
+        if (e is NoSuchFileException) {
+            error("no such file: ${e.message}")
+            return
+        }
+        errorCount.incrementAndGet()
+        e.printStackTrace(env.standardError)
+    }
+
+    override fun startProcessing(taskSize: Int) {
+        totalShards = taskSize
+        processedShards.set(0)
+        lastReportedDecile.set(0)
+        if (taskSize >= PROGRESS_THRESHOLD) {
+            env.standardError.println("Aggregating $taskSize SCIP shards...")
+        }
+    }
+
+    override fun processedOneItem() {
+        val total = totalShards
+        if (total < PROGRESS_THRESHOLD) return
+        val current = processedShards.incrementAndGet()
+        val decile = current * 10 / total
+        val previous = lastReportedDecile.get()
+        if (decile > previous && lastReportedDecile.compareAndSet(previous, decile)) {
+            env.standardError.println(
+                "Aggregated $current/$total SCIP shards (${current * 100 / total}%)"
+            )
+        }
+    }
+
+    override fun hasErrors(): Boolean = errorCount.get() > 0
+
+    fun exitCode(): Int = if (hasErrors()) 1 else 0
+
+    fun reset() {
+        errorCount.set(0)
+        totalShards = 0
+        processedShards.set(0)
+        lastReportedDecile.set(0)
+    }
+
+    private companion object {
+        const val PROGRESS_THRESHOLD = 100
+    }
+}

scip-java/src/main/kotlin/com/sourcegraph/scip_java/Embedded.kt

@@ -0,0 +1,105 @@
+package com.sourcegraph.scip_java
+
+import com.sourcegraph.scip_java.buildtools.ProcessResult
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardCopyOption
+
+object Embedded {
+
+    fun scipJar(tmpDir: Path): Path = copyFile(tmpDir, "scip-plugin.jar")
+
+    fun gradlePluginJar(tmpDir: Path): Path = copyFile(tmpDir, "gradle-plugin.jar")
+
+    fun scipKotlincJar(tmpDir: Path): Path = copyFile(tmpDir, "scip-kotlinc.jar")
+
+    private fun javacErrorpath(tmp: Path): Path = tmp.resolve("errorpath.txt")
+
+    fun customJavac(sourceroot: Path, targetroot: Path, tmp: Path): Path {
+        val bin = tmp.resolve("bin")
+        val javac = bin.resolve("javac")
+        val java = bin.resolve("java")
+        val pluginpath = scipJar(tmp)
+        val errorpath = javacErrorpath(tmp)
+        val javacopts = targetroot.resolve("javacopts.txt")
+        Files.createDirectories(targetroot)
+        Files.createDirectories(bin)
+        Files.write(
+            java,
+            ("#!/usr/bin/env bash\n" +
+                "java \"\$@\"\n").toByteArray(StandardCharsets.UTF_8),
+        )
+        val newJavacopts = tmp.resolve("javac_newarguments")
+        // --add-exports flags required to access internal javac APIs from our
+        // SCIP plugin. Always set; Java 11+ is the supported baseline.
+        val javacModuleOptions = BuildInfo.javacModuleOptions.joinToString(" ")
+        val injectScipArguments =
+            listOf(
+                "java",
+                "-Dscip.errorpath=$errorpath",
+                "-Dscip.pluginpath=$pluginpath",
+                "-Dscip.sourceroot=$sourceroot",
+                "-Dscip.targetroot=$targetroot",
+                "-Dscip.output=\$NEW_JAVAC_OPTS",
+                "-Dscip.old-output=$javacopts",
+                "-classpath $pluginpath",
+                "com.sourcegraph.scip_javac.InjectScipOptions",
+                "\"\$@\"",
+            ).joinToString(" ")
+        val script = buildString {
+            append("#!/usr/bin/env bash\n")
+            append("set -eu\n")
+            append("LAUNCHER_ARGS=()\n")
+            append("NEW_JAVAC_OPTS=\"$newJavacopts-\$RANDOM\"\n")
+            append("for arg in \"\$@\"; do\n")
+            append("  if [[ \$arg == -J* ]]; then\n")
+            append("    LAUNCHER_ARGS+=(\"\$arg\")\n")
+            append("  fi\n")
+            append("done\n")
+            append(injectScipArguments).append('\n')
+            append("if [ \${#LAUNCHER_ARGS[@]} -eq 0 ]; then\n")
+            append("  javac $javacModuleOptions \"@\$NEW_JAVAC_OPTS\"\n")
+            append("else\n")
+            append("  javac $javacModuleOptions \"@\$NEW_JAVAC_OPTS\" \"\${LAUNCHER_ARGS[@]}\"\n")
+            append("fi\n")
+        }
+        Files.write(javac, script.toByteArray(StandardCharsets.UTF_8))
+        javac.toFile().setExecutable(true)
+        java.toFile().setExecutable(true)
+        return javac
+    }
+
+    /**
+     * The custom javac wrapper reports errors to a specific file if unexpected
+     * errors happen. The javac wrapper gets invoked by builds tools like
+     * Gradle/Maven, which hide the actual errors from the script because they
+     * assume the standard output is from javac. This file is used a side-channel
+     * to avoid relying on the error reporting from Gradle/Maven.
+     */
+    fun reportUnexpectedJavacErrors(reporter: CliReporter, tmp: Path): ProcessResult? {
+        val errorpath = javacErrorpath(tmp)
+        if (!Files.isRegularFile(errorpath)) return null
+        reporter.error("unexpected javac compile errors")
+        Files.readAllLines(errorpath).forEach { reporter.error(it) }
+        return ProcessResult(1)
+    }
+
+    /** Returns the string contents of the scip_java.bzl file on disk. */
+    fun bazelAspectFile(tmpDir: Path): String {
+        val tmpFile = copyFile(tmpDir, "scip-java/scip_java.bzl")
+        val contents = String(Files.readAllBytes(tmpFile), StandardCharsets.UTF_8)
+        Files.deleteIfExists(tmpFile)
+        return contents
+    }
+
+    private fun copyFile(tmpDir: Path, filename: String): Path {
+        val input =
+            Embedded::class.java.getResourceAsStream("/$filename")
+                ?: error("missing embedded resource: /$filename")
+        val out = tmpDir.resolve(filename)
+        Files.createDirectories(out.parent)
+        input.use { Files.copy(it, out, StandardCopyOption.REPLACE_EXISTING) }
+        return out
+    }
+}