Restructure Kotlin/JS and Kotlin/JS/Wasm testing, fix #3329 (#3954)

* Use the `FrameworkAdapter` interface provided by the Kotlin/JS test
infra, so that failed tests are correctly reported for the JS target on
Node.js (does not apply to Wasm/JS, which does not require "adapter
transformation").
* Use JS Promise instead of callbacks to interact with the JS test
framework.
* Clean up the code to avoid API layers changing from Mocha/Jasmine
style ("describe", "it) to Kotlin test style ("suite", "test) and back.

kotest-common/api/kotest-common.api

@@ -45,7 +45,6 @@ public final class io/kotest/common/ResultsKt {
 public final class io/kotest/common/RunBlockingKt {
 	public static final fun runBlocking (Lkotlin/jvm/functions/Function1;)Ljava/lang/Object;
 	public static final fun runPromise (Lkotlin/jvm/functions/Function1;)V
-	public static final fun runPromiseIgnoringErrors (Lkotlin/jvm/functions/Function1;)V
 }
 
 public abstract interface annotation class io/kotest/common/SoftDeprecated : java/lang/annotation/Annotation {

kotest-common/src/commonMain/kotlin/io/kotest/common/runBlocking.kt

@@ -3,5 +3,3 @@ package io.kotest.common
 expect fun <T> runBlocking(f: suspend () -> T): T
 
 expect fun runPromise(f: suspend () -> Unit)
-
-expect fun runPromiseIgnoringErrors(f: suspend () -> Unit)

kotest-common/src/desktopMain/kotlin/io/kotest/common/runBlocking.kt

@@ -5,7 +5,3 @@ actual fun <T> runBlocking(f: suspend () -> T): T = kotlinx.coroutines.runBlocki
 actual fun runPromise(f: suspend () -> Unit) {
    error("Promise is only available on kotest/js")
 }
-
-actual fun runPromiseIgnoringErrors(f: suspend () -> Unit) {
-   error("Promise is only available on kotest/js")
-}

kotest-common/src/jsMain/kotlin/io/kotest/common/runBlocking.js.kt

@@ -11,8 +11,3 @@ actual fun runPromise(f: suspend () -> Unit) {
       throw it
    }
 }
-
-@OptIn(DelicateCoroutinesApi::class)
-actual fun runPromiseIgnoringErrors(f: suspend () -> Unit) {
-   GlobalScope.promise { f() }
-}

kotest-common/src/jvmMain/kotlin/io/kotest/common/runBlocking.kt

@@ -5,7 +5,3 @@ actual fun <T> runBlocking(f: suspend () -> T): T = kotlinx.coroutines.runBlocki
 actual fun runPromise(f: suspend () -> Unit) {
    error("Promise is only available on kotest/js")
 }
-
-actual fun runPromiseIgnoringErrors(f: suspend () -> Unit) {
-   error("Promise is only available on kotest/js")
-}

kotest-common/src/wasmJsMain/kotlin/io/kotest/common/runBlocking.wasmJs.kt

@@ -11,8 +11,3 @@ actual fun runPromise(f: suspend () -> Unit) {
       throw Throwable("$jsException")
    }
 }
-
-@OptIn(DelicateCoroutinesApi::class)
-actual fun runPromiseIgnoringErrors(f: suspend () -> Unit) {
-   GlobalScope.promise { f() }
-}

kotest-framework/kotest-framework-engine/src/jsHostedMain/kotlin/io/kotest/engine/KotlinJsTestFramework.kt

@@ -0,0 +1,44 @@
+package io.kotest.engine
+
+import kotlinx.coroutines.CoroutineScope
+
+/**
+ * A view of the test infrastructure API provided by the Kotlin Gradle plugins.
+ *
+ * API description:
+ * - https://github.com/JetBrains/kotlin/blob/v1.9.23/libraries/kotlin.test/js/src/main/kotlin/kotlin/test/TestApi.kt#L38
+ * NOTE: This API does not require `kotlin.test` as a dependency. It is actually provided by
+ * - https://github.com/JetBrains/kotlin/tree/v1.9.23/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/targets/js/testing/mocha/KotlinMocha.kt
+ * - https://github.com/JetBrains/kotlin/tree/v1.9.23/libraries/tools/kotlin-test-js-runner
+ *
+ * Nesting of test suites may not be supported by TeamCity reporters of kotlin-test-js-runner.
+ */
+internal interface KotlinJsTestFramework {
+   /**
+    * Declares a test suite. (Theoretically, suites may be nested and may contain tests at each level.)
+    *
+    * [suiteFn] declares one or more tests (and/or sub-suites, theoretically).
+    * Due to [limitations of JS test frameworks](https://github.com/mochajs/mocha/issues/2975) supported by
+    * Kotlin's test infra, [suiteFn] cannot handle asynchronous invocations.
+    */
+   fun suite(name: String, ignored: Boolean, suiteFn: () -> Unit)
+
+   /**
+    * Declares a test.
+    *
+    * [testFn] may return a `Promise`-like object for asynchronous invocation. Otherwise, the underlying JS test
+    * framework will invoke [testFn] synchronously.
+    */
+   fun test(name: String, ignored: Boolean, testFn: () -> Any?)
+}
+
+internal expect val kotlinJsTestFramework: KotlinJsTestFramework
+
+/**
+ * Returns an invocation of [testFunction] as a Promise.
+ *
+ * As there is no common `Promise` type in kotlinx-coroutines, implementations return a target-specific type.
+ * The Promise is passed to the type-agnostic JS test framework, which discovers a Promise-like object by
+ * [probing for a `then` method](https://jasmine.github.io/tutorials/async).
+ */
+internal expect fun CoroutineScope.testFunctionPromise(testFunction: suspend () -> Unit): Any?

kotest-framework/kotest-framework-engine/src/jsHostedMain/kotlin/io/kotest/engine/spec/KotlinJsTestSpecExecutorDelegate.kt

@@ -0,0 +1,61 @@
+package io.kotest.engine.spec
+
+import io.kotest.common.ExperimentalKotest
+import io.kotest.core.spec.Spec
+import io.kotest.core.test.TestCase
+import io.kotest.core.test.TestResult
+import io.kotest.engine.concurrency.NoopCoroutineDispatcherFactory
+import io.kotest.engine.interceptors.EngineContext
+import io.kotest.engine.kotlinJsTestFramework
+import io.kotest.engine.test.NoopTestCaseExecutionListener
+import io.kotest.engine.test.TestCaseExecutor
+import io.kotest.engine.test.interceptors.testNameEscape
+import io.kotest.engine.test.names.getFallbackDisplayNameFormatter
+import io.kotest.engine.test.scopes.TerminalTestScope
+import io.kotest.engine.test.status.isEnabledInternal
+import io.kotest.engine.testFunctionPromise
+import io.kotest.mpp.bestName
+import kotlinx.coroutines.DelicateCoroutinesApi
+import kotlinx.coroutines.GlobalScope
+import kotlin.coroutines.coroutineContext
+
+/**
+ * A [SpecExecutorDelegate] running tests via the Kotlin test infra for JS-hosted targets.
+ */
+@ExperimentalKotest
+internal class KotlinJsTestSpecExecutorDelegate(private val context: EngineContext) : SpecExecutorDelegate {
+
+   private val formatter = getFallbackDisplayNameFormatter(
+      context.configuration.registry,
+      context.configuration,
+   )
+
+   private val materializer = Materializer(context.configuration)
+
+   override suspend fun execute(spec: Spec): Map<TestCase, TestResult> {
+      val cc = coroutineContext
+
+      // This implementation supports a two-level test hierarchy with the spec itself as the test `suite`,
+      // which declares a single level of `test`s.
+      kotlinJsTestFramework.suite(testNameEscape(spec::class.bestName()), ignored = false) {
+         materializer.materialize(spec).forEach { testCase ->
+            kotlinJsTestFramework.test(
+               testNameEscape(formatter.format(testCase)),
+               ignored = testCase.isEnabledInternal(context.configuration).isDisabled
+            ) {
+               // We rely on JS Promise to interact with the JS test framework. We cannot use callbacks here
+               // because we pass our function through the Kotlin/JS test infra via its interface `FrameworkAdapter`,
+               // which does not support callbacks. It does, however, allow the test function to return a Promise-like
+               // type for asynchronous invocations. See `KotlinJsTestFramework` for details.
+               @OptIn(DelicateCoroutinesApi::class)
+               GlobalScope.testFunctionPromise {
+                  TestCaseExecutor(NoopTestCaseExecutionListener, NoopCoroutineDispatcherFactory, context)
+                     .execute(testCase, TerminalTestScope(testCase, cc))
+                     .errorOrNull?.let { throw it }
+               }
+            }
+         }
+      }
+      return emptyMap()
+   }
+}

kotest-framework/kotest-framework-engine/src/jsMain/kotlin/io/kotest/engine/KotlinJsTestFramework.js.kt

@@ -0,0 +1,87 @@
+package io.kotest.engine
+
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.promise
+
+internal actual val kotlinJsTestFramework: KotlinJsTestFramework = object : KotlinJsTestFramework {
+   override fun suite(name: String, ignored: Boolean, suiteFn: () -> Unit) {
+      frameworkAdapter.suite(name, ignored, suiteFn)
+   }
+
+   override fun test(name: String, ignored: Boolean, testFn: () -> Any?) {
+      frameworkAdapter.test(name, ignored, testFn)
+   }
+}
+
+internal actual fun CoroutineScope.testFunctionPromise(testFunction: suspend () -> Unit): Any? =
+   promise { testFunction() }
+
+/**
+ * JS test framework adapter interface defined by the Kotlin/JS test infra.
+ *
+ * This interface allows framework function invocations to be conditionally transformed as required for proper
+ * reporting of [failing JS tests on Node.js](https://youtrack.jetbrains.com/issue/KT-64533).
+ *
+ * Inside the Kotlin/JS test infra, the interface is actually known as `KotlinTestRunner`:
+ *     https://github.com/JetBrains/kotlin/blob/v1.9.23/libraries/tools/kotlin-test-js-runner/src/KotlinTestRunner.ts
+ * Proper test reporting depends on using kotlinTest.adapterTransformer, which is defined here for Node.js:
+ *     https://github.com/JetBrains/kotlin/blob/v1.9.23/libraries/tools/kotlin-test-js-runner/nodejs.ts
+ */
+private external interface FrameworkAdapter {
+   /** Declares a test suite. */
+   fun suite(name: String, ignored: Boolean, suiteFn: () -> Unit)
+
+   /** Declares a test. */
+   fun test(name: String, ignored: Boolean, testFn: () -> Any?)
+}
+
+// Conditional transformation required by the Kotlin/JS test infra.
+private val frameworkAdapter: FrameworkAdapter by lazy {
+   val originalAdapter = JasmineLikeAdapter()
+   if (jsTypeOf(kotlinTestNamespace) != "undefined") {
+      kotlinTestNamespace.adapterTransformer?.invoke(originalAdapter) ?: originalAdapter
+   } else {
+      originalAdapter
+   }
+}
+
+// Part of the Kotlin/JS test infra.
+private external interface KotlinTestNamespace {
+   val adapterTransformer: ((FrameworkAdapter) -> FrameworkAdapter)?
+}
+
+// Part of the Kotlin/JS test infra.
+@JsName("kotlinTest")
+private external val kotlinTestNamespace: KotlinTestNamespace
+
+private class JasmineLikeAdapter : FrameworkAdapter {
+   override fun suite(name: String, ignored: Boolean, suiteFn: () -> Unit) {
+      if (ignored) {
+         xdescribe(name, suiteFn)
+      } else {
+         describe(name, suiteFn)
+      }
+   }
+
+   override fun test(name: String, ignored: Boolean, testFn: () -> Any?) {
+      if (ignored) {
+         xit(name, testFn)
+      } else {
+         it(name, testFn)
+      }
+   }
+}
+
+// Jasmine/Mocha/Jest test API
+
+@Suppress("UNUSED_PARAMETER")
+private fun describe(description: String, suiteFn: () -> Unit) {
+   // Here we disable the default 2s timeout and use the timeout support which Kotest provides via coroutines.
+   // The strange invocation is necessary to avoid using a JS arrow function which would bind `this` to a
+   // wrong scope: https://stackoverflow.com/a/23492442/2529022
+   js("describe(description, function () { this.timeout(0); suiteFn(); })")
+}
+
+private external fun xdescribe(name: String, testFn: () -> Unit)
+private external fun it(name: String, testFn: () -> Any?)
+private external fun xit(name: String, testFn: () -> Any?)

kotest-framework/kotest-framework-engine/src/jsMain/kotlin/io/kotest/engine/spec/createSpecExecutorDelegate.js.kt

@@ -6,4 +6,4 @@ import io.kotest.engine.interceptors.EngineContext
 internal actual fun createSpecExecutorDelegate(
     defaultCoroutineDispatcherFactory: CoroutineDispatcherFactory,
     context: EngineContext
-): SpecExecutorDelegate = JasmineTestSpecExecutorDelegate(context)
+): SpecExecutorDelegate = KotlinJsTestSpecExecutorDelegate(context)