Make translators run in parallel.

Kotlin · Sep 28, 2020 · e534ec0 · e534ec0
1 parent bba7977
commit e534ec0
Show file tree

Hide file tree

Showing 9 changed files with 660 additions and 523 deletions.
diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml
diff --git a/core/build.gradle.kts b/core/build.gradle.kts
@@ -12,6 +12,9 @@ dependencies {
     implementation("org.jsoup:jsoup:1.12.1")
     implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.11.1")
 
+    val coroutines_version: String by project
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutines_version")
+
     testImplementation(project(":core:test-api"))
     testImplementation(kotlin("test-junit"))
 }

diff --git a/core/src/main/kotlin/DokkaGenerator.kt b/core/src/main/kotlin/DokkaGenerator.kt
@@ -9,6 +9,8 @@ import org.jetbrains.dokka.plugability.DokkaContext
 import org.jetbrains.dokka.plugability.DokkaPlugin
 import org.jetbrains.dokka.utilities.DokkaLogger
 import org.jetbrains.dokka.utilities.report
+import kotlinx.coroutines.*
+import org.jetbrains.dokka.utilities.parallelMap
 
 
 /**
@@ -73,9 +75,10 @@ class DokkaGenerator(
 
     fun createDocumentationModels(
         context: DokkaContext
-    ) = context.configuration.sourceSets
-        .flatMap { sourceSet -> translateSources(sourceSet, context) }
-        .also { modules -> if (modules.isEmpty()) exitGenerationGracefully("Nothing to document") }
+    ) = runBlocking(Dispatchers.Default) {
+        context.configuration.sourceSets.parallelMap { sourceSet -> translateSources(sourceSet, context) }.flatten()
+            .also { modules -> if (modules.isEmpty()) exitGenerationGracefully("Nothing to document") }
+    }
 
     fun transformDocumentationModelBeforeMerge(
         modulesFromPlatforms: List<DModule>,
@@ -133,8 +136,8 @@ class DokkaGenerator(
         }
     }
 
-    private fun translateSources(sourceSet: DokkaSourceSet, context: DokkaContext) =
-        context[CoreExtensions.sourceToDocumentableTranslator].map {
+    private suspend fun translateSources(sourceSet: DokkaSourceSet, context: DokkaContext) =
+        context[CoreExtensions.sourceToDocumentableTranslator].parallelMap {
             it.invoke(sourceSet, context)
         }
 }

diff --git a/core/src/main/kotlin/plugability/LazyEvaluated.kt b/core/src/main/kotlin/plugability/LazyEvaluated.kt
@@ -2,6 +2,7 @@ package org.jetbrains.dokka.plugability
 
 internal class LazyEvaluated<T : Any> private constructor(private val recipe: ((DokkaContext) -> T)? = null, private var value: T? = null) {
 
+    @Synchronized
     internal fun get(context: DokkaContext): T {
         if(value == null) {
             value = recipe?.invoke(context)
@@ -13,4 +14,4 @@ internal class LazyEvaluated<T : Any> private constructor(private val recipe: ((
         fun <T : Any> fromInstance(value: T) = LazyEvaluated(value = value)
         fun <T : Any> fromRecipe(recipe: (DokkaContext) -> T) = LazyEvaluated(recipe = recipe)
     }
-}
+}
diff --git a/core/src/main/kotlin/transformers/sources/SourceToDocumentableTranslator.kt b/core/src/main/kotlin/transformers/sources/SourceToDocumentableTranslator.kt
@@ -5,5 +5,5 @@ import org.jetbrains.dokka.model.DModule
 import org.jetbrains.dokka.plugability.DokkaContext
 
 interface SourceToDocumentableTranslator {
-    fun invoke(sourceSet: DokkaSourceSet, context: DokkaContext): DModule
-}
+    suspend fun invoke(sourceSet: DokkaSourceSet, context: DokkaContext): DModule
+}
diff --git a/core/src/main/kotlin/utilities/parallelCollectionOperations.kt b/core/src/main/kotlin/utilities/parallelCollectionOperations.kt
@@ -0,0 +1,17 @@
+package org.jetbrains.dokka.utilities
+
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.coroutineScope
+
+suspend inline fun <A, B> Iterable<A>.parallelMap(crossinline f: suspend (A) -> B): List<B> = coroutineScope {
+    map { async { f(it) } }.awaitAll()
+}
+
+suspend inline fun <A, B> Iterable<A>.parallelMapNotNull(crossinline f: suspend (A) -> B?): List<B> = coroutineScope {
+    map { async { f(it) } }.awaitAll().filterNotNull()
+}
+
+suspend inline fun <A> Iterable<A>.parallelForEach(crossinline f: suspend (A) -> Unit): Unit = coroutineScope {
+    map { async { f(it) } }.awaitAll()
+}
diff --git a/plugins/base/src/main/kotlin/renderers/DefaultRenderer.kt b/plugins/base/src/main/kotlin/renderers/DefaultRenderer.kt
@@ -228,4 +228,4 @@ fun ContentPage.sourceSets() = this.content.sourceSets
 fun ContentEmbeddedResource.isImage(): Boolean {
     val imageExtensions = setOf("png", "jpg", "jpeg", "gif", "bmp", "tif", "webp", "svg")
     return File(address).extension.toLowerCase() in imageExtensions
-}
+}
diff --git a/...base/src/main/kotlin/translators/descriptors/DefaultDescriptorToDocumentableTranslator.kt b/...base/src/main/kotlin/translators/descriptors/DefaultDescriptorToDocumentableTranslator.kt