From bff6e9205300adeecaab3f746fe3f5fc025533ce Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Thu, 25 Aug 2022 22:49:58 +0200 Subject: [PATCH 1/7] introduce Kotlinx Benchmark tests (copied from #763 - credit to @tmszdmsk) refactor test projects to be in separate directory --- .../buildsrc/convention/kotlin-jvm.gradle.kts | 2 -- settings.gradle.kts | 3 +- .../client-tests/build.gradle.kts | 0 .../kotlin/io/mockk/test/AnotherClientTest.kt | 0 .../kotlin/io/mockk/test/BasicClientTest.kt | 0 .../kotlin/io/mockk/test/CompileTimeTest.kt | 0 test-modules/performance-tests/README.md | 31 +++++++++++++++++++ .../performance-tests/build.gradle.kts | 30 ++++++++++++++++++ .../kotlin/io/mockk/performance/JmhTest.kt | 23 ++++++++++++++ 9 files changed, 86 insertions(+), 3 deletions(-) rename {modules => test-modules}/client-tests/build.gradle.kts (100%) rename {modules => test-modules}/client-tests/src/jvmTest/kotlin/io/mockk/test/AnotherClientTest.kt (100%) rename {modules => test-modules}/client-tests/src/jvmTest/kotlin/io/mockk/test/BasicClientTest.kt (100%) rename {modules => test-modules}/client-tests/src/jvmTest/kotlin/io/mockk/test/CompileTimeTest.kt (100%) create mode 100644 test-modules/performance-tests/README.md create mode 100644 test-modules/performance-tests/build.gradle.kts create mode 100644 test-modules/performance-tests/src/main/kotlin/io/mockk/performance/JmhTest.kt diff --git a/buildSrc/src/main/kotlin/buildsrc/convention/kotlin-jvm.gradle.kts b/buildSrc/src/main/kotlin/buildsrc/convention/kotlin-jvm.gradle.kts index cafa67580..30f278083 100644 --- a/buildSrc/src/main/kotlin/buildsrc/convention/kotlin-jvm.gradle.kts +++ b/buildSrc/src/main/kotlin/buildsrc/convention/kotlin-jvm.gradle.kts @@ -11,8 +11,6 @@ plugins { id("buildsrc.convention.toolchain-jvm") } -// note: all subprojects are currently Kotlin Multiplatform, so this convention plugin is unused - java { withJavadocJar() withSourcesJar() diff --git a/settings.gradle.kts b/settings.gradle.kts index 8beac794c..d53d10207 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -16,7 +16,8 @@ include( ":modules:mockk-agent", ":modules:mockk-dsl", - ":modules:client-tests", + ":test-modules:client-tests", + ":test-modules:performance-tests", ) val androidSdkDetected: Boolean? by extra diff --git a/modules/client-tests/build.gradle.kts b/test-modules/client-tests/build.gradle.kts similarity index 100% rename from modules/client-tests/build.gradle.kts rename to test-modules/client-tests/build.gradle.kts diff --git a/modules/client-tests/src/jvmTest/kotlin/io/mockk/test/AnotherClientTest.kt b/test-modules/client-tests/src/jvmTest/kotlin/io/mockk/test/AnotherClientTest.kt similarity index 100% rename from modules/client-tests/src/jvmTest/kotlin/io/mockk/test/AnotherClientTest.kt rename to test-modules/client-tests/src/jvmTest/kotlin/io/mockk/test/AnotherClientTest.kt diff --git a/modules/client-tests/src/jvmTest/kotlin/io/mockk/test/BasicClientTest.kt b/test-modules/client-tests/src/jvmTest/kotlin/io/mockk/test/BasicClientTest.kt similarity index 100% rename from modules/client-tests/src/jvmTest/kotlin/io/mockk/test/BasicClientTest.kt rename to test-modules/client-tests/src/jvmTest/kotlin/io/mockk/test/BasicClientTest.kt diff --git a/modules/client-tests/src/jvmTest/kotlin/io/mockk/test/CompileTimeTest.kt b/test-modules/client-tests/src/jvmTest/kotlin/io/mockk/test/CompileTimeTest.kt similarity index 100% rename from modules/client-tests/src/jvmTest/kotlin/io/mockk/test/CompileTimeTest.kt rename to test-modules/client-tests/src/jvmTest/kotlin/io/mockk/test/CompileTimeTest.kt diff --git a/test-modules/performance-tests/README.md b/test-modules/performance-tests/README.md new file mode 100644 index 000000000..7ec1ed7a6 --- /dev/null +++ b/test-modules/performance-tests/README.md @@ -0,0 +1,31 @@ +This is a performance testing setup for `mockk` using [JMH](https://github.com/openjdk/jmh). + +### How to run + +There are 2 ways to execute the benchmark code + +Basic with Gradle + +```shell +./gradlew benchmark +``` + +It will use default configuration set in [build.gradle.kts](build.gradle.kts) and print out the results + +#### Advanced with JMH binary + +If you want access to more JMH features you can + +```shell +./gradlew mainBenchmarkJar +``` + +to build the `.jar` file containing the benchmark code. It will be saved to `./build/benchmarks/main/jars` directory. + +From there you can run JMH with all the JMH-specific configuration options form CLI (even on different machine). + +```shell +java -jar ./build/benchmarks/main/jars/mockk-performance-main-jmh-.jar -w 1 -i 2 -r 60s -prof jfr +``` + +which will attach the JFR profiler (not possible with `kontlinx-benchmark` executed from Gradle) diff --git a/test-modules/performance-tests/build.gradle.kts b/test-modules/performance-tests/build.gradle.kts new file mode 100644 index 000000000..bd8a43269 --- /dev/null +++ b/test-modules/performance-tests/build.gradle.kts @@ -0,0 +1,30 @@ +import kotlinx.benchmark.gradle.JvmBenchmarkTarget +import kotlinx.benchmark.gradle.benchmark + +plugins { + buildsrc.convention.`kotlin-jvm` + id("org.jetbrains.kotlinx.benchmark") version "0.4.5" +} + +dependencies { + implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.4.5") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4") + implementation(projects.modules.mockk) +} + +benchmark { + configurations { + named("main") { + iterationTime = 60 + iterationTimeUnit = "sec" + iterations = 2 + warmups = 1 + } + } + targets { + register("main") { + this as JvmBenchmarkTarget + jmhVersion = "1.35" + } + } +} diff --git a/test-modules/performance-tests/src/main/kotlin/io/mockk/performance/JmhTest.kt b/test-modules/performance-tests/src/main/kotlin/io/mockk/performance/JmhTest.kt new file mode 100644 index 000000000..5d38f52b2 --- /dev/null +++ b/test-modules/performance-tests/src/main/kotlin/io/mockk/performance/JmhTest.kt @@ -0,0 +1,23 @@ +package io.mockk.performance + +import io.mockk.every +import io.mockk.mockk +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.infra.Blackhole + +@BenchmarkMode(Mode.Throughput) +open class JmhTest { + + @Benchmark + fun simpleMockAndStub(blackhole: Blackhole) { + val mockedClass: MockedClass = mockk() + every { mockedClass.mockedFun() } returns "Hello, mockk!" + blackhole.consume(mockedClass) + } + + class MockedClass { + fun mockedFun(): String = "Hello, world!" + } +} From 101675252a8001392a1345cfb9397dc22c07f2ee Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Thu, 25 Aug 2022 22:59:42 +0200 Subject: [PATCH 2/7] format benchmark readme --- test-modules/performance-tests/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test-modules/performance-tests/README.md b/test-modules/performance-tests/README.md index 7ec1ed7a6..90b53ffe2 100644 --- a/test-modules/performance-tests/README.md +++ b/test-modules/performance-tests/README.md @@ -2,9 +2,9 @@ This is a performance testing setup for `mockk` using [JMH](https://github.com/o ### How to run -There are 2 ways to execute the benchmark code +There are two ways to execute the benchmark code -Basic with Gradle +#### 1. Basic with Gradle ```shell ./gradlew benchmark @@ -12,7 +12,7 @@ Basic with Gradle It will use default configuration set in [build.gradle.kts](build.gradle.kts) and print out the results -#### Advanced with JMH binary +#### 2. Advanced with JMH binary If you want access to more JMH features you can From 302f37170898cfb390717d0d8a0383f95923bbb0 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Thu, 25 Aug 2022 23:10:09 +0200 Subject: [PATCH 3/7] update benchmark jar command --- test-modules/performance-tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-modules/performance-tests/README.md b/test-modules/performance-tests/README.md index 90b53ffe2..85a8214de 100644 --- a/test-modules/performance-tests/README.md +++ b/test-modules/performance-tests/README.md @@ -25,7 +25,7 @@ to build the `.jar` file containing the benchmark code. It will be saved to `./b From there you can run JMH with all the JMH-specific configuration options form CLI (even on different machine). ```shell -java -jar ./build/benchmarks/main/jars/mockk-performance-main-jmh-.jar -w 1 -i 2 -r 60s -prof jfr +java -jar ./build/benchmarks/main/jars/performance-tests-main-jmh-.jar -w 1 -i 2 -r 60s -prof jfr ``` which will attach the JFR profiler (not possible with `kontlinx-benchmark` executed from Gradle) From 0da0629b14d80cca7a42dea1d70b7c8f8a13d0d0 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Thu, 25 Aug 2022 23:22:47 +0200 Subject: [PATCH 4/7] use Deps dependencies --- test-modules/performance-tests/build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-modules/performance-tests/build.gradle.kts b/test-modules/performance-tests/build.gradle.kts index bd8a43269..cd92aa193 100644 --- a/test-modules/performance-tests/build.gradle.kts +++ b/test-modules/performance-tests/build.gradle.kts @@ -1,3 +1,4 @@ +import buildsrc.config.Deps import kotlinx.benchmark.gradle.JvmBenchmarkTarget import kotlinx.benchmark.gradle.benchmark @@ -8,7 +9,8 @@ plugins { dependencies { implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.4.5") - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4") + implementation(platform(Deps.Libs.kotlinCoroutinesBom)) + implementation(Deps.Libs.kotlinCoroutinesCore) implementation(projects.modules.mockk) } From e5bc53348e456a186cf57fcf4ff19d4b3b93c8f7 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Thu, 25 Aug 2022 23:26:12 +0200 Subject: [PATCH 5/7] typo kontlinx->kotlinx --- test-modules/performance-tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-modules/performance-tests/README.md b/test-modules/performance-tests/README.md index 85a8214de..d97767847 100644 --- a/test-modules/performance-tests/README.md +++ b/test-modules/performance-tests/README.md @@ -28,4 +28,4 @@ From there you can run JMH with all the JMH-specific configuration options form java -jar ./build/benchmarks/main/jars/performance-tests-main-jmh-.jar -w 1 -i 2 -r 60s -prof jfr ``` -which will attach the JFR profiler (not possible with `kontlinx-benchmark` executed from Gradle) +which will attach the JFR profiler (not possible with `kotlinx-benchmark` executed from Gradle) From 8c36f1fa315dc8c1ab57212652bdfba7aa109eb4 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Thu, 25 Aug 2022 23:37:53 +0200 Subject: [PATCH 6/7] additional tests, so there's a frame of reference --- .../src/main/kotlin/io/mockk/performance/JmhTest.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test-modules/performance-tests/src/main/kotlin/io/mockk/performance/JmhTest.kt b/test-modules/performance-tests/src/main/kotlin/io/mockk/performance/JmhTest.kt index 5d38f52b2..ff6e6595a 100644 --- a/test-modules/performance-tests/src/main/kotlin/io/mockk/performance/JmhTest.kt +++ b/test-modules/performance-tests/src/main/kotlin/io/mockk/performance/JmhTest.kt @@ -10,6 +10,18 @@ import org.openjdk.jmh.infra.Blackhole @BenchmarkMode(Mode.Throughput) open class JmhTest { + @Benchmark + fun noMockOrStub(blackhole: Blackhole) { + val mockedClass = MockedClass() + blackhole.consume(mockedClass) + } + + @Benchmark + fun simpleMock(blackhole: Blackhole) { + val mockedClass: MockedClass = mockk() + blackhole.consume(mockedClass) + } + @Benchmark fun simpleMockAndStub(blackhole: Blackhole) { val mockedClass: MockedClass = mockk() From 3c7c5362a5693ca8b76e5a72a478c1255e860b71 Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Thu, 25 Aug 2022 23:39:02 +0200 Subject: [PATCH 7/7] reduce iteration time, increase iterations (my noob instinct says 60 seconds per test seems quite long, more iterations is better) --- test-modules/performance-tests/build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-modules/performance-tests/build.gradle.kts b/test-modules/performance-tests/build.gradle.kts index cd92aa193..dc79b6d68 100644 --- a/test-modules/performance-tests/build.gradle.kts +++ b/test-modules/performance-tests/build.gradle.kts @@ -17,9 +17,9 @@ dependencies { benchmark { configurations { named("main") { - iterationTime = 60 + iterationTime = 10 iterationTimeUnit = "sec" - iterations = 2 + iterations = 3 warmups = 1 } }