-
Notifications
You must be signed in to change notification settings - Fork 4.6k
/
TestLauncherSpec.groovy
510 lines (440 loc) · 19.9 KB
/
TestLauncherSpec.groovy
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
/*
* Copyright 2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.gradle.integtests.tooling
import groovy.transform.CompileStatic
import org.gradle.integtests.tooling.fixture.GradleBuildCancellation
import org.gradle.integtests.tooling.fixture.ProgressEvents
import org.gradle.integtests.tooling.fixture.ToolingApiSpecification
import org.gradle.integtests.tooling.fixture.WithOldConfigurationsSupport
import org.gradle.test.fixtures.ConcurrentTestUtil
import org.gradle.tooling.BuildException
import org.gradle.tooling.CancellationToken
import org.gradle.tooling.ProjectConnection
import org.gradle.tooling.ResultHandler
import org.gradle.tooling.TestLauncher
import org.gradle.tooling.events.OperationDescriptor
import org.gradle.tooling.events.OperationType
import org.gradle.tooling.events.task.TaskFinishEvent
import org.gradle.tooling.events.task.TaskOperationDescriptor
import org.gradle.tooling.events.test.JvmTestKind
import org.gradle.tooling.events.test.JvmTestOperationDescriptor
import org.gradle.tooling.events.test.TestOperationDescriptor
import org.gradle.util.GradleVersion
import org.junit.Rule
import static org.gradle.integtests.tooling.fixture.ContinuousBuildToolingApiSpecification.getWaitingMessage
abstract class TestLauncherSpec extends ToolingApiSpecification implements WithOldConfigurationsSupport {
ProgressEvents events = ProgressEvents.create()
@Rule
GradleBuildCancellation cancellationTokenSource
def setup() {
// Avoid mixing JUnit dependencies with the ones from the JVM running this test
// For example, when using PTS/TD for running this test, the JUnit Platform Launcher classes from the GE plugin take precedence
toolingApi.requireDaemons()
testCode()
}
boolean supportsEfficientClassFiltering() {
return getTargetVersion() >= GradleVersion.version('4.7')
}
void launchTests(Collection<TestOperationDescriptor> testsToLaunch) {
launchTests { TestLauncher testLauncher ->
testLauncher.withTests(testsToLaunch)
}
}
void launchTests(Closure configurationClosure) {
launchTests(null, configurationClosure)
}
void launchTests(ResultHandler<Void> resultHandler, Closure configurationClosure) {
withConnection { ProjectConnection connection ->
launchTests(connection, resultHandler, cancellationTokenSource.token(), configurationClosure)
}
}
void launchTests(ProjectConnection connection, ResultHandler<Void> resultHandler, CancellationToken cancellationToken, Closure configurationClosure) {
TestLauncher testLauncher = connection.newTestLauncher()
.withCancellationToken(cancellationToken)
.addProgressListener(events, OperationType.TASK, OperationType.TEST)
collectOutputs(testLauncher)
configurationClosure.call(testLauncher)
events.clear()
if (resultHandler == null) {
testLauncher.run()
} else {
testLauncher.run(resultHandler)
}
}
def assertBuildCancelled() {
stdout.toString().contains("Build cancelled.")
true
}
void waitingForBuild() {
ConcurrentTestUtil.poll(30) {
assert stdout.toString().contains(getWaitingMessage(targetVersion))
}
stdout.reset()
stderr.reset()
}
boolean assertTaskExecuted(String taskPath) {
assert events.all.findAll { it instanceof TaskFinishEvent }.any { it.descriptor.taskPath == taskPath }
true
}
def assertTaskNotExecuted(String taskPath) {
assert !events.all.findAll { it instanceof TaskFinishEvent }.any { it.descriptor.taskPath == taskPath }
true
}
def assertTaskNotUpToDate(String taskPath) {
assert events.all.findAll { it instanceof TaskFinishEvent }.any { it.descriptor.taskPath == taskPath && !it.result.upToDate }
true
}
def assertTestNotExecuted(Map testInfo) {
assert !hasTestDescriptor(testInfo)
true
}
def assertTestExecuted(Map testInfo) {
assert hasTestDescriptor(testInfo)
true
}
private static boolean matchIfPresent(String actual, String requested) {
if (requested == null) {
return true
}
actual == requested
}
Collection<TestOperationDescriptor> testDescriptors(String className, String methodName = null, String taskPath = null, String displayName = null) {
findTestDescriptors(events.tests.collect { it.descriptor }, className, methodName, taskPath, displayName)
}
private static Collection<TestOperationDescriptor> findTestDescriptors(List<TestOperationDescriptor> descriptors, String className, String methodName = null, String taskpath = null, String displayName = null) {
def descriptorByClassAndMethod = descriptors.findAll {
it.className == className &&
it.methodName == methodName &&
matchIfPresent(it.displayName, displayName)
}
if (taskpath == null) {
return descriptorByClassAndMethod
}
return descriptorByClassAndMethod.findAll {
def parent = it
while (parent.parent != null) {
parent = parent.parent
if (parent instanceof TaskOperationDescriptor) {
return parent.taskPath == taskpath
}
}
false
}
}
boolean hasTestDescriptor(testInfo) {
def collect = events.tests.collect { it.descriptor }
!findTestDescriptors(collect, testInfo.className, testInfo.methodName, testInfo.task, testInfo.displayName).isEmpty()
}
void collectDescriptorsFromBuild() {
try {
withConnection {
ProjectConnection connection ->
connection.newBuild().forTasks('build')
.withArguments("--continue")
.addProgressListener(events)
.setStandardOutput(System.out)
.setStandardError(System.err)
.run()
}
} catch (BuildException e) {
}
}
def testCode() {
settingsFile << "rootProject.name = 'testproject'\n"
buildFile.text = simpleJavaProject()
def classesDir = 'file("build/classes/moreTests")'
buildFile << """
sourceSets {
moreTests {
java.srcDir "src/test"
${destinationDirectoryCode(classesDir)}
compileClasspath = compileClasspath + sourceSets.test.compileClasspath
runtimeClasspath = runtimeClasspath + sourceSets.test.runtimeClasspath
}
}
task secondTest(type:Test) {
classpath = sourceSets.moreTests.runtimeClasspath
${separateClassesDirs(targetVersion) ? "testClassesDirs" : "testClassesDir"} = sourceSets.moreTests.output.${separateClassesDirs(targetVersion) ? "classesDirs" : "classesDir"}
}
build.dependsOn secondTest
"""
addDefaultTests()
}
void addDefaultTests() {
file("src/test/java/example/MyTest.java") << """
package example;
public class MyTest {
@org.junit.Test public void foo() throws Exception {
org.junit.Assert.assertEquals(1, 1);
}
@org.junit.Test public void foo2() throws Exception {
org.junit.Assert.assertEquals(1, 1);
}
}
"""
file("src/test/java/example2/MyOtherTest.java") << """
package example2;
public class MyOtherTest {
@org.junit.Test public void bar() throws Exception {
org.junit.Assert.assertEquals(2, 2);
}
}
"""
file("src/test/java/example2/MyOtherTest2.java") << """
package example2;
public class MyOtherTest2 {
@org.junit.Test public void baz() throws Exception {
org.junit.Assert.assertEquals(2, 2);
}
}
"""
}
static boolean separateClassesDirs(GradleVersion version) {
version.baseVersion >= GradleVersion.version("4.0")
}
String destinationDirectoryCode(String destinationDirectory) {
//${separateClassesDirs(targetVersion) ? "java.outputDir" : "output.classesDir"} = file("build/classes/moreTests")
if (!separateClassesDirs(targetVersion)) {
return "output.classesDir = $destinationDirectory"
}
if (targetVersion.baseVersion < GradleVersion.version("6.1")) {
return "java.outputDir = $destinationDirectory"
}
return "java.destinationDirectory.set($destinationDirectory)"
}
def changeTestSource() {
// adding two more test methods
file("src/test/java/example/MyTest.java").text = """
package example;
public class MyTest {
@org.junit.Test public void foo() throws Exception {
org.junit.Assert.assertEquals(1, 1);
}
@org.junit.Test public void foo2() throws Exception {
org.junit.Assert.assertEquals(1, 1);
}
@org.junit.Test public void foo3() throws Exception {
org.junit.Assert.assertEquals(1, 1);
}
@org.junit.Test public void foo4() throws Exception {
org.junit.Assert.assertEquals(1, 1);
}
}
"""
}
def withFailingTest() {
file("src/test/java/example/MyFailingTest.java").text = """
package example;
public class MyFailingTest {
@org.junit.Test public void fail() throws Exception {
org.junit.Assert.assertEquals(1, 2);
}
@org.junit.Test public void fail2() throws Exception {
org.junit.Assert.assertEquals(1, 2);
}
}"""
}
String simpleJavaProject() {
"""
allprojects{
apply plugin: 'java'
${mavenCentralRepository()}
dependencies { ${testImplementationConfiguration} 'junit:junit:4.13' }
}
"""
}
void jvmTestEvents(@DelegatesTo(value = TestEventsSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> assertionSpec) {
DefaultTestEventsSpec spec = new DefaultTestEventsSpec()
assertionSpec.delegate = spec
assertionSpec.resolveStrategy = Closure.DELEGATE_FIRST
assertionSpec()
def remainingEvents = spec.testEvents - spec.verifiedEvents
if (remainingEvents) {
ErrorMessageBuilder err = new ErrorMessageBuilder()
err.title("The following test events were received but not verified")
remainingEvents.each { err.candidate("${it} : Kind=${it.jvmTestKind} suiteName=${it.suiteName} className=${it.className} methodName=${it.methodName} displayName=${it.displayName}") }
throw err.build()
}
}
interface TestEventsSpec {
void task(String path, @DelegatesTo(value = TestEventSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> rootSpec)
}
interface TestEventSpec {
void operationDisplayName(String displayName)
void testDisplayName(String displayName)
void suite(String name, @DelegatesTo(value = TestEventSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> spec)
void testClass(String name, @DelegatesTo(value = TestEventSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> spec)
void test(String name, @DelegatesTo(value = TestEventSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> spec)
void testMethodSuite(String name, @DelegatesTo(value = TestEventSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> spec)
}
class DefaultTestEventsSpec implements TestEventsSpec {
final List<JvmTestOperationDescriptor> testEvents = events.tests.collect { (JvmTestOperationDescriptor) it.descriptor }
final Set<OperationDescriptor> verifiedEvents = []
@Override
void task(String path, @DelegatesTo(value = TestEventSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> rootSpec) {
def task = testEvents.find {
it.jvmTestKind == JvmTestKind.SUITE &&
(it.parent instanceof TaskOperationDescriptor) &&
it.parent.taskPath == path
}
if (task == null) {
throw new AssertionError("Expected to find a test task $path but none was found")
}
DefaultTestEventSpec.assertSpec(task.parent, testEvents, verifiedEvents, "Task $path", rootSpec)
}
}
@CompileStatic
static class DefaultTestEventSpec implements TestEventSpec {
private final List<JvmTestOperationDescriptor> testEvents
private final Set<OperationDescriptor> verifiedEvents
private final OperationDescriptor parent
private String testDisplayName
private String operationDisplayName
static void assertSpec(OperationDescriptor descriptor, List<JvmTestOperationDescriptor> testEvents, Set<OperationDescriptor> verifiedEvents, String expectedOperationDisplayName, @DelegatesTo(value = TestEventSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> spec) {
verifiedEvents.add(descriptor)
DefaultTestEventSpec childSpec = new DefaultTestEventSpec(descriptor, testEvents, verifiedEvents)
spec.delegate = childSpec
spec.resolveStrategy = Closure.DELEGATE_FIRST
spec()
childSpec.validate(expectedOperationDisplayName)
}
DefaultTestEventSpec(OperationDescriptor parent, List<JvmTestOperationDescriptor> testEvents, Set<OperationDescriptor> verifiedEvents) {
this.parent = parent
this.testEvents = testEvents
this.verifiedEvents = verifiedEvents
}
@Override
void operationDisplayName(String displayName) {
this.operationDisplayName = displayName
}
@Override
void testDisplayName(String displayName) {
this.testDisplayName = displayName
}
private static String normalizeExecutor(String name) {
if (name.startsWith("Gradle Test Executor")) {
return "Gradle Test Executor"
}
return name
}
@Override
void suite(String name, @DelegatesTo(value = TestEventSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> spec) {
def child = testEvents.find {
it.parent == parent &&
it.jvmTestKind == JvmTestKind.SUITE &&
normalizeExecutor(it.suiteName) == name &&
it.className == null &&
it.methodName == null &&
normalizeExecutor(it.name) == name
}
if (child == null) {
failWith("test suite", name)
}
String expectedOperationDisplayName = name.startsWith("Gradle Test") ? normalizeExecutor(child.displayName) : "Test suite '$name'"
assertSpec(child, testEvents, verifiedEvents, expectedOperationDisplayName, spec)
}
@Override
void testClass(String name, @DelegatesTo(value = TestEventSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> spec) {
def child = testEvents.find {
it.parent == parent &&
it.jvmTestKind == JvmTestKind.SUITE &&
it.suiteName == name &&
it.className == name &&
it.methodName == null &&
it.name == name
}
if (child == null) {
failWith("test class", name)
}
assertSpec(child, testEvents, verifiedEvents, "Test class $name", spec)
}
@Override
void test(String name, @DelegatesTo(value = TestEventSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> spec) {
def expectedClassName = ((JvmTestOperationDescriptor) parent).className
assert expectedClassName != null
def child = testEvents.find {
it.parent == parent &&
it.jvmTestKind == JvmTestKind.ATOMIC &&
it.suiteName == null &&
it.className == expectedClassName &&
it.methodName == name &&
it.name == name
}
if (child == null) {
failWith("test", name)
}
assertSpec(child, testEvents, verifiedEvents, "Test $name($expectedClassName)", spec)
}
@Override
void testMethodSuite(String name, @DelegatesTo(value = TestEventSpec, strategy = Closure.DELEGATE_FIRST) Closure<?> spec) {
def expectedClassName = ((JvmTestOperationDescriptor) parent).className
assert expectedClassName != null
def child = testEvents.find {
it.parent == parent &&
it.jvmTestKind == JvmTestKind.SUITE &&
it.suiteName == name &&
it.className == expectedClassName &&
it.methodName == name &&
it.name == name
}
if (child == null) {
failWith("test method suite", name)
}
assertSpec(child, testEvents, verifiedEvents, "Test method $name", spec)
}
private void failWith(String what, String name) {
ErrorMessageBuilder err = new ErrorMessageBuilder()
def remaining = testEvents.findAll { it.parent == parent && !verifiedEvents.contains(it) }
if (remaining) {
err.title("Expected to find a '$what' named '$name' under '${parent.displayName}' and none was found. Possible events are:")
remaining.each {
err.candidate("${it} : Kind=${it.jvmTestKind} suiteName=${it.suiteName} className=${it.className} methodName=${it.methodName} displayName=${it.displayName}")
}
} else {
err.title("Expected to find a '$what' named '$name' under '${parent.displayName}' and none was found. There are no more events available for this parent.")
}
throw err.build()
}
void validate(String expectedOperationDisplayName) {
if (testDisplayName != null && parent.respondsTo("getTestDisplayName")) {
assert testDisplayName == ((TestOperationDescriptor) parent).testDisplayName
}
if (operationDisplayName != null) {
assert operationDisplayName == normalizeExecutor(parent.displayName)
} else {
assert expectedOperationDisplayName == normalizeExecutor(parent.displayName)
}
}
}
@CompileStatic
static class ErrorMessageBuilder {
private final StringBuilder builder = new StringBuilder()
boolean inCandidates = false
void title(String title) {
builder.append(title)
}
void candidate(String candidate) {
if (!inCandidates) {
builder.append(":\n")
}
inCandidates = true
builder.append(" - ").append(candidate).append("\n")
}
AssertionError build() {
new AssertionError(builder)
}
}
}