Upgrade gradle wrapper to 4.8 (#31525)

* Move to Gradle 4.8 RC1

* Use latest version of plugin

The current does not work with Gradle 4.8 RC1

* Switch to Gradle GA

* Add and configure build compare plugin

* add work-around for gradle/gradle#5692

* work around gradle/gradle#5696

* Make use of Gradle build compare with reference project

* Make the manifest more compare friendly

* Clear the manifest in compare friendly mode

* Remove animalsniffer from buildscript classpath

* Fix javadoc errors

* Fix doc issues

* reference Gradle issues in comments

* Conditionally configure build compare

* Fix some more doclint issues

* fix typo in build script

* Add sanity check to make sure the test task was replaced

Relates to #31324. It seems like Gradle has an inconsistent behavior and
the taks is not always replaced.

* Include number of non conforming tasks in the exception.

* No longer replace test task, create implicit instead

Closes #31324. The issue has full context in comments.

With this change the `test` task becomes nothing more than an alias for `utest`.
Some of the stand alone tests that had a `test` task now have `integTest`, and a
few of them that used to have `integTest` to run multiple tests now only
have `check`.
This will also help separarate unit/micro tests from integration tests.

* Revert "No longer replace test task, create implicit instead"

This reverts commit f1ebaf7.

* Fix replacement of the test task

Based on information from gradle/gradle#5730 replace the task taking
into account the task providres.
Closes #31324.

* Only apply build comapare plugin if needed

* Make sure test runs before integTest

* Fix doclint aftter merge

* PR review comments

* Switch to Gradle 4.8.1 and remove workaround

* PR review comments

* Consolidate task ordering

benchmarks/build.gradle

@@ -29,10 +29,11 @@ buildscript {
 }
 
 apply plugin: 'elasticsearch.build'
-// build an uberjar with all benchmarks
-apply plugin: 'com.github.johnrengelman.shadow'
-// have the shadow plugin provide the runShadow task
-apply plugin: 'application'
+
+// order of this section matters, see: https://github.com/johnrengelman/shadow/issues/336
+apply plugin: 'application' // have the shadow plugin provide the runShadow task
+mainClassName = 'org.openjdk.jmh.Main'
+apply plugin: 'com.github.johnrengelman.shadow' // build an uberjar with all benchmarks
 
 // Not published so no need to assemble
 tasks.remove(assemble)

build.gradle

@@ -320,6 +320,9 @@ gradle.projectsEvaluated {
       // :test:framework:test cannot run before and after :server:test
       return
     }
+    if (tasks.findByPath('test') != null && tasks.findByPath('integTest') != null) {
+      integTest.mustRunAfter test
+    }
     configurations.all { Configuration configuration ->
       /*
        * The featureAwarePlugin configuration has a dependency on x-pack:plugin:core and x-pack:plugin:core has a dependency on the
@@ -569,3 +572,28 @@ gradle.projectsEvaluated {
     }
   }
 }
+
+if (System.properties.get("build.compare") != null) {
+  apply plugin: 'compare-gradle-builds'
+  compareGradleBuilds {
+    ext.referenceProject = System.properties.get("build.compare")
+    doFirst {
+      if (file(referenceProject).exists() == false) {
+        throw new GradleException(
+                "Use git worktree to check out a version to compare against to ../elasticsearch_build_reference"
+        )
+      }
+    }
+    sourceBuild {
+      gradleVersion = "4.7" // does not default to gradle weapper of project dir, but current version
+      projectDir = referenceProject
+      tasks = ["clean", "assemble"]
+      arguments = ["-Dbuild.compare_friendly=true"]
+    }
+    targetBuild {
+      tasks = ["clean", "assemble"]
+      // use -Dorg.gradle.java.home= to alter jdk versions
+      arguments = ["-Dbuild.compare_friendly=true"]
+    }
+  }
+}

buildSrc/build.gradle

@@ -106,7 +106,6 @@ GradleVersion logVersion = GradleVersion.current() > GradleVersion.version('4.3'
 
 dependencies {
   compileOnly "org.gradle:gradle-logging:${logVersion.getVersion()}"
-  compile 'ru.vyarus:gradle-animalsniffer-plugin:1.2.0' // Gradle 2.14 requires a version > 1.0.1
 }
 
 /*****************************************************************************

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy

@@ -1,20 +1,44 @@
 package com.carrotsearch.gradle.junit4
 
 import com.carrotsearch.ant.tasks.junit4.JUnit4
-import org.gradle.api.AntBuilder
+import org.gradle.api.GradleException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.UnknownTaskException
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.tasks.TaskContainer
+import org.gradle.api.tasks.TaskProvider
 import org.gradle.api.tasks.testing.Test
 
+import java.util.concurrent.atomic.AtomicBoolean
+
 class RandomizedTestingPlugin implements Plugin<Project> {
 
+    static private AtomicBoolean sanityCheckConfigured = new AtomicBoolean(false)
+
     void apply(Project project) {
         setupSeed(project)
         replaceTestTask(project.tasks)
         configureAnt(project.ant)
+        configureSanityCheck(project)
+    }
+
+    private static void configureSanityCheck(Project project) {
+        // Check the task graph to confirm tasks were indeed replaced
+        // https://github.com/elastic/elasticsearch/issues/31324
+        if (sanityCheckConfigured.getAndSet(true) == false) {
+            project.rootProject.getGradle().getTaskGraph().whenReady {
+                List<Task> nonConforming = project.getGradle().getTaskGraph().allTasks
+                        .findAll { it.name == "test" }
+                        .findAll { (it instanceof RandomizedTestingTask) == false}
+                        .collect { "${it.path} -> ${it.class}" }
+                if (nonConforming.isEmpty() == false) {
+                    throw new GradleException("Found the ${nonConforming.size()} `test` tasks:" +
+                            "\n  ${nonConforming.join("\n  ")}")
+                }
+            }
+        }
     }
 
     /**
@@ -45,29 +69,32 @@ class RandomizedTestingPlugin implements Plugin<Project> {
     }
 
     static void replaceTestTask(TaskContainer tasks) {
-        Test oldTestTask = tasks.findByPath('test')
-        if (oldTestTask == null) {
+        // Gradle 4.8 introduced lazy tasks, thus we deal both with the `test` task as well as it's provider
+        // https://github.com/gradle/gradle/issues/5730#issuecomment-398822153
+        // since we can't be sure if the task was ever realized, we remove both the provider and the task
+        TaskProvider<Test> oldTestProvider
+        try {
+            oldTestProvider = tasks.getByNameLater(Test, 'test')
+        } catch (UnknownTaskException unused) {
             // no test task, ok, user will use testing task on their own
             return
         }
-        tasks.remove(oldTestTask)
+        Test oldTestTask = oldTestProvider.get()
 
-        Map properties = [
-            name: 'test',
-            type: RandomizedTestingTask,
-            dependsOn: oldTestTask.dependsOn,
-            group: JavaBasePlugin.VERIFICATION_GROUP,
-            description: 'Runs unit tests with the randomized testing framework'
-        ]
-        RandomizedTestingTask newTestTask = tasks.create(properties)
-        newTestTask.classpath = oldTestTask.classpath
-        newTestTask.testClassesDir = oldTestTask.project.sourceSets.test.output.classesDir
-        // since gradle 4.5, tasks immutable dependencies are "hidden" (do not show up in dependsOn)
-        // so we must explicitly add a dependency on generating the test classpath
-        newTestTask.dependsOn('testClasses')
+        // we still have to use replace here despite the remove above because the task container knows about the provider
+        // by the same name
+        RandomizedTestingTask newTestTask = tasks.replace('test', RandomizedTestingTask)
+        newTestTask.configure{
+            group =  JavaBasePlugin.VERIFICATION_GROUP
+            description = 'Runs unit tests with the randomized testing framework'
+            dependsOn oldTestTask.dependsOn, 'testClasses'
+            classpath = oldTestTask.classpath
+            testClassesDir = oldTestTask.project.sourceSets.test.output.classesDir
+        }
 
         // hack so check task depends on custom test
-        Task checkTask = tasks.findByPath('check')
+        Task checkTask = tasks.getByName('check')
+        checkTask.dependsOn.remove(oldTestProvider)
         checkTask.dependsOn.remove(oldTestTask)
         checkTask.dependsOn.add(newTestTask)
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -348,7 +348,9 @@ class BuildPlugin implements Plugin<Project> {
                 // just a self contained test-fixture configuration, likely transitive and hellacious
                 return
             }
-            configuration.resolutionStrategy.failOnVersionConflict()
+            configuration.resolutionStrategy { 
+                failOnVersionConflict()
+            }
         })
 
         // force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself
@@ -475,13 +477,17 @@ class BuildPlugin implements Plugin<Project> {
                 }
             }
 
-            project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
-                // place the pom next to the jar it is for
-                t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
-                // build poms with assemble (if the assemble task exists)
-                Task assemble = project.tasks.findByName('assemble')
-                if (assemble) {
-                    assemble.dependsOn(t)
+            // Work around Gradle 4.8 issue until we `enableFeaturePreview('STABLE_PUBLISHING')`
+            // https://github.com/gradle/gradle/issues/5696#issuecomment-396965185
+            project.getGradle().getTaskGraph().whenReady {
+                project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
+                    // place the pom next to the jar it is for
+                    t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
+                    // build poms with assemble (if the assemble task exists)
+                    Task assemble = project.tasks.findByName('assemble')
+                    if (assemble) {
+                        assemble.dependsOn(t)
+                    }
                 }
             }
         }
@@ -625,6 +631,10 @@ class BuildPlugin implements Plugin<Project> {
                         jarTask.manifest.attributes('Change': shortHash)
                     }
                 }
+                // Force manifest entries that change by nature to a constant to be able to compare builds more effectively
+                if (System.properties.getProperty("build.compare_friendly", "false") == "true") {
+                    jarTask.manifest.getAttributes().clear()
+                }
             }
             // add license/notice files
             project.afterEvaluate {

client/rest/src/main/java/org/elasticsearch/client/RestClient.java

@@ -1010,8 +1010,8 @@ public int compareTo(DeadNode rhs) {
     }
 
     /**
-     * Adapts an <code>Iterator<DeadNodeAndRevival></code> into an
-     * <code>Iterator<Node></code>.
+     * Adapts an <code>Iterator&lt;DeadNodeAndRevival&gt;</code> into an
+     * <code>Iterator&lt;Node&gt;</code>.
      */
     private static class DeadNodeIteratorAdapter implements Iterator<Node> {
         private final Iterator<DeadNode> itr;

client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java

@@ -314,7 +314,7 @@ public void testBody() throws IOException {
     }
 
     /**
-     * @deprecated will remove method in 7.0 but needs tests until then. Replaced by {@link RequestTests#testAddHeaders()}.
+     * @deprecated will remove method in 7.0 but needs tests until then. Replaced by {@link RequestTests}.
      */
     @Deprecated
     public void tesPerformRequestOldStyleNullHeaders() throws IOException {

client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java

@@ -141,7 +141,7 @@ public void onFailure(Exception exception) {
     }
 
     /**
-     * @deprecated will remove method in 7.0 but needs tests until then. Replaced by {@link RequestTests#testAddHeader()}.
+     * @deprecated will remove method in 7.0 but needs tests until then. Replaced by {@link RequestTests}.
      */
     @Deprecated
     public void testPerformOldStyleAsyncWithNullHeaders() throws Exception {

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.7-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.8.1-all.zip
 zipStoreBase=GRADLE_USER_HOME
-distributionSha256Sum=203f4537da8b8075e38c036a6d14cb71b1149de5bf0a8f6db32ac2833a1d1294
+zipStorePath=wrapper/dists
+distributionSha256Sum=ce1645ff129d11aad62dab70d63426fdce6cfd646fa309dc5dc5255dd03c7c11

server/src/main/java/org/elasticsearch/search/SearchHit.java

@@ -497,8 +497,8 @@ public XContentBuilder toInnerXContent(XContentBuilder builder, Params params) t
      * This parser outputs a temporary map of the objects needed to create the
      * SearchHit instead of directly creating the SearchHit. The reason for this
      * is that this way we can reuse the parser when parsing xContent from
-     * {@link CompletionSuggestion.Entry.Option} which unfortunately inlines the
-     * output of
+     * {@link org.elasticsearch.search.suggest.completion.CompletionSuggestion.Entry.Option} which unfortunately inlines
+     * the output of
      * {@link #toInnerXContent(XContentBuilder, org.elasticsearch.common.xcontent.ToXContent.Params)}
      * of the included search hit. The output of the map is used to create the
      * actual SearchHit instance via {@link #createFromMap(Map)}

server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregator.java

@@ -200,7 +200,7 @@ public void collect(int doc, long bucket) throws IOException {
 
     /**
      * Replay the documents that might contain a top bucket and pass top buckets to
-     * the {@link this#deferredCollectors}.
+     * the {@link #deferredCollectors}.
      */
     private void runDeferredCollections() throws IOException {
         final boolean needsScores = needsScores();

server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java

@@ -49,7 +49,7 @@ final class CompositeValuesCollectorQueue implements Releasable {
      *
      * @param sources The list of {@link CompositeValuesSourceConfig} to build the composite buckets.
      * @param size The number of composite buckets to keep.
-     * @param afterKey
+     * @param afterKey composite key
      */
     CompositeValuesCollectorQueue(BigArrays bigArrays, SingleDimensionValuesSource<?>[] sources, int size, CompositeKey afterKey) {
         this.bigArrays = bigArrays;

server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java

@@ -79,7 +79,7 @@ abstract class SingleDimensionValuesSource<T extends Comparable<T>> implements R
      * The current value is filled by a {@link LeafBucketCollector} that visits all the
      * values of each document. This method saves this current value in a slot and should only be used
      * in the context of a collection.
-     * See {@link this#getLeafCollector}.
+     * See {@link #getLeafCollector}.
      */
     abstract void copyCurrent(int slot);
 
@@ -92,15 +92,15 @@ abstract class SingleDimensionValuesSource<T extends Comparable<T>> implements R
      * The current value is filled by a {@link LeafBucketCollector} that visits all the
      * values of each document. This method compares this current value with the value present in
      * the provided slot and should only be used in the context of a collection.
-     * See {@link this#getLeafCollector}.
+     * See {@link #getLeafCollector}.
      */
     abstract int compareCurrent(int slot);
 
     /**
      * The current value is filled by a {@link LeafBucketCollector} that visits all the
      * values of each document. This method compares this current value with the after value
      * set on this source and should only be used in the context of a collection.
-     * See {@link this#getLeafCollector}.
+     * See {@link #getLeafCollector}.
      */
     abstract int compareCurrentWithAfter();
 
@@ -125,7 +125,7 @@ T getAfter() {
      * Creates a {@link LeafBucketCollector} that extracts all values from a document and invokes
      * {@link LeafBucketCollector#collect} on the provided <code>next</code> collector for each of them.
      * The current value of this source is set on each call and can be accessed by <code>next</code> via
-     * the {@link this#copyCurrent(int)} and {@link this#compareCurrent(int)} methods. Note that these methods
+     * the {@link #copyCurrent(int)} and {@link #compareCurrent(int)} methods. Note that these methods
      * are only valid when invoked from the {@link LeafBucketCollector} created in this source.
      */
     abstract LeafBucketCollector getLeafCollector(LeafReaderContext context, LeafBucketCollector next) throws IOException;

server/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java

@@ -854,7 +854,7 @@ public void testEnsureNoSelfReferences() throws IOException {
 
     /**
      * Test that the same map written multiple times do not trigger the self-reference check in
-     * {@link CollectionUtils#ensureNoSelfReferences(Object)}
+     * {@link CollectionUtils#ensureNoSelfReferences(Object, String)} (Object)}
      */
     public void testRepeatedMapsAndNoSelfReferences() throws Exception {
         Map<String, Object> mapB = singletonMap("b", "B");

server/src/test/java/org/elasticsearch/index/mapper/package-info.java

@@ -19,9 +19,7 @@
 
 /**
  * Mappings. Mappings define the way that documents should be translated to
- * Lucene indices, for instance how the
- * {@link org.elasticsearch.index.mapper.UidFieldMapper document identifier}
- * should be indexed, whether a string field should be indexed as a
+ * Lucene indices, for instance whether a string field should be indexed as a
  * {@link org.elasticsearch.index.mapper.TextFieldMapper text} or
  * {@link org.elasticsearch.index.mapper.KeywordFieldMapper keyword} field,
  * etc. This parsing is done by the

server/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java

@@ -1345,8 +1345,6 @@ public void testExceptionOnNegativeInterval() {
      * https://github.com/elastic/elasticsearch/issues/31392 demonstrates an edge case where a date field mapping with
      * "format" = "epoch_millis" can lead for the date histogram aggregation to throw an error if a non-UTC time zone
      * with daylight savings time is used. This test was added to check this is working now
-     * @throws ExecutionException
-     * @throws InterruptedException
      */
     public void testRewriteTimeZone_EpochMillisFormat() throws InterruptedException, ExecutionException {
         String index = "test31392";

test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java

@@ -275,7 +275,7 @@ protected IndexShard newShard(ShardRouting routing, IndexMetaData indexMetaData,
      * @param indexMetaData          indexMetaData for the shard, including any mapping
      * @param indexSearcherWrapper   an optional wrapper to be used during searchers
      * @param globalCheckpointSyncer callback for syncing global checkpoints
-     * @param indexEventListener
+     * @param indexEventListener     index even listener
      * @param listeners              an optional set of listeners to add to the shard
      */
     protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetaData indexMetaData,

test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java

@@ -132,7 +132,6 @@ protected String[] shuffleProtectedFields() {
      * To find the right position in the root query, we add a marker as `queryName` which
      * all query builders support. The added bogus field after that should trigger the exception.
      * Queries that allow arbitrary field names at this level need to override this test.
-     * @throws IOException
      */
     public void testUnknownField() throws IOException {
         String marker = "#marker#";

x-pack/plugin/core/src/main/java/org/elasticsearch/license/ExpirationCallback.java

@@ -135,8 +135,8 @@ final TimeValue delay(long expirationDate, long now) {
     }
 
     /**
-     * {@link SchedulerEngine.Schedule#nextScheduledTimeAfter(long, long)} with respect to
-     * license expiry date
+     * {@link org.elasticsearch.xpack.core.scheduler.SchedulerEngine.Schedule#nextScheduledTimeAfter(long, long)}
+     * with respect to license expiry date
      */
     public final long nextScheduledTimeForExpiry(long expiryDate, long startTime, long time) {
         TimeValue delay = delay(expiryDate, time);