From ea11a979eb82326a2fabf5129c4ccfe33200f961 Mon Sep 17 00:00:00 2001 From: Cedric Champeau Date: Thu, 10 Feb 2022 18:30:39 +0100 Subject: [PATCH] Use argument file when invoking `native-image` With this, it is no longer necessary to build a fat jar (or shaded jar) under Windows in case of long classpath issues. Using the argument file is enabled by default. If the user uses a version older than 21.3, then they can use the fat jar support (and disable arg-file usage). Fixes #203 --- .../buildtools/utils/NativeImageUtils.java | 16 ++++++++++++++++ docs/src/docs/asciidoc/gradle-plugin.adoc | 16 +++++++--------- docs/src/docs/asciidoc/index.adoc | 2 ++ docs/src/docs/asciidoc/maven-plugin.adoc | 8 ++++++-- .../src/docs/snippets/gradle/groovy/build.gradle | 7 ++++--- .../docs/snippets/gradle/kotlin/build.gradle.kts | 7 ++++--- .../buildtools/gradle/NativeImagePlugin.java | 2 ++ .../buildtools/gradle/dsl/GraalVMExtension.java | 8 ++++++++ .../gradle/internal/BaseNativeImageOptions.java | 3 +-- .../internal/NativeImageCommandLineProvider.java | 10 +++++++++- .../gradle/tasks/BuildNativeImageTask.java | 7 ++++++- .../maven/JavaLibraryFunctionalTest.groovy | 2 +- .../buildtools/maven/NativeBuildMojo.java | 15 +++++++++++++-- samples/java-application-with-tests/pom.xml | 1 + samples/java-application/pom.xml | 1 + 15 files changed, 81 insertions(+), 24 deletions(-) diff --git a/common/utils/src/main/java/org/graalvm/buildtools/utils/NativeImageUtils.java b/common/utils/src/main/java/org/graalvm/buildtools/utils/NativeImageUtils.java index 2959e7ae8..d2a83c588 100644 --- a/common/utils/src/main/java/org/graalvm/buildtools/utils/NativeImageUtils.java +++ b/common/utils/src/main/java/org/graalvm/buildtools/utils/NativeImageUtils.java @@ -42,8 +42,12 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.List; import static org.graalvm.buildtools.utils.SharedConstants.GRAALVM_EXE_EXTENSION; @@ -66,4 +70,16 @@ public static void maybeCreateConfigureUtilSymlink(File configureUtilFile, Path public static String nativeImageConfigureFileName() { return "native-image-configure" + GRAALVM_EXE_EXTENSION; } + + public static List convertToArgsFile(List cliArgs) { + try { + File tmpFile = File.createTempFile("native-image", "args"); + tmpFile.deleteOnExit(); + Files.write(tmpFile.toPath(), cliArgs, StandardCharsets.UTF_8, StandardOpenOption.CREATE); + return Collections.singletonList("@" + tmpFile.getAbsolutePath()); + } catch (IOException e) { + + return Collections.unmodifiableList(cliArgs); + } + } } diff --git a/docs/src/docs/asciidoc/gradle-plugin.adoc b/docs/src/docs/asciidoc/gradle-plugin.adoc index 248cf4178..1a23caa6a 100644 --- a/docs/src/docs/asciidoc/gradle-plugin.adoc +++ b/docs/src/docs/asciidoc/gradle-plugin.adoc @@ -174,27 +174,25 @@ include::../snippets/gradle/kotlin/build.gradle.kts[tags=all-config-options] NOTE: For options that can be set using command-line, if both DSL and command-line options are present, command-line options take precedence. [[long_classpath_and_fat_jar_support]] -==== Long classpath and fat jar support +==== Long classpath, @argument file and fat jar support -Under Windows, https://github.com/graalvm/native-build-tools/issues/85[it is possible that the length of the classpath exceeds what the operating system supports] when invoking the CLI to build a native image. -As a consequence, if you are running under Windows, the plugin will automatically shorten the classpath of your project by building a so called "fat jar", which includes all entries from the classpath automatically. +Since release 0.9.10, the plugin will automatically pass arguments to the `native-image` tool using an argument file, which should prevent all https://github.com/graalvm/native-build-tools/issues/85[long classpath issues] under Windows. +However, if you are using an older GraalVM release (older than 21.3) which doesn't support argument files, you will need to rely on creating a "fat jar", which includes all entries from the classpath automatically, to workaround the problem: -In case this behavior is not required, you can disable the fat jar creation by calling: - -.Disabling the fat jar creation +.Enabling the fat jar creation [source, groovy, role="multi-language-sample"] ---- -include::../snippets/gradle/groovy/build.gradle[tags=disable-fatjar] +include::../snippets/gradle/groovy/build.gradle[tags=enable-fatjar] ---- [source,kotlin,role="multi-language-sample"] ---- -include::../snippets/gradle/kotlin/build.gradle.kts[tags=disable-fatjar] +include::../snippets/gradle/kotlin/build.gradle.kts[tags=enable-fatjar] ---- Alternatively, it is possible to use your own fat jar (for example created using the https://imperceptiblethoughts.com/shadow/[Shadow plugin]) by setting the `classpathJar` property directly on the _task_: -.Disabling the fat jar creation +.Enabling a custom fat jar creation [source, groovy, role="multi-language-sample"] ---- include::../snippets/gradle/groovy/build.gradle[tags=custom-fatjar] diff --git a/docs/src/docs/asciidoc/index.adoc b/docs/src/docs/asciidoc/index.adoc index 7ac018b7e..911b6a123 100644 --- a/docs/src/docs/asciidoc/index.adoc +++ b/docs/src/docs/asciidoc/index.adoc @@ -24,11 +24,13 @@ If you are interested in contributing, please refer to our https://github.com/gr * Fixed race condition which prevented the agent files to be generated properly if tests were executed concurrently * Documented version compatibility for the JUnit Platform and Maven Surefire plugin. - See <> for details. +* Add support for long classpath by using an argument file when invoking `native-image` ==== Gradle plugin * Fixed `nativeRun` not working properly under Windows * Fixed race condition which prevented the agent files to be generated properly if tests were executed concurrently +* Add support for long classpath by using an argument file when invoking `native-image` === Release 0.9.9 diff --git a/docs/src/docs/asciidoc/maven-plugin.adoc b/docs/src/docs/asciidoc/maven-plugin.adoc index b0cabd393..129b5f07c 100644 --- a/docs/src/docs/asciidoc/maven-plugin.adoc +++ b/docs/src/docs/asciidoc/maven-plugin.adoc @@ -258,11 +258,15 @@ use cases such as the following: wish to run those same tests in native mode. [[long_classpath_and_shading_support]] -== Long classpath and shading support +== Long classpath, @argument file and shading support Under Windows, https://github.com/graalvm/native-build-tools/issues/85[it is possible that the length of the classpath exceeds what the operating system supports] when invoking the CLI to build a native image. -If this happens, one option is to use a https://maven.apache.org/plugins/maven-shade-plugin[shaded jar] and use it instead of individual jars on classpath. +To avoid this, since release 0.9.10, the plugin will use an argument file to pass the arguments to the `native-image` tool, instead of passing them directly. + +In case you are using a GraalVM version older than 21.3, you will however have to use a workaround, since the argument file wasn't supported. + +One option is to use a https://maven.apache.org/plugins/maven-shade-plugin[shaded jar] and use it instead of individual jars on classpath. First, you'll need to setup the https://maven.apache.org/plugins/maven-shade-plugin[Maven Shade plugin]: diff --git a/docs/src/docs/snippets/gradle/groovy/build.gradle b/docs/src/docs/snippets/gradle/groovy/build.gradle index ca0fa69ff..c7ba25b7e 100644 --- a/docs/src/docs/snippets/gradle/groovy/build.gradle +++ b/docs/src/docs/snippets/gradle/groovy/build.gradle @@ -98,15 +98,16 @@ graalvmNative { } // end::all-config-options[] -// tag::disable-fatjar[] +// tag::enable-fatjar[] graalvmNative { + useArgFile = false // required for older GraalVM releases binaries { main { - useFatJar = false + useFatJar = true } } } -// end::disable-fatjar[] +// end::enable-fatjar[] def myFatJar = tasks.register("myFatJar", Jar) diff --git a/docs/src/docs/snippets/gradle/kotlin/build.gradle.kts b/docs/src/docs/snippets/gradle/kotlin/build.gradle.kts index 2e99c247a..7d26539d0 100644 --- a/docs/src/docs/snippets/gradle/kotlin/build.gradle.kts +++ b/docs/src/docs/snippets/gradle/kotlin/build.gradle.kts @@ -99,15 +99,16 @@ graalvmNative { } // end::all-config-options[] -// tag::disable-fatjar[] +// tag::enable-fatjar[] graalvmNative { + useFatJar.set(false) // required for older GraalVM releases binaries { named("main") { - useFatJar.set(false) + useFatJar.set(true) } } } -// end::disable-fatjar[] +// end::enable-fatjar[] val myFatJar = tasks.register("myFatJar") diff --git a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/NativeImagePlugin.java b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/NativeImagePlugin.java index 8013cff84..72ff61bd3 100644 --- a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/NativeImagePlugin.java +++ b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/NativeImagePlugin.java @@ -171,6 +171,7 @@ public void apply(Project project) { logger = GraalVMLogger.of(project.getLogger()); DefaultGraalVmExtension graalExtension = (DefaultGraalVmExtension) registerGraalVMExtension(project); + graalExtension.getUseArgFile().convention(true); project.getPlugins() .withType(JavaPlugin.class, javaPlugin -> configureJavaProject(project, nativeImageServiceProvider, graalExtension)); project.afterEvaluate(p -> { @@ -253,6 +254,7 @@ private void configureAutomaticTaskCreation(Project project, builder.setGroup(LifecycleBasePlugin.BUILD_GROUP); builder.getOptions().convention(options); builder.getAgentEnabled().set(agent); + builder.getUseArgFile().convention(graalExtension.getUseArgFile()); }); String runTaskName = deriveTaskName(binaryName, "native", "Run"); if ("main".equals(binaryName)) { diff --git a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/dsl/GraalVMExtension.java b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/dsl/GraalVMExtension.java index 456e50497..fb574b07b 100644 --- a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/dsl/GraalVMExtension.java +++ b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/dsl/GraalVMExtension.java @@ -90,6 +90,14 @@ public interface GraalVMExtension { */ Property getToolchainDetection(); + /** + * Property driving the use of @-arg files when invoking native image. + * This is enabled by default. For older native-image versions, this + * needs to be disabled. + * @return the argument file property + */ + Property getUseArgFile(); + interface TestBinaryConfig { /** diff --git a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/internal/BaseNativeImageOptions.java b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/internal/BaseNativeImageOptions.java index e320eef3b..cc4a82d6c 100644 --- a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/internal/BaseNativeImageOptions.java +++ b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/internal/BaseNativeImageOptions.java @@ -43,7 +43,6 @@ import org.graalvm.buildtools.gradle.dsl.NativeImageOptions; import org.graalvm.buildtools.gradle.dsl.NativeResourcesOptions; -import org.graalvm.buildtools.utils.SharedConstants; import org.gradle.api.Action; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.model.ObjectFactory; @@ -215,7 +214,7 @@ public BaseNativeImageOptions(String name, getAgent().getEnabled().convention(false); getSharedLibrary().convention(false); getImageName().convention(defaultImageName); - getUseFatJar().convention(SharedConstants.IS_WINDOWS); + getUseFatJar().convention(false); } private static Provider property(ProviderFactory providers, String name) { diff --git a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/internal/NativeImageCommandLineProvider.java b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/internal/NativeImageCommandLineProvider.java index 1ea22d346..9d0b80394 100644 --- a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/internal/NativeImageCommandLineProvider.java +++ b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/internal/NativeImageCommandLineProvider.java @@ -42,6 +42,7 @@ package org.graalvm.buildtools.gradle.internal; import org.graalvm.buildtools.gradle.dsl.NativeImageOptions; +import org.graalvm.buildtools.utils.NativeImageUtils; import org.gradle.api.Transformer; import org.gradle.api.file.FileSystemLocation; import org.gradle.api.file.RegularFile; @@ -65,17 +66,20 @@ public class NativeImageCommandLineProvider implements CommandLineArgumentProvid private final Provider executableName; private final Provider outputDirectory; private final Provider classpathJar; + private final Provider useArgFile; public NativeImageCommandLineProvider(Provider options, Provider agentEnabled, Provider executableName, Provider outputDirectory, - Provider classpathJar) { + Provider classpathJar, + Provider useArgFile) { this.options = options; this.agentEnabled = agentEnabled; this.executableName = executableName; this.outputDirectory = outputDirectory; this.classpathJar = classpathJar; + this.useArgFile = useArgFile; } @Nested @@ -145,7 +149,11 @@ public List asArguments() { cliArgs.add("-H:Class=" + options.getMainClass().get()); } cliArgs.addAll(options.getBuildArgs().get()); + if (useArgFile.getOrElse(true)) { + return NativeImageUtils.convertToArgsFile(cliArgs); + } return Collections.unmodifiableList(cliArgs); + } /** diff --git a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/tasks/BuildNativeImageTask.java b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/tasks/BuildNativeImageTask.java index c36ac585e..4a75130e6 100644 --- a/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/tasks/BuildNativeImageTask.java +++ b/native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/tasks/BuildNativeImageTask.java @@ -133,6 +133,10 @@ public Provider getOutputFile() { @Optional public abstract RegularFileProperty getClasspathJar(); + @Input + @Optional + public abstract Property getUseArgFile(); + public BuildNativeImageTask() { DirectoryProperty buildDir = getProject().getLayout().getBuildDirectory(); Provider outputDir = buildDir.dir("native/" + getName()); @@ -154,7 +158,8 @@ private List buildActualCommandLineArgs() { // Can't use getOutputDirectory().map(...) because Gradle would complain that we use // a mapped value before the task was called, when we are actually calling it... getProviders().provider(() -> getOutputDirectory().getAsFile().get().getAbsolutePath()), - getClasspathJar()).asArguments(); + getClasspathJar(), + getUseArgFile()).asArguments(); } // This property provides access to the service instance diff --git a/native-maven-plugin/src/functionalTest/groovy/org/graalvm/buildtools/maven/JavaLibraryFunctionalTest.groovy b/native-maven-plugin/src/functionalTest/groovy/org/graalvm/buildtools/maven/JavaLibraryFunctionalTest.groovy index b8c0a1dc7..0534bfd09 100644 --- a/native-maven-plugin/src/functionalTest/groovy/org/graalvm/buildtools/maven/JavaLibraryFunctionalTest.groovy +++ b/native-maven-plugin/src/functionalTest/groovy/org/graalvm/buildtools/maven/JavaLibraryFunctionalTest.groovy @@ -57,7 +57,7 @@ class JavaLibraryFunctionalTest extends AbstractGraalVMMavenFunctionalTest { def library = file("target/java-library" + libExt) when: - mvn '-Pnative', '-DskipTests', 'package' + mvn '-Pnative', '-DskipTests', 'package', '-DuseArgFile=false' then: buildSucceeded diff --git a/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/NativeBuildMojo.java b/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/NativeBuildMojo.java index ca7113ae6..ac6b94608 100644 --- a/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/NativeBuildMojo.java +++ b/native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/NativeBuildMojo.java @@ -55,6 +55,7 @@ import org.codehaus.plexus.component.configurator.expression.ExpressionEvaluationException; import org.codehaus.plexus.util.xml.Xpp3Dom; import org.graalvm.buildtools.Utils; +import org.graalvm.buildtools.utils.NativeImageUtils; import java.io.File; import java.io.IOException; @@ -98,6 +99,9 @@ public class NativeBuildMojo extends AbstractNativeMojo { @Parameter(property = "classpath") private List classpath; + @Parameter(property = "useArgFile", defaultValue = "true") + private boolean useArgFile; + private final List imageClasspath = new ArrayList<>(); private PluginParameterExpressionEvaluator evaluator; @@ -128,8 +132,15 @@ public void execute() throws MojoExecutionException { maybeAddGeneratedResourcesConfig(buildArgs); try { - ProcessBuilder processBuilder = new ProcessBuilder(nativeImageExecutable.toString(), "-cp", classpathStr); - processBuilder.command().addAll(getBuildArgs()); + List cliArgs = new ArrayList<>(); + cliArgs.add("-cp"); + cliArgs.add(classpathStr); + cliArgs.addAll(getBuildArgs()); + if (useArgFile) { + cliArgs = NativeImageUtils.convertToArgsFile(cliArgs); + } + ProcessBuilder processBuilder = new ProcessBuilder(nativeImageExecutable.toString()); + processBuilder.command().addAll(cliArgs); processBuilder.directory(getWorkingDirectory().toFile()); processBuilder.inheritIO(); diff --git a/samples/java-application-with-tests/pom.xml b/samples/java-application-with-tests/pom.xml index 33b9a6a2b..66ddf36b1 100644 --- a/samples/java-application-with-tests/pom.xml +++ b/samples/java-application-with-tests/pom.xml @@ -205,6 +205,7 @@ false + false ${imageName} --no-fallback diff --git a/samples/java-application/pom.xml b/samples/java-application/pom.xml index 330efaf10..d09afa043 100644 --- a/samples/java-application/pom.xml +++ b/samples/java-application/pom.xml @@ -127,6 +127,7 @@ false + false ${imageName} --no-fallback