Merge pull request #240 from graalvm/metadata_maven

.gitattributes

@@ -0,0 +1,21 @@
+# Auto detect text files and perform LF normalization
+*        text=auto
+
+*.java   text
+*.html   text
+*.kt     text
+*.kts    text
+*.md     text diff=markdown
+*.py     text diff=python executable
+*.pl     text diff=perl executable
+*.pm     text diff=perl
+*.css    text diff=css
+*.js     text
+*.sql    text
+*.q      text
+
+*.sh     text eol=lf
+gradlew  text eol=lf
+
+*.bat    text eol=crlf
+*.cmd    text eol=crlf

common/jvm-reachability-metadata/src/main/java/org/graalvm/reachability/internal/FileSystemRepository.java

@@ -56,6 +56,7 @@
 import java.util.stream.Collectors;
 
 public class FileSystemRepository implements JvmReachabilityMetadataRepository {
+
     private final FileSystemModuleToConfigDirectoryIndex moduleIndex;
     private final Logger logger;
     private final Map<Path, VersionToConfigDirectoryIndex> artifactIndexes;
@@ -72,6 +73,11 @@ public FileSystemRepository(Path rootDirectory, Logger logger) {
         this.rootDirectory = rootDirectory;
     }
 
+    public static boolean isSupportedArchiveFormat(String path) {
+        String normalizedPath = path.toLowerCase();
+        return normalizedPath.endsWith(".zip") || normalizedPath.endsWith(".tar.gz") || normalizedPath.endsWith(".tar.bz2");
+    }
+
     @Override
     public Set<Path> findConfigurationDirectoriesFor(Consumer<? super Query> queryBuilder) {
         DefaultQuery query = new DefaultQuery();

common/utils/src/main/java/org/graalvm/buildtools/utils/FileUtils.java

@@ -41,8 +41,103 @@
 
 package org.graalvm.buildtools.utils;
 
-public class FileUtils {
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+public final class FileUtils {
+
+    public static final int CONNECT_TIMEOUT = 5000;
+    public static final int READ_TIMEOUT = 5000;
+
     public static String normalizePathSeparators(String path) {
         return path.replace('\\', '/');
     }
+
+    public static Optional<Path> download(URL url, Path destination, Consumer<String> errorLogger) {
+        try {
+            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+            connection.setRequestMethod("GET");
+            connection.setConnectTimeout(CONNECT_TIMEOUT);
+            connection.setReadTimeout(READ_TIMEOUT);
+
+            if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
+                errorLogger.accept("Failed to download from " + url + ": " + connection.getResponseCode() + " " + connection.getResponseMessage());
+            } else {
+                String fileName = "";
+                String disposition = connection.getHeaderField("Content-Disposition");
+
+                if (disposition != null) {
+                    int index = disposition.indexOf("filename=");
+                    if (index > 0) {
+                        fileName = disposition.substring(index + 9);
+                    }
+                } else {
+                    fileName = url.getFile().substring(url.getFile().lastIndexOf("/") + 1);
+                }
+
+                if (!Files.exists(destination)) {
+                    Files.createDirectories(destination);
+                }
+                Path result = destination.resolve(fileName);
+                Files.copy(connection.getInputStream(), result);
+
+                connection.disconnect();
+                return Optional.of(result);
+            }
+        } catch (IOException e) {
+            errorLogger.accept("Failed to download from " + url + ": " + e.getMessage());
+        }
+
+        return Optional.empty();
+    }
+
+    public static void extract(Path archive, Path destination, Consumer<String> errorLogger) {
+        if (isZip(archive)) {
+            try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(archive.toFile().toPath()))) {
+                for (ZipEntry entry = zis.getNextEntry(); entry != null; entry = zis.getNextEntry()) {
+                    Optional<Path> sanitizedPath = sanitizePath(entry, destination);
+                    if (sanitizedPath.isPresent()) {
+                        Path zipEntryPath = sanitizedPath.get();
+                        if (entry.isDirectory()) {
+                            Files.createDirectories(zipEntryPath);
+                        } else {
+                            if (zipEntryPath.getParent() != null && !Files.exists(zipEntryPath.getParent())) {
+                                Files.createDirectories(zipEntryPath.getParent());
+                            }
+
+                            Files.copy(zis, zipEntryPath, StandardCopyOption.REPLACE_EXISTING);
+                        }
+                    } else {
+                        errorLogger.accept("Wrong entry " + entry.getName() + " in " + archive);
+                    }
+                    zis.closeEntry();
+                }
+            } catch (IOException e) {
+                errorLogger.accept("Failed to extract " + archive + ": " + e.getMessage());
+            }
+        } else {
+            errorLogger.accept("Unsupported archive format: " + archive + ". Only ZIP files are supported");
+        }
+    }
+
+    public static boolean isZip(Path archive) {
+        return archive.toString().toLowerCase().endsWith(".zip");
+    }
+
+    private static Optional<Path> sanitizePath(ZipEntry entry, Path destination) {
+        Path normalized = destination.resolve(entry.getName()).normalize();
+        if (normalized.startsWith(destination)) {
+            return Optional.of(normalized);
+        } else {
+            return Optional.empty();
+        }
+    }
 }

common/utils/src/test/java/org/graalvm/buildtools/utils/FileUtilsTest.java

@@ -0,0 +1,134 @@
+package org.graalvm.buildtools.utils;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class FileUtilsTest {
+
+    @Test
+    @DisplayName("It can download a file from a URL that exists")
+    void testDownloadOk(@TempDir Path tempDir) throws IOException {
+        URL url = new URL("https://github.com/graalvm/native-build-tools/archive/refs/heads/master.zip");
+        List<String> errorLogs = new ArrayList<>();
+
+        Optional<Path> download = FileUtils.download(url, tempDir, errorLogs::add);
+        System.out.println("errorLogs = " + errorLogs);
+
+        assertTrue(download.isPresent());
+        assertEquals("native-build-tools-master.zip", download.get().getFileName().toString());
+        assertEquals(0, errorLogs.size());
+    }
+
+    @Test
+    @DisplayName("It doesn't blow up with a URL that isn't a file download")
+    void testDownloadNoFile(@TempDir Path tempDir) throws IOException {
+        URL url = new URL("https://httpstat.us/200");
+        List<String> errorLogs = new ArrayList<>();
+
+        Optional<Path> download = FileUtils.download(url, tempDir, errorLogs::add);
+        System.out.println("errorLogs = " + errorLogs);
+
+        assertTrue(download.isPresent());
+        assertEquals("200", download.get().getFileName().toString());
+        assertEquals(0, errorLogs.size());
+    }
+
+    @Test
+    @DisplayName("It doesn't blow up with a URL that does not exist")
+    void testDownloadNotFound(@TempDir Path tempDir) throws IOException {
+        URL url = new URL("https://httpstat.us/404");
+        List<String> errorLogs = new ArrayList<>();
+
+        Optional<Path> download = FileUtils.download(url, tempDir, errorLogs::add);
+        System.out.println("errorLogs = " + errorLogs);
+
+        assertFalse(download.isPresent());
+        assertEquals(1, errorLogs.size());
+    }
+
+    @Test
+    @DisplayName("It doesn't blow up with connection timeouts")
+    void testDownloadTimeout(@TempDir Path tempDir) throws IOException {
+        URL url = new URL("https://httpstat.us/200?sleep=" + (FileUtils.READ_TIMEOUT + 1000));
+        List<String> errorLogs = new ArrayList<>();
+
+        Optional<Path> download = FileUtils.download(url, tempDir, errorLogs::add);
+        System.out.println("errorLogs = " + errorLogs);
+
+        assertFalse(download.isPresent());
+        assertEquals(1, errorLogs.size());
+    }
+
+
+    @Test
+    @DisplayName("It can unzip a file")
+    void testExtract(@TempDir Path tempDir) throws IOException {
+        Path zipFile = new File("src/test/resources/graalvm-reachability-metadata.zip").toPath();
+        List<String> errorLogs = new ArrayList<>();
+
+        FileUtils.extract(zipFile, tempDir, errorLogs::add);
+
+        assertEquals(0, errorLogs.size());
+
+        assertTrue(Files.exists(tempDir.resolve("index.json")));
+        assertEquals("[]", String.join("\n", Files.readAllLines(tempDir.resolve("index.json"))));
+
+        assertTrue(Files.isDirectory(tempDir.resolve("org")));
+        assertTrue(Files.isDirectory(tempDir.resolve("org/graalvm")));
+        assertTrue(Files.isDirectory(tempDir.resolve("org/graalvm/internal")));
+        assertTrue(Files.isDirectory(tempDir.resolve("org/graalvm/internal/library-with-reflection")));
+        assertTrue(Files.exists(tempDir.resolve("org/graalvm/internal/library-with-reflection/index.json")));
+        assertTrue(Files.isDirectory(tempDir.resolve("org/graalvm/internal/library-with-reflection/1")));
+
+        assertTrue(Files.exists(tempDir.resolve("org/graalvm/internal/library-with-reflection/1/reflect-config.json")));
+        assertEquals("[  {    \"name\": \"org.graalvm.internal.reflect.Message\",    \"allDeclaredFields\": true,    \"allDeclaredMethods\": true  }]", String.join("", Files.readAllLines(tempDir.resolve("org/graalvm/internal/library-with-reflection/1/reflect-config.json"))));
+    }
+    @Test
+    @DisplayName("It is protected against ZIP slip attacks")
+    void testZipSlip(@TempDir Path tempDir) throws IOException {
+        Path zipFile = new File("src/test/resources/zip-slip.zip").toPath();
+        List<String> errorLogs = new ArrayList<>();
+
+        FileUtils.extract(zipFile, tempDir, errorLogs::add);
+
+        assertEquals(1, errorLogs.size());
+        assertEquals("Wrong entry ../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../../tmp/evil.txt in src/test/resources/zip-slip.zip", errorLogs.get(0));
+
+        assertTrue(Files.exists(tempDir.resolve("good.txt")));
+        assertTrue(Files.notExists(tempDir.resolve("evil.txt")));
+        assertTrue(Files.notExists(Paths.get("/tmp/evil.txt")));
+
+        Stream<Path> stream = Files.list(tempDir);
+        assertEquals(1, stream.count());
+        stream.close();
+    }
+
+    @ParameterizedTest(name = "Archives with format {0} are not supported")
+    @ValueSource(strings = {"tar.gz", "tar.bz2"})
+    void testExtractNonZip(String format, @TempDir Path tempDir) {
+        Path archive = new File("src/test/resources/graalvm-reachability-metadata." + format).toPath();
+        List<String> errorLogs = new ArrayList<>();
+
+        FileUtils.extract(archive, tempDir, errorLogs::add);
+
+        assertEquals(1, errorLogs.size());
+        assertEquals("Unsupported archive format: src/test/resources/graalvm-reachability-metadata." + format + ". Only ZIP files are supported", errorLogs.get(0));
+    }
+
+}

docs/src/docs/asciidoc/gradle-plugin.adoc

@@ -362,7 +362,7 @@ include::../snippets/gradle/kotlin/build.gradle.kts[tags=add-agent-options]
 ----
 
 [[metadata-support]]
-== JVM Reachability Metadata support
+== JVM Reachability Metadata Support
 
 Since release 0.9.11, the plugin adds experimental support for the https://github.com/graalvm/jvm-reachability-metadata/[JVM reachability metadata repository].
 This repository provides GraalVM configuration for libraries which do not officially support GraalVM native.
@@ -411,7 +411,7 @@ include::../snippets/gradle/kotlin/build.gradle.kts[tags=specify-metadata-reposi
 
 === Configuring the metadata repository
 
-Once activated, for each library included in the native image, the plugin will automatically search for GraalVM JVM reachability metadata in the repository.
+Once activated, for each library included in the native image, the plugin will automatically search for GraalVM reachability metadata in the repository.
 In some cases, you may need to exclude a particular module from the search.
 This can be done by adding it to the exclude list:
 

docs/src/docs/asciidoc/maven-plugin.adoc

@@ -502,6 +502,54 @@ mvn -Pnative -Dagent=true -DskipTests package exec:exec@native
 WARNING: If the agent is enabled, the `--allow-incomplete-classpath` option is
 automatically added to your native build options.
 
+[[metadata-support]]
+== GraalVM Reachability Metadata Support
+
+Since release 0.9.12, the plugin adds experimental support for the https://github.com/graalvm/jvm-reachability-metadata/[JVM reachability metadata repository].
+This repository provides GraalVM metadata for libraries which do not officially support GraalVM native.
+
+A metadata repository consists of configuration files for GraalVM.
+
+=== Enabling the metadata repository
+
+Support needs to be enabled explicitly:. It is possible to use a _local repository_, in which case you can specify the path to the repository:
+
+.Enabling the metadata repository
+[source,xml,indent=0]
+----
+include::../../../../samples/native-config-integration/pom.xml[tag=metadata-local]
+----
+<1> The local path can point to an _exploded_ directory, or to a compressed ZIP file.
+
+Alternatively, you can use a _remote repository_, in which case you can specify the URL of the ZIP file:
+
+.Enabling a remote repository
+[source,xml,indent=0]
+----
+include::../../../../samples/native-config-integration/pom.xml[tag=metadata-url]
+----
+
+=== Configuring the metadata repository
+
+Once activated, for each library included in the native image, the plugin will automatically search for GraalVM reachability metadata in the repository.
+In some cases, you may need to exclude a particular module from the search.
+This can be done by configuring that particular dependency:
+
+.Excluding a module from search
+[source,xml,indent=0]
+----
+include::../../../../samples/native-config-integration/pom.xml[tag=metadata-exclude]
+----
+
+Last, it is possible for you to override the _metadata version_ of a particular module.
+This may be interesting if there's no specific metadata available for the particular version of the library that you use, but that you know that a version works:
+
+.Specifying the metadata version to use for a particular library
+[source,xml,indent=0]
+----
+include::../../../../samples/native-config-integration/pom.xml[tag=metadata-force-version]
+----
+
 [[javadocs]]
 == Javadocs
 

gradle/libs.versions.toml

@@ -16,6 +16,7 @@ aether = "1.1.0"
 slf4j = "1.7.9"
 groovy = "3.0.8"
 jgit = "5.12.0.202106070339-r"
+jetty = "11.0.9"
 
 [libraries]
 # Local projects
@@ -55,3 +56,5 @@ slf4j-simple = { module = "org.slf4j:slf4j-simple", version.ref = "slf4j" }
 
 jgit = { module = "org.eclipse.jgit:org.eclipse.jgit", version.ref = "jgit" }
 jsch = { module = "org.eclipse.jgit:org.eclipse.jgit.ssh.jsch", version.ref="jgit" }
+
+jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref = "jetty" }

native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/internal/JvmReachabilityMetadataService.java

@@ -117,12 +117,12 @@ private JvmReachabilityMetadataRepository newRepository(URI uri) throws URISynta
         LogLevel logLevel = getParameters().getLogLevel().get();
         if (uri.getScheme().equals("file")) {
             File localFile = new File(uri);
-            if (isSupportedZipFormat(path)) {
+            if (FileSystemRepository.isSupportedArchiveFormat(path)) {
                 return newRepositoryFromZipFile(cacheKey, localFile, logLevel);
             }
             return newRepositoryFromDirectory(localFile.toPath(), logLevel);
         }
-        if (isSupportedZipFormat(path)) {
+        if (FileSystemRepository.isSupportedArchiveFormat(path)) {
             File zipped = getParameters().getCacheDir().file(cacheKey + "/archive").get().getAsFile();
             if (!zipped.exists()) {
                 try (ReadableByteChannel readableByteChannel = Channels.newChannel(uri.toURL().openStream())) {
@@ -138,10 +138,6 @@ private JvmReachabilityMetadataRepository newRepository(URI uri) throws URISynta
         throw new UnsupportedOperationException("Remote URI must point to a zip, a tar.gz or tar.bz2 file");
     }
 
-    private static boolean isSupportedZipFormat(String path) {
-        return path.endsWith(".zip") || path.endsWith(".tar.gz") || path.endsWith(".tar.bz2");
-    }
-
     private FileSystemRepository newRepositoryFromZipFile(String cacheKey, File localFile, LogLevel logLevel) {
         File explodedEntry = getParameters().getCacheDir().file(cacheKey + "/exploded").get().getAsFile();
         if (!explodedEntry.exists()) {