[JENKINS-65911] Correct JNA method signature for fcntl(2) (#9026)

core/src/main/java/hudson/util/jna/GNUCLibrary.java

@@ -77,7 +77,7 @@ public interface GNUCLibrary extends Library {
 
     int fcntl(int fd, int command);
 
-    int fcntl(int fd, int command, int flags);
+    int fcntl(int fd, int command, Object... flags);
 
     // obtained from Linux. Needs to be checked if these values are portable.
     int F_GETFD = 1;

core/src/test/java/hudson/util/jna/GNUCLibraryTest.java

@@ -0,0 +1,103 @@
+package hudson.util.jna;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
+import com.sun.jna.Native;
+import hudson.Functions;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.junit.Test;
+
+public class GNUCLibraryTest {
+
+    private static final int EBADF = 9;
+    private static final int O_CREAT = "Linux".equals(System.getProperty("os.name")) ? 64 : 512;
+    private static final int O_RDWR = 2;
+
+    @Test
+    public void openTest() throws IOException {
+        assumeTrue(Functions.isGlibcSupported());
+
+        int fd = GNUCLibrary.LIBC.open("/dev/null", 0);
+        assertNotEquals(-1, fd);
+
+        int result = GNUCLibrary.LIBC.close(fd);
+        assertEquals(0, result);
+
+        result = GNUCLibrary.LIBC.close(fd);
+        assertEquals(-1, result);
+        assertEquals(EBADF, Native.getLastError());
+
+        Path tmpFile = Files.createTempFile("openTest", null);
+        Files.delete(tmpFile);
+        assertFalse(Files.exists(tmpFile));
+        try {
+            fd = GNUCLibrary.LIBC.open(tmpFile.toString(), O_CREAT | O_RDWR);
+            assertTrue(Files.exists(tmpFile));
+
+            result = GNUCLibrary.LIBC.close(fd);
+            assertEquals(0, result);
+        } finally {
+            Files.deleteIfExists(tmpFile);
+        }
+    }
+
+    @Test
+    public void closeTest() {
+        assumeTrue(Functions.isGlibcSupported());
+
+        int fd = GNUCLibrary.LIBC.open("/dev/null", 0);
+        assertNotEquals(-1, fd);
+
+        int result = GNUCLibrary.LIBC.close(fd);
+        assertEquals(0, result);
+
+        result = GNUCLibrary.LIBC.close(fd);
+        assertEquals(-1, result);
+        assertEquals(EBADF, Native.getLastError());
+    }
+
+    @Test
+    public void fcntlTest() {
+        assumeTrue(Functions.isGlibcSupported());
+
+        int fd = GNUCLibrary.LIBC.open("/dev/null", 0);
+        assertNotEquals(-1, fd);
+        try {
+            int flags = GNUCLibrary.LIBC.fcntl(fd, GNUCLibrary.F_GETFD);
+            assertEquals(0, flags);
+            int result =
+                    GNUCLibrary.LIBC.fcntl(fd, GNUCLibrary.F_SETFD, flags | GNUCLibrary.FD_CLOEXEC);
+            assertEquals(0, result);
+            result = GNUCLibrary.LIBC.fcntl(fd, GNUCLibrary.F_GETFD);
+            assertEquals(GNUCLibrary.FD_CLOEXEC, result);
+        } finally {
+            GNUCLibrary.LIBC.close(fd);
+        }
+    }
+
+    @Test
+    public void renameTest() throws IOException {
+        assumeTrue(Functions.isGlibcSupported());
+
+        Path oldFile = Files.createTempFile("renameTest", null);
+        Path newFile = Files.createTempFile("renameTest", null);
+        Files.delete(newFile);
+        assertTrue(Files.exists(oldFile));
+        assertFalse(Files.exists(newFile));
+        try {
+            GNUCLibrary.LIBC.rename(oldFile.toString(), newFile.toString());
+
+            assertFalse(Files.exists(oldFile));
+            assertTrue(Files.exists(newFile));
+        } finally {
+            Files.deleteIfExists(oldFile);
+            Files.deleteIfExists(newFile);
+        }
+    }
+}