From cd2b458a4b13db3afb7f2432dc5f0eeea3c994d8 Mon Sep 17 00:00:00 2001 From: Lutz Neugebauer Date: Fri, 12 Jan 2024 17:04:15 +0100 Subject: [PATCH 1/2] allow specification of achrive's file name encoding in Archiver#create() For JENKINS-72540 only needed for tar, but changed zip for consistency as well. --- .../java/hudson/util/io/ArchiverFactory.java | 23 +++++++++++++++---- .../main/java/hudson/util/io/TarArchiver.java | 5 ++-- .../main/java/hudson/util/io/ZipArchiver.java | 7 +++--- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/hudson/util/io/ArchiverFactory.java b/core/src/main/java/hudson/util/io/ArchiverFactory.java index 0193721523de..6220f945d1d8 100644 --- a/core/src/main/java/hudson/util/io/ArchiverFactory.java +++ b/core/src/main/java/hudson/util/io/ArchiverFactory.java @@ -30,6 +30,7 @@ import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; +import java.nio.charset.Charset; import java.nio.file.OpenOption; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.NoExternalUse; @@ -43,9 +44,21 @@ public abstract class ArchiverFactory implements Serializable { /** * Creates an archiver on top of the given stream. + * File names in the archive are encoded with default character set. */ @NonNull - public abstract Archiver create(OutputStream out) throws IOException; + public Archiver create(OutputStream out) throws IOException { + return create(out, Charset.defaultCharset()); + } + + /** + * Creates an archiver on top of the given stream. + * + * @param filenamesEncoding the encoding to be used in the archive for file names + * @since TODO + */ + @NonNull + public abstract Archiver create(OutputStream out, Charset filenamesEncoding) throws IOException; /** * Uncompressed tar format. @@ -85,8 +98,8 @@ private TarArchiverFactory(TarCompression method) { @NonNull @Override - public Archiver create(OutputStream out) throws IOException { - return new TarArchiver(method.compress(out)); + public Archiver create(OutputStream out, Charset filenamesEncoding) throws IOException { + return new TarArchiver(method.compress(out), filenamesEncoding); } private static final long serialVersionUID = 1L; @@ -108,8 +121,8 @@ private static final class ZipArchiverFactory extends ArchiverFactory { @NonNull @Override - public Archiver create(OutputStream out) { - return new ZipArchiver(out, prefix, openOptions); + public Archiver create(OutputStream out, Charset filenamesEncoding) { + return new ZipArchiver(out, prefix, filenamesEncoding, openOptions); } private static final long serialVersionUID = 1L; diff --git a/core/src/main/java/hudson/util/io/TarArchiver.java b/core/src/main/java/hudson/util/io/TarArchiver.java index 6e656e856a74..00b3d1b55d12 100644 --- a/core/src/main/java/hudson/util/io/TarArchiver.java +++ b/core/src/main/java/hudson/util/io/TarArchiver.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.attribute.BasicFileAttributes; @@ -49,8 +50,8 @@ final class TarArchiver extends Archiver { private final byte[] buf = new byte[8192]; private final TarArchiveOutputStream tar; - TarArchiver(OutputStream out) { - tar = new TarArchiveOutputStream(out); + TarArchiver(OutputStream out, Charset filenamesEncoding) { + tar = new TarArchiveOutputStream(out, filenamesEncoding.name()); tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR); tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU); } diff --git a/core/src/main/java/hudson/util/io/ZipArchiver.java b/core/src/main/java/hudson/util/io/ZipArchiver.java index a189457a2861..abdc0025c42c 100644 --- a/core/src/main/java/hudson/util/io/ZipArchiver.java +++ b/core/src/main/java/hudson/util/io/ZipArchiver.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.OpenOption; @@ -55,12 +56,12 @@ final class ZipArchiver extends Archiver { private final String prefix; ZipArchiver(OutputStream out) { - this(out, ""); + this(out, "", Charset.defaultCharset()); } // Restriction added for clarity, it's a package class, you should not use it outside of Jenkins core @Restricted(NoExternalUse.class) - ZipArchiver(OutputStream out, String prefix, OpenOption... openOptions) { + ZipArchiver(OutputStream out, String prefix, Charset filenamesEncoding, OpenOption... openOptions) { this.openOptions = openOptions; if (StringUtils.isBlank(prefix)) { this.prefix = ""; @@ -69,7 +70,7 @@ final class ZipArchiver extends Archiver { } zip = new ZipOutputStream(out); - zip.setEncoding(System.getProperty("file.encoding")); + zip.setEncoding(filenamesEncoding.name()); zip.setUseZip64(Zip64Mode.AsNeeded); } From 154923b8c7cd82e8e3d6b11cfb90916b2329db55 Mon Sep 17 00:00:00 2001 From: Lutz Neugebauer Date: Fri, 12 Jan 2024 17:05:32 +0100 Subject: [PATCH 2/2] revise copyRecursiveTo to use same file name encoding locally and remote such that local and remote understand each other Otherwise, if remote is z/OS with native EBCDIC encoding the file names will be in EBCDIC and fails reading --- core/src/main/java/hudson/FilePath.java | 39 ++++++++++++++++--------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/hudson/FilePath.java b/core/src/main/java/hudson/FilePath.java index 1da338d6e549..b071c2b55a3f 100644 --- a/core/src/main/java/hudson/FilePath.java +++ b/core/src/main/java/hudson/FilePath.java @@ -87,6 +87,7 @@ import java.net.URL; import java.net.URLConnection; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.CopyOption; import java.nio.file.FileSystemException; import java.nio.file.FileSystems; @@ -2848,8 +2849,8 @@ public int copyRecursiveTo(final DirScanner scanner, final FilePath target, fina // local -> remote copy final Pipe pipe = Pipe.createLocalToRemote(); - Future future = target.actAsync(new ReadFromTar(target, pipe, description, compression)); - Future future2 = actAsync(new WriteToTar(scanner, pipe, compression)); + Future future = target.actAsync(new ReadFromTar(target, pipe, description, compression, StandardCharsets.UTF_8)); + Future future2 = actAsync(new WriteToTar(scanner, pipe, compression, StandardCharsets.UTF_8)); try { // JENKINS-9540 in case the reading side failed, report that error first future.get(); @@ -2861,9 +2862,9 @@ public int copyRecursiveTo(final DirScanner scanner, final FilePath target, fina // remote -> local copy final Pipe pipe = Pipe.createRemoteToLocal(); - Future future = actAsync(new CopyRecursiveRemoteToLocal(pipe, scanner, compression)); + Future future = actAsync(new CopyRecursiveRemoteToLocal(pipe, scanner, compression, StandardCharsets.UTF_8)); try { - readFromTar(remote + '/' + description, new File(target.remote), compression.extract(pipe.getIn())); + readFromTar(remote + '/' + description, new File(target.remote), compression.extract(pipe.getIn()), StandardCharsets.UTF_8); } catch (IOException e) { // BuildException or IOException try { future.get(3, TimeUnit.SECONDS); @@ -2976,12 +2977,14 @@ private static class ReadFromTar extends MasterToSlaveFileCallable { private final String description; private final TarCompression compression; private final FilePath target; + private final String filenamesEncoding; - ReadFromTar(FilePath target, Pipe pipe, String description, @NonNull TarCompression compression) { + ReadFromTar(FilePath target, Pipe pipe, String description, @NonNull TarCompression compression, Charset filenamesEncoding) { this.target = target; this.pipe = pipe; this.description = description; this.compression = compression; + this.filenamesEncoding = filenamesEncoding.name(); } private static final long serialVersionUID = 1L; @@ -2989,7 +2992,7 @@ private static class ReadFromTar extends MasterToSlaveFileCallable { @Override public Void invoke(File f, VirtualChannel channel) throws IOException { try (InputStream in = pipe.getIn()) { - readFromTar(target.remote + '/' + description, f, compression.extract(in)); + readFromTar(target.remote + '/' + description, f, compression.extract(in), Charset.forName(filenamesEncoding)); return null; } } @@ -2999,18 +3002,20 @@ private static class WriteToTar extends MasterToSlaveFileCallable { private final DirScanner scanner; private final Pipe pipe; private final TarCompression compression; + private final String filenamesEncoding; - WriteToTar(DirScanner scanner, Pipe pipe, @NonNull TarCompression compression) { + WriteToTar(DirScanner scanner, Pipe pipe, @NonNull TarCompression compression, Charset filenamesEncoding) { this.scanner = scanner; this.pipe = pipe; this.compression = compression; + this.filenamesEncoding = filenamesEncoding.name(); } private static final long serialVersionUID = 1L; @Override public Integer invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { - return writeToTar(f, scanner, compression.compress(pipe.getOut())); + return writeToTar(f, scanner, compression.compress(pipe.getOut()), Charset.forName(filenamesEncoding)); } } @@ -3019,17 +3024,19 @@ private static class CopyRecursiveRemoteToLocal extends MasterToSlaveFileCallabl private final Pipe pipe; private final DirScanner scanner; private final TarCompression compression; + private final String filenamesEncoding; - CopyRecursiveRemoteToLocal(Pipe pipe, DirScanner scanner, @NonNull TarCompression compression) { + CopyRecursiveRemoteToLocal(Pipe pipe, DirScanner scanner, @NonNull TarCompression compression, Charset filenamesEncoding) { this.pipe = pipe; this.scanner = scanner; this.compression = compression; + this.filenamesEncoding = filenamesEncoding.name(); } @Override public Integer invoke(File f, VirtualChannel channel) throws IOException { try (OutputStream out = pipe.getOut()) { - return writeToTar(f, scanner, compression.compress(out)); + return writeToTar(f, scanner, compression.compress(out), Charset.forName(filenamesEncoding)); } } } @@ -3061,22 +3068,26 @@ public int tar(OutputStream out, DirScanner scanner) throws IOException, Interru * @return * number of files/directories that are written. */ - private static Integer writeToTar(File baseDir, DirScanner scanner, OutputStream out) throws IOException { - Archiver tw = ArchiverFactory.TAR.create(out); + private static Integer writeToTar(File baseDir, DirScanner scanner, OutputStream out, Charset filenamesEncoding) throws IOException { + Archiver tw = ArchiverFactory.TAR.create(out, filenamesEncoding); try (tw) { scanner.scan(baseDir, tw); } return tw.countEntries(); } + private static void readFromTar(String name, File baseDir, InputStream in) throws IOException { + readFromTar(name, baseDir, in, Charset.defaultCharset()); + } + /** * Reads from a tar stream and stores obtained files to the base dir. * Supports large files > 10 GB since 1.627 when this was migrated to use commons-compress. */ - private static void readFromTar(String name, File baseDir, InputStream in) throws IOException { + private static void readFromTar(String name, File baseDir, InputStream in, Charset filenamesEncoding) throws IOException { // TarInputStream t = new TarInputStream(in); - try (TarArchiveInputStream t = new TarArchiveInputStream(in)) { + try (TarArchiveInputStream t = new TarArchiveInputStream(in, filenamesEncoding.name())) { TarArchiveEntry te; while ((te = t.getNextTarEntry()) != null) { File f = new File(baseDir, te.getName());