Skip to content

Commit

Permalink
[JENKINS-72540] FilePath.copyRecursiveTo() copying now also if local …
Browse files Browse the repository at this point in the history 
…and remote have incompatible character sets at binary level (#8860)

* allow specification of achrive's file name encoding in Archiver#create()

For JENKINS-72540 only needed for tar, but changed zip for consistency as well.

* revise copyRecursiveTo to use same file name encoding locally and remote

such that local and remote understand each other
Otherwise, if remote is z/OS with native EBCDIC encoding the file names
will be in EBCDIC and fails reading
  • Loading branch information
@lne3
lne3 committed Mar 5, 2024
1 parent b0a251a commit 9f68c18
Show file tree
Hide file tree
Showing 4 changed files with 50 additions and 24 deletions.
39 changes: 25 additions & 14 deletions core/src/main/java/hudson/FilePath.java
View file
Expand Up @@ -87,6 +87,7 @@
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.CopyOption;
import java.nio.file.FileSystemException;
import java.nio.file.FileSystems;
Expand Down Expand Up @@ -2834,8 +2835,8 @@ public int copyRecursiveTo(final DirScanner scanner, final FilePath target, fina
// local -> remote copy
final Pipe pipe = Pipe.createLocalToRemote();

Future<Void> future = target.actAsync(new ReadFromTar(target, pipe, description, compression));
Future<Integer> future2 = actAsync(new WriteToTar(scanner, pipe, compression));
Future<Void> future = target.actAsync(new ReadFromTar(target, pipe, description, compression, StandardCharsets.UTF_8));
Future<Integer> future2 = actAsync(new WriteToTar(scanner, pipe, compression, StandardCharsets.UTF_8));
try {
// JENKINS-9540 in case the reading side failed, report that error first
future.get();
Expand All @@ -2847,9 +2848,9 @@ public int copyRecursiveTo(final DirScanner scanner, final FilePath target, fina
// remote -> local copy
final Pipe pipe = Pipe.createRemoteToLocal();

Future<Integer> future = actAsync(new CopyRecursiveRemoteToLocal(pipe, scanner, compression));
Future<Integer> future = actAsync(new CopyRecursiveRemoteToLocal(pipe, scanner, compression, StandardCharsets.UTF_8));
try {
readFromTar(remote + '/' + description, new File(target.remote), compression.extract(pipe.getIn()));
readFromTar(remote + '/' + description, new File(target.remote), compression.extract(pipe.getIn()), StandardCharsets.UTF_8);
} catch (IOException e) { // BuildException or IOException
try {
future.get(3, TimeUnit.SECONDS);
Expand Down Expand Up @@ -2962,20 +2963,22 @@ private static class ReadFromTar extends MasterToSlaveFileCallable<Void> {
private final String description;
private final TarCompression compression;
private final FilePath target;
private final String filenamesEncoding;

ReadFromTar(FilePath target, Pipe pipe, String description, @NonNull TarCompression compression) {
ReadFromTar(FilePath target, Pipe pipe, String description, @NonNull TarCompression compression, Charset filenamesEncoding) {
this.target = target;
this.pipe = pipe;
this.description = description;
this.compression = compression;
this.filenamesEncoding = filenamesEncoding.name();
}

private static final long serialVersionUID = 1L;

@Override
public Void invoke(File f, VirtualChannel channel) throws IOException {
try (InputStream in = pipe.getIn()) {
readFromTar(target.remote + '/' + description, f, compression.extract(in));
readFromTar(target.remote + '/' + description, f, compression.extract(in), Charset.forName(filenamesEncoding));
return null;
}
}
Expand All @@ -2985,18 +2988,20 @@ private static class WriteToTar extends MasterToSlaveFileCallable<Integer> {
private final DirScanner scanner;
private final Pipe pipe;
private final TarCompression compression;
private final String filenamesEncoding;

WriteToTar(DirScanner scanner, Pipe pipe, @NonNull TarCompression compression) {
WriteToTar(DirScanner scanner, Pipe pipe, @NonNull TarCompression compression, Charset filenamesEncoding) {
this.scanner = scanner;
this.pipe = pipe;
this.compression = compression;
this.filenamesEncoding = filenamesEncoding.name();
}

private static final long serialVersionUID = 1L;

@Override
public Integer invoke(File f, VirtualChannel channel) throws IOException, InterruptedException {
return writeToTar(f, scanner, compression.compress(pipe.getOut()));
return writeToTar(f, scanner, compression.compress(pipe.getOut()), Charset.forName(filenamesEncoding));
}
}

Expand All @@ -3005,17 +3010,19 @@ private static class CopyRecursiveRemoteToLocal extends MasterToSlaveFileCallabl
private final Pipe pipe;
private final DirScanner scanner;
private final TarCompression compression;
private final String filenamesEncoding;

CopyRecursiveRemoteToLocal(Pipe pipe, DirScanner scanner, @NonNull TarCompression compression) {
CopyRecursiveRemoteToLocal(Pipe pipe, DirScanner scanner, @NonNull TarCompression compression, Charset filenamesEncoding) {
this.pipe = pipe;
this.scanner = scanner;
this.compression = compression;
this.filenamesEncoding = filenamesEncoding.name();
}

@Override
public Integer invoke(File f, VirtualChannel channel) throws IOException {
try (OutputStream out = pipe.getOut()) {
return writeToTar(f, scanner, compression.compress(out));
return writeToTar(f, scanner, compression.compress(out), Charset.forName(filenamesEncoding));
}
}
}
Expand Down Expand Up @@ -3047,22 +3054,26 @@ public int tar(OutputStream out, DirScanner scanner) throws IOException, Interru
* @return
* number of files/directories that are written.
*/
private static Integer writeToTar(File baseDir, DirScanner scanner, OutputStream out) throws IOException {
Archiver tw = ArchiverFactory.TAR.create(out);
private static Integer writeToTar(File baseDir, DirScanner scanner, OutputStream out, Charset filenamesEncoding) throws IOException {
Archiver tw = ArchiverFactory.TAR.create(out, filenamesEncoding);
try (tw) {
scanner.scan(baseDir, tw);
}
return tw.countEntries();
}

private static void readFromTar(String name, File baseDir, InputStream in) throws IOException {
readFromTar(name, baseDir, in, Charset.defaultCharset());
}

/**
* Reads from a tar stream and stores obtained files to the base dir.
* Supports large files > 10 GB since 1.627 when this was migrated to use commons-compress.
*/
private static void readFromTar(String name, File baseDir, InputStream in) throws IOException {
private static void readFromTar(String name, File baseDir, InputStream in, Charset filenamesEncoding) throws IOException {

// TarInputStream t = new TarInputStream(in);
try (TarArchiveInputStream t = new TarArchiveInputStream(in)) {
try (TarArchiveInputStream t = new TarArchiveInputStream(in, filenamesEncoding.name())) {
TarArchiveEntry te;
while ((te = t.getNextTarEntry()) != null) {
File f = new File(baseDir, te.getName());
Expand Down
23 changes: 18 additions & 5 deletions core/src/main/java/hudson/util/io/ArchiverFactory.java
View file
Expand Up @@ -30,6 +30,7 @@
import java.io.IOException;
import java.io.OutputStream;
import java.io.Serializable;
import java.nio.charset.Charset;
import java.nio.file.OpenOption;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
Expand All @@ -43,9 +44,21 @@
public abstract class ArchiverFactory implements Serializable {
/**
* Creates an archiver on top of the given stream.
* File names in the archive are encoded with default character set.
*/
@NonNull
public abstract Archiver create(OutputStream out) throws IOException;
public Archiver create(OutputStream out) throws IOException {
return create(out, Charset.defaultCharset());
}

/**
* Creates an archiver on top of the given stream.
*
* @param filenamesEncoding the encoding to be used in the archive for file names
* @since TODO
*/
@NonNull
public abstract Archiver create(OutputStream out, Charset filenamesEncoding) throws IOException;

/**
* Uncompressed tar format.
Expand Down Expand Up @@ -85,8 +98,8 @@ private TarArchiverFactory(TarCompression method) {

@NonNull
@Override
public Archiver create(OutputStream out) throws IOException {
return new TarArchiver(method.compress(out));
public Archiver create(OutputStream out, Charset filenamesEncoding) throws IOException {
return new TarArchiver(method.compress(out), filenamesEncoding);
}

private static final long serialVersionUID = 1L;
Expand All @@ -108,8 +121,8 @@ private static final class ZipArchiverFactory extends ArchiverFactory {

@NonNull
@Override
public Archiver create(OutputStream out) {
return new ZipArchiver(out, prefix, openOptions);
public Archiver create(OutputStream out, Charset filenamesEncoding) {
return new ZipArchiver(out, prefix, filenamesEncoding, openOptions);
}

private static final long serialVersionUID = 1L;
Expand Down
5 changes: 3 additions & 2 deletions core/src/main/java/hudson/util/io/TarArchiver.java
View file
Expand Up @@ -32,6 +32,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.attribute.BasicFileAttributes;
Expand All @@ -49,8 +50,8 @@ final class TarArchiver extends Archiver {
private final byte[] buf = new byte[8192];
private final TarArchiveOutputStream tar;

TarArchiver(OutputStream out) {
tar = new TarArchiveOutputStream(out);
TarArchiver(OutputStream out, Charset filenamesEncoding) {
tar = new TarArchiveOutputStream(out, filenamesEncoding.name());
tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);
tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
}
Expand Down
7 changes: 4 additions & 3 deletions core/src/main/java/hudson/util/io/ZipArchiver.java
View file
Expand Up @@ -32,6 +32,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.OpenOption;
Expand All @@ -54,12 +55,12 @@ final class ZipArchiver extends Archiver {
private final String prefix;

ZipArchiver(OutputStream out) {
this(out, "");
this(out, "", Charset.defaultCharset());
}

// Restriction added for clarity, it's a package class, you should not use it outside of Jenkins core
@Restricted(NoExternalUse.class)
ZipArchiver(OutputStream out, String prefix, OpenOption... openOptions) {
ZipArchiver(OutputStream out, String prefix, Charset filenamesEncoding, OpenOption... openOptions) {
this.openOptions = openOptions;
if (prefix == null || prefix.isBlank()) {
this.prefix = "";
Expand All @@ -68,7 +69,7 @@ final class ZipArchiver extends Archiver {
}

zip = new ZipOutputStream(out);
zip.setEncoding(System.getProperty("file.encoding"));
zip.setEncoding(filenamesEncoding.name());
zip.setUseZip64(Zip64Mode.AsNeeded);
}

Expand Down

0 comments on commit 9f68c18

Please sign in to comment.