Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Copying fifos hangs. #727

Open
wants to merge 27 commits into
base: master
Choose a base branch
from
Open

Copying fifos hangs. #727

Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
e17d624
Don't attempt to copy fifos.
bruce-one May 31, 2017
52c412f
Avoid fifos via symlinks.
bruce-one May 31, 2017
e4a911d
Better replicate cp during direct fifo copies.
bruce-one May 31, 2017
eaaa40a
Bugfix: don't break offset during copies.
bruce-one May 31, 2017
476d4b4
Test fix, use printf for osx not echo.
bruce-one Jun 1, 2017
c71edc2
Check exec exit code to fix tests on Windows.
bruce-one Jun 2, 2017
d469212
Skip fifo tests on Windows.
bruce-one Jun 2, 2017
5689dc0
Typo.
bruce-one Jun 2, 2017
777fe04
Shell out to create fifos during recursive copy.
bruce-one Jun 19, 2017
8fa217b
Handle "recursive" flag for special non-files.
bruce-one Jun 19, 2017
18c513e
Cleanup.
bruce-one Jun 19, 2017
be93375
Error messages.
bruce-one Jun 19, 2017
599a2c5
mknod fails without root.
bruce-one Jun 19, 2017
2d3af53
Linting.
bruce-one Jun 19, 2017
d942f3f
Permissions on special files.
bruce-one Jun 19, 2017
fad43aa
Logging.
bruce-one Jun 19, 2017
e6760bb
Lint.
bruce-one Jun 19, 2017
045c290
chown and chmod special files.
bruce-one Jun 19, 2017
40518f7
Use execFileSync for safety and simplicity.
bruce-one Jun 20, 2017
c69ddc3
Use mkfifo for fifo creation due for OSX compat.
bruce-one Jun 20, 2017
b2932cf
Replace unix specific code with warning.
bruce-one Jun 21, 2017
6a4a6a0
Log that special files were skipped.
bruce-one Jun 22, 2017
f7957bf
Revert "Log that special files were skipped."
bruce-one Jun 22, 2017
a4e5411
Doc.
bruce-one Jun 22, 2017
117894c
Treat special files the same whether recursive or not.
bruce-one Jun 22, 2017
48a0198
Doc.
bruce-one Jun 22, 2017
0d70322
Build doc.
bruce-one Jun 22, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
22 changes: 22 additions & 0 deletions src/common.js
View file
Open in desktop
Expand Up @@ -84,6 +84,28 @@ function log() {
}
exports.log = log;

var _logLater = [];
// facilate logging informational messages at the end of a command
function logLater(message) {
if (!_logLater.length) {
// only logs if outputLaterLog hasn't been called
process.on('exit', outputLaterLog);
}
if (_logLater.indexOf(message) === -1) {
_logLater = _logLater.concat(message);
}
}
exports.logLater = logLater;

function outputLaterLog() {
_logLater.forEach(function (m) {
log(m);
});
_logLater = [];
process.removeListener('exit', outputLaterLog);
}
exports.outputLaterLog = outputLaterLog;

// Converts strings to be equivalent across all platforms. Primarily responsible
// for making sure we use '/' instead of '\' as path separators, but this may be
// expanded in the future if necessary
Expand Down
36 changes: 26 additions & 10 deletions src/cp.js
View file
Open in desktop
Expand Up @@ -34,16 +34,31 @@ function copyFileSync(srcFile, destFile, options) {
// If we're here, destFile probably doesn't exist, so just do a normal copy
}

if (fs.lstatSync(srcFile).isSymbolicLink() && !options.followsymlink) {
try {
fs.lstatSync(destFile);
common.unlinkSync(destFile); // re-link it
} catch (e) {
// it doesn't exist, so no work needs to be done
}
var srcFileStat = fs.lstatSync(srcFile);

var symlinkFull = fs.readlinkSync(srcFile);
fs.symlinkSync(symlinkFull, destFile, isWindows ? 'junction' : null);
if (srcFileStat.isSymbolicLink()) {
if (options.followsymlink) {
// recurse to copy the target of the symlink to the dest to ensure we handle complex scenarios
return copyFileSync(fs.realpathSync(srcFile), destFile, options);
} else {
try {
fs.lstatSync(destFile);
common.unlinkSync(destFile); // re-link it
} catch (e) {
// it doesn't exist, so no work needs to be done
}

var symlinkFull = fs.readlinkSync(srcFile);
fs.symlinkSync(symlinkFull, destFile, isWindows ? 'junction' : null);
}
// recursive is a special case for non-files that means "create an equivalent at the dest" rather than reading and writing
} else if (options.recursive && (srcFileStat.isFIFO() || srcFileStat.isCharacterDevice() || srcFileStat.isBlockDevice())) {
var type = 'unknown file';
if (srcFileStat.isFIFO()) type = 'fifo';
if (srcFileStat.isCharacterDevice()) type = 'block device';
if (srcFileStat.isBlockDevice()) type = 'character device';
common.error('copyFileSync: ' + type + ' is not supported (' + srcFile + ')', { continue: true });
common.logLater('copyFileSync: Special files were encountered during this operation. Please investigate shelljs plugins if you would like to add support for these.');
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the common.error() log message is sufficient. Let's get rid of logLater and move something like this into the docs. You can edit the //@ section in this file, and then run npm run gendocs.

} else {
var buf = common.buffer();
var bufLength = buf.length;
Expand All @@ -67,7 +82,7 @@ function copyFileSync(srcFile, destFile, options) {
}

while (bytesRead === bufLength) {
bytesRead = fs.readSync(fdr, buf, 0, bufLength, pos);
bytesRead = fs.readSync(fdr, buf, 0, bufLength, pos || -1);
fs.writeSync(fdw, buf, 0, bytesRead);
pos += bytesRead;
}
Expand Down Expand Up @@ -294,6 +309,7 @@ function _cp(options, sources, dest) {
}
}); // forEach(src)

common.outputLaterLog();
return new common.ShellString('', common.state.error, common.state.errorCode);
}
module.exports = _cp;
79 changes: 79 additions & 0 deletions test/cp.js
View file
Open in desktop
Expand Up @@ -7,6 +7,7 @@ import utils from './utils/utils';

const oldMaxDepth = shell.config.maxdepth;
const CWD = process.cwd();
const isRoot = process.getuid && process.getuid() === 0;

test.beforeEach(t => {
t.context.tmp = utils.getTempDir();
Expand Down Expand Up @@ -755,3 +756,81 @@ test('should not overwrite recently created files (not give error no-force mode)
// Ensure First file is copied
t.is(shell.cat(`${t.context.tmp}/file1`).toString(), 'test1');
});

test('should copy fifo contents', t => {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this case should also produce a warning. We'll completely drop fifo support and add it back in the plugin.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This fits in the "act like cp" sense.

Being, without the recursive flag cp will treat these files as just "dumb files" rather than special files.

From what I'd read, that's the context of where this came from in posix's cp; i.e. originally it would just treat everything as a file and do a dumb copy (creating a normal-file rather than a special-file), but then later on support was added for being clever with special-files. Hence, my brain goes "native shelljs can treat them like normal-files" and "plugin'd shelljs gains the knowledge to treat them as special-files" parallels "normal cp is dumb" and "modern cp -R is smart". If that helps frame why I'd left this in?

There's a conflict between "act like cp" and "pull all the unix out" :-s
I was inclined to "act like cp" (and treat them like normal files, and hence hang while copying) because it's simple and native? Although that could just be me :-)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's a conflict between "act like cp" and "pull all the unix out" :-s

I agree these goals are in conflict. I think in this case we should try to stay internally consistent. We should output a warning when copying any fifos and delegate to the plugin to provide actual posix behavior.

If the plugin aims for posix compatibility, what arguments does it need passed? It sounds like it needs options.recursive, srcfile, & destFile, is that correct?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, sounds right :-) (Assuming the srcFile symlink has already been resolved if options.followsymlink === true.) Perhaps it's worth passing the whole options object though? (Thinking in terms of flexibility, eg if more flags are added (eg -p))

(And I agree with your perspective on this: the whole reason I opened this PR in the first place was because it tripped me up, and staying internally consistent would have saved me whereas "acting like cp" wouldn't. Just hadn't quite come to that realisation yet :-p )

if (process.platform !== 'win32') { // fs.exists doesn't support fifos on windows
shell.exec(`mkfifo ${t.context.tmp}/fifo`);
shell.exec(`printf test1 > ${t.context.tmp}/fifo`, { async: true });
t.truthy(fs.existsSync(`${t.context.tmp}/fifo`));
const result = shell.cp(`${t.context.tmp}/fifo`, `${t.context.tmp}/newFifo`);
t.falsy(shell.error());
t.is(result.code, 0);
t.is(shell.cat(`${t.context.tmp}/newFifo`).toString(), 'test1');
t.falsy(fs.statSync(`${t.context.tmp}/newFifo`).isFIFO());
}
});

test('should warn about fifos in recursive mode', t => {
if (process.platform !== 'win32') { // fs.exists doesn't support fifos on windows
shell.exec(`mkfifo ${t.context.tmp}/fifo`);
shell.exec(`printf test1 > ${t.context.tmp}/fifo`, { async: true });
t.truthy(fs.existsSync(`${t.context.tmp}/fifo`));
const result = shell.cp('-r', `${t.context.tmp}/fifo`, `${t.context.tmp}/newFifo`);
t.truthy(shell.error());
t.is(result.code, 1);
t.falsy(fs.existsSync(`${t.context.tmp}/newFifo`));
}
});

test('should copy fifo contents via symlinks', t => {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For consistency, should this also be a warning?

if (process.platform !== 'win32') { // fs.exists doesn't support fifos on windows
shell.exec(`mkfifo ${t.context.tmp}/fifo`);
shell.exec(`printf test1 > ${t.context.tmp}/fifo`, { async: true });
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd prefer to see shx echo -n test1 here. Appveyor may have the command, but it may not be consistent with unix printf, and I don't think it's available by default on Windows systems.

Copy link
Author

@bruce-one bruce-one Jun 2, 2017 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Iirc, osx's echo doesn't support the -n... (Which was why I changed it initially)

Also worth noting named pipes aren't supported directly by Windows (only via cygwin etc, which didn't appear reliable)

(Could match the \n though if echo is preferred)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Iirc, osx's echo doesn't support the -n... (Which was why I changed it initially)

shx echo is platform agnostic, so it runs on OS X, Win, and Linux. The behavior is guaranteed to be the same across all machines, which is what you want out of an external dependency.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh yeah :-) good point :-)

Is it possible to easily shell out to shelljs's echo implementation? (I've shelled out here because the event loop is blocked by the cp, so need a way to get data into the fifo in the background; and atm this is shelling out to the /bin/sh impl (iirc?) and using it's echo/printf (although maybe there's a better way?)).

shell.pushd(t.context.tmp);
fs.symlinkSync('fifo', 'symFifo');
shell.popd();
t.truthy(fs.existsSync(`${t.context.tmp}/symFifo`));
const result = shell.cp(`${t.context.tmp}/symFifo`, `${t.context.tmp}/newFifo`);
t.falsy(shell.error());
t.is(result.code, 0);
t.is(shell.cat(`${t.context.tmp}/newFifo`).toString(), 'test1');
t.falsy(fs.statSync(`${t.context.tmp}/newFifo`).isFIFO());
}
});

test('should warn about fifos when set to recursive mode, via symlinks', t => {
if (process.platform !== 'win32') { // fs.exists doesn't support fifos on windows
shell.exec(`mkfifo ${t.context.tmp}/fifo`);
shell.exec(`printf test1 > ${t.context.tmp}/fifo`, { async: true });
shell.pushd(t.context.tmp);
fs.symlinkSync('fifo', 'symFifo');
shell.popd();
t.truthy(fs.existsSync(`${t.context.tmp}/symFifo`));
const result = shell.cp('-rL', `${t.context.tmp}/symFifo`, `${t.context.tmp}/newFifo`);
t.truthy(shell.error());
t.is(result.code, 1);
t.falsy(fs.existsSync(`${t.context.tmp}/newFifo`));
}
});

test('should warn about character devices when set to recursive mode', t => {
if (process.platform !== 'win32' && isRoot) {
shell.exec(`mknod ${t.context.tmp}/zero c 1 5`);
t.truthy(fs.existsSync(`${t.context.tmp}/zero`));
const result = shell.cp('-r', `${t.context.tmp}/zero`, `${t.context.tmp}/newZero`);
t.truthy(shell.error());
t.is(result.code, 1);
t.falsy(fs.existsSync(`${t.context.tmp}/newZero`));
}
});

test('should warn about block devices when set to recursive mode', t => {
if (process.platform !== 'win32' && isRoot) {
shell.exec(`mknod ${t.context.tmp}/sda1 b 8 1`);
t.truthy(fs.existsSync(`${t.context.tmp}/sda1`));
const result = shell.cp('-r', `${t.context.tmp}/sda1`, `${t.context.tmp}/newSda1`);
t.truthy(shell.error());
t.is(result.code, 1);
t.falsy(fs.existsSync(`${t.context.tmp}/newSda1`));
}
});