From d3a62afd602c6629ca944b38d9f9230870252885 Mon Sep 17 00:00:00 2001 From: Vidar Tonaas Fauske Date: Fri, 8 Feb 2019 16:52:07 +0100 Subject: [PATCH] Use yarn-deduplicate Use yarn-deduplicate instead of cleaning out yarn lock every build. --- buildutils/src/update-core-mode.ts | 6 +++++ dev_mode/package.json | 3 ++- jupyterlab/commands.py | 37 +++++++++++++++++++++++++----- jupyterlab/labapp.py | 4 ++-- jupyterlab/staging/package.json | 3 ++- package.json | 2 +- yarn.lock | 11 ++++++++- 7 files changed, 54 insertions(+), 12 deletions(-) diff --git a/buildutils/src/update-core-mode.ts b/buildutils/src/update-core-mode.ts index 5e684c0fdc28..cd5d0bd14f1d 100644 --- a/buildutils/src/update-core-mode.ts +++ b/buildutils/src/update-core-mode.ts @@ -35,6 +35,12 @@ utils.writePackageData(path.join(staging, 'package.json'), data); // Create a new yarn.lock file to ensure it is correct. fs.removeSync(path.join(staging, 'yarn.lock')); utils.run('jlpm', { cwd: staging }); +try { + utils.run('jlpm yarn-deduplicate -s fewer --fail', { cwd: staging }); +} catch { + // re-run install if we deduped packages! + utils.run('jlpm', { cwd: staging }); +} // Build the core assets. utils.run('jlpm run build:prod', { cwd: staging }); diff --git a/dev_mode/package.json b/dev_mode/package.json index 75d3b10675d7..91316a681bfb 100644 --- a/dev_mode/package.json +++ b/dev_mode/package.json @@ -136,7 +136,8 @@ "webpack": "~4.29.6", "webpack-cli": "^3.3.0", "webpack-merge": "^4.2.1", - "webpack-visualizer-plugin": "^0.1.11" + "webpack-visualizer-plugin": "^0.1.11", + "yarn-deduplicate": "^1.1.1" }, "engines": { "node": ">=6.11.5" diff --git a/jupyterlab/commands.py b/jupyterlab/commands.py index 16907423bf85..a5ed174089da 100644 --- a/jupyterlab/commands.py +++ b/jupyterlab/commands.py @@ -99,6 +99,13 @@ def ensure_dev(logger=None): yarn_proc = Process(['node', YARN_PATH], cwd=parent, logger=logger) yarn_proc.wait() + yarn_proc = Process(['node', YARN_PATH, 'yarn-deduplicate', '-s', 'fewer', '--fail'], + cwd=parent, logger=logger) + had_dupes = yarn_proc.wait() != 0 + if had_dupes: + yarn_proc = Process(['node', YARN_PATH], cwd=parent, logger=logger) + yarn_proc.wait() + if not osp.exists(pjoin(parent, 'dev_mode', 'static')): yarn_proc = Process(['node', YARN_PATH, 'build'], cwd=parent, logger=logger) @@ -118,6 +125,13 @@ def ensure_core(logger=None): yarn_proc = Process(['node', YARN_PATH], cwd=staging, logger=logger) yarn_proc.wait() + yarn_proc = Process(['node', YARN_PATH, 'yarn-deduplicate', '-s', 'fewer', '--fail'], + cwd=staging, logger=logger) + had_dupes = yarn_proc.wait() != 0 + if had_dupes: + yarn_proc = Process(['node', YARN_PATH], cwd=staging, logger=logger) + yarn_proc.wait() + if not osp.exists(pjoin(HERE, 'static')): yarn_proc = Process(['node', YARN_PATH, 'build'], cwd=staging, logger=logger) @@ -142,6 +156,13 @@ def watch_packages(logger=None): yarn_proc = Process(['node', YARN_PATH], cwd=parent, logger=logger) yarn_proc.wait() + yarn_proc = Process(['node', YARN_PATH, 'yarn-deduplicate', '-s', 'fewer', '--fail'], + cwd=parent, logger=logger) + had_dupes = yarn_proc.wait() != 0 + if had_dupes: + yarn_proc = Process(['node', YARN_PATH], cwd=parent, logger=logger) + yarn_proc.wait() + logger = _ensure_logger(logger) ts_dir = osp.abspath(osp.join(HERE, '..', 'packages', 'metapackage')) @@ -447,6 +468,11 @@ def build(self, name=None, version=None, public_url=None, msg = 'npm dependencies failed to install' self.logger.error(msg) raise RuntimeError(msg) + had_dupes = 0 != self._run( + ['node', YARN_PATH, 'yarn-deduplicate', '-s', 'fewer', '--fail'], + cwd=staging) + if had_dupes: + self._run(['node', YARN_PATH, 'install'], cwd=staging) # Build the app. ret = self._run(['node', YARN_PATH, 'run', command], cwd=staging) @@ -465,6 +491,11 @@ def watch(self): # Make sure packages are installed. self._run(['node', YARN_PATH, 'install'], cwd=staging) + had_dupes = 0 != self._run( + ['node', YARN_PATH, 'yarn-deduplicate', '-s', 'fewer', '--fail'], + cwd=staging) + if had_dupes: + self._run(['node', YARN_PATH, 'install'], cwd=staging) proc = WatchHelper(['node', YARN_PATH, 'run', 'watch'], cwd=pjoin(self.app_dir, 'staging'), @@ -882,12 +913,6 @@ def _populate_staging(self, name=None, version=None, public_url=None, target = pjoin(staging, fname) shutil.copy(pjoin(HERE, 'staging', fname), target) - # Remove an existing yarn.lock file - # Because otherwise we can end up with unwanted duplicates - # cf https://github.com/yarnpkg/yarn/issues/3967 - if osp.exists(pjoin(staging, 'yarn.lock')): - os.remove(pjoin(staging, 'yarn.lock')) - # Ensure a clean templates directory templates = pjoin(staging, 'templates') if osp.exists(templates): diff --git a/jupyterlab/labapp.py b/jupyterlab/labapp.py index bb0ae7d3a74b..83f59e1f9d9d 100644 --- a/jupyterlab/labapp.py +++ b/jupyterlab/labapp.py @@ -60,8 +60,8 @@ class LabBuildApp(JupyterApp): dev_build = Bool(True, config=True, help="Whether to build in dev mode (defaults to dev mode)") - pre_clean = Bool(True, config=True, - help="Whether to clean before building (defaults to True)") + pre_clean = Bool(False, config=True, + help="Whether to clean before building (defaults to False)") def start(self): command = 'build:prod' if not self.dev_build else 'build' diff --git a/jupyterlab/staging/package.json b/jupyterlab/staging/package.json index b7dbdea49ef8..fa71dc937098 100644 --- a/jupyterlab/staging/package.json +++ b/jupyterlab/staging/package.json @@ -136,7 +136,8 @@ "webpack": "~4.29.6", "webpack-cli": "^3.3.0", "webpack-merge": "^4.2.1", - "webpack-visualizer-plugin": "^0.1.11" + "webpack-visualizer-plugin": "^0.1.11", + "yarn-deduplicate": "^1.1.1" }, "engines": { "node": ">=6.11.5" diff --git a/package.json b/package.json index 4fa3ac4c65d6..f70ee99f2b92 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "analyze:dev": "cd dev_mode && jlpm run build --analyze && opn static/stats.html", "analyze:prod": "cd dev_mode && jlpm run build:prod --analyze && opn static/stats.html", "build": "jlpm run build:dev", - "build:core": "cd jupyterlab/staging && jlpm && jlpm run build", + "build:core": "cd jupyterlab/staging && jlpm && (jlpm yarn-deduplicate -s fewer --fail || jlpm) && jlpm run build", "build:dev": "jlpm run integrity && jlpm run build:packages && cd dev_mode && jlpm run build", "build:dev:prod": "jlpm run integrity && jlpm run build:packages && cd dev_mode && jlpm run build:prod", "build:examples": "lerna run build --scope \"@jupyterlab/example-*\"", diff --git a/yarn.lock b/yarn.lock index d41f4a2206c3..bad949c01fa4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3264,7 +3264,7 @@ combined-stream@1.0.6, combined-stream@^1.0.5, combined-stream@~1.0.5, combined- dependencies: delayed-stream "~1.0.0" -commander@2, commander@^2.12.1, commander@^2.14.1, commander@^2.19.0, commander@^2.5.0, commander@^2.9.0, commander@~2.20.0: +commander@2, commander@^2.10.0, commander@^2.12.1, commander@^2.14.1, commander@^2.19.0, commander@^2.5.0, commander@^2.9.0, commander@~2.20.0: version "2.20.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422" integrity sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ== @@ -11851,6 +11851,15 @@ yargs@~13.2.2: y18n "^4.0.0" yargs-parser "^13.0.0" +yarn-deduplicate@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/yarn-deduplicate/-/yarn-deduplicate-1.1.1.tgz#19b4a87654b66f55bf3a4bd6b153b4e4ab1b6e6d" + integrity sha512-2FDJ1dFmtvqhRmfja89ohYzpaheCYg7BFBSyaUq+kxK0y61C9oHv1XaQovCWGJtP2WU8PksQOgzMVV7oQOobzw== + dependencies: + "@yarnpkg/lockfile" "^1.1.0" + commander "^2.10.0" + semver "^5.3.0" + yarn@1.15.2: version "1.15.2" resolved "https://registry.yarnpkg.com/yarn/-/yarn-1.15.2.tgz#7a064ca81ca34235f16376ad2f796ed432f9e285"