Merge pull request #5775 from vidartf/yarn-opt

Use yarn-deduplicate instead of cleaning
jupyterlab · Apr 25, 2019 · f40af6c · f40af6c
2 parents 7844502 + d3a62af
commit f40af6c
Show file tree

Hide file tree

Showing 7 changed files with 54 additions and 12 deletions.
diff --git a/buildutils/src/update-core-mode.ts b/buildutils/src/update-core-mode.ts
@@ -35,6 +35,12 @@ utils.writePackageData(path.join(staging, 'package.json'), data);
 // Create a new yarn.lock file to ensure it is correct.
 fs.removeSync(path.join(staging, 'yarn.lock'));
 utils.run('jlpm', { cwd: staging });
+try {
+  utils.run('jlpm yarn-deduplicate -s fewer --fail', { cwd: staging });
+} catch {
+  // re-run install if we deduped packages!
+  utils.run('jlpm', { cwd: staging });
+}
 
 // Build the core assets.
 utils.run('jlpm run build:prod', { cwd: staging });
diff --git a/dev_mode/package.json b/dev_mode/package.json
@@ -136,7 +136,8 @@
     "webpack": "~4.29.6",
     "webpack-cli": "^3.3.0",
     "webpack-merge": "^4.2.1",
-    "webpack-visualizer-plugin": "^0.1.11"
+    "webpack-visualizer-plugin": "^0.1.11",
+    "yarn-deduplicate": "^1.1.1"
   },
   "engines": {
     "node": ">=6.11.5"

diff --git a/jupyterlab/commands.py b/jupyterlab/commands.py
@@ -99,6 +99,13 @@ def ensure_dev(logger=None):
         yarn_proc = Process(['node', YARN_PATH], cwd=parent, logger=logger)
         yarn_proc.wait()
 
+        yarn_proc = Process(['node', YARN_PATH, 'yarn-deduplicate', '-s', 'fewer', '--fail'],
+                            cwd=parent, logger=logger)
+        had_dupes = yarn_proc.wait() != 0
+        if had_dupes:
+            yarn_proc = Process(['node', YARN_PATH], cwd=parent, logger=logger)
+            yarn_proc.wait()
+
     if not osp.exists(pjoin(parent, 'dev_mode', 'static')):
         yarn_proc = Process(['node', YARN_PATH, 'build'], cwd=parent,
                             logger=logger)
@@ -118,6 +125,13 @@ def ensure_core(logger=None):
         yarn_proc = Process(['node', YARN_PATH], cwd=staging, logger=logger)
         yarn_proc.wait()
 
+        yarn_proc = Process(['node', YARN_PATH, 'yarn-deduplicate', '-s', 'fewer', '--fail'],
+                            cwd=staging, logger=logger)
+        had_dupes = yarn_proc.wait() != 0
+        if had_dupes:
+            yarn_proc = Process(['node', YARN_PATH], cwd=staging, logger=logger)
+            yarn_proc.wait()
+
     if not osp.exists(pjoin(HERE, 'static')):
         yarn_proc = Process(['node', YARN_PATH, 'build'], cwd=staging,
                             logger=logger)
@@ -142,6 +156,13 @@ def watch_packages(logger=None):
         yarn_proc = Process(['node', YARN_PATH], cwd=parent, logger=logger)
         yarn_proc.wait()
 
+        yarn_proc = Process(['node', YARN_PATH, 'yarn-deduplicate', '-s', 'fewer', '--fail'],
+                            cwd=parent, logger=logger)
+        had_dupes = yarn_proc.wait() != 0
+        if had_dupes:
+            yarn_proc = Process(['node', YARN_PATH], cwd=parent, logger=logger)
+            yarn_proc.wait()
+
     logger = _ensure_logger(logger)
     ts_dir = osp.abspath(osp.join(HERE, '..', 'packages', 'metapackage'))
 
@@ -447,6 +468,11 @@ def build(self, name=None, version=None, public_url=None,
             msg = 'npm dependencies failed to install'
             self.logger.error(msg)
             raise RuntimeError(msg)
+        had_dupes = 0 != self._run(
+            ['node', YARN_PATH, 'yarn-deduplicate', '-s', 'fewer', '--fail'],
+            cwd=staging)
+        if had_dupes:
+            self._run(['node', YARN_PATH, 'install'], cwd=staging)
 
         # Build the app.
         ret = self._run(['node', YARN_PATH, 'run', command], cwd=staging)
@@ -465,6 +491,11 @@ def watch(self):
 
         # Make sure packages are installed.
         self._run(['node', YARN_PATH, 'install'], cwd=staging)
+        had_dupes = 0 != self._run(
+            ['node', YARN_PATH, 'yarn-deduplicate', '-s', 'fewer', '--fail'],
+            cwd=staging)
+        if had_dupes:
+            self._run(['node', YARN_PATH, 'install'], cwd=staging)
 
         proc = WatchHelper(['node', YARN_PATH, 'run', 'watch'],
                            cwd=pjoin(self.app_dir, 'staging'),
@@ -882,12 +913,6 @@ def _populate_staging(self, name=None, version=None, public_url=None,
             target = pjoin(staging, fname)
             shutil.copy(pjoin(HERE, 'staging', fname), target)
 
-        # Remove an existing yarn.lock file
-        # Because otherwise we can end up with unwanted duplicates
-        # cf https://github.com/yarnpkg/yarn/issues/3967
-        if osp.exists(pjoin(staging, 'yarn.lock')):
-            os.remove(pjoin(staging, 'yarn.lock'))
-
         # Ensure a clean templates directory
         templates = pjoin(staging, 'templates')
         if osp.exists(templates):

diff --git a/jupyterlab/labapp.py b/jupyterlab/labapp.py
@@ -60,8 +60,8 @@ class LabBuildApp(JupyterApp):
     dev_build = Bool(True, config=True,
         help="Whether to build in dev mode (defaults to dev mode)")
 
-    pre_clean = Bool(True, config=True,
-        help="Whether to clean before building (defaults to True)")
+    pre_clean = Bool(False, config=True,
+        help="Whether to clean before building (defaults to False)")
 
     def start(self):
         command = 'build:prod' if not self.dev_build else 'build'

diff --git a/jupyterlab/staging/package.json b/jupyterlab/staging/package.json
@@ -136,7 +136,8 @@
     "webpack": "~4.29.6",
     "webpack-cli": "^3.3.0",
     "webpack-merge": "^4.2.1",
-    "webpack-visualizer-plugin": "^0.1.11"
+    "webpack-visualizer-plugin": "^0.1.11",
+    "yarn-deduplicate": "^1.1.1"
   },
   "engines": {
     "node": ">=6.11.5"

diff --git a/package.json b/package.json
@@ -7,7 +7,7 @@
     "analyze:dev": "cd dev_mode && jlpm run build --analyze && opn static/stats.html",
     "analyze:prod": "cd dev_mode && jlpm run build:prod --analyze && opn static/stats.html",
     "build": "jlpm run build:dev",
-    "build:core": "cd jupyterlab/staging && jlpm && jlpm run build",
+    "build:core": "cd jupyterlab/staging && jlpm && (jlpm yarn-deduplicate -s fewer --fail || jlpm) && jlpm run build",
     "build:dev": "jlpm run integrity && jlpm run build:packages && cd dev_mode && jlpm run build",
     "build:dev:prod": "jlpm run integrity && jlpm run build:packages && cd dev_mode && jlpm run build:prod",
     "build:examples": "lerna run build --scope \"@jupyterlab/example-*\"",

diff --git a/yarn.lock b/yarn.lock
@@ -3264,7 +3264,7 @@ combined-stream@1.0.6, combined-stream@^1.0.5, combined-stream@~1.0.5, combined-
   dependencies:
     delayed-stream "~1.0.0"
 
-commander@2, commander@^2.12.1, commander@^2.14.1, commander@^2.19.0, commander@^2.5.0, commander@^2.9.0, commander@~2.20.0:
+commander@2, commander@^2.10.0, commander@^2.12.1, commander@^2.14.1, commander@^2.19.0, commander@^2.5.0, commander@^2.9.0, commander@~2.20.0:
   version "2.20.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422"
   integrity sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==
@@ -11851,6 +11851,15 @@ yargs@~13.2.2:
     y18n "^4.0.0"
     yargs-parser "^13.0.0"
 
+yarn-deduplicate@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/yarn-deduplicate/-/yarn-deduplicate-1.1.1.tgz#19b4a87654b66f55bf3a4bd6b153b4e4ab1b6e6d"
+  integrity sha512-2FDJ1dFmtvqhRmfja89ohYzpaheCYg7BFBSyaUq+kxK0y61C9oHv1XaQovCWGJtP2WU8PksQOgzMVV7oQOobzw==
+  dependencies:
+    "@yarnpkg/lockfile" "^1.1.0"
+    commander "^2.10.0"
+    semver "^5.3.0"
+
 yarn@1.15.2:
   version "1.15.2"
   resolved "https://registry.yarnpkg.com/yarn/-/yarn-1.15.2.tgz#7a064ca81ca34235f16376ad2f796ed432f9e285"