Fix yarn registry with command argument (#7109)

* Use `yarn config list` to update the registry

* Move registry to `AppOptions`

* Always replace yarn registry if != default

* Force utf-8 for yarn.lock

jupyterlab/commands.py

@@ -44,6 +44,11 @@
 # If we are pinning the package, rename it `pin@<alias>`
 PIN_PREFIX = 'pin@'
 
+
+# Default Yarn registry used in default yarn.lock
+YARN_DEFAULT_REGISTRY = 'https://registry.yarnpkg.com'
+
+
 class ProgressProcess(Process):
 
     def __init__(self, cmd, logger=None, cwd=None, kill_event=None,
@@ -312,6 +317,8 @@ def __init__(self, logger=None, core_config=None, **kwargs):
 
     kill_event = Instance(Event, args=(), help='Event for aborting call')
 
+    registry = Unicode(help="NPM packages registry URL")
+
     @default('logger')
     def _default_logger(self):
         return logging.getLogger('jupyterlab')
@@ -326,6 +333,11 @@ def _default_app_dir(self):
     def _default_core_config(self):
         return CoreConfig()
 
+    @default('registry')
+    def _default_registry(self):
+        config = _yarn_config(self.logger)["yarn config"]
+        return config.get("registry", YARN_DEFAULT_REGISTRY)
+
 
 def _ensure_options(options, **kwargs):
     """Helper to use deprecated kwargs for AppOption"""
@@ -397,10 +409,8 @@ def uninstall_extension(name=None, app_dir=None, logger=None, all_=False, core_c
 
 def update_extension(name=None, all_=False, app_dir=None, logger=None, core_config=None, app_options=None):
     """Update an extension by name, or all extensions.
-
     Either `name` must be given as a string, or `all_` must be `True`.
     If `all_` is `True`, the value of `name` is ignored.
-
     Returns `True` if a rebuild is recommended, `False` otherwise.
     """
     app_options = _ensure_options(
@@ -564,8 +574,7 @@ def __init__(self, options):
         self.core_data = options.core_config._data
         self.info = self._get_app_info()
         self.kill_event = options.kill_event
-        # TODO: Make this configurable
-        self.registry = 'https://registry.npmjs.org'
+        self.registry = options.registry
 
     def install_extension(self, extension, existing=None, pin=None):
         """Install an extension package into JupyterLab.
@@ -1155,9 +1164,16 @@ def _populate_staging(self, name=None, version=None, static_url=None,
             json.dump(data, fid, indent=4)
 
         # copy known-good yarn.lock if missing
-        lock_path = pjoin(staging, 'yarn.lock')
-        if not osp.exists(lock_path):
-            shutil.copy(pjoin(HERE, 'staging', 'yarn.lock'), lock_path)
+        lock_path = pjoin(staging, 'yarn.lock')        
+        lock_template = pjoin(HERE, 'staging', 'yarn.lock')
+        if self.registry != YARN_DEFAULT_REGISTRY:  # Replace on the fly the yarn repository see #3658
+            with open(lock_template, encoding='utf-8') as f:
+                template = f.read()
+            template = template.replace(YARN_DEFAULT_REGISTRY, self.registry.strip("/"))
+            with open(lock_path, 'w', encoding='utf-8') as f:
+                f.write(template)
+        elif not osp.exists(lock_path):
+            shutil.copy(lock_template, lock_path)
 
     def _get_package_template(self, silent=False):
         """Get the template the for staging package.json file.
@@ -1734,6 +1750,37 @@ def _node_check(logger):
         msg = 'Please install nodejs %s before continuing. nodejs may be installed using conda or directly from the nodejs website.' % ver
         raise ValueError(msg)
 
+def _yarn_config(logger):
+    """Get the yarn configuration.
+    
+    Returns
+    -------
+    {"yarn config": dict, "npm config": dict} if unsuccessfull the subdictionary are empty
+    """
+    node = which('node')
+    configuration = {"yarn config": {}, "npm config": {}}
+    try:
+        output_binary = subprocess.check_output([node, YARN_PATH, 'config', 'list', '--json'], stderr=subprocess.PIPE, cwd=HERE)
+        output = output_binary.decode('utf-8')
+        lines = iter(output.splitlines())
+        try:
+            for line in lines:
+                info = json.loads(line)
+                if info["type"] == "info":
+                    key = info["data"]
+                    inspect = json.loads(next(lines))
+                    if inspect["type"] == "inspect":
+                        configuration[key] = inspect["data"]
+        except StopIteration:
+            pass
+        logger.debug("Yarn configuration loaded.")
+    except subprocess.CalledProcessError as e:
+        logger.error("Fail to get yarn configuration. {!s}{!s}".format(e.stderr.decode('utf-8'), e.output.decode('utf-8')))
+    except Exception as e:
+        logger.error("Fail to get yarn configuration. {!s}".format(e))
+    finally:
+        return configuration
+
 
 def _ensure_logger(logger=None):
     """Ensure that we have a logger"""

jupyterlab/tests/test_jupyterlab.py

@@ -52,7 +52,7 @@ def touch(file, mtime=None):
     return os.stat(file).st_mtime
 
 
-class TestExtension(TestCase):
+class AppHandlerTest(TestCase):
 
     def tempdir(self):
         td = TemporaryDirectory()
@@ -135,6 +135,9 @@ def ignore(dname, files):
         # Set pinned extension names
         self.pinned_packages = ['jupyterlab-test-extension@1.0', 'jupyterlab-test-extension@2.0']
 
+
+class TestExtension(AppHandlerTest):
+
     def test_install_extension(self):
         assert install_extension(self.mock_extension) is True
         path = pjoin(self.app_dir, 'extensions', '*.tgz')

jupyterlab/tests/test_registry.py

@@ -0,0 +1,84 @@
+"""Test yarn registry replacement"""
+
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+import os
+from os.path import join as pjoin
+import subprocess
+from tempfile import TemporaryDirectory
+from unittest import TestCase
+from unittest.mock import patch
+
+from jupyter_core import paths
+
+from jupyterlab import commands
+
+from .test_jupyterlab import AppHandlerTest
+
+
+class TestAppHandlerRegistry(AppHandlerTest):
+
+    def test_yarn_config(self):
+        with patch("subprocess.check_output") as check_output:
+            yarn_registry = "https://private.yarn/manager"
+            check_output.return_value = b'\n'.join([
+                b'{"type":"info","data":"yarn config"}',
+                b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}',
+                b'{"type":"info","data":"npm config"}',
+                b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}'
+            ])
+            logger = logging.getLogger('jupyterlab')
+            config = commands._yarn_config(logger)
+
+            self.assertDictEqual(config, 
+                {"yarn config": {"registry": yarn_registry}, 
+                "npm config": {"registry": yarn_registry}}
+            )
+
+    def test_yarn_config_failure(self):
+        with patch("subprocess.check_output") as check_output:
+            check_output.side_effect = subprocess.CalledProcessError(1, ['yarn', 'config', 'list'], stderr=b"yarn config failed.")
+
+            logger = logging.getLogger('jupyterlab')
+            config = commands._yarn_config(logger)
+
+            self.assertDictEqual(config, 
+                {"yarn config": {}, 
+                "npm config": {}}
+            )
+
+    def test_get_registry(self):
+        with patch("subprocess.check_output") as check_output:
+            yarn_registry = "https://private.yarn/manager"
+            check_output.return_value = b'\n'.join([
+                b'{"type":"info","data":"yarn config"}',
+                b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}',
+                b'{"type":"info","data":"npm config"}',
+                b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}'
+            ])
+
+            handler = commands.AppOptions()
+
+            self.assertEqual(handler.registry, yarn_registry)
+
+    def test_populate_staging(self):
+        with patch("subprocess.check_output") as check_output:
+            yarn_registry = "https://private.yarn/manager"
+            check_output.return_value = b'\n'.join([
+                b'{"type":"info","data":"yarn config"}',
+                b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}',
+                b'{"type":"info","data":"npm config"}',
+                b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}'
+            ])
+
+            staging = pjoin(self.app_dir, 'staging')
+            handler = commands._AppHandler(commands.AppOptions())
+            handler._populate_staging()
+
+            lock_path = pjoin(staging, 'yarn.lock')
+            with open(lock_path) as f:
+                lock = f.read()
+
+            self.assertNotIn(commands.YARN_DEFAULT_REGISTRY, lock)
+            self.assertIn(yarn_registry, lock)