From 9aa420582a59d81f7cde2abc16aa870bf8060ceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Collonval?= Date: Fri, 20 Sep 2019 16:07:08 +0200 Subject: [PATCH] Fix yarn registry with command argument (#7109) * Use `yarn config list` to update the registry * Move registry to `AppOptions` * Always replace yarn registry if != default * Force utf-8 for yarn.lock --- jupyterlab/commands.py | 61 ++++++++++++++++++--- jupyterlab/tests/test_jupyterlab.py | 5 +- jupyterlab/tests/test_registry.py | 84 +++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 jupyterlab/tests/test_registry.py diff --git a/jupyterlab/commands.py b/jupyterlab/commands.py index fec2ca990bf4..ff77c9a3da26 100644 --- a/jupyterlab/commands.py +++ b/jupyterlab/commands.py @@ -44,6 +44,11 @@ # If we are pinning the package, rename it `pin@` PIN_PREFIX = 'pin@' + +# Default Yarn registry used in default yarn.lock +YARN_DEFAULT_REGISTRY = 'https://registry.yarnpkg.com' + + class ProgressProcess(Process): def __init__(self, cmd, logger=None, cwd=None, kill_event=None, @@ -312,6 +317,8 @@ def __init__(self, logger=None, core_config=None, **kwargs): kill_event = Instance(Event, args=(), help='Event for aborting call') + registry = Unicode(help="NPM packages registry URL") + @default('logger') def _default_logger(self): return logging.getLogger('jupyterlab') @@ -326,6 +333,11 @@ def _default_app_dir(self): def _default_core_config(self): return CoreConfig() + @default('registry') + def _default_registry(self): + config = _yarn_config(self.logger)["yarn config"] + return config.get("registry", YARN_DEFAULT_REGISTRY) + def _ensure_options(options, **kwargs): """Helper to use deprecated kwargs for AppOption""" @@ -397,10 +409,8 @@ def uninstall_extension(name=None, app_dir=None, logger=None, all_=False, core_c def update_extension(name=None, all_=False, app_dir=None, logger=None, core_config=None, app_options=None): """Update an extension by name, or all extensions. - Either `name` must be given as a string, or `all_` must be `True`. If `all_` is `True`, the value of `name` is ignored. - Returns `True` if a rebuild is recommended, `False` otherwise. """ app_options = _ensure_options( @@ -564,8 +574,7 @@ def __init__(self, options): self.core_data = options.core_config._data self.info = self._get_app_info() self.kill_event = options.kill_event - # TODO: Make this configurable - self.registry = 'https://registry.npmjs.org' + self.registry = options.registry def install_extension(self, extension, existing=None, pin=None): """Install an extension package into JupyterLab. @@ -1155,9 +1164,16 @@ def _populate_staging(self, name=None, version=None, static_url=None, json.dump(data, fid, indent=4) # copy known-good yarn.lock if missing - lock_path = pjoin(staging, 'yarn.lock') - if not osp.exists(lock_path): - shutil.copy(pjoin(HERE, 'staging', 'yarn.lock'), lock_path) + lock_path = pjoin(staging, 'yarn.lock') + lock_template = pjoin(HERE, 'staging', 'yarn.lock') + if self.registry != YARN_DEFAULT_REGISTRY: # Replace on the fly the yarn repository see #3658 + with open(lock_template, encoding='utf-8') as f: + template = f.read() + template = template.replace(YARN_DEFAULT_REGISTRY, self.registry.strip("/")) + with open(lock_path, 'w', encoding='utf-8') as f: + f.write(template) + elif not osp.exists(lock_path): + shutil.copy(lock_template, lock_path) def _get_package_template(self, silent=False): """Get the template the for staging package.json file. @@ -1734,6 +1750,37 @@ def _node_check(logger): msg = 'Please install nodejs %s before continuing. nodejs may be installed using conda or directly from the nodejs website.' % ver raise ValueError(msg) +def _yarn_config(logger): + """Get the yarn configuration. + + Returns + ------- + {"yarn config": dict, "npm config": dict} if unsuccessfull the subdictionary are empty + """ + node = which('node') + configuration = {"yarn config": {}, "npm config": {}} + try: + output_binary = subprocess.check_output([node, YARN_PATH, 'config', 'list', '--json'], stderr=subprocess.PIPE, cwd=HERE) + output = output_binary.decode('utf-8') + lines = iter(output.splitlines()) + try: + for line in lines: + info = json.loads(line) + if info["type"] == "info": + key = info["data"] + inspect = json.loads(next(lines)) + if inspect["type"] == "inspect": + configuration[key] = inspect["data"] + except StopIteration: + pass + logger.debug("Yarn configuration loaded.") + except subprocess.CalledProcessError as e: + logger.error("Fail to get yarn configuration. {!s}{!s}".format(e.stderr.decode('utf-8'), e.output.decode('utf-8'))) + except Exception as e: + logger.error("Fail to get yarn configuration. {!s}".format(e)) + finally: + return configuration + def _ensure_logger(logger=None): """Ensure that we have a logger""" diff --git a/jupyterlab/tests/test_jupyterlab.py b/jupyterlab/tests/test_jupyterlab.py index 297d05365508..a4fd6002b35f 100644 --- a/jupyterlab/tests/test_jupyterlab.py +++ b/jupyterlab/tests/test_jupyterlab.py @@ -52,7 +52,7 @@ def touch(file, mtime=None): return os.stat(file).st_mtime -class TestExtension(TestCase): +class AppHandlerTest(TestCase): def tempdir(self): td = TemporaryDirectory() @@ -135,6 +135,9 @@ def ignore(dname, files): # Set pinned extension names self.pinned_packages = ['jupyterlab-test-extension@1.0', 'jupyterlab-test-extension@2.0'] + +class TestExtension(AppHandlerTest): + def test_install_extension(self): assert install_extension(self.mock_extension) is True path = pjoin(self.app_dir, 'extensions', '*.tgz') diff --git a/jupyterlab/tests/test_registry.py b/jupyterlab/tests/test_registry.py new file mode 100644 index 000000000000..8c7712992800 --- /dev/null +++ b/jupyterlab/tests/test_registry.py @@ -0,0 +1,84 @@ +"""Test yarn registry replacement""" + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import logging +import os +from os.path import join as pjoin +import subprocess +from tempfile import TemporaryDirectory +from unittest import TestCase +from unittest.mock import patch + +from jupyter_core import paths + +from jupyterlab import commands + +from .test_jupyterlab import AppHandlerTest + + +class TestAppHandlerRegistry(AppHandlerTest): + + def test_yarn_config(self): + with patch("subprocess.check_output") as check_output: + yarn_registry = "https://private.yarn/manager" + check_output.return_value = b'\n'.join([ + b'{"type":"info","data":"yarn config"}', + b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}', + b'{"type":"info","data":"npm config"}', + b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}' + ]) + logger = logging.getLogger('jupyterlab') + config = commands._yarn_config(logger) + + self.assertDictEqual(config, + {"yarn config": {"registry": yarn_registry}, + "npm config": {"registry": yarn_registry}} + ) + + def test_yarn_config_failure(self): + with patch("subprocess.check_output") as check_output: + check_output.side_effect = subprocess.CalledProcessError(1, ['yarn', 'config', 'list'], stderr=b"yarn config failed.") + + logger = logging.getLogger('jupyterlab') + config = commands._yarn_config(logger) + + self.assertDictEqual(config, + {"yarn config": {}, + "npm config": {}} + ) + + def test_get_registry(self): + with patch("subprocess.check_output") as check_output: + yarn_registry = "https://private.yarn/manager" + check_output.return_value = b'\n'.join([ + b'{"type":"info","data":"yarn config"}', + b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}', + b'{"type":"info","data":"npm config"}', + b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}' + ]) + + handler = commands.AppOptions() + + self.assertEqual(handler.registry, yarn_registry) + + def test_populate_staging(self): + with patch("subprocess.check_output") as check_output: + yarn_registry = "https://private.yarn/manager" + check_output.return_value = b'\n'.join([ + b'{"type":"info","data":"yarn config"}', + b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}', + b'{"type":"info","data":"npm config"}', + b'{"type":"inspect","data":{"registry":"' + bytes(yarn_registry, 'utf-8') + b'"}}' + ]) + + staging = pjoin(self.app_dir, 'staging') + handler = commands._AppHandler(commands.AppOptions()) + handler._populate_staging() + + lock_path = pjoin(staging, 'yarn.lock') + with open(lock_path) as f: + lock = f.read() + + self.assertNotIn(commands.YARN_DEFAULT_REGISTRY, lock) + self.assertIn(yarn_registry, lock)