Feature public extensions (#218)

* now using public NPM registry * added extension verification with code-signer * additional output to ext:list * refactored extension discovery to use cascading defaults from package.json if not in manifest * Allow installing tarballs from URL - bugfix broken code signature for static packages - remove cache entry when installing from file or URL * fix broken TestExtensions * changes from review * address case where version as path is passed but no name * fix flaky windows tests
neo4j-devtools · Oct 29, 2020 · c0c5f6a · c0c5f6a
1 parent 2d5fd6b
commit c0c5f6a
Show file tree

Hide file tree

Showing 26 changed files with 431 additions and 286 deletions.
diff --git a/package.json b/package.json
@@ -16,7 +16,7 @@
         "lint": "eslint . --ext .ts",
         "test:teardown": "node scripts/tests/teardown.js",
         "test:setup": "node scripts/tests/setup.js",
-        "lerna:test": "lerna run test --concurrency 1",
+        "lerna:test": "lerna run test",
         "test": "run-s --continue-on-error test:setup lerna:test test:teardown",
         "test:pr": "run-s lint test",
         "lerna:clean": "lerna clean -y",

diff --git a/packages/cli/docs/extension.md b/packages/cli/docs/extension.md
@@ -17,7 +17,7 @@ USAGE
   $ relate extension:install [NAME]
 
 OPTIONS
-  -V, --version=version          Version to install
+  -V, --version=version          Version to install (semver), or path to tarball
   -e, --environment=environment  Name of the environment to run the command against
 
 ALIASES

diff --git a/packages/cli/src/commands/extension/install.ts b/packages/cli/src/commands/extension/install.ts
@@ -31,7 +31,7 @@ export default class InstallCommand extends BaseCommand {
         ...FLAGS.ENVIRONMENT,
         version: flags.string({
             char: 'V',
-            description: 'Version to install',
+            description: 'Version to install (semver), or path to tarball',
         }),
     };
 }
diff --git a/packages/cli/src/modules/extension/install.module.ts b/packages/cli/src/modules/extension/install.module.ts
@@ -1,5 +1,6 @@
 import {OnApplicationBootstrap, Module, Inject} from '@nestjs/common';
 import {
+    arrayHasItems,
     EXTENSION_ORIGIN,
     IExtensionVersion,
     InvalidArgumentError,
@@ -12,6 +13,7 @@ import path from 'path';
 import fse from 'fs-extra';
 import _ from 'lodash';
 import cli from 'cli-ux';
+import semver from 'semver';
 
 import InstallCommand from '../../commands/extension/install';
 import {selectPrompt} from '../../prompts';
@@ -57,9 +59,10 @@ export class InstallModule implements OnApplicationBootstrap {
         let {version = ''} = flags;
         const {environment: environmentId} = flags;
         const environment = await this.systemProvider.getEnvironment(environmentId);
+
         this.registerHookListeners();
 
-        if (!(name && version)) {
+        if (!version || (semver.valid(version) && !name)) {
             const versions = (await environment.extensions.versions()).toArray();
             const cached = _.filter(versions, ({origin}) => origin === EXTENSION_ORIGIN.CACHED);
             const onlineNotCached = _.filter(versions, (v) => {
@@ -80,6 +83,11 @@ export class InstallModule implements OnApplicationBootstrap {
             const maybeWithName = name
                 ? _.filter([...choices.values()], (v) => v.name === name)
                 : [...choices.values()];
+
+            if (!arrayHasItems(maybeWithName)) {
+                throw new InvalidArgumentError(`Could not find installable extension with name: ${name}`);
+            }
+
             const selected = await selectPrompt(
                 'Select a version to install',
                 _.map(maybeWithName, (v) => ({

diff --git a/packages/cli/src/modules/extension/list.module.ts b/packages/cli/src/modules/extension/list.module.ts
@@ -28,6 +28,8 @@ export class ListModule implements OnApplicationBootstrap {
             {
                 name: {},
                 type: {},
+                official: {},
+                verification: {header: 'Verified'},
                 version: {},
             },
             {

diff --git a/packages/cli/src/modules/extension/uninstall.module.ts b/packages/cli/src/modules/extension/uninstall.module.ts
@@ -1,5 +1,5 @@
 import {OnApplicationBootstrap, Module, Inject} from '@nestjs/common';
-import {IExtensionMeta, SystemModule, SystemProvider, InvalidArgumentError} from '@relate/common';
+import {IExtensionInfo, SystemModule, SystemProvider, InvalidArgumentError} from '@relate/common';
 import _ from 'lodash';
 
 import UninstallCommand from '../../commands/extension/uninstall';
@@ -45,7 +45,7 @@ export class UninstallModule implements OnApplicationBootstrap {
         return environment.extensions.uninstall(name).then((exts) => {
             // @todo: will we have more than 1 version installed? If so, will
             // need to pass in / select version to uninstall
-            const extFormatter = (ext: IExtensionMeta): string => `${ext.name}@${ext.version}`;
+            const extFormatter = (ext: IExtensionInfo): string => `${ext.name}@${ext.version}`;
 
             this.utils.log(`Uninstalled ${_.join(_.map(exts.toArray(), extFormatter), ', ')}`);
         });

diff --git a/packages/common/package-lock.json b/packages/common/package-lock.json
diff --git a/packages/common/package.json b/packages/common/package.json
@@ -65,6 +65,7 @@
     },
     "dependencies": {
         "@huboneo/tapestry": "1.0.1-1",
+        "@neo4j/code-signer": "1.1.1",
         "apollo-link": "1.2.14",
         "apollo-link-http": "1.5.17",
         "class-validator": "0.12.1",

diff --git a/packages/common/src/constants.ts b/packages/common/src/constants.ts
@@ -32,11 +32,10 @@ export const DISCOVER_DBMS_THROTTLE_MS = 500;
 
 export const EXTENSION_DIR_NAME = 'extensions';
 export const PACKAGE_JSON = 'package.json';
-export const EXTENSION_MANIFEST_FILE_LEGACY = 'relate.manifest.json';
 export const EXTENSION_MANIFEST_KEY = 'relate';
 export const EXTENSION_SHA_ALGORITHM = 'sha1';
-export const EXTENSION_NPM_PREFIX = '@relate-ext/';
-export const EXTENSION_URL_PATH = `https://neo.jfrog.io/artifactory/api/npm/npm-local-private/${EXTENSION_NPM_PREFIX}`;
+export const RELATE_NPM_PREFIX = '@relate/';
+export const EXTENSION_URL_PATH = `https://registry.npmjs.org/`;
 
 export const BOLT_DEFAULT_PORT = ':7687';
 
@@ -184,3 +183,11 @@ export enum FILTER_CONNECTORS {
 
 export const HEALTH_BASE_ENDPOINT = '/health';
 export const STATIC_APP_BASE_ENDPOINT = '/static';
+
+export enum EXTENSION_VERIFICATION_STATUS {
+    UNKNOWN = 'UNKNOWN',
+    UNSIGNED = 'UNSIGNED',
+    TRUSTED = 'TRUSTED',
+    UNTRUSTED = 'UNTRUSTED',
+    REVOKED = 'REVOKED',
+}
diff --git a/packages/common/src/entities/dbmss/dbmss.local.ts b/packages/common/src/entities/dbmss/dbmss.local.ts
@@ -122,8 +122,13 @@ export class LocalDbmss extends DbmssAbstract<LocalEnvironment> {
 
         // version as a file path.
         if ((await fse.pathExists(version)) && (await fse.stat(version)).isFile()) {
-            const {extractedDistPath} = await extractNeo4j(version, this.environment.dirPaths.dbmssCache);
-            return this.installNeo4j(name, this.getDbmsRootPath(), extractedDistPath, credentials);
+            const tmpPath = path.join(this.environment.dirPaths.tmp, uuidv4());
+            const {extractedDistPath} = await extractNeo4j(version, tmpPath);
+            const dbms = await this.installNeo4j(name, this.getDbmsRootPath(), extractedDistPath, credentials);
+
+            await fse.remove(tmpPath);
+
+            return dbms;
         }
 
         // @todo: version as a URL.

diff --git a/packages/common/src/entities/environments/environment.constants.ts b/packages/common/src/entities/environments/environment.constants.ts
@@ -1,3 +1,5 @@
+import {EXTENSION_URL_PATH} from '../../constants';
+
 export const ENVIRONMENTS_DIR_NAME = 'environments';
 export const RUNTIME_DIR_NAME = 'runtime';
 
@@ -88,8 +90,5 @@ export const NEO4J_JWT_CERT_ATTRS = [
     },
 ];
 
-// @todo: needs to be removed and handled by env vars
-export const EXTENSION_SEARCH_PATH = 'https://neo.jfrog.io/artifactory/api/search/aql';
-export const EXTENSION_REPO_NAME = 'npm-local-private';
-export const JFROG_PRIVATE_REGISTRY_PASSWORD = 'zaFwod-rypvyh-3mohka';
-export const JFROG_PRIVATE_REGISTRY_USERNAME = 'devtools-reader';
+export const EXTENSION_SEARCH_PATH = `${EXTENSION_URL_PATH}-/v1/search`;
+export const EXTENSION_KEYWORD_NAME = 'neo4j-relate-extension';
diff --git a/packages/common/src/entities/extensions/extensions.abstract.ts b/packages/common/src/entities/extensions/extensions.abstract.ts
@@ -1,9 +1,9 @@
 import {List} from '@relate/types';
 
-import {IExtensionMeta, IExtensionVersion} from '../../utils/extensions';
+import {IExtensionVersion} from '../../utils/extensions';
 import {EnvironmentAbstract} from '../environments';
 import {IRelateFilter} from '../../utils/generic';
-import {IAppLaunchToken} from '../../models';
+import {IAppLaunchToken, IExtensionInfo} from '../../models';
 
 export abstract class ExtensionsAbstract<Env extends EnvironmentAbstract> {
     /**
@@ -22,19 +22,19 @@ export abstract class ExtensionsAbstract<Env extends EnvironmentAbstract> {
      * List all installed extensions
      * @param   filters     Filters to apply
      */
-    abstract list(filters?: List<IRelateFilter> | IRelateFilter[]): Promise<List<IExtensionMeta>>;
+    abstract list(filters?: List<IRelateFilter> | IRelateFilter[]): Promise<List<IExtensionInfo>>;
 
     /**
      * List all installed apps
      * @param   filters     Filters to apply
      */
-    abstract listApps(filters?: List<IRelateFilter> | IRelateFilter[]): Promise<List<IExtensionMeta>>;
+    abstract listApps(filters?: List<IRelateFilter> | IRelateFilter[]): Promise<List<IExtensionInfo>>;
 
     /**
      * Link local extension (useful for development)
      * @param   filePath
      */
-    abstract link(filePath: string): Promise<IExtensionMeta>;
+    abstract link(filePath: string): Promise<IExtensionInfo>;
 
     /**
      * List all available extensions to install
@@ -47,13 +47,13 @@ export abstract class ExtensionsAbstract<Env extends EnvironmentAbstract> {
      * @param   name
      * @param   version
      */
-    abstract install(name: string, version: string): Promise<IExtensionMeta>;
+    abstract install(name: string, version: string): Promise<IExtensionInfo>;
 
     /**
      * Uninstall given extension
      * @param   name
      */
-    abstract uninstall(name: string): Promise<List<IExtensionMeta>>;
+    abstract uninstall(name: string): Promise<List<IExtensionInfo>>;
 
     /**
      * Creates an app launch token, for passing DBMS info and credentials to app

diff --git a/packages/common/src/entities/extensions/extensions.constants.ts b/packages/common/src/entities/extensions/extensions.constants.ts
@@ -0,0 +1,73 @@
+// @todo: Create cert for relate and decide where it should live
+export const RELATE_ROOT_CERT = `
+-----BEGIN CERTIFICATE-----
+MIIF6zCCA9OgAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwYjELMAkGA1UEBhMCU0Ux
+DzANBgNVBAgMBlN3ZWRlbjETMBEGA1UECgwKTmVvNGogSW5jLjERMA8GA1UECwwI
+RGV2VG9vbHMxGjAYBgNVBAMMEU5FbzRqIERldlRvb2xzIENBMB4XDTIwMDMwNjE0
+MTcwOFoXDTMwMDMwNDE0MTcwOFowbDELMAkGA1UEBhMCU0UxDzANBgNVBAgMBlN3
+ZWRlbjETMBEGA1UECgwKTmVvNGogSW5jLjERMA8GA1UECwwIRGV2VG9vbHMxJDAi
+BgNVBAMMG05lbzRqIERldlRvb2xzIEludGVybWVkaWF0ZTCCAiIwDQYJKoZIhvcN
+AQEBBQADggIPADCCAgoCggIBANQqpPMJ1dY6oKOmDqGGlTE8YzAQ3gOlG0DwY55n
+6IC6y/kd0Ia85ka6KZCMWMeJpap0Lfq57P27i7nDTCA9UDSLDxZSlQHTqdCQGscB
+BBOUaGrzUUQR6fU4/oqp2qBEkz59wFzw2Vu4uyB81Pb1jgO59+NvrA204yNkFySP
+HwV9+mxTk/O/LXkuqpZuYPK+71DS5Lfey/cmTccOuANLRMIAOdX9p7AecpCsvna4
+j3QfhUo2SyQAlUB9BfbDTtGIUwsixFCeI+lOz5tb1nbd8OjXs4pU1Wr1Zw51sjhG
+woaMtc8kM4MbjsaUxQoY6VqNgIu06peFNmDRhNO18fY+aIH75NPCnLB+9+7pPN+r
+ftJyxrEK6rDEsdv/Q/lyyhcwIX71Wtmx/c+TpHhPGiO0RU+TNaKiSvXOxTj21hwr
+mbrJ/8++U2awRNs1exlLefcaXc/ZccHJWqGUSR7FlzPTV3LPwQTZ3Q1Qg1zMQ4vr
+2QdHK+SnPuzlwCNSz9SVOBqodotaS6POQZDm1YblwVosGzRr5E9ulRhfNp8bg5/r
+VJefmr6w3Oqasi2hcXO/ZtX54PPMykWwe9ZgxUoAaU76y1ZpBhvq7RG6AWkGBYKY
+rDvw7xiQ/3JJnzuEXhI2IcShOWu3IgyoUMvne6baQKD82VEVK5sdOlE6hCdOc1/c
+itVrAgMBAAGjgaAwgZ0wHQYDVR0OBBYEFCGaMCJI4diXPtBIle77LZf8lYFWMB8G
+A1UdIwQYMBaAFKKKOlHTfV9EYWXon9rEWzZKp4MeMBIGA1UdEwEB/wQIMAYBAf8C
+AQAwDgYDVR0PAQH/BAQDAgGGMDcGA1UdHwQwMC4wLKAqoCiGJmh0dHBzOi8vdHJ1
+c3QubmVvNGouY29tL2NybC9jYS5jcmwucGVtMA0GCSqGSIb3DQEBCwUAA4ICAQAs
+h7wXThcd8liy23CfswxoWnrBEj/rKbLPs0RtROoNTt+CXnVeKPOaFQz+37bW0CcD
+xlcHRnENnubkGJdMF6VR4yAzNgAZAhOOEiKu+xV+41vbaDkWWDT3bXTMXBxHV54s
+zXaYW77iGGbZXtcFL0BMkLLEq/R/Gvi7gEpVLeEl8M0+R7RiG+bqF6Ry7GHzjVPE
+Y0pjTbX7eRrWxANpxgK8Wf4XR9IrX1dlrc2nS6Z7mx3ovRXHmqnAe34pf19zqfGv
+4GAeSCgtRw6WkBTYP0/JzIswdIgmTxOA8A2T/17pFG1jeFLtZ6KXVKttvlb0gCDi
+Sdea5AlTa/i5ZT9XoBi6ewqWxG91UGQVE3c28bjQ17zG2++cbbxj3coZXBqMEl/T
+4Hv25E6jPy7GduDlgjNxnAfAarIDR9oazMr3GTkZgNlUnTXesTDcgee6r7ckt/ht
+zNgStMEBnqUeg/+73KuyRcVr4klxoXyDx4RIxL71WlGut490bdLc6AxwpdSDB1WV
+GzCyxE2UMBWMGOnb0VcbUgao7RRnM0PGIJzP6m1Z3t59q1LeTkXEbnKEXI6/bx3x
+KgT9MOG41XqG4oMZUOBBbGHKQxskqPv1IFkEP3xAnn/FkE4Y2GXm020mwxrhc4Cv
+VMaJmJn1TMzvAxj3fmSK3BgvuJFIJXSVMl4vcQBHwA==
+-----END CERTIFICATE-----
+
+-----BEGIN CERTIFICATE-----
+MIIF5TCCA82gAwIBAgIJAKgfdcBkZ/+JMA0GCSqGSIb3DQEBCwUAMGIxCzAJBgNV
+BAYTAlNFMQ8wDQYDVQQIDAZTd2VkZW4xEzARBgNVBAoMCk5lbzRqIEluYy4xETAP
+BgNVBAsMCERldlRvb2xzMRowGAYDVQQDDBFORW80aiBEZXZUb29scyBDQTAeFw0y
+MDAzMDYxNDE2MjVaFw00MDAzMDExNDE2MjVaMGIxCzAJBgNVBAYTAlNFMQ8wDQYD
+VQQIDAZTd2VkZW4xEzARBgNVBAoMCk5lbzRqIEluYy4xETAPBgNVBAsMCERldlRv
+b2xzMRowGAYDVQQDDBFORW80aiBEZXZUb29scyBDQTCCAiIwDQYJKoZIhvcNAQEB
+BQADggIPADCCAgoCggIBANlfN3yiA01RiF4aZjczBosfA4Cvzap9QM8kMtEeBl/N
+A1ObLx3BQlUedr8HQFeO1yZmJuiMJUTrlZogZSZMZ8Quo5W+wEG5va5bJHXD2Gnn
+EuJ5Z26WA560QQKywhwIa3Tybkocqy9CLofDCJSHDwNUUWJjRP/uT2uLUHxdCTAD
+yHn3sCYsgBDDSmvV9RilaLivdfZGCRcqeDFifFbc37JvZhNs5cIz/OT6fLMXxuuv
+J27UJySNmH95yLygmVy49ChbwbSVpDub9NLMdRVFYd+9nm5Ee+yCbvZhOPo7Z1QU
+60Q6BnrptlxhyEqAqUYnwyZCoY1c8o3nnarpuwxTXMRP8zp7LglMowqp6bdqS3e+
+fTWN/MBEg0M5gkED4ap1bV76dE0ybsoxCqlwGT5plEwVemlbeb/bMUBEFc4F/TSF
+ukMGXrDrKQ/4aarTuLb/q3LLIH2rydvdxG4H/HPg++lddGxC9zlGRBBgvA3ORbdi
+hhBKm+QoFNdvxJzGtU3PGPkcpGQGzWvy8jVDjD7hz+55FrhSyQz6f/+QiCpHpQdd
+CrVHp22eHdpAX1IEurLhH9E1jlJQ7uYVwVd8hnPmqcemZ1Y4iqj4AszsTKLAJR4t
+KnBL+rq/av4x7CZkuxVcT2Zm1//kdwzkNn0her7rfNkpi2zzDxWHjZKguK+BY75H
+AgMBAAGjgZ0wgZowHQYDVR0OBBYEFKKKOlHTfV9EYWXon9rEWzZKp4MeMB8GA1Ud
+IwQYMBaAFKKKOlHTfV9EYWXon9rEWzZKp4MeMA8GA1UdEwEB/wQFMAMBAf8wDgYD
+VR0PAQH/BAQDAgGGMDcGA1UdHwQwMC4wLKAqoCiGJmh0dHBzOi8vdHJ1c3QubmVv
+NGouY29tL2NybC9jYS5jcmwucGVtMA0GCSqGSIb3DQEBCwUAA4ICAQAslPTIaYXb
+w5Bx4lL87R6+DP5duUGtkfFLomtl3TJIvKK7Kl7PInFhV5IxVaaJFb8kV6tv1MnF
+S/kCzFDwrOSn9egWmRLz2ekIZdYzc0RtzByuev65GpGk7aCddYqgq3dXwVAm5jnc
+/WVk4ZZPqGy0ru4RU7G4H0XZC/xOZczdu7ypRr8OQvXzAB72OTCGWtdKFev+KRJl
+7s6Urk7IbLhqtuNBvPiJck3EhN3mK41jSvaFYwFLOwYgqcRPXntRttaMAFezSS7n
+IOaibG9QiN06z8yyKsBBZ19Z+YbFe1x79zS0EATRsSx8udXzPUSKffnLWybFqLvD
+koePh4QBhPXJdd4IJJSbBArKeWBWH5KOjhBWegccTsVx5kt/mPVYCJFMfRLzauro
+IzOrPuApWjQzbYEBdvVlp1VoCNYifFhnhiBsgt5pzrZCk3nRlar8Mm0DaS5QYW7r
+XoXQml2nWt5hCk6Cs10Xju6PY1SYX+AJAi1TTglwvWXYQQU/SPRQqI//7IzV8HfA
+eFubz6I8jYbkCp0/uQlX/1j3nXDamuzpZXKi/oiCTgvdcteYvT+e5KbC53K3hlPZ
+k2PFQTyfqaUFuOBsZV+NBiPncvIG5OWNciijbxoTHau8hDkN118F+C8Z2yk0I1DV
+wdVgXZfslcdTgR+nQFCl1CK40/rolEPO5A==
+-----END CERTIFICATE-----
+
+`;