diff --git a/.circleci/config.yml b/.circleci/config.yml index e5b55b4d..4bdacb4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ version: 2.1 jobs: Build: docker: - - image: cimg/node:14.13.1 + - image: cimg/python:3.8.8-node steps: - checkout - run: @@ -23,19 +23,22 @@ jobs: Addon unit tests: docker: - - image: cimg/node:14.13.1 + - image: cimg/python:3.8.8-node steps: - checkout - run: name: Install the dependencies command: npm install + - run: + name: Build the Addon + command: npm run build-addon - run: name: MochaJS unit tests command: npm run test-addon Addon code linting: docker: - - image: cimg/node:14.13.1 + - image: cimg/python:3.8.8-node steps: - checkout - run: @@ -53,12 +56,15 @@ jobs: Firefox integration tests: docker: - - image: cimg/node:14.13.1 + - image: cimg/python:3.8.8-node steps: - checkout - run: name: Install the dependencies command: npm install && sudo add-apt-repository ppa:ubuntu-mozilla-daily/ppa && sudo apt update && sudo apt install firefox-trunk + - run: + name: Build the Addon + command: npm run build-addon - run: name: Run Selenium tests command: export PATH=.:$PATH && npm run test-integration diff --git a/.eslintrc.cjs b/.eslintrc.cjs index b45fa181..10e3c5c6 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -22,6 +22,7 @@ module.exports = { __DISABLE_REMOTE_SETTINGS__: false, __DISABLE_LOCALE_CHECK__: false, __ENABLE_DATA_SUBMISSION__: false, + __ENABLE_GLEAN__: false, __WEBSITE_URL__: false, }, overrides: [ diff --git a/.gitignore b/.gitignore index 4e54387c..f58a0b00 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,8 @@ public/protocol.css .DS_Store /web-ext-artifacts/ + +# Ignore Glean's virtual environment and +# generated files. +.venv +generated/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b8c7737..27f43559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ [Full changelog](https://github.com/mozilla-rally/core-addon/compare/v1.1.0...master) * [#295](https://github.com/mozilla-rally/rally-core-addon/pull/295): Enable watching the repository and a better developer workflow using `npm run watch`. +* [#505](https://github.com/mozilla-rally/rally-core-addon/pull/505): Integrate Glean.js (disabled by default) and provide a way to enable it, as a developer, using the `config-enable-glean` option. # v1.1.0 (2021-03-10) diff --git a/core-addon/Core.js b/core-addon/Core.js index d1ba5611..7fcff23f 100644 --- a/core-addon/Core.js +++ b/core-addon/Core.js @@ -4,6 +4,7 @@ import Storage from "./Storage.js"; import DataCollection from "./DataCollection.js"; +import * as rallyMetrics from "../public/generated/rally.js"; // The path of the embedded resource used to control options. const OPTIONS_PAGE_PATH = "public/index.html"; @@ -27,6 +28,11 @@ export default class Core { this._storage = new Storage(); this._dataCollection = new DataCollection(); + // Initialize the collection engine once we know if + // user is enrolled or not. + this._storage.getRallyID().finally(id => + this._dataCollection.initialize(id !== undefined)); + // Asynchronously get the available studies. We don't need to wait // for this to finish, the UI can handle the wait. this._availableStudies = this._fetchAvailableStudies().then((studies) => @@ -397,6 +403,8 @@ export default class Core { await this._storage.setRallyID(rallyId); await this._storage.setDeletionID(deletionId); + rallyMetrics.id.set(rallyId); + // Override the uninstall URL to include the rallyID, for deleting data without exposing the Rally ID. await this.setUninstallURL(); diff --git a/core-addon/DataCollection.js b/core-addon/DataCollection.js index 332c67b6..4aea78ab 100644 --- a/core-addon/DataCollection.js +++ b/core-addon/DataCollection.js @@ -2,6 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import Glean from "@mozilla/glean/webext"; +import PingEncryptionPlugin from "@mozilla/glean/webext/plugins/encryption"; + // The encryption key id and JWK to encrypt data that go // to the "core" environment (i.e. `pioneer-core`). See // bug 1677761 for additional details. @@ -15,6 +18,29 @@ const CORE_ENCRYPTION_JWK = { }; export default class DataCollection { + /** + * Initializes the data collection engine. + * + * @param {boolean} userEnrolled + * Whether or not user has enrolled in the platform. + */ + initialize(userEnrolled) { + if (!__ENABLE_GLEAN__) { + console.warn("DataCollection - Glean disabled by the build configuration."); + return; + } + + // Initialize Glean. Note that we always set 'uploadEnabled=true' if user + // consented to join Rally. Upload is always enabled unless the web-extension + // is uninstalled. + Glean.initialize("rally-core", userEnrolled, { + plugins: [ + new PingEncryptionPlugin(CORE_ENCRYPTION_JWK) + ] + } + ); + } + /** * Sends an otherwise-empty ping with the deletion ID other provided info. * diff --git a/docs/metrics.md b/docs/metrics.md new file mode 100644 index 00000000..62f13319 --- /dev/null +++ b/docs/metrics.md @@ -0,0 +1,41 @@ + + +# Metrics + +This document enumerates the metrics collected by this project using the [Glean SDK](https://mozilla.github.io/glean/book/index.html). +This project may depend on other projects which also collect metrics. +This means you might have to go searching through the dependency tree to get a full picture of everything collected by this project. + +# Pings + +- [deletion-request](#deletion-request) +- [onboarding](#onboarding) + +## deletion-request + +This is a built-in ping that is assembled out of the box by the Glean SDK. + +See the Glean SDK documentation for the [`deletion-request` ping](https://mozilla.github.io/glean/book/user/pings/deletion-request.html). + +All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections. + +In addition to those built-in metrics, the following metrics are added to the ping: + +| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | +| --- | --- | --- | --- | --- | --- | --- | +| rally.id |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |The id of the Rally client. |[Review 1](TODO)||never | | + +## onboarding + +All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections. + +In addition to those built-in metrics, the following metrics are added to the ping: + +| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) | +| --- | --- | --- | --- | --- | --- | --- | +| rally.id |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |The id of the Rally client. |[Review 1](TODO)||never | | + +Data categories are [defined here](https://wiki.mozilla.org/Firefox/Data_Collection). + + + diff --git a/metrics.yaml b/metrics.yaml new file mode 100644 index 00000000..8f89420c --- /dev/null +++ b/metrics.yaml @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file defines the metrics that are recorded by the Glean SDK. +# APIs to use these pings are automatically generated at build time using +# the `glean_parser` PyPI package. + +# Metrics in this file may make use of SDK reserved ping names. See +# https://mozilla.github.io/glean/book/dev/core/internal/reserved-ping-names.html +# for additional information. + +--- +$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0 + +rally: + id: + type: uuid + lifetime: user + send_in_pings: + - deletion-request + - onboarding + description: | + The id of the Rally client. + bugs: + - https://github.com/mozilla-rally/rally-core-addon/issues/117 + data_reviews: + - TODO + notification_emails: + - than@mozilla.com + expires: never diff --git a/package-lock.json b/package-lock.json index 362a1044..ef8574b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "devDependencies": { "@babel/core": "^7.13.10", "@mozilla-protocol/core": "^12.1.0", + "@mozilla/glean": "^0.7.0", "@rollup/plugin-commonjs": "^17.1.0", "@rollup/plugin-node-resolve": "^11.2.0", "@rollup/plugin-replace": "^2.4.1", @@ -2533,6 +2534,28 @@ "integrity": "sha512-KiuTc6HFBXcfo5dUBsXX4MP6C5wjBsDt4WbKsUO8jDQz4g/wx7T7++Q9wc45gu7lS0QKklJ0FWsqJpGrXVFY3Q==", "dev": true }, + "node_modules/@mozilla/glean": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@mozilla/glean/-/glean-0.7.0.tgz", + "integrity": "sha512-yG0+OfNRv2ctnr/aka+mIs9ATf93Ni7jA1MWu6FEt9K4BTtjs+VR7h7KIzAeaTvmTv6Xdp57/+gedxGYZqq23w==", + "dev": true, + "dependencies": { + "jose": "^3.7.0", + "uuid": "^8.3.2" + }, + "bin": { + "glean": "dist/cli/cli.js" + } + }, + "node_modules/@mozilla/glean/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", @@ -15957,6 +15980,15 @@ "node": ">=8" } }, + "node_modules/jose": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/jose/-/jose-3.10.0.tgz", + "integrity": "sha512-rbVUNWkGSsvbGPdpBjnLgw0YNYr2zc50T3vutT8Mx/7QOTyilt5urqgyxySRR6aMzxCEh7N+y8PO16H4P57Chg==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/js-string-escape": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz", @@ -30385,6 +30417,24 @@ "integrity": "sha512-KiuTc6HFBXcfo5dUBsXX4MP6C5wjBsDt4WbKsUO8jDQz4g/wx7T7++Q9wc45gu7lS0QKklJ0FWsqJpGrXVFY3Q==", "dev": true }, + "@mozilla/glean": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@mozilla/glean/-/glean-0.7.0.tgz", + "integrity": "sha512-yG0+OfNRv2ctnr/aka+mIs9ATf93Ni7jA1MWu6FEt9K4BTtjs+VR7h7KIzAeaTvmTv6Xdp57/+gedxGYZqq23w==", + "dev": true, + "requires": { + "jose": "^3.7.0", + "uuid": "^8.3.2" + }, + "dependencies": { + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true + } + } + }, "@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", @@ -41327,6 +41377,12 @@ } } }, + "jose": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/jose/-/jose-3.10.0.tgz", + "integrity": "sha512-rbVUNWkGSsvbGPdpBjnLgw0YNYr2zc50T3vutT8Mx/7QOTyilt5urqgyxySRR6aMzxCEh7N+y8PO16H4P57Chg==", + "dev": true + }, "js-string-escape": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz", diff --git a/package.json b/package.json index e41e1b88..56748e8a 100644 --- a/package.json +++ b/package.json @@ -3,10 +3,14 @@ "version": "1.1.0", "type": "module", "scripts": { + "prebuild": "node scripts/setupTaskcluster.js", "build": "rollup -c && npm run build-addon && web-ext --config=web-ext-config.cjs build --overwrite-dest && mv web-ext-artifacts/*.zip web-ext-artifacts/rally_core.xpi", - "build-addon": "rollup -c rollup.config.addon.js --config-enable-data-submission", - "build-local-addon": "rollup -c rollup.config.addon.js --config-disable-remote-settings --config-studies-list-url=/public/locally-available-studies.json", + "build-addon": "npm run glean && rollup -c rollup.config.addon.js --config-enable-data-submission", + "build-local-addon": "npm run glean && rollup -c rollup.config.addon.js --config-disable-remote-settings --config-studies-list-url=/public/locally-available-studies.json", "build-storybook": "build-storybook -s ./public", + "glean": "npm run glean-metrics && npm run glean-docs", + "glean-metrics": "glean translate ./metrics.yaml -f javascript -o public/generated", + "glean-docs": "glean translate ./metrics.yaml -f markdown -o docs", "lint": "npm run build && npm-run-all lint-*", "lint-addon": "web-ext --config=web-ext-config.cjs lint", "lint-css": "stylelint 'public/*.css' '.storybook/*.css' 'src/**/*.svelte' 'stories/**/*.svelte'", @@ -25,6 +29,7 @@ "devDependencies": { "@babel/core": "^7.13.10", "@mozilla-protocol/core": "^12.1.0", + "@mozilla/glean": "^0.7.0", "@rollup/plugin-commonjs": "^17.1.0", "@rollup/plugin-node-resolve": "^11.2.0", "@rollup/plugin-replace": "^2.4.1", diff --git a/rollup.config.addon.js b/rollup.config.addon.js index bed39bfc..aa508a41 100644 --- a/rollup.config.addon.js +++ b/rollup.config.addon.js @@ -29,12 +29,18 @@ export default (cliArgs) => { // to enable it for testing until https://github.com/mozilla-rally/core-addon/issues/304 // is fixed. __ENABLE_DATA_SUBMISSION__: !!cliArgs["config-enable-data-submission"], + // In order to ease the integration, Glean will be embedded in the code but disabled + // until the integration is fully complete. + __ENABLE_GLEAN__: !!cliArgs["config-enable-glean"], __WEBSITE_URL__: cliArgs['config-website'] ? `'${cliArgs['config-website']}'` : "'https://rally.mozilla.org'", }), resolve({ - browser: true, + exportConditions: ["browser"], + // This is required in order for rollup to pick up + // the correct dependencies for ping encryption. + preferBuiltins: false, }), commonjs(), ], diff --git a/scripts/setupTaskcluster.js b/scripts/setupTaskcluster.js new file mode 100644 index 00000000..8832accb --- /dev/null +++ b/scripts/setupTaskcluster.js @@ -0,0 +1,18 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import * as exec from "child_process"; + +// If this is not running on TaskCluster, exit +// without an error code. +if ('TASK_ID' in process.env) { + console.log("Running in TaskCluster. Running set-up."); + + // The TaskCluster instance on the Mozilla add-ons pipeline + // does not allow image customization, so python3 venv cannot + // be installed. That package is required for Glean to build. + // We make sure to have all the dependencies in place when on + // TaskCluster by running the commands manually. + exec.exec("sudo apt-get -y install python3-venv"); +} diff --git a/tests/core-addon/unit/Core.test.js b/tests/core-addon/unit/Core.test.js index 8d88d512..e733f212 100644 --- a/tests/core-addon/unit/Core.test.js +++ b/tests/core-addon/unit/Core.test.js @@ -6,7 +6,11 @@ import { strict as assert } from 'assert'; // eslint-disable-next-line node/no-extraneous-import import sinon from 'sinon'; +import Glean from "@mozilla/glean/webext"; + import Core from '../../../core-addon/Core.js'; +import * as rallyMetrics from "../../../public/generated/rally.js"; + // The website to post deletion IDs to. const OFFBOARD_URL = "https://production.rally.mozilla.org/offboard"; @@ -27,7 +31,12 @@ const FAKE_UUID = "c0ffeec0-ffee-c0ff-eec0-ffeec0ffeec0"; const FAKE_WEBSITE = "https://test.website"; describe('Core', function () { - beforeEach(function () { + // eslint-disable-next-line mocha/no-setup-in-describe + const testAppId = `core.test.${this.title}`; + + beforeEach(async function() { + await Glean.testResetGlean(testAppId); + // Force the sinon-chrome stubbed API to resolve its promise // in tests. Without the next two lines, tests querying the // `browser.management.getAll` API will be stuck and timeout. @@ -203,6 +212,8 @@ describe('Core', function () { {type: "enrollment", data: {}} ); + assert.equal(await rallyMetrics.id.testGetValue(), FAKE_UUID); + // We expect to store the fake ion ID. assert.ok(this.core._storage.setRallyID.withArgs(FAKE_UUID).calledOnce); assert.ok(this.core._dataCollection.sendEnrollmentPing.calledOnce); diff --git a/tests/hooks.cjs b/tests/hooks.cjs index 5c4f019b..b8c0d01d 100644 --- a/tests/hooks.cjs +++ b/tests/hooks.cjs @@ -18,6 +18,7 @@ exports.mochaHooks = { // TODO: remove the next line once https://github.com/mozilla-rally/core-addon/issues/304 // is merged. global.__ENABLE_DATA_SUBMISSION__ = true; + global.__ENABLE_GLEAN__ = true; }, afterAll() { chrome.flush(); @@ -26,5 +27,6 @@ exports.mochaHooks = { // TODO: remove the next line once https://github.com/mozilla-rally/core-addon/issues/304 // is merged. delete global.__ENABLE_DATA_SUBMISSION__; + delete global.__ENABLE_GLEAN__; }, };