Merge pull request #505 from Dexterp37/gleanjs

Introduce Glean.js for collecting data

.circleci/config.yml

@@ -8,7 +8,7 @@ version: 2.1
 jobs:
   Build:
     docker:
-      - image: cimg/node:14.13.1
+      - image: cimg/python:3.8.8-node
     steps:
       - checkout
       - run:
@@ -23,19 +23,22 @@ jobs:
 
   Addon unit tests:
     docker:
-      - image: cimg/node:14.13.1
+      - image: cimg/python:3.8.8-node
     steps:
       - checkout
       - run:
           name: Install the dependencies
           command: npm install
+      - run:
+          name: Build the Addon
+          command: npm run build-addon
       - run:
           name: MochaJS unit tests
           command: npm run test-addon
 
   Addon code linting:
     docker:
-      - image: cimg/node:14.13.1
+      - image: cimg/python:3.8.8-node
     steps:
       - checkout
       - run:
@@ -53,12 +56,15 @@ jobs:
 
   Firefox integration tests:
     docker:
-      - image: cimg/node:14.13.1
+      - image: cimg/python:3.8.8-node
     steps:
       - checkout
       - run:
           name: Install the dependencies
           command: npm install && sudo add-apt-repository ppa:ubuntu-mozilla-daily/ppa && sudo apt update && sudo apt install firefox-trunk
+      - run:
+          name: Build the Addon
+          command: npm run build-addon
       - run:
           name: Run Selenium tests
           command: export PATH=.:$PATH && npm run test-integration

.eslintrc.cjs

@@ -22,6 +22,7 @@ module.exports = {
     __DISABLE_REMOTE_SETTINGS__: false,
     __DISABLE_LOCALE_CHECK__: false,
     __ENABLE_DATA_SUBMISSION__: false,
+    __ENABLE_GLEAN__: false,
     __WEBSITE_URL__: false,
   },
   overrides: [

.gitignore

@@ -11,3 +11,8 @@ public/protocol.css
 .DS_Store
 
 /web-ext-artifacts/
+
+# Ignore Glean's virtual environment and
+# generated files.
+.venv
+generated/

CHANGELOG.md

@@ -3,6 +3,7 @@
 [Full changelog](https://github.com/mozilla-rally/core-addon/compare/v1.1.0...master)
 
 * [#295](https://github.com/mozilla-rally/rally-core-addon/pull/295): Enable watching the repository and a better developer workflow using `npm run watch`.
+* [#505](https://github.com/mozilla-rally/rally-core-addon/pull/505): Integrate Glean.js (disabled by default) and provide a way to enable it, as a developer, using the `config-enable-glean` option.
 
 # v1.1.0 (2021-03-10)
 

core-addon/Core.js

@@ -4,6 +4,7 @@
 
 import Storage from "./Storage.js";
 import DataCollection from "./DataCollection.js";
+import * as rallyMetrics from "../public/generated/rally.js";
 
 // The path of the embedded resource used to control options.
 const OPTIONS_PAGE_PATH = "public/index.html";
@@ -27,6 +28,11 @@ export default class Core {
     this._storage = new Storage();
     this._dataCollection = new DataCollection();
 
+    // Initialize the collection engine once we know if
+    // user is enrolled or not.
+    this._storage.getRallyID().finally(id =>
+      this._dataCollection.initialize(id !== undefined));
+
     // Asynchronously get the available studies. We don't need to wait
     // for this to finish, the UI can handle the wait.
     this._availableStudies = this._fetchAvailableStudies().then((studies) =>
@@ -397,6 +403,8 @@ export default class Core {
     await this._storage.setRallyID(rallyId);
     await this._storage.setDeletionID(deletionId);
 
+    rallyMetrics.id.set(rallyId);
+
     // Override the uninstall URL to include the rallyID, for deleting data without exposing the Rally ID.
     await this.setUninstallURL();
 

core-addon/DataCollection.js

@@ -2,6 +2,9 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+import Glean from "@mozilla/glean/webext";
+import PingEncryptionPlugin from "@mozilla/glean/webext/plugins/encryption";
+
 // The encryption key id and JWK to encrypt data that go
 // to the "core" environment (i.e. `pioneer-core`). See
 // bug 1677761 for additional details.
@@ -15,6 +18,29 @@ const CORE_ENCRYPTION_JWK = {
 };
 
 export default class DataCollection {
+  /**
+   * Initializes the data collection engine.
+   *
+   * @param {boolean} userEnrolled
+   *        Whether or not user has enrolled in the platform.
+   */
+  initialize(userEnrolled) {
+    if (!__ENABLE_GLEAN__) {
+      console.warn("DataCollection - Glean disabled by the build configuration.");
+      return;
+    }
+
+    // Initialize Glean. Note that we always set 'uploadEnabled=true' if user
+    // consented to join Rally. Upload is always enabled unless the web-extension
+    // is uninstalled.
+    Glean.initialize("rally-core", userEnrolled, {
+        plugins: [
+          new PingEncryptionPlugin(CORE_ENCRYPTION_JWK)
+        ]
+      }
+    );
+  }
+
   /**
    * Sends an otherwise-empty ping with the deletion ID other provided info.
    *

docs/metrics.md

@@ -0,0 +1,41 @@
+<!-- AUTOGENERATED BY glean_parser.  DO NOT EDIT. -->
+
+# Metrics
+
+This document enumerates the metrics collected by this project using the [Glean SDK](https://mozilla.github.io/glean/book/index.html).
+This project may depend on other projects which also collect metrics.
+This means you might have to go searching through the dependency tree to get a full picture of everything collected by this project.
+
+# Pings
+
+- [deletion-request](#deletion-request)
+- [onboarding](#onboarding)
+
+## deletion-request
+
+This is a built-in ping that is assembled out of the box by the Glean SDK.
+
+See the Glean SDK documentation for the [`deletion-request` ping](https://mozilla.github.io/glean/book/user/pings/deletion-request.html).
+
+All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections.
+
+In addition to those built-in metrics, the following metrics are added to the ping:
+
+| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) |
+| --- | --- | --- | --- | --- | --- | --- |
+| rally.id |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |The id of the Rally client.  |[Review 1](TODO)||never | |
+
+## onboarding
+
+All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections.
+
+In addition to those built-in metrics, the following metrics are added to the ping:
+
+| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) |
+| --- | --- | --- | --- | --- | --- | --- |
+| rally.id |[uuid](https://mozilla.github.io/glean/book/user/metrics/uuid.html) |The id of the Rally client.  |[Review 1](TODO)||never | |
+
+Data categories are [defined here](https://wiki.mozilla.org/Firefox/Data_Collection).
+
+<!-- AUTOGENERATED BY glean_parser.  DO NOT EDIT. -->
+

metrics.yaml

@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file defines the metrics that are recorded by the Glean SDK.
+# APIs to use these pings are automatically generated at build time using
+# the `glean_parser` PyPI package.
+
+# Metrics in this file may make use of SDK reserved ping names. See
+# https://mozilla.github.io/glean/book/dev/core/internal/reserved-ping-names.html
+# for additional information.
+
+---
+$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
+
+rally:
+  id:
+    type: uuid
+    lifetime: user
+    send_in_pings:
+      - deletion-request
+      - onboarding
+    description: |
+      The id of the Rally client.
+    bugs:
+      - https://github.com/mozilla-rally/rally-core-addon/issues/117
+    data_reviews:
+      - TODO
+    notification_emails:
+      - than@mozilla.com
+    expires: never

package.json

@@ -3,10 +3,14 @@
   "version": "1.1.0",
   "type": "module",
   "scripts": {
+    "prebuild": "node scripts/setupTaskcluster.js",
     "build": "rollup -c && npm run build-addon && web-ext --config=web-ext-config.cjs build --overwrite-dest && mv web-ext-artifacts/*.zip web-ext-artifacts/rally_core.xpi",
-    "build-addon": "rollup -c rollup.config.addon.js --config-enable-data-submission",
-    "build-local-addon": "rollup -c rollup.config.addon.js --config-disable-remote-settings --config-studies-list-url=/public/locally-available-studies.json",
+    "build-addon": "npm run glean && rollup -c rollup.config.addon.js --config-enable-data-submission",
+    "build-local-addon": "npm run glean && rollup -c rollup.config.addon.js --config-disable-remote-settings --config-studies-list-url=/public/locally-available-studies.json",
     "build-storybook": "build-storybook -s ./public",
+    "glean": "npm run glean-metrics && npm run glean-docs",
+    "glean-metrics": "glean translate ./metrics.yaml -f javascript -o public/generated",
+    "glean-docs": "glean translate ./metrics.yaml -f markdown -o docs",
     "lint": "npm run build && npm-run-all lint-*",
     "lint-addon": "web-ext --config=web-ext-config.cjs lint",
     "lint-css": "stylelint 'public/*.css' '.storybook/*.css' 'src/**/*.svelte' 'stories/**/*.svelte'",
@@ -25,6 +29,7 @@
   "devDependencies": {
     "@babel/core": "^7.13.10",
     "@mozilla-protocol/core": "^12.1.0",
+    "@mozilla/glean": "^0.7.0",
     "@rollup/plugin-commonjs": "^17.1.0",
     "@rollup/plugin-node-resolve": "^11.2.0",
     "@rollup/plugin-replace": "^2.4.1",

rollup.config.addon.js

@@ -29,12 +29,18 @@ export default (cliArgs) => {
         // to enable it for testing until https://github.com/mozilla-rally/core-addon/issues/304
         // is fixed.
         __ENABLE_DATA_SUBMISSION__: !!cliArgs["config-enable-data-submission"],
+        // In order to ease the integration, Glean will be embedded in the code but disabled
+        // until the integration is fully complete.
+        __ENABLE_GLEAN__: !!cliArgs["config-enable-glean"],
         __WEBSITE_URL__: cliArgs['config-website'] ?
           `'${cliArgs['config-website']}'` :
           "'https://rally.mozilla.org'",
       }),
       resolve({
-        browser: true,
+        exportConditions: ["browser"],
+        // This is required in order for rollup to pick up
+        // the correct dependencies for ping encryption.
+        preferBuiltins: false,
       }),
       commonjs(),
     ],

scripts/setupTaskcluster.js

@@ -0,0 +1,18 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import * as exec from "child_process";
+
+// If this is not running on TaskCluster, exit
+// without an error code.
+if ('TASK_ID' in process.env) {
+  console.log("Running in TaskCluster. Running set-up.");
+
+  // The TaskCluster instance on the Mozilla add-ons pipeline
+  // does not allow image customization, so python3 venv cannot
+  // be installed. That package is required for Glean to build.
+  // We make sure to have all the dependencies in place when on
+  // TaskCluster by running the commands manually.
+  exec.exec("sudo apt-get -y install python3-venv");
+}