From 01f158695b22d721320a77a9a0b68b166c63dc3f Mon Sep 17 00:00:00 2001
From: Mariusz Nowak <medyk@medikoo.com>
Date: Wed, 30 Jun 2021 16:49:16 +0200
Subject: [PATCH] feat(Telemetry): Report configuration validation result

---
 lib/classes/ConfigSchemaHandler/index.js             | 12 ++++++++++++
 lib/utils/telemetry/generatePayload.js               |  2 ++
 .../lib/classes/ConfigSchemaHandler/index.test.js    |  8 ++++++--
 .../unit/lib/utils/telemetry/generatePayload.test.js |  4 ++++
 4 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/lib/classes/ConfigSchemaHandler/index.js b/lib/classes/ConfigSchemaHandler/index.js
index c81372523ce..b1c8ac40dfd 100644
--- a/lib/classes/ConfigSchemaHandler/index.js
+++ b/lib/classes/ConfigSchemaHandler/index.js
@@ -2,6 +2,7 @@
 
 const Ajv = require('ajv');
 const _ = require('lodash');
+const ensurePlainObject = require('type/plain-object/ensure');
 const schema = require('../../configSchema');
 const ServerlessError = require('../../serverless-error');
 const normalizeAjvErrors = require('./normalizeAjvErrors');
@@ -56,6 +57,8 @@ const denormalizeUserConfig = (userConfig, { removedValuesMap }) => {
   }
 };
 
+const configurationValidationResults = new WeakMap();
+
 class ConfigSchemaHandler {
   constructor(serverless) {
     this.serverless = serverless;
@@ -67,8 +70,14 @@ class ConfigSchemaHandler {
     deepFreeze(this.schema.properties.package);
   }
 
+  static getConfigurationValidationResult(configuration) {
+    if (!configurationValidationResults.has(ensurePlainObject(configuration))) return null;
+    return configurationValidationResults.get(ensurePlainObject(configuration));
+  }
+
   validateConfig(userConfig) {
     if (!this.schema.properties.provider.properties.name) {
+      configurationValidationResults.set(this.serverless.configurationInput, false);
       if (this.serverless.service.configValidationMode !== 'off') {
         this.serverless.cli.log(
           `${WARNING_PREFIX}: Unrecognized provider '${this.serverless.service.provider.name}'`,
@@ -108,6 +117,9 @@ class ConfigSchemaHandler {
     const denormalizeOptions = normalizeUserConfig(userConfig);
     validate(userConfig);
     denormalizeUserConfig(userConfig, denormalizeOptions);
+    if (!configurationValidationResults.has(this.serverless.configurationInput)) {
+      configurationValidationResults.set(this.serverless.configurationInput, !validate.errors);
+    }
     if (validate.errors && this.serverless.service.configValidationMode !== 'off') {
       const messages = normalizeAjvErrors(validate.errors).map((err) => err.message);
       this.handleErrorMessages(messages);
diff --git a/lib/utils/telemetry/generatePayload.js b/lib/utils/telemetry/generatePayload.js
index 5c701909d55..3678d8f50f3 100644
--- a/lib/utils/telemetry/generatePayload.js
+++ b/lib/utils/telemetry/generatePayload.js
@@ -9,6 +9,7 @@ const isPlainObject = require('type/plain-object/is');
 const isObject = require('type/object/is');
 const userConfig = require('@serverless/utils/config');
 const isStandalone = require('../isStandaloneExecutable');
+const { getConfigurationValidationResult } = require('../../classes/ConfigSchemaHandler');
 const { triggeredDeprecations } = require('../logDeprecation');
 const isNpmGlobal = require('../npmPackage/isGlobal');
 const resolveLocalServerlessPath = require('../../cli/resolve-local-serverless-path');
@@ -279,6 +280,7 @@ module.exports = async ({
     payload.hasLocalCredentials = isAwsProvider && Boolean(new AWS.Config().credentials);
     payload.npmDependencies = npmDependencies;
     payload.config = getServiceConfig({ configuration, options });
+    payload.isConfigValid = getConfigurationValidationResult(configuration);
     payload.dashboard.orgUid = serverless && serverless.service.orgUid;
   }
 
diff --git a/test/unit/lib/classes/ConfigSchemaHandler/index.test.js b/test/unit/lib/classes/ConfigSchemaHandler/index.test.js
index c0ab6fa5935..75896d41417 100644
--- a/test/unit/lib/classes/ConfigSchemaHandler/index.test.js
+++ b/test/unit/lib/classes/ConfigSchemaHandler/index.test.js
@@ -2,6 +2,9 @@
 
 const chai = require('chai');
 const runServerless = require('../../../../utils/run-serverless');
+const {
+  getConfigurationValidationResult,
+} = require('../../../../../lib/classes/ConfigSchemaHandler');
 
 chai.use(require('chai-as-promised'));
 
@@ -69,11 +72,12 @@ describe('test/unit/lib/classes/ConfigSchemaHandler/index.test.js', () => {
   });
 
   describe('#validateConfig', () => {
-    it('should run without errors for valid config', () => {
-      return runServerless({
+    it('should run without errors for valid config', async () => {
+      const { serverless } = await runServerless({
         fixture: 'configSchemaExtensions',
         command: 'info',
       });
+      expect(getConfigurationValidationResult(serverless.configurationInput)).to.be.true;
     });
   });
 
diff --git a/test/unit/lib/utils/telemetry/generatePayload.test.js b/test/unit/lib/utils/telemetry/generatePayload.test.js
index d0b919ba4e2..411525068ca 100644
--- a/test/unit/lib/utils/telemetry/generatePayload.test.js
+++ b/test/unit/lib/utils/telemetry/generatePayload.test.js
@@ -119,6 +119,7 @@ describe('test/unit/lib/utils/telemetry/generatePayload.test.js', () => {
       cliName: 'serverless',
       command: 'print',
       commandOptionNames: [],
+      isConfigValid: true,
       config: {
         configValidationMode: 'error',
         provider: {
@@ -181,6 +182,7 @@ describe('test/unit/lib/utils/telemetry/generatePayload.test.js', () => {
       cliName: 'serverless',
       command: 'print',
       commandOptionNames: [],
+      isConfigValid: false, // No schema for custom provider
       config: {
         configValidationMode: 'warn',
         provider: {
@@ -244,6 +246,7 @@ describe('test/unit/lib/utils/telemetry/generatePayload.test.js', () => {
       cliName: 'serverless',
       command: 'print',
       commandOptionNames: [],
+      isConfigValid: null,
       config: {
         configValidationMode: 'error',
         provider: {
@@ -332,6 +335,7 @@ describe('test/unit/lib/utils/telemetry/generatePayload.test.js', () => {
       command: '',
       commandOptionNames: [],
       cliName: 'serverless',
+      isConfigValid: null,
       config: {
         configValidationMode: 'warn',
         provider: {