.github/workflows/python-deps.yml

@@ -18,6 +18,11 @@ jobs:
           os: [ubuntu-latest, macos-latest]
           python_deps_type: [pipenv, poetry, requirements, setup_py]
           python_version: [2, 3]
+          exclude:
+            # Python2 and poetry are not supported. See https://github.com/actions/setup-python/issues/374
+            - python_version: 2
+              python_deps_type: poetry
+
 
     env:
       PYTHON_DEPS_TYPE: ${{ matrix.python_deps_type }}
@@ -115,6 +120,10 @@ jobs:
         matrix:
           python_deps_type: [pipenv, poetry, requirements, setup_py]
           python_version: [2, 3]
+          exclude:
+            # Python2 and poetry are not supported. See https://github.com/actions/setup-python/issues/374
+            - python_version: 2
+              python_deps_type: poetry
 
     env:
       PYTHON_DEPS_TYPE: ${{ matrix.python_deps_type }}

CHANGELOG.md

@@ -1,5 +1,11 @@
 # CodeQL Action Changelog
 
+## 1.1.15 - 28 Jun 2022
+
+- CodeQL query packs listed in the `packs` configuration field will be skipped if their target language is not being analyzed in the current Actions job. Previously, this would throw an error. [#1116](https://github.com/github/codeql-action/pull/1116)
+- The combination of python2 and poetry is no longer supported. See https://github.com/actions/setup-python/issues/374 for more details. [#1124](https://github.com/github/codeql-action/pull/1124)
+- Update default CodeQL bundle version to 2.10.0. [#1123](https://github.com/github/codeql-action/pull/1123)
+
 ## 1.1.14 - 22 Jun 2022
 
 No user facing changes.

lib/defaults.json

@@ -1,3 +1,3 @@
 {
-    "bundleVersion": "codeql-bundle-20220615"
+    "bundleVersion": "codeql-bundle-20220623"
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "1.1.14",
+  "version": "1.1.15",
   "private": true,
   "description": "CodeQL action",
   "scripts": {

runner/package.json

@@ -7,11 +7,10 @@
     "build-runner": "webpack --mode production && pkg dist/codeql-runner.js  --out-path dist"
   },
   "license": "MIT",
-  "dependencies": {},
   "devDependencies": {
     "pkg": "^5.3.1",
-    "ts-loader": "9.2.5",
-    "webpack": "^5.50.0",
-    "webpack-cli": "^4.7.2"
+    "ts-loader": "^9.3.1",
+    "webpack": "^5.73.0",
+    "webpack-cli": "^4.10.0"
   }
 }

src/analyze-action.ts

@@ -12,9 +12,11 @@ import {
   runQueries,
   runFinalize,
 } from "./analyze";
+import { getGitHubVersionActionsOnly } from "./api-client";
 import { CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
 import { uploadDatabases } from "./database-upload";
+import { GitHubFeatureFlags } from "./feature-flags";
 import { getActionsLogger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 import * as upload_lib from "./upload-lib";
@@ -112,7 +114,16 @@ async function run() {
       util.getRequiredEnvParam("GITHUB_REPOSITORY")
     );
 
-    await runFinalize(outputDir, threads, memory, config, logger);
+    const gitHubVersion = await getGitHubVersionActionsOnly();
+
+    const featureFlags = new GitHubFeatureFlags(
+      gitHubVersion,
+      apiDetails,
+      repositoryNwo,
+      logger
+    );
+
+    await runFinalize(outputDir, threads, memory, config, logger, featureFlags);
     if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
       runStats = await runQueries(
         outputDir,

src/analyze.test.ts

@@ -5,10 +5,12 @@ import test from "ava";
 import * as yaml from "js-yaml";
 import * as sinon from "sinon";
 
-import { runQueries } from "./analyze";
-import { setCodeQL } from "./codeql";
+import { runQueries, createdDBForScannedLanguages } from "./analyze";
+import { setCodeQL, getCodeQLForTesting } from "./codeql";
+import { stubToolRunnerConstructor } from "./codeql.test";
 import { Config } from "./config-utils";
 import * as count from "./count-loc";
+import { createFeatureFlags, FeatureFlag } from "./feature-flags";
 import { Language } from "./languages";
 import { getRunnerLogger } from "./logging";
 import { setupTests, setupActionsVars } from "./testing-utils";
@@ -249,3 +251,99 @@ test("status report fields and search path setting", async (t) => {
     }
   }
 });
+
+const stubConfig: Config = {
+  languages: [Language.cpp, Language.go],
+  queries: {},
+  pathsIgnore: [],
+  paths: [],
+  originalUserInput: {},
+  tempDir: "",
+  toolCacheDir: "",
+  codeQLCmd: "",
+  gitHubVersion: {
+    type: util.GitHubVariant.DOTCOM,
+  } as util.GitHubVersion,
+  dbLocation: "",
+  packs: {},
+  debugMode: false,
+  debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
+  debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
+  injectedMlQueries: false,
+};
+
+for (const options of [
+  {
+    name: "Lua feature flag enabled, but old CLI",
+    version: "2.9.0",
+    featureFlags: [FeatureFlag.LuaTracerConfigEnabled],
+    yesFlagSet: false,
+    noFlagSet: false,
+  },
+  {
+    name: "Lua feature flag disabled, with old CLI",
+    version: "2.9.0",
+    featureFlags: [],
+    yesFlagSet: false,
+    noFlagSet: false,
+  },
+  {
+    name: "Lua feature flag enabled, with new CLI",
+    version: "2.10.0",
+    featureFlags: [FeatureFlag.LuaTracerConfigEnabled],
+    yesFlagSet: true,
+    noFlagSet: false,
+  },
+  {
+    name: "Lua feature flag disabled, with new CLI",
+    version: "2.10.0",
+    featureFlags: [],
+    yesFlagSet: false,
+    noFlagSet: true,
+  },
+]) {
+  test(`createdDBForScannedLanguages() ${options.name}`, async (t) => {
+    const runnerConstructorStub = stubToolRunnerConstructor();
+    const codeqlObject = await getCodeQLForTesting("codeql/for-testing");
+    sinon.stub(codeqlObject, "getVersion").resolves(options.version);
+
+    const promise = createdDBForScannedLanguages(
+      codeqlObject,
+      stubConfig,
+      getRunnerLogger(true),
+      createFeatureFlags(options.featureFlags)
+    );
+    // call listener on `codeql resolve extractor`
+    const mockToolRunner = runnerConstructorStub.getCall(0);
+    mockToolRunner.args[2].listeners.stdout('"/path/to/extractor"');
+    await promise;
+    if (options.yesFlagSet)
+      t.true(
+        runnerConstructorStub.secondCall.args[1].includes(
+          "--internal-use-lua-tracing"
+        ),
+        "--internal-use-lua-tracing should be present, but it is absent"
+      );
+    else
+      t.false(
+        runnerConstructorStub.secondCall.args[1].includes(
+          "--internal-use-lua-tracing"
+        ),
+        "--internal-use-lua-tracing should be absent, but it is present"
+      );
+    if (options.noFlagSet)
+      t.true(
+        runnerConstructorStub.secondCall.args[1].includes(
+          "--no-internal-use-lua-tracing"
+        ),
+        "--no-internal-use-lua-tracing should be present, but it is absent"
+      );
+    else
+      t.false(
+        runnerConstructorStub.secondCall.args[1].includes(
+          "--no-internal-use-lua-tracing"
+        ),
+        "--no-internal-use-lua-tracing should be absent, but it is present"
+      );
+  });
+}

src/analyze.ts

@@ -7,12 +7,14 @@ import * as yaml from "js-yaml";
 
 import * as analysisPaths from "./analysis-paths";
 import {
+  CodeQL,
   CODEQL_VERSION_COUNTS_LINES,
   CODEQL_VERSION_NEW_TRACING,
   getCodeQL,
 } from "./codeql";
 import * as configUtils from "./config-utils";
 import { countLoc } from "./count-loc";
+import { FeatureFlags } from "./feature-flags";
 import { isScannedLanguage, Language } from "./languages";
 import { Logger } from "./logging";
 import * as sharedEnv from "./shared-environment";
@@ -114,15 +116,16 @@ async function setupPythonExtractor(logger: Logger) {
   process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
 }
 
-async function createdDBForScannedLanguages(
+export async function createdDBForScannedLanguages(
+  codeql: CodeQL,
   config: configUtils.Config,
-  logger: Logger
+  logger: Logger,
+  featureFlags: FeatureFlags
 ) {
   // Insert the LGTM_INDEX_X env vars at this point so they are set when
   // we extract any scanned languages.
   analysisPaths.includeAndExcludeAnalysisPaths(config);
 
-  const codeql = await getCodeQL(config.codeQLCmd);
   for (const language of config.languages) {
     if (
       isScannedLanguage(language) &&
@@ -136,7 +139,8 @@ async function createdDBForScannedLanguages(
 
       await codeql.extractScannedLanguage(
         util.getCodeQLDatabasePath(config, language),
-        language
+        language,
+        featureFlags
       );
       logger.endGroup();
     }
@@ -166,11 +170,12 @@ async function finalizeDatabaseCreation(
   config: configUtils.Config,
   threadsFlag: string,
   memoryFlag: string,
-  logger: Logger
+  logger: Logger,
+  featureFlags: FeatureFlags
 ) {
-  await createdDBForScannedLanguages(config, logger);
-
   const codeql = await getCodeQL(config.codeQLCmd);
+  await createdDBForScannedLanguages(codeql, config, logger, featureFlags);
+
   for (const language of config.languages) {
     if (dbIsFinalized(config, language, logger)) {
       logger.info(
@@ -425,7 +430,8 @@ export async function runFinalize(
   threadsFlag: string,
   memoryFlag: string,
   config: configUtils.Config,
-  logger: Logger
+  logger: Logger,
+  featureFlags: FeatureFlags
 ) {
   const codeql = await getCodeQL(config.codeQLCmd);
   if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
@@ -445,7 +451,13 @@ export async function runFinalize(
   }
   await fs.promises.mkdir(outputDir, { recursive: true });
 
-  await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger);
+  await finalizeDatabaseCreation(
+    config,
+    threadsFlag,
+    memoryFlag,
+    logger,
+    featureFlags
+  );
 }
 
 export async function runCleanup(

src/codeql.test.ts

@@ -540,7 +540,7 @@ test("databaseInitCluster() Lua feature flag disabled, compatible CLI", async (t
   );
 });
 
-function stubToolRunnerConstructor(): sinon.SinonStub<
+export function stubToolRunnerConstructor(): sinon.SinonStub<
   any[],
   toolrunner.ToolRunner
 > {

src/codeql.ts

@@ -95,7 +95,11 @@ export interface CodeQL {
    * Extract code for a scanned language using 'codeql database trace-command'
    * and running the language extractor.
    */
-  extractScannedLanguage(database: string, language: Language): Promise<void>;
+  extractScannedLanguage(
+    database: string,
+    language: Language,
+    featureFlags: FeatureFlags
+  ): Promise<void>;
   /**
    * Finalize a database using 'codeql database finalize'.
    */
@@ -630,8 +634,10 @@ export function getCachedCodeQL(): CodeQL {
  * a non-existent placeholder codeql command, so tests that use this function
  * should also stub the toolrunner.ToolRunner constructor.
  */
-export async function getCodeQLForTesting(): Promise<CodeQL> {
-  return getCodeQLForCmd("codeql-for-testing", false);
+export async function getCodeQLForTesting(
+  cmd = "codeql-for-testing"
+): Promise<CodeQL> {
+  return getCodeQLForCmd(cmd, false);
 }
 
 /**
@@ -789,7 +795,11 @@ async function getCodeQLForCmd(
 
       await runTool(autobuildCmd);
     },
-    async extractScannedLanguage(databasePath: string, language: Language) {
+    async extractScannedLanguage(
+      databasePath: string,
+      language: Language,
+      featureFlags: FeatureFlags
+    ) {
       // Get extractor location
       let extractorPath = "";
       await new toolrunner.ToolRunner(
@@ -821,13 +831,24 @@ async function getCodeQLForCmd(
         "tools",
         `autobuild${ext}`
       );
+      const extraArgs: string[] = [];
+      if (
+        await util.codeQlVersionAbove(this, CODEQL_VERSION_LUA_TRACER_CONFIG)
+      ) {
+        if (await featureFlags.getValue(FeatureFlag.LuaTracerConfigEnabled)) {
+          extraArgs.push("--internal-use-lua-tracing");
+        } else {
+          extraArgs.push("--no-internal-use-lua-tracing");
+        }
+      }
 
       // Run trace command
       await toolrunnerErrorCatcher(
         cmd,
         [
           "database",
           "trace-command",
+          ...extraArgs,
           ...getExtraOptionsFromEnv(["database", "trace-command"]),
           databasePath,
           "--",

src/config-utils.test.ts

@@ -10,7 +10,7 @@ import { getCachedCodeQL, setCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
 import { createFeatureFlags, FeatureFlag } from "./feature-flags";
 import { Language } from "./languages";
-import { getRunnerLogger } from "./logging";
+import { getRunnerLogger, Logger } from "./logging";
 import { setupTests } from "./testing-utils";
 import * as util from "./util";
 
@@ -1424,7 +1424,12 @@ const parsePacksMacro = test.macro({
     expected: Partial<Record<Language, string[]>>
   ) =>
     t.deepEqual(
-      configUtils.parsePacksFromConfig(packsByLanguage, languages, "/a/b"),
+      configUtils.parsePacksFromConfig(
+        packsByLanguage,
+        languages,
+        "/a/b",
+        mockLogger
+      ),
       expected
     ),
 
@@ -1446,7 +1451,8 @@ const parsePacksErrorMacro = test.macro({
         configUtils.parsePacksFromConfig(
           packsByLanguage as string[] | Record<string, string[]>,
           languages,
-          "/a/b"
+          "/a/b",
+          {} as Logger
         ),
       {
         message: expected,
@@ -1499,6 +1505,19 @@ test(
   }
 );
 
+test(
+  "two packs with unused language in config",
+  parsePacksMacro,
+  {
+    [Language.cpp]: ["a/b", "c/d@1.2.3"],
+    [Language.java]: ["d/e", "f/g@1.2.3"],
+  },
+  [Language.cpp, Language.csharp],
+  {
+    [Language.cpp]: ["a/b", "c/d@1.2.3"],
+  }
+);
+
 test(
   "packs with other valid names",
   parsePacksMacro,
@@ -1544,13 +1563,6 @@ test(
   [Language.java, Language.python],
   /The configuration file "\/a\/b" is invalid: property "packs" must split packages by language/
 );
-test(
-  "invalid language",
-  parsePacksErrorMacro,
-  { [Language.java]: ["c/d"] },
-  [Language.cpp],
-  /The configuration file "\/a\/b" is invalid: property "packs" has "java", but it is not one of the languages to analyze/
-);
 test(
   "not an array",
   parsePacksErrorMacro,
@@ -1583,13 +1595,25 @@ function parseInputAndConfigMacro(
   expected
 ) {
   t.deepEqual(
-    configUtils.parsePacks(packsFromConfig, packsFromInput, languages, "/a/b"),
+    configUtils.parsePacks(
+      packsFromConfig,
+      packsFromInput,
+      languages,
+      "/a/b",
+      mockLogger
+    ),
     expected
   );
 }
 parseInputAndConfigMacro.title = (providedTitle: string) =>
   `Parse Packs input and config: ${providedTitle}`;
 
+const mockLogger = {
+  info: (message: string) => {
+    console.log(message);
+  },
+} as Logger;
+
 function parseInputAndConfigErrorMacro(
   t: ExecutionContext<unknown>,
   packsFromConfig: string[] | Record<string, string[]>,
@@ -1603,7 +1627,8 @@ function parseInputAndConfigErrorMacro(
         packsFromConfig,
         packsFromInput,
         languages,
-        "/a/b"
+        "/a/b",
+        mockLogger
       );
     },
     {

src/config-utils.ts

@@ -629,14 +629,11 @@ export function getPathsInvalid(configFile: string): string {
   );
 }
 
-export function getPacksRequireLanguage(
-  lang: string,
-  configFile: string
-): string {
+function getPacksRequireLanguage(lang: string, configFile: string): string {
   return getConfigFilePropertyError(
     configFile,
     PACKS_PROPERTY,
-    `has "${lang}", but it is not one of the languages to analyze`
+    `has "${lang}", but it is not a valid language.`
   );
 }
 
@@ -1026,7 +1023,8 @@ async function loadConfig(
     parsedYAML[PACKS_PROPERTY] ?? {},
     packsInput,
     languages,
-    configFile
+    configFile,
+    logger
   );
 
   // If queries were provided using `with` in the action configuration,
@@ -1146,7 +1144,8 @@ const PACK_IDENTIFIER_PATTERN = (function () {
 export function parsePacksFromConfig(
   packsByLanguage: string[] | Record<string, string[]>,
   languages: Language[],
-  configFile: string
+  configFile: string,
+  logger: Logger
 ): Packs {
   const packs = {};
 
@@ -1168,7 +1167,16 @@ export function parsePacksFromConfig(
       throw new Error(getPacksInvalid(configFile));
     }
     if (!languages.includes(lang as Language)) {
-      throw new Error(getPacksRequireLanguage(lang, configFile));
+      // This particular language is not being analyzed in this run.
+      if (Language[lang as Language]) {
+        logger.info(
+          `Ignoring packs for ${lang} since this language is not being analyzed in this run.`
+        );
+        continue;
+      } else {
+        // This language is invalid, probably a misspelling
+        throw new Error(getPacksRequireLanguage(configFile, lang));
+      }
     }
     packs[lang] = [];
     for (const packStr of packsArr) {
@@ -1296,13 +1304,15 @@ export function parsePacks(
   rawPacksFromConfig: string[] | Record<string, string[]>,
   rawPacksInput: string | undefined,
   languages: Language[],
-  configFile: string
+  configFile: string,
+  logger: Logger
 ) {
   const packsFromInput = parsePacksFromInput(rawPacksInput, languages);
   const packsFomConfig = parsePacksFromConfig(
     rawPacksFromConfig,
     languages,
-    configFile
+    configFile,
+    logger
   );
 
   if (!packsFromInput) {

src/defaults.json

@@ -1,3 +1,3 @@
 {
-  "bundleVersion": "codeql-bundle-20220615"
+  "bundleVersion": "codeql-bundle-20220623"
 }

src/runner.ts

@@ -501,7 +501,14 @@ program
         logger
       );
       const memory = getMemoryFlag(cmd.ram || initEnv["CODEQL_RAM"]);
-      await runFinalize(outputDir, threads, memory, config, logger);
+      await runFinalize(
+        outputDir,
+        threads,
+        memory,
+        config,
+        logger,
+        createFeatureFlags([])
+      );
       await runQueries(
         outputDir,
         memory,

tests/multi-language-repo/.github/codeql/codeql-config-packaging.yml

@@ -6,6 +6,9 @@ packs:
     - dsp-testing/codeql-pack1@1.0.0
     - dsp-testing/codeql-pack2
     - dsp-testing/codeql-pack3:other-query.ql
+  ruby:
+    - dsp-testing/hucairz
+    - dsp-testing/i-dont-exist@1.0.0
 
 paths-ignore:
   - tests