.github/workflows/script/update-node-modules.sh

@@ -1,9 +1,12 @@
-if [ "$1" != "update" && "$1" != "check-only" ]; then
+#!/bin/bash
+set -eu
+
+if [ "$1" != "update" ] && [ "$1" != "check-only" ]; then
     >&2 echo "Failed: Invalid argument. Must be 'update' or 'check-only'"
     exit 1
 fi
 
-sudo npm install --force -g npm@9.2.0
+npm install --force -g npm@9.2.0
 
 # clean the npm cache to ensure we don't have any files owned by root
 sudo npm cache clean --force

CHANGELOG.md

@@ -4,6 +4,11 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
 
 Note that the only difference between `v2` and `v3` of the CodeQL Action is the node version they support, with `v3` running on node 20 while we continue to release `v2` to support running on node 16. For example `3.22.11` was the first `v3` release and is functionally identical to `2.22.11`. This approach ensures an easy way to track exactly which features are included in different versions, indicated by the minor and patch version numbers.
 
+## 3.25.9 - 12 Jun 2024
+
+- Avoid failing database creation if the database folder already exists and contains some unexpected files. Requires CodeQL 2.18.0 or higher. [#2330](https://github.com/github/codeql-action/pull/2330)
+- The init Action will attempt to clean up the database cluster directory before creating a new database and at the end of the job. This will help to avoid issues where the database cluster directory is left in an inconsistent state. [#2332](https://github.com/github/codeql-action/pull/2332)
+
 ## 3.25.8 - 04 Jun 2024
 
 - Update default CodeQL bundle version to 2.17.4. [#2321](https://github.com/github/codeql-action/pull/2321)

lib/api-compatibility.json

@@ -1 +1 @@
-{ "maximumVersion": "3.13", "minimumVersion": "3.9" }
+{ "maximumVersion": "3.14", "minimumVersion": "3.9" }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeql",
-  "version": "3.25.8",
+  "version": "3.25.9",
   "private": true,
   "description": "CodeQL action",
   "scripts": {
@@ -33,7 +33,7 @@
     "@octokit/types": "^13.5.0",
     "@schemastore/package": "0.0.10",
     "@types/uuid": "^9.0.8",
-    "adm-zip": "^0.5.13",
+    "adm-zip": "^0.5.14",
     "check-disk-space": "^3.4.0",
     "console-log-level": "^1.4.1",
     "del": "^6.1.1",
@@ -46,7 +46,7 @@
     "long": "^5.2.3",
     "path": "^0.12.7",
     "semver": "^7.6.2",
-    "uuid": "^9.0.1",
+    "uuid": "^10.0.0",
     "zlib": "^1.0.5"
   },
   "//": [
@@ -60,13 +60,13 @@
     "@types/node": "20.9.0",
     "@types/semver": "^7.5.8",
     "@types/sinon": "^17.0.3",
-    "@typescript-eslint/eslint-plugin": "^7.12.0",
-    "@typescript-eslint/parser": "^7.12.0",
+    "@typescript-eslint/eslint-plugin": "^7.13.0",
+    "@typescript-eslint/parser": "^7.13.0",
     "ava": "^5.3.1",
     "eslint": "^8.57.0",
     "eslint-import-resolver-typescript": "^3.6.1",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-github": "^5.0.0",
+    "eslint-plugin-github": "^5.0.1",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-no-async-foreach": "^0.1.1",
     "micromatch": "4.0.7",

pr-checks/checks/cleanup-db-cluster-dir.yml

@@ -0,0 +1,25 @@
+name: "Clean up database cluster directory"
+description: "The database cluster directory is cleaned up if it is not empty."
+operatingSystems: ["ubuntu"]
+versions: ["linked"]
+steps:
+  - name: Add a file to the database cluster directory
+    run: |
+      mkdir -p "${{ runner.temp }}/customDbLocation/javascript"
+      touch "${{ runner.temp }}/customDbLocation/javascript/a-file-to-clean-up.txt"
+
+  - uses: ./../action/init
+    id: init
+    with:
+      build-mode: none
+      db-location: "${{ runner.temp }}/customDbLocation"
+      languages: javascript
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+
+  - name: Validate file cleaned up
+    run: |
+      if [[ -f "${{ runner.temp }}/customDbLocation/javascript/a-file-to-clean-up.txt" ]]; then
+        echo "File was not cleaned up"
+        exit 1
+      fi
+      echo "File was cleaned up"

src/actions-util.ts

@@ -476,3 +476,7 @@ export const getFileType = async (filePath: string): Promise<string> => {
     throw e;
   }
 };
+
+export function isSelfHostedRunner() {
+  return process.env.RUNNER_ENVIRONMENT === "self-hosted";
+}

src/api-compatibility.json

@@ -1 +1 @@
-{"maximumVersion": "3.13", "minimumVersion": "3.9"}
+{"maximumVersion": "3.14", "minimumVersion": "3.9"}

src/codeql.ts

@@ -612,12 +612,19 @@ export async function getCodeQLForCmd(
         extraArgs.push("--no-sublanguage-file-coverage");
       }
 
+      const overwriteFlag = isSupportedToolsFeature(
+        await this.getVersion(),
+        ToolsFeature.ForceOverwrite,
+      )
+        ? "--force-overwrite"
+        : "--overwrite";
+
       await runTool(
         cmd,
         [
           "database",
           "init",
-          "--overwrite",
+          overwriteFlag,
           "--db-cluster",
           config.dbLocation,
           `--source-root=${sourceRoot}`,

src/init-action-post-helper.ts

@@ -1,3 +1,5 @@
+import * as fs from "fs";
+
 import * as core from "@actions/core";
 import * as github from "@actions/github";
 
@@ -217,6 +219,28 @@ export async function run(
     await printDebugLogs(config);
   }
 
+  if (actionsUtil.isSelfHostedRunner()) {
+    try {
+      fs.rmSync(config.dbLocation, {
+        recursive: true,
+        force: true,
+        maxRetries: 3,
+      });
+      logger.info(
+        `Cleaned up database cluster directory ${config.dbLocation}.`,
+      );
+    } catch (e) {
+      logger.warning(
+        `Failed to clean up database cluster directory ${config.dbLocation}. Details: ${e}`,
+      );
+    }
+  } else {
+    logger.debug(
+      "Skipping cleanup of database cluster directory since we are running on a GitHub-hosted " +
+        "runner which will be automatically cleaned up.",
+    );
+  }
+
   return uploadFailedSarifResult;
 }
 

src/init-action.ts

@@ -26,6 +26,7 @@ import { EnvVar } from "./environment";
 import { Feature, Features } from "./feature-flags";
 import {
   checkInstallPython311,
+  cleanupDatabaseClusterDirectory,
   initCodeQL,
   initConfig,
   isSipEnabled,
@@ -321,6 +322,8 @@ async function run() {
   }
 
   try {
+    cleanupDatabaseClusterDirectory(config, logger);
+
     // Forward Go flags
     const goFlags = process.env["GOFLAGS"];
     if (goFlags) {

src/init.test.ts

@@ -1,9 +1,21 @@
+import * as fs from "fs";
+import path from "path";
+
 import test from "ava";
 
 import { Config } from "./config-utils";
-import { printPathFiltersWarning } from "./init";
+import {
+  cleanupDatabaseClusterDirectory,
+  printPathFiltersWarning,
+} from "./init";
 import { Language } from "./languages";
-import { LoggedMessage, getRecordingLogger, setupTests } from "./testing-utils";
+import {
+  LoggedMessage,
+  createTestConfig,
+  getRecordingLogger,
+  setupTests,
+} from "./testing-utils";
+import { ConfigurationError, withTmpDir } from "./util";
 
 setupTests(test);
 
@@ -30,3 +42,96 @@ test("printPathFiltersWarning does not trigger when 'paths' and 'paths-ignore' a
   );
   t.is(messages.length, 0);
 });
+
+test("cleanupDatabaseClusterDirectory cleans up where possible", async (t) => {
+  await withTmpDir(async (tmpDir: string) => {
+    const dbLocation = path.resolve(tmpDir, "dbs");
+    fs.mkdirSync(dbLocation, { recursive: true });
+
+    const fileToCleanUp = path.resolve(dbLocation, "something-to-cleanup.txt");
+    fs.writeFileSync(fileToCleanUp, "");
+
+    const messages: LoggedMessage[] = [];
+    cleanupDatabaseClusterDirectory(
+      createTestConfig({ dbLocation }),
+      getRecordingLogger(messages),
+    );
+
+    t.is(messages.length, 2);
+    t.is(messages[0].type, "warning");
+    t.is(
+      messages[0].message,
+      `The database cluster directory ${dbLocation} must be empty. Attempting to clean it up.`,
+    );
+    t.is(messages[1].type, "info");
+    t.is(
+      messages[1].message,
+      `Cleaned up database cluster directory ${dbLocation}.`,
+    );
+
+    t.false(fs.existsSync(fileToCleanUp));
+  });
+});
+
+for (const { runnerEnv, ErrorConstructor, message } of [
+  {
+    runnerEnv: "self-hosted",
+    ErrorConstructor: ConfigurationError,
+    message: (dbLocation) =>
+      "The CodeQL Action requires an empty database cluster directory. By default, this is located " +
+      `at ${dbLocation}. You can customize it using the 'db-location' input to the init Action. An ` +
+      "attempt was made to clean up the directory, but this failed. This can happen if another " +
+      "process is using the directory or the directory is owned by a different user. Please clean " +
+      "up the directory manually and rerun the job.",
+  },
+  {
+    runnerEnv: "github-hosted",
+    ErrorConstructor: Error,
+    message: (dbLocation) =>
+      "The CodeQL Action requires an empty database cluster directory. By default, this is located " +
+      `at ${dbLocation}. You can customize it using the 'db-location' input to the init Action. An ` +
+      "attempt was made to clean up the directory, but this failed. This shouldn't typically " +
+      "happen on hosted runners. If you are using an advanced setup, please check your workflow, " +
+      "otherwise we recommend rerunning the job.",
+  },
+]) {
+  test(`cleanupDatabaseClusterDirectory throws a ${ErrorConstructor.name} when cleanup fails on ${runnerEnv} runner`, async (t) => {
+    await withTmpDir(async (tmpDir: string) => {
+      process.env["RUNNER_ENVIRONMENT"] = runnerEnv;
+
+      const dbLocation = path.resolve(tmpDir, "dbs");
+      fs.mkdirSync(dbLocation, { recursive: true });
+
+      const fileToCleanUp = path.resolve(
+        dbLocation,
+        "something-to-cleanup.txt",
+      );
+      fs.writeFileSync(fileToCleanUp, "");
+
+      const rmSyncError = `Failed to clean up file ${fileToCleanUp}`;
+
+      const messages: LoggedMessage[] = [];
+      t.throws(
+        () =>
+          cleanupDatabaseClusterDirectory(
+            createTestConfig({ dbLocation }),
+            getRecordingLogger(messages),
+            () => {
+              throw new Error(rmSyncError);
+            },
+          ),
+        {
+          instanceOf: ErrorConstructor,
+          message: `${message(dbLocation)} Details: ${rmSyncError}`,
+        },
+      );
+
+      t.is(messages.length, 1);
+      t.is(messages[0].type, "warning");
+      t.is(
+        messages[0].message,
+        `The database cluster directory ${dbLocation} must be empty. Attempting to clean it up.`,
+      );
+    });
+  });
+}

src/init.ts

@@ -5,6 +5,7 @@ import * as exec from "@actions/exec/lib/exec";
 import * as toolrunner from "@actions/exec/lib/toolrunner";
 import * as safeWhich from "@chrisgavin/safe-which";
 
+import { getOptionalInput, isSelfHostedRunner } from "./actions-util";
 import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
 import { CodeQL, setupCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
@@ -171,3 +172,57 @@ export async function isSipEnabled(logger): Promise<boolean | undefined> {
     return undefined;
   }
 }
+
+export function cleanupDatabaseClusterDirectory(
+  config: configUtils.Config,
+  logger: Logger,
+  // We can't stub the fs module in tests, so we allow the caller to override the rmSync function
+  // for testing.
+  rmSync = fs.rmSync,
+): void {
+  if (
+    fs.existsSync(config.dbLocation) &&
+    (fs.statSync(config.dbLocation).isFile() ||
+      fs.readdirSync(config.dbLocation).length)
+  ) {
+    logger.warning(
+      `The database cluster directory ${config.dbLocation} must be empty. Attempting to clean it up.`,
+    );
+    try {
+      rmSync(config.dbLocation, {
+        force: true,
+        maxRetries: 3,
+        recursive: true,
+      });
+
+      logger.info(
+        `Cleaned up database cluster directory ${config.dbLocation}.`,
+      );
+    } catch (e) {
+      const blurb = `The CodeQL Action requires an empty database cluster directory. ${
+        getOptionalInput("db-location")
+          ? `This is currently configured to be ${config.dbLocation}. `
+          : `By default, this is located at ${config.dbLocation}. ` +
+            "You can customize it using the 'db-location' input to the init Action. "
+      }An attempt was made to clean up the directory, but this failed.`;
+
+      // Hosted runners are automatically cleaned up, so this error should not occur for hosted runners.
+      if (isSelfHostedRunner()) {
+        throw new util.ConfigurationError(
+          `${blurb} This can happen if another process is using the directory or the directory is owned by a different user. ` +
+            `Please clean up the directory manually and rerun the job. Details: ${
+              util.wrapError(e).message
+            }`,
+        );
+      } else {
+        throw new Error(
+          `${blurb} This shouldn't typically happen on hosted runners. ` +
+            "If you are using an advanced setup, please check your workflow, otherwise we " +
+            `recommend rerunning the job. Details: ${
+              util.wrapError(e).message
+            }`,
+        );
+      }
+    }
+  }
+}

src/tools-features.ts

@@ -8,6 +8,7 @@ export enum ToolsFeature {
   SetsCodeqlRunnerEnvVar = "setsCodeqlRunnerEnvVar",
   TraceCommandUseBuildMode = "traceCommandUseBuildMode",
   SarifMergeRunsFromEqualCategory = "sarifMergeRunsFromEqualCategory",
+  ForceOverwrite = "forceOverwrite",
 }
 
 /**