refactor(manager/cargo): use zod to parse manifest/config (#31260)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Rhys Arkins <rhys@arkins.net>

Author: 3 people

lib/modules/manager/cargo/__snapshots__/extract.spec.ts.snap

@@ -461,6 +461,7 @@ exports[`modules/manager/cargo/extract extractPackageFile() extracts registry ur
     "depType": "dependencies",
     "managerData": {
       "nestedVersion": true,
+      "registryName": "private-crates",
     },
     "registryUrls": [
       "https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git",
@@ -473,6 +474,7 @@ exports[`modules/manager/cargo/extract extractPackageFile() extracts registry ur
     "depType": "dependencies",
     "managerData": {
       "nestedVersion": true,
+      "registryName": "mcorbin",
     },
     "registryUrls": [
       "https://github.com/mcorbin/testregistry",
@@ -499,6 +501,7 @@ exports[`modules/manager/cargo/extract extractPackageFile() extracts registry ur
     "depType": "dependencies",
     "managerData": {
       "nestedVersion": true,
+      "registryName": "private-crates",
     },
     "registryUrls": [
       "https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git",
@@ -511,6 +514,7 @@ exports[`modules/manager/cargo/extract extractPackageFile() extracts registry ur
     "depType": "dependencies",
     "managerData": {
       "nestedVersion": true,
+      "registryName": "mcorbin",
     },
     "registryUrls": [
       "https://github.com/mcorbin/testregistry",
@@ -537,6 +541,7 @@ exports[`modules/manager/cargo/extract extractPackageFile() fails to parse cargo
     "depType": "dependencies",
     "managerData": {
       "nestedVersion": true,
+      "registryName": "private-crates",
     },
     "skipReason": "unknown-registry",
   },
@@ -547,6 +552,7 @@ exports[`modules/manager/cargo/extract extractPackageFile() fails to parse cargo
     "depType": "dependencies",
     "managerData": {
       "nestedVersion": true,
+      "registryName": "mcorbin",
     },
     "skipReason": "unknown-registry",
   },
@@ -718,6 +724,7 @@ exports[`modules/manager/cargo/extract extractPackageFile() ignore cargo config
     "depType": "dependencies",
     "managerData": {
       "nestedVersion": true,
+      "registryName": "private-crates",
     },
     "skipReason": "unknown-registry",
   },
@@ -728,6 +735,7 @@ exports[`modules/manager/cargo/extract extractPackageFile() ignore cargo config
     "depType": "dependencies",
     "managerData": {
       "nestedVersion": true,
+      "registryName": "mcorbin",
     },
     "skipReason": "unknown-registry",
   },
@@ -752,6 +760,7 @@ exports[`modules/manager/cargo/extract extractPackageFile() skips unknown regist
     "depType": "dependencies",
     "managerData": {
       "nestedVersion": true,
+      "registryName": "not-listed",
     },
     "skipReason": "unknown-registry",
   },

lib/modules/manager/cargo/extract.spec.ts

@@ -136,6 +136,7 @@ replace-with = "private-crates"`,
           depType: 'dependencies',
           managerData: {
             nestedVersion: true,
+            registryName: 'private-crates',
           },
           registryUrls: [
             'https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git',
@@ -148,6 +149,7 @@ replace-with = "private-crates"`,
           depType: 'dependencies',
           managerData: {
             nestedVersion: true,
+            registryName: 'mcorbin',
           },
           registryUrls: [
             'https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git',
@@ -212,6 +214,7 @@ replace-with = "mcorbin"`,
           depType: 'dependencies',
           managerData: {
             nestedVersion: true,
+            registryName: 'private-crates',
           },
         },
         {
@@ -221,6 +224,7 @@ replace-with = "mcorbin"`,
           depType: 'dependencies',
           managerData: {
             nestedVersion: true,
+            registryName: 'mcorbin',
           },
         },
         {
@@ -302,6 +306,7 @@ replace-with = "mcorbin"`,
           depType: 'dependencies',
           managerData: {
             nestedVersion: true,
+            registryName: 'private-crates',
           },
           registryUrls: [
             'https://dl.cloudsmith.io/basic/my-org/my-repo/cargo/index.git',
@@ -314,6 +319,7 @@ replace-with = "mcorbin"`,
           depType: 'dependencies',
           managerData: {
             nestedVersion: true,
+            registryName: 'mcorbin',
           },
           registryUrls: ['https://github.com/mcorbin/testregistry'],
         },
@@ -437,6 +443,7 @@ replace-with = "mine"`,
           depType: 'dependencies',
           managerData: {
             nestedVersion: true,
+            registryName: 'private-crates',
           },
           skipReason: 'unknown-registry',
         },
@@ -447,6 +454,7 @@ replace-with = "mine"`,
           depType: 'dependencies',
           managerData: {
             nestedVersion: true,
+            registryName: 'mcorbin',
           },
           skipReason: 'unknown-registry',
         },
@@ -493,6 +501,7 @@ replace-with = "mcorbin"
           depType: 'dependencies',
           managerData: {
             nestedVersion: true,
+            registryName: 'private-crates',
           },
           skipReason: 'unknown-registry',
         },
@@ -503,6 +512,7 @@ replace-with = "mcorbin"
           depType: 'dependencies',
           managerData: {
             nestedVersion: true,
+            registryName: 'mcorbin',
           },
           skipReason: 'unknown-registry',
         },

lib/modules/manager/cargo/extract.ts

@@ -1,22 +1,22 @@
 import { logger } from '../../../logger';
-import type { SkipReason } from '../../../types';
 import { coerceArray } from '../../../util/array';
 import { findLocalSiblingOrParent, readLocalFile } from '../../../util/fs';
-import { parse as parseToml } from '../../../util/toml';
-import { CrateDatasource } from '../../datasource/crate';
 import { api as versioning } from '../../versioning/cargo';
 import type {
   ExtractConfig,
   PackageDependency,
   PackageFileContent,
 } from '../types';
 import { extractLockFileVersions } from './locked-version';
+import {
+  type CargoConfig,
+  CargoConfigSchema,
+  CargoManifestSchema,
+} from './schema';
 import type {
-  CargoConfig,
-  CargoManifest,
+  CargoManagerData,
   CargoRegistries,
   CargoRegistryUrl,
-  CargoSection,
 } from './types';
 import { DEFAULT_REGISTRY_URL } from './utils';
 
@@ -28,75 +28,32 @@ function getCargoIndexEnv(registryName: string): string | null {
 }
 
 function extractFromSection(
-  parsedContent: CargoSection,
-  section: keyof CargoSection,
+  dependencies: PackageDependency<CargoManagerData>[] | undefined,
   cargoRegistries: CargoRegistries,
   target?: string,
-  depTypeOverride?: string,
 ): PackageDependency[] {
-  const deps: PackageDependency[] = [];
-  const sectionContent = parsedContent[section];
-  if (!sectionContent) {
+  if (!dependencies) {
     return [];
   }
-  Object.keys(sectionContent).forEach((depName) => {
-    let skipReason: SkipReason | undefined;
-    let currentValue = sectionContent[depName];
-    let nestedVersion = false;
-    let registryUrls: string[] | undefined;
-    let packageName: string | undefined;
-
-    if (typeof currentValue !== 'string') {
-      const version = currentValue.version;
-      const path = currentValue.path;
-      const git = currentValue.git;
-      const registryName = currentValue.registry;
-      const workspace = currentValue.workspace;
 
-      packageName = currentValue.package;
+  const deps: PackageDependency<CargoManagerData>[] = [];
 
-      if (version) {
-        currentValue = version;
-        nestedVersion = true;
-        if (registryName) {
-          const registryUrl =
-            getCargoIndexEnv(registryName) ?? cargoRegistries[registryName];
+  for (const dep of Object.values(dependencies)) {
+    let registryUrls: string[] | undefined;
 
-          if (registryUrl) {
-            if (registryUrl !== DEFAULT_REGISTRY_URL) {
-              registryUrls = [registryUrl];
-            }
-          } else {
-            skipReason = 'unknown-registry';
-          }
-        }
-        if (path) {
-          skipReason = 'path-dependency';
-        }
-        if (git) {
-          skipReason = 'git-dependency';
+    if (dep.managerData?.registryName) {
+      const registryUrl =
+        getCargoIndexEnv(dep.managerData.registryName) ??
+        cargoRegistries[dep.managerData?.registryName];
+      if (registryUrl) {
+        if (registryUrl !== DEFAULT_REGISTRY_URL) {
+          registryUrls = [registryUrl];
         }
-      } else if (path) {
-        currentValue = '';
-        skipReason = 'path-dependency';
-      } else if (git) {
-        currentValue = '';
-        skipReason = 'git-dependency';
-      } else if (workspace) {
-        currentValue = '';
-        skipReason = 'inherited-dependency';
       } else {
-        currentValue = '';
-        skipReason = 'invalid-dependency-specification';
+        dep.skipReason = 'unknown-registry';
       }
     }
-    const dep: PackageDependency = {
-      depName,
-      depType: section,
-      currentValue: currentValue as any,
-      managerData: { nestedVersion },
-      datasource: CrateDatasource.id,
-    };
+
     if (registryUrls) {
       dep.registryUrls = registryUrls;
     } else {
@@ -108,24 +65,16 @@ function extractFromSection(
       } else {
         // we always expect to have DEFAULT_REGISTRY_ID set, if it's not it means the config defines an alternative
         // registry that we couldn't resolve.
-        skipReason = 'unknown-registry';
+        dep.skipReason = 'unknown-registry';
       }
     }
 
-    if (skipReason) {
-      dep.skipReason = skipReason;
-    }
     if (target) {
       dep.target = target;
     }
-    if (packageName) {
-      dep.packageName = packageName;
-    }
-    if (depTypeOverride) {
-      dep.depType = depTypeOverride;
-    }
     deps.push(dep);
-  });
+  }
+
   return deps;
 }
 
@@ -135,12 +84,15 @@ async function readCargoConfig(): Promise<CargoConfig | null> {
     const path = `.cargo/${configName}`;
     const payload = await readLocalFile(path, 'utf8');
     if (payload) {
-      try {
-        return parseToml(payload) as CargoConfig;
-      } catch (err) {
-        logger.debug({ err }, `Error parsing ${path}`);
+      const parsedCargoConfig = CargoConfigSchema.safeParse(payload);
+      if (parsedCargoConfig.success) {
+        return parsedCargoConfig.data;
+      } else {
+        logger.debug(
+          { err: parsedCargoConfig.error, path },
+          `Error parsing cargo config`,
+        );
       }
-      break;
     }
   }
 
@@ -217,19 +169,23 @@ export async function extractPackageFile(
   content: string,
   packageFile: string,
   _config?: ExtractConfig,
-): Promise<PackageFileContent | null> {
+): Promise<PackageFileContent<CargoManagerData> | null> {
   logger.trace(`cargo.extractPackageFile(${packageFile})`);
 
   const cargoConfig = (await readCargoConfig()) ?? {};
   const cargoRegistries = extractCargoRegistries(cargoConfig);
 
-  let cargoManifest: CargoManifest;
-  try {
-    cargoManifest = parseToml(content) as CargoManifest;
-  } catch (err) {
-    logger.debug({ err, packageFile }, 'Error parsing Cargo.toml file');
+  const parsedCargoManifest = CargoManifestSchema.safeParse(content);
+  if (!parsedCargoManifest.success) {
+    logger.debug(
+      { err: parsedCargoManifest.error, packageFile },
+      'Error parsing Cargo.toml file',
+    );
     return null;
   }
+
+  const cargoManifest = parsedCargoManifest.data;
+
   /*
     There are the following sections in Cargo.toml:
     [package]
@@ -249,20 +205,17 @@ export async function extractPackageFile(
       // Dependencies for `${target}`
       const deps = [
         ...extractFromSection(
-          targetContent,
-          'dependencies',
+          targetContent.dependencies,
           cargoRegistries,
           target,
         ),
         ...extractFromSection(
-          targetContent,
-          'dev-dependencies',
+          targetContent['dev-dependencies'],
           cargoRegistries,
           target,
         ),
         ...extractFromSection(
-          targetContent,
-          'build-dependencies',
+          targetContent['build-dependencies'],
           cargoRegistries,
           target,
         ),
@@ -275,18 +228,16 @@ export async function extractPackageFile(
   let workspaceDeps: PackageDependency[] = [];
   if (workspaceSection) {
     workspaceDeps = extractFromSection(
-      workspaceSection,
-      'dependencies',
+      workspaceSection.dependencies,
       cargoRegistries,
       undefined,
-      'workspace.dependencies',
     );
   }
 
   const deps = [
-    ...extractFromSection(cargoManifest, 'dependencies', cargoRegistries),
-    ...extractFromSection(cargoManifest, 'dev-dependencies', cargoRegistries),
-    ...extractFromSection(cargoManifest, 'build-dependencies', cargoRegistries),
+    ...extractFromSection(cargoManifest.dependencies, cargoRegistries),
+    ...extractFromSection(cargoManifest['dev-dependencies'], cargoRegistries),
+    ...extractFromSection(cargoManifest['build-dependencies'], cargoRegistries),
     ...targetDeps,
     ...workspaceDeps,
   ];

lib/modules/manager/cargo/schema.ts

@@ -1,5 +1,133 @@
 import { z } from 'zod';
-import { Toml } from '../../../util/schema-utils';
+import type { SkipReason } from '../../../types';
+import { Toml, withDepType } from '../../../util/schema-utils';
+import { CrateDatasource } from '../../datasource/crate';
+import type { PackageDependency } from '../types';
+import type { CargoManagerData } from './types';
+
+const CargoDep = z.union([
+  z
+    .object({
+      /** Path on disk to the crate sources */
+      path: z.string().optional(),
+      /** Git URL for the dependency */
+      git: z.string().optional(),
+      /** Semver version */
+      version: z.string().optional(),
+      /** Name of a registry whose URL is configured in `.cargo/config.toml` or `.cargo/config` */
+      registry: z.string().optional(),
+      /** Name of a package to look up */
+      package: z.string().optional(),
+      /** Whether the dependency is inherited from the workspace */
+      workspace: z.boolean().optional(),
+    })
+    .transform(
+      ({
+        path,
+        git,
+        version,
+        registry,
+        package: pkg,
+        workspace,
+      }): PackageDependency<CargoManagerData> => {
+        let skipReason: SkipReason | undefined;
+        let currentValue: string | undefined;
+        let nestedVersion = false;
+
+        if (version) {
+          currentValue = version;
+          nestedVersion = true;
+        } else {
+          currentValue = '';
+          skipReason = 'invalid-dependency-specification';
+        }
+
+        if (path) {
+          skipReason = 'path-dependency';
+        } else if (git) {
+          skipReason = 'git-dependency';
+        } else if (workspace) {
+          skipReason = 'inherited-dependency';
+        }
+
+        const dep: PackageDependency<CargoManagerData> = {
+          currentValue,
+          managerData: { nestedVersion },
+          datasource: CrateDatasource.id,
+        };
+
+        if (skipReason) {
+          dep.skipReason = skipReason;
+        }
+        if (pkg) {
+          dep.packageName = pkg;
+        }
+        if (registry) {
+          dep.managerData!.registryName = registry;
+        }
+
+        return dep;
+      },
+    ),
+  z.string().transform(
+    (version): PackageDependency<CargoManagerData> => ({
+      currentValue: version,
+      managerData: { nestedVersion: false },
+      datasource: CrateDatasource.id,
+    }),
+  ),
+]);
+
+const CargoDeps = z.record(z.string(), CargoDep).transform((record) => {
+  const deps: PackageDependency[] = [];
+
+  for (const [depName, dep] of Object.entries(record)) {
+    dep.depName = depName;
+    deps.push(dep);
+  }
+
+  return deps;
+});
+
+export type CargoDeps = z.infer<typeof CargoDeps>;
+
+const CargoSection = z.object({
+  dependencies: withDepType(CargoDeps, 'dependencies').optional(),
+  'dev-dependencies': withDepType(CargoDeps, 'dev-dependencies').optional(),
+  'build-dependencies': withDepType(CargoDeps, 'build-dependencies').optional(),
+});
+
+const CargoWorkspace = z.object({
+  dependencies: withDepType(CargoDeps, 'workspace.dependencies').optional(),
+});
+
+const CargoTarget = z.record(z.string(), CargoSection);
+
+export const CargoManifestSchema = Toml.pipe(
+  CargoSection.extend({
+    package: z.object({ version: z.string().optional() }).optional(),
+    workspace: CargoWorkspace.optional(),
+    target: CargoTarget.optional(),
+  }),
+);
+
+const CargoConfigRegistry = z.object({
+  index: z.string().optional(),
+});
+
+const CargoConfigSource = z.object({
+  'replace-with': z.string().optional(),
+  registry: z.string().optional(),
+});
+
+export const CargoConfigSchema = Toml.pipe(
+  z.object({
+    registries: z.record(z.string(), CargoConfigRegistry).optional(),
+    source: z.record(z.string(), CargoConfigSource).optional(),
+  }),
+);
+
+export type CargoConfig = z.infer<typeof CargoConfigSchema>;
 
 const CargoLockPackageSchema = z.object({
   name: z.string(),

lib/modules/manager/cargo/types.ts

@@ -1,57 +1,14 @@
 import type { DEFAULT_REGISTRY_URL } from './utils';
 
-export interface CargoPackage {
-  /** Semver version */
-  version: string;
-}
-
-export interface CargoDep {
-  /** Path on disk to the crate sources */
-  path?: string;
-  /** Git URL for the dependency */
-  git?: string;
-  /** Semver version */
-  version?: string;
-  /** Name of a registry whose URL is configured in `.cargo/config.toml` */
-  registry?: string;
-  /** Name of a package to look up */
-  package?: string;
-  /** Whether the dependency is inherited from the workspace*/
-  workspace?: boolean;
-}
-
-export type CargoDeps = Record<string, CargoDep | string>;
-
-export interface CargoSection {
-  dependencies?: CargoDeps;
-  'dev-dependencies'?: CargoDeps;
-  'build-dependencies'?: CargoDeps;
-}
-
-export interface CargoManifest extends CargoSection {
-  target?: Record<string, CargoSection>;
-  workspace?: CargoSection;
-  package?: CargoPackage;
-}
-
-export interface CargoConfig {
-  registries?: Record<string, CargoRegistry>;
-  source?: Record<string, CargoSource>;
-}
-
-export interface CargoRegistry {
-  index?: string;
-}
-
-export interface CargoSource {
-  'replace-with'?: string;
-  registry?: string;
-}
-
 /**
  * null means a registry was defined, but we couldn't find a valid URL
  */
 export type CargoRegistryUrl = string | typeof DEFAULT_REGISTRY_URL | null;
 export interface CargoRegistries {
   [key: string]: CargoRegistryUrl;
 }
+
+export interface CargoManagerData {
+  nestedVersion?: boolean;
+  registryName?: string;
+}

lib/modules/manager/poetry/schema.ts

@@ -1,9 +1,13 @@
-import type { ZodEffects, ZodType, ZodTypeDef } from 'zod';
 import { z } from 'zod';
 import { logger } from '../../../logger';
 import { parseGitUrl } from '../../../util/git/url';
 import { regEx } from '../../../util/regex';
-import { LooseArray, LooseRecord, Toml } from '../../../util/schema-utils';
+import {
+  LooseArray,
+  LooseRecord,
+  Toml,
+  withDepType,
+} from '../../../util/schema-utils';
 import { uniq } from '../../../util/uniq';
 import { GitRefsDatasource } from '../../datasource/git-refs';
 import { GitTagsDatasource } from '../../datasource/git-tags';
@@ -172,18 +176,6 @@ export const PoetryDependencies = LooseRecord(
   return deps;
 });
 
-function withDepType<
-  Output extends PackageDependency[],
-  Schema extends ZodType<Output, ZodTypeDef, unknown>,
->(schema: Schema, depType: string): ZodEffects<Schema> {
-  return schema.transform((deps) => {
-    for (const dep of deps) {
-      dep.depType = depType;
-    }
-    return deps;
-  });
-}
-
 export const PoetryGroupDependencies = LooseRecord(
   z.string(),
   z

lib/util/schema-utils.ts

@@ -2,7 +2,8 @@ import JSON5 from 'json5';
 import * as JSONC from 'jsonc-parser';
 import { DateTime } from 'luxon';
 import type { JsonArray, JsonValue } from 'type-fest';
-import { z } from 'zod';
+import { type ZodEffects, type ZodType, type ZodTypeDef, z } from 'zod';
+import type { PackageDependency } from '../modules/manager/types';
 import { parse as parseToml } from './toml';
 import { parseSingleYaml, parseYaml } from './yaml';
 
@@ -263,3 +264,15 @@ export const Toml = z.string().transform((str, ctx) => {
     return z.NEVER;
   }
 });
+
+export function withDepType<
+  Output extends PackageDependency[],
+  Schema extends ZodType<Output, ZodTypeDef, unknown>,
+>(schema: Schema, depType: string): ZodEffects<Schema> {
+  return schema.transform((deps) => {
+    for (const dep of deps) {
+      dep.depType = depType;
+    }
+    return deps;
+  });
+}