feat: add inherit to override

bin/generate_schema.py

@@ -17,6 +17,14 @@
 additionalProperties: false
 description: cibuildwheel's settings.
 type: object
+defines:
+  inherit:
+    enum:
+      - none
+      - prepend
+      - append
+    default: none
+    description: How to inherit the parent's value.
 properties:
   archs:
     description: Change the architectures built on your machine by default.
@@ -153,9 +161,6 @@
 
 schema = yaml.safe_load(starter)
 
-if args.schemastore:
-    schema["$id"] += "#"
-
 string_array = yaml.safe_load(
     """
 - type: string
@@ -214,19 +219,28 @@
   additionalProperties: false
   properties:
     select: {}
+    inherit:
+      type: object
+      additionalProperties: false
+      properties:
+        before-all: {"$ref": "#/defines/inherit"}
+        before-build: {"$ref": "#/defines/inherit"}
+        before-test: {"$ref": "#/defines/inherit"}
+        config-settings: {"$ref": "#/defines/inherit"}
+        container-engine: {"$ref": "#/defines/inherit"}
+        environment: {"$ref": "#/defines/inherit"}
+        environment-pass: {"$ref": "#/defines/inherit"}
+        repair-wheel-command: {"$ref": "#/defines/inherit"}
+        test-command: {"$ref": "#/defines/inherit"}
+        test-extras: {"$ref": "#/defines/inherit"}
+        test-requires: {"$ref": "#/defines/inherit"}
 """
 )
 
 for key, value in schema["properties"].items():
     value["title"] = f'CIBW_{key.replace("-", "_").upper()}'
 
-if args.schemastore:
-    non_global_options = {
-        k: {"$ref": f"#/properties/tool/properties/cibuildwheel/properties/{k}"}
-        for k in schema["properties"]
-    }
-else:
-    non_global_options = {k: {"$ref": f"#/properties/{k}"} for k in schema["properties"]}
+non_global_options = {k: {"$ref": f"#/properties/{k}"} for k in schema["properties"]}
 del non_global_options["build"]
 del non_global_options["skip"]
 del non_global_options["container-engine"]
@@ -273,27 +287,11 @@ def as_object(d: dict[str, Any]) -> dict[str, Any]:
 schema["properties"]["overrides"] = overrides
 schema["properties"] |= oses
 
-if not args.schemastore:
-    print(json.dumps(schema, indent=2))
-    raise SystemExit(0)
-
-schema_store_txt = """
-$id: https://json.schemastore.org/cibuildwheel.json
-$schema: http://json-schema.org/draft-07/schema#
-additionalProperties: false
-description: cibuildwheel's toml file, generated with ./bin/generate_schema.py --schemastore from cibuildwheel.
-type: object
-properties:
-    tool:
-        type: object
-        properties:
-            cibuildwheel:
-                type: object
-"""
-schema_store = yaml.safe_load(schema_store_txt)
-
-schema_store["properties"]["tool"]["properties"]["cibuildwheel"]["properties"] = schema[
-    "properties"
-]
+if args.schemastore:
+    schema["$id"] = "https://json.schemastore.org/partial-cibuildwheel.json"
+    schema["$id"] = "http://json-schema.org/draft-07/schema#"
+    schema[
+        "description"
+    ] = "cibuildwheel's toml file, generated with ./bin/generate_schema.py --schemastore from cibuildwheel."
 
-print(json.dumps(schema_store, indent=2))
+print(json.dumps(schema, indent=2))

cibuildwheel/options.py

@@ -5,14 +5,15 @@
 import contextlib
 import dataclasses
 import difflib
+import enum
 import functools
 import shlex
 import sys
 import textwrap
 import traceback
-from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Set
+from collections.abc import Callable, Generator, Iterable, Iterator, Set
 from pathlib import Path
-from typing import Any, Dict, List, Literal, TypedDict, Union
+from typing import Any, Literal, Mapping, Sequence, TypedDict, Union  # noqa: TID251
 
 from packaging.specifiers import SpecifierSet
 
@@ -116,13 +117,14 @@ def architectures(self) -> set[Architecture]:
         return self.globals.architectures
 
 
-Setting = Union[Dict[str, str], List[str], str, int]
+Setting = Union[Mapping[str, str], Sequence[str], str, int]
 
 
 @dataclasses.dataclass(frozen=True)
 class Override:
     select_pattern: str
     options: dict[str, Setting]
+    inherit: dict[str, Inherit]
 
 
 MANYLINUX_OPTIONS = {f"manylinux-{build_platform}-image" for build_platform in MANYLINUX_ARCHS}
@@ -150,25 +152,78 @@ class ConfigOptionError(KeyError):
     pass
 
 
-def _dig_first(*pairs: tuple[Mapping[str, Setting], str], ignore_empty: bool = False) -> Setting:
+class Inherit(enum.Enum):
+    NONE = enum.auto()
+    APPEND = enum.auto()
+    PREPEND = enum.auto()
+
+
+def _dig_first(
+    *pairs: tuple[Mapping[str, Setting], str, Inherit], ignore_empty: bool = False
+) -> Setting:
     """
-    Return the first dict item that matches from pairs of dicts and keys.
-    Will throw a KeyError if missing.
+    Return the dict items that match from pairs of dicts and keys. The third
+    value (enum) indicates if values should merge.
+
+    The following idiom can be used to get the first matching value:
 
-    _dig_first((dict1, "key1"), (dict2, "key2"), ...)
+        _dig_first((dict1, "key1", Inherit.NONE), (dict2, "key2", Inherit.NONE), ...)))
     """
     if not pairs:
         msg = "pairs cannot be empty"
         raise ValueError(msg)
 
-    for dict_like, key in pairs:
+    commands: list[tuple[Setting, Inherit]] = []
+
+    for dict_like, key, merge in pairs:
         if key in dict_like:
             value = dict_like[key]
-
             if ignore_empty and value == "":
                 continue
 
-            return value
+            commands.append((value, merge))
+            if merge == Inherit.NONE:
+                break
+
+    if len(commands) == 1:
+        return commands[0][0]
+
+    if len(commands) > 1:
+        if isinstance(commands[0][0], list):
+            retlist: list[str] = []
+            for c in commands:
+                if c[1] == Inherit.PREPEND:
+                    assert isinstance(c[0], list)
+                    retlist.extend(c[0])
+            for c in commands:
+                if c[1] == Inherit.NONE:
+                    assert isinstance(c[0], list)
+                    retlist.extend(c[0])
+            for c in commands[::-1]:
+                if c[1] == Inherit.APPEND:
+                    assert isinstance(c[0], list)
+                    retlist.extend(c[0])
+            return retlist
+
+        if isinstance(commands[0][0], dict):
+            retdict: dict[str, str] = {}
+            for c in commands:
+                if c[1] == Inherit.PREPEND:
+                    assert isinstance(c[0], dict)
+                    retdict.update(c[0])
+            for c in commands:
+                if c[1] == Inherit.NONE:
+                    assert isinstance(c[0], dict)
+                    retdict.update(c[0])
+            for c in commands[::-1]:
+                if c[1] == Inherit.APPEND:
+                    assert isinstance(c[0], dict)
+                    retdict.update(c[0])
+            return retdict
+
+        else:
+            msg = f"Must be list or dict, got {type(commands[0])}"
+            raise TypeError(msg)
 
     last_key = pairs[-1][1]
     raise KeyError(last_key)
@@ -244,7 +299,16 @@ def __init__(
                 if isinstance(select, list):
                     select = " ".join(select)
 
-                self.overrides.append(Override(select, config_override))
+                inherit = config_override.pop("inherit", {})
+                if not isinstance(inherit, dict) or not all(
+                    i in {"none", "append", "prepend"} for i in inherit.values()
+                ):
+                    msg = "'inherit' must be a dict containing only {'none', 'append', 'prepend'} values"
+                    raise ConfigOptionError(msg)
+
+                inherit_enum = {k: Inherit[v.upper()] for k, v in inherit.items()}
+
+                self.overrides.append(Override(select, config_override, inherit_enum))
 
     def _validate_global_option(self, name: str) -> None:
         """
@@ -341,25 +405,28 @@ def get(
         # get the option from the environment, then the config file, then finally the default.
         # platform-specific options are preferred, if they're allowed.
         result = _dig_first(
-            (self.env if env_plat else {}, plat_envvar),
-            (self.env, envvar),
-            *[(o.options, name) for o in active_config_overrides],
-            (self.config_platform_options, name),
-            (self.config_options, name),
-            (self.default_platform_options, name),
-            (self.default_options, name),
+            (self.env if env_plat else {}, plat_envvar, Inherit.NONE),
+            (self.env, envvar, Inherit.NONE),
+            *[
+                (o.options, name, o.inherit.get(name, Inherit.NONE))
+                for o in active_config_overrides
+            ],
+            (self.config_platform_options, name, Inherit.NONE),
+            (self.config_options, name, Inherit.NONE),
+            (self.default_platform_options, name, Inherit.NONE),
+            (self.default_options, name, Inherit.NONE),
             ignore_empty=ignore_empty,
         )
 
-        if isinstance(result, dict):
+        if isinstance(result, Mapping):
             if table is None:
                 msg = f"{name!r} does not accept a table"
                 raise ConfigOptionError(msg)
             return table["sep"].join(
                 item for k, v in result.items() for item in _inner_fmt(k, v, table)
             )
 
-        if isinstance(result, list):
+        if not isinstance(result, str) and isinstance(result, Sequence):
             if sep is None:
                 msg = f"{name!r} does not accept a list"
                 raise ConfigOptionError(msg)

cibuildwheel/resources/cibuildwheel.schema.json

@@ -4,6 +4,17 @@
   "additionalProperties": false,
   "description": "cibuildwheel's settings.",
   "type": "object",
+  "defines": {
+    "inherit": {
+      "enum": [
+        "none",
+        "prepend",
+        "append"
+      ],
+      "default": "none",
+      "description": "How to inherit the parent's value."
+    }
+  },
   "properties": {
     "archs": {
       "description": "Change the architectures built on your machine by default.",
@@ -423,6 +434,45 @@
               }
             ]
           },
+          "inherit": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "before-all": {
+                "$ref": "#/defines/inherit"
+              },
+              "before-build": {
+                "$ref": "#/defines/inherit"
+              },
+              "before-test": {
+                "$ref": "#/defines/inherit"
+              },
+              "config-settings": {
+                "$ref": "#/defines/inherit"
+              },
+              "container-engine": {
+                "$ref": "#/defines/inherit"
+              },
+              "environment": {
+                "$ref": "#/defines/inherit"
+              },
+              "environment-pass": {
+                "$ref": "#/defines/inherit"
+              },
+              "repair-wheel-command": {
+                "$ref": "#/defines/inherit"
+              },
+              "test-command": {
+                "$ref": "#/defines/inherit"
+              },
+              "test-extras": {
+                "$ref": "#/defines/inherit"
+              },
+              "test-requires": {
+                "$ref": "#/defines/inherit"
+              }
+            }
+          },
           "before-all": {
             "$ref": "#/properties/before-all"
           },

docs/options.md

@@ -135,6 +135,10 @@ trigger new containers, one per image. Some commands are not supported;
 `output-dir`, build/skip/test_skip selectors, and architectures cannot be
 overridden.
 
+You can specify a table of overrides in `inherit={}`, any list or table in this
+list will inherit from previous overrides or the main configuration. The valid
+options are `"none"` (the default), `"append"`, and `"prepend"`.
+
 ##### Examples:
 
 ```toml
@@ -169,6 +173,28 @@ This example will build CPython 3.6 wheels on manylinux1, CPython 3.7-3.9
 wheels on manylinux2010, and manylinux2014 wheels for any newer Python
 (like 3.10).
 
+```toml
+[tool.cibuildwheel]
+environment = {FOO="BAR", "HAM"="EGGS"}
+test-command = ["pyproject"]
+
+[[tool.cibuildwheel.overrides]]
+select = "cp311*"
+
+inherit.test-command="prepend"
+test-command = ["pyproject-before"]
+
+inherit.environment="append"
+environment = {FOO="BAZ", "PYTHON"="MONTY"}
+
+[[tool.cibuildwheel.overrides]]
+select = "cp311*"
+inherit.test-command="append"
+test-command = ["pyproject-after"]
+```
+
+This example will provide the command `"pyproject-before && pyproject && pyproject-after"`
+on Python 3.11, and will have `environment = {FOO="BAZ", "PYTHON"="MONTY", "HAM"="EGGS"}`.
 
 ## Options summary