From 8070fe10f1ccc346539f87c88fb65ea55d001b9a Mon Sep 17 00:00:00 2001 From: Nulano Date: Sat, 23 Dec 2023 14:41:50 +0100 Subject: [PATCH 1/7] pass build config before setuptools command; add build_editable to custom build backend --- .github/workflows/wheels.yml | 2 ++ _custom_build/backend.py | 57 +++++++++++++----------------------- pyproject.toml | 2 +- setup.py | 34 ++++++++++++++++++--- 4 files changed, 54 insertions(+), 41 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 5adff7ec1c4..0c8a941de30 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -5,6 +5,7 @@ on: paths: - ".ci/requirements-cibw.txt" - ".github/workflows/wheel*" + - "setup.py" - "wheels/*" - "winbuild/build_prepare.py" - "winbuild/fribidi.cmake" @@ -14,6 +15,7 @@ on: paths: - ".ci/requirements-cibw.txt" - ".github/workflows/wheel*" + - "setup.py" - "wheels/*" - "winbuild/build_prepare.py" - "winbuild/fribidi.cmake" diff --git a/_custom_build/backend.py b/_custom_build/backend.py index d1537b80987..0b183a5870b 100644 --- a/_custom_build/backend.py +++ b/_custom_build/backend.py @@ -11,41 +11,16 @@ class _CustomBuildMetaBackend(backend_class): def run_setup(self, setup_script="setup.py"): if self.config_settings: - - def config_has(key, value): - settings = self.config_settings.get(key) - if settings: - if not isinstance(settings, list): - settings = [settings] - return value in settings - - flags = [] - for dependency in ( - "zlib", - "jpeg", - "tiff", - "freetype", - "raqm", - "lcms", - "webp", - "webpmux", - "jpeg2000", - "imagequant", - "xcb", - ): - if config_has(dependency, "enable"): - flags.append("--enable-" + dependency) - elif config_has(dependency, "disable"): - flags.append("--disable-" + dependency) - for dependency in ("raqm", "fribidi"): - if config_has(dependency, "vendor"): - flags.append("--vendor-" + dependency) - if self.config_settings.get("platform-guessing") == "disable": - flags.append("--disable-platform-guessing") - if self.config_settings.get("debug") == "true": - flags.append("--debug") - if flags: - sys.argv = sys.argv[:1] + ["build_ext"] + flags + sys.argv[1:] + params = [] + for k, v in self.config_settings.items(): + if isinstance(v, list): + msg = "Conflicting options: " + ", ".join( + f"'--config-setting {k}={v_}'" for v_ in v + ) + raise ValueError(msg) + params.append(f"--pillow-configuration={k}={v}") + + sys.argv = sys.argv[:1] + params + sys.argv[1:] return super().run_setup(setup_script) def build_wheel( @@ -54,5 +29,15 @@ def build_wheel( self.config_settings = config_settings return super().build_wheel(wheel_directory, config_settings, metadata_directory) + def build_editable( + self, wheel_directory, config_settings=None, metadata_directory=None + ): + self.config_settings = config_settings + return super().build_editable( + wheel_directory, config_settings, metadata_directory + ) + -build_wheel = _CustomBuildMetaBackend().build_wheel +_backend = _CustomBuildMetaBackend() +build_wheel = _backend.build_wheel +build_editable = _backend.build_editable diff --git a/pyproject.toml b/pyproject.toml index da2537b2137..d63e401af1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ version = {attr = "PIL.__version__"} [tool.cibuildwheel] before-all = ".github/workflows/wheels-dependencies.sh" build-verbosity = 1 -config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable" +config-settings = "raqm=vendor fribidi=vendor imagequant=disable" test-command = "cd {project} && .github/workflows/wheels-test.sh" test-extras = "tests" diff --git a/setup.py b/setup.py index 1bf0bcff558..c74165fb73a 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,9 @@ def get_version(): return locals()["__version__"] +configuration = {} + + PILLOW_VERSION = get_version() FREETYPE_ROOT = None HARFBUZZ_ROOT = None @@ -334,15 +337,24 @@ def __iter__(self): + [("add-imaging-libs=", None, "Add libs to _imaging build")] ) + @staticmethod + def check_configuration(option, value): + return True if configuration.get(option) == value else None + def initialize_options(self): - self.disable_platform_guessing = None + self.disable_platform_guessing = self.check_configuration( + "platform-guessing", "disable" + ) self.add_imaging_libs = "" build_ext.initialize_options(self) for x in self.feature: - setattr(self, f"disable_{x}", None) - setattr(self, f"enable_{x}", None) + setattr(self, f"disable_{x}", self.check_configuration(x, "disable")) + setattr(self, f"enable_{x}", self.check_configuration(x, "enable")) for x in ("raqm", "fribidi"): - setattr(self, f"vendor_{x}", None) + setattr(self, f"vendor_{x}", self.check_configuration(x, "vendor")) + if self.check_configuration("debug", "true"): + self.debug = True + self.parallel = configuration.get("parallel") def finalize_options(self): build_ext.finalize_options(self) @@ -390,6 +402,9 @@ def finalize_options(self): raise ValueError(msg) _dbg("Using vendored version of %s", x) self.feature.vendor.add(x) + if x == "raqm": + _dbg("--vendor-raqm implies --enable-raqm") + self.feature.required.add(x) def _update_extension(self, name, libraries, define_macros=None, sources=None): for extension in self.extensions: @@ -985,6 +1000,17 @@ def debug_build(): Extension("PIL._imagingmorph", ["src/_imagingmorph.c"]), ] + +# parse configuration from _custom_build/backend.py +while len(sys.argv[1]) >= 2 and sys.argv[1].startswith("--pillow-configuration="): + _, key, value = sys.argv[1].split("=", 2) + old = configuration.get(key) + if old is not None: + msg = f"Conflicting options: '-C {key}={old}' and '-C {key}={value}'" + raise ValueError(msg) + configuration[key] = value + del sys.argv[1] + try: setup( cmdclass={"build_ext": pil_build_ext}, From b4e690049d81dc2569d6f02c5b0c1b9eb29a07b2 Mon Sep 17 00:00:00 2001 From: Nulano Date: Sun, 31 Dec 2023 02:22:05 +0100 Subject: [PATCH 2/7] document config setting "-C parallel=n" for number of CPUs to use for compilation --- docs/installation.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index fbcfbb90724..03011619ff7 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -390,9 +390,10 @@ After navigating to the Pillow directory, run:: Build Options """"""""""""" -* Environment variable: ``MAX_CONCURRENCY=n``. Pillow can use - multiprocessing to build the extension. Setting ``MAX_CONCURRENCY`` - sets the number of CPUs to use, or can disable parallel building by +* Config setting: ``-C parallel=n``. Can also be given + with environment variable: ``MAX_CONCURRENCY=n``. Pillow can use + multiprocessing to build the extension. Setting ``-C parallel=n`` + sets the number of CPUs to use to ``n``, or can disable parallel building by using a setting of 1. By default, it uses 4 CPUs, or if 4 are not available, as many as are present. @@ -417,14 +418,13 @@ Build Options used to compile the standard Pillow wheels. Compiling libraqm requires a C99-compliant compiler. -* Build flag: ``-C platform-guessing=disable``. Skips all of the +* Config setting: ``-C platform-guessing=disable``. Skips all of the platform dependent guessing of include and library directories for automated build systems that configure the proper paths in the environment variables (e.g. Buildroot). -* Build flag: ``-C debug=true``. Adds a debugging flag to the include and - library search process to dump all paths searched for and found to - stdout. +* Config setting: ``-C debug=true``. Adds a debugging flag to the include and + library search process to dump all paths searched for and found to stdout. Sample usage:: From f27b838a451d0b20befea03da7b2d58dd1da7821 Mon Sep 17 00:00:00 2001 From: Nulano Date: Tue, 2 Jan 2024 15:47:47 +0100 Subject: [PATCH 3/7] support multiple --config-settings --- _custom_build/backend.py | 12 +++++------- pyproject.toml | 2 +- setup.py | 13 +++---------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/_custom_build/backend.py b/_custom_build/backend.py index 0b183a5870b..2c670ff0ac3 100644 --- a/_custom_build/backend.py +++ b/_custom_build/backend.py @@ -12,13 +12,11 @@ class _CustomBuildMetaBackend(backend_class): def run_setup(self, setup_script="setup.py"): if self.config_settings: params = [] - for k, v in self.config_settings.items(): - if isinstance(v, list): - msg = "Conflicting options: " + ", ".join( - f"'--config-setting {k}={v_}'" for v_ in v - ) - raise ValueError(msg) - params.append(f"--pillow-configuration={k}={v}") + for key, values in self.config_settings.items(): + if not isinstance(values, list): + values = [values] + for value in values: + params.append(f"--pillow-configuration={key}={value}") sys.argv = sys.argv[:1] + params + sys.argv[1:] return super().run_setup(setup_script) diff --git a/pyproject.toml b/pyproject.toml index d63e401af1e..da2537b2137 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ version = {attr = "PIL.__version__"} [tool.cibuildwheel] before-all = ".github/workflows/wheels-dependencies.sh" build-verbosity = 1 -config-settings = "raqm=vendor fribidi=vendor imagequant=disable" +config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable" test-command = "cd {project} && .github/workflows/wheels-test.sh" test-extras = "tests" diff --git a/setup.py b/setup.py index c74165fb73a..686e8889fd9 100755 --- a/setup.py +++ b/setup.py @@ -339,7 +339,7 @@ def __iter__(self): @staticmethod def check_configuration(option, value): - return True if configuration.get(option) == value else None + return True if value in configuration.get(option, []) else None def initialize_options(self): self.disable_platform_guessing = self.check_configuration( @@ -354,7 +354,7 @@ def initialize_options(self): setattr(self, f"vendor_{x}", self.check_configuration(x, "vendor")) if self.check_configuration("debug", "true"): self.debug = True - self.parallel = configuration.get("parallel") + self.parallel = configuration.get("parallel", [None])[-1] def finalize_options(self): build_ext.finalize_options(self) @@ -402,9 +402,6 @@ def finalize_options(self): raise ValueError(msg) _dbg("Using vendored version of %s", x) self.feature.vendor.add(x) - if x == "raqm": - _dbg("--vendor-raqm implies --enable-raqm") - self.feature.required.add(x) def _update_extension(self, name, libraries, define_macros=None, sources=None): for extension in self.extensions: @@ -1004,11 +1001,7 @@ def debug_build(): # parse configuration from _custom_build/backend.py while len(sys.argv[1]) >= 2 and sys.argv[1].startswith("--pillow-configuration="): _, key, value = sys.argv[1].split("=", 2) - old = configuration.get(key) - if old is not None: - msg = f"Conflicting options: '-C {key}={old}' and '-C {key}={value}'" - raise ValueError(msg) - configuration[key] = value + configuration.setdefault(key, []).append(value) del sys.argv[1] try: From 01e5f06da055490f70b1294f39a2718297ee8067 Mon Sep 17 00:00:00 2001 From: Nulano Date: Tue, 2 Jan 2024 16:12:37 +0100 Subject: [PATCH 4/7] document editable mode installation in winbuild/build.rst --- winbuild/build.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/winbuild/build.rst b/winbuild/build.rst index a8e4ebaa6cc..26d0da0a35a 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -87,11 +87,18 @@ are set by running ``winbuild\build\build_env.cmd`` and install Pillow with pip: winbuild\build\build_env.cmd python.exe -m pip install -v -C raqm=vendor -C fribidi=vendor . +You can also install Pillow in `editable mode`_:: + + winbuild\build\build_env.cmd + python.exe -m pip install -v -C raqm=vendor -C fribidi=vendor -e . + To build a wheel instead, run:: winbuild\build\build_env.cmd python.exe -m pip wheel -v -C raqm=vendor -C fribidi=vendor . +.. _editable mode: https://setuptools.pypa.io/en/latest/userguide/development_mode.html + Testing Pillow -------------- From 74af933a9f0441e1285c398675dc3eca1de5406c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ondrej=20Baranovi=C4=8D?= Date: Sat, 20 Jan 2024 10:08:14 +0100 Subject: [PATCH 5/7] Link to stable setuptools documentation Co-authored-by: Andrew Murray <3112309+radarhere@users.noreply.github.com> --- winbuild/build.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winbuild/build.rst b/winbuild/build.rst index 26d0da0a35a..f40982cd546 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -97,7 +97,7 @@ To build a wheel instead, run:: winbuild\build\build_env.cmd python.exe -m pip wheel -v -C raqm=vendor -C fribidi=vendor . -.. _editable mode: https://setuptools.pypa.io/en/latest/userguide/development_mode.html +.. _editable mode: https://setuptools.pypa.io/en/stable/userguide/development_mode.html Testing Pillow -------------- From 420150f0e251c9519a5aea1c24546a44d489828c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ondrej=20Baranovi=C4=8D?= Date: Sat, 20 Jan 2024 14:56:20 +0100 Subject: [PATCH 6/7] Update winbuild/build.rst --- winbuild/build.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winbuild/build.rst b/winbuild/build.rst index f40982cd546..c980d9c7567 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -92,7 +92,7 @@ You can also install Pillow in `editable mode`_:: winbuild\build\build_env.cmd python.exe -m pip install -v -C raqm=vendor -C fribidi=vendor -e . -To build a wheel instead, run:: +To build a binary wheel instead, run:: winbuild\build\build_env.cmd python.exe -m pip wheel -v -C raqm=vendor -C fribidi=vendor . From e6fa5dff631d7da4020889d889a42c835a4c8e70 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Tue, 19 Mar 2024 19:14:17 +1100 Subject: [PATCH 7/7] Append custom parameter to the end of sys.argv --- _custom_build/backend.py | 4 +--- setup.py | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/_custom_build/backend.py b/_custom_build/backend.py index 2c670ff0ac3..44ec6efe5c6 100644 --- a/_custom_build/backend.py +++ b/_custom_build/backend.py @@ -11,14 +11,12 @@ class _CustomBuildMetaBackend(backend_class): def run_setup(self, setup_script="setup.py"): if self.config_settings: - params = [] for key, values in self.config_settings.items(): if not isinstance(values, list): values = [values] for value in values: - params.append(f"--pillow-configuration={key}={value}") + sys.argv.append(f"--pillow-configuration={key}={value}") - sys.argv = sys.argv[:1] + params + sys.argv[1:] return super().run_setup(setup_script) def build_wheel( diff --git a/setup.py b/setup.py index 9eb95771675..5effe657fe2 100644 --- a/setup.py +++ b/setup.py @@ -998,10 +998,9 @@ def debug_build(): # parse configuration from _custom_build/backend.py -while len(sys.argv[1]) >= 2 and sys.argv[1].startswith("--pillow-configuration="): - _, key, value = sys.argv[1].split("=", 2) +while sys.argv[-1].startswith("--pillow-configuration="): + _, key, value = sys.argv.pop().split("=", 2) configuration.setdefault(key, []).append(value) - del sys.argv[1] try: setup(