From d3ef00b6c4a131eb56ef20a73ba12f4045dd1a27 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 22 Apr 2023 11:26:43 +0200 Subject: [PATCH 1/3] Ability to install armv7l manylinux/musllinux wheels on armv8l aarch64 systems running under linux32 emulation will report an armv8l machine rather than armv7l. This shall not prevent packaging to report musllinux/manylinux armv7l compatibiility as long as the running python has a compatible ABI. --- src/packaging/_manylinux.py | 9 +++++++-- src/packaging/_musllinux.py | 4 ++++ src/packaging/tags.py | 2 +- tests/test_tags.py | 20 ++++++++++++++------ 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/packaging/_manylinux.py b/src/packaging/_manylinux.py index d307eb1b..d904b7c5 100644 --- a/src/packaging/_manylinux.py +++ b/src/packaging/_manylinux.py @@ -204,6 +204,11 @@ def _is_compatible(arch: str, version: _GLibCVersion) -> bool: def platform_tags(linux: str, arch: str) -> Iterator[str]: + if arch == "armv8l": + # armv8l wheels are not accepted on PyPI + # As long as we pass the the ABI check below, + # the armv7l wheels can be installed. + arch = "armv7l" if not _have_compatible_abi(sys.executable, arch): return # Oldest glibc to be supported regardless of architecture is (2, 17). @@ -232,9 +237,9 @@ def platform_tags(linux: str, arch: str) -> Iterator[str]: glibc_version = _GLibCVersion(glibc_max.major, glibc_minor) tag = "manylinux_{}_{}".format(*glibc_version) if _is_compatible(arch, glibc_version): - yield linux.replace("linux", tag) + yield f"{tag}_{arch}" # Handle the legacy manylinux1, manylinux2010, manylinux2014 tags. if glibc_version in _LEGACY_MANYLINUX_MAP: legacy_tag = _LEGACY_MANYLINUX_MAP[glibc_version] if _is_compatible(arch, glibc_version): - yield linux.replace("linux", legacy_tag) + yield f"{legacy_tag}_{arch}" diff --git a/src/packaging/_musllinux.py b/src/packaging/_musllinux.py index 706ba600..b9e72f8f 100644 --- a/src/packaging/_musllinux.py +++ b/src/packaging/_musllinux.py @@ -63,6 +63,10 @@ def platform_tags(arch: str) -> Iterator[str]: sys_musl = _get_musl_version(sys.executable) if sys_musl is None: # Python not dynamically linked against musl. return + if arch == "armv8l": + # armv8l wheels are not accepted on PyPI + # The armv7l wheels can be installed. + arch = "armv7l" for minor in range(sys_musl.minor, -1, -1): yield f"musllinux_{sys_musl.major}_{minor}_{arch}" diff --git a/src/packaging/tags.py b/src/packaging/tags.py index 76d24341..a9f4f01d 100644 --- a/src/packaging/tags.py +++ b/src/packaging/tags.py @@ -473,7 +473,7 @@ def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]: if linux == "linux_x86_64": linux = "linux_i686" elif linux == "linux_aarch64": - linux = "linux_armv7l" + linux = "linux_armv8l" _, arch = linux.split("_", 1) yield from _manylinux.platform_tags(linux, arch) yield from _musllinux.platform_tags(arch) diff --git a/tests/test_tags.py b/tests/test_tags.py index 96a78197..892f153f 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -363,7 +363,7 @@ def test_get_config_var_does_log(self, monkeypatch): ("linux-x86_64", False, "linux_x86_64"), ("linux-x86_64", True, "linux_i686"), ("linux-aarch64", False, "linux_aarch64"), - ("linux-aarch64", True, "linux_armv7l"), + ("linux-aarch64", True, "linux_armv8l"), ], ) def test_linux_platforms_32_64bit_on_64bit_os( @@ -445,14 +445,20 @@ def test_linux_platforms_manylinux2014(self, monkeypatch): ] assert platforms == expected - def test_linux_platforms_manylinux2014_armhf_abi(self, monkeypatch): + @pytest.mark.parametrize( + "native_arch, cross_arch", + [("armv7l", "armv7l"), ("armv8l", "armv8l"), ("aarch64", "armv8l")], + ) + def test_linux_platforms_manylinux2014_armhf_abi( + self, native_arch, cross_arch, monkeypatch + ): monkeypatch.setattr(tags._manylinux, "_glibc_version_string", lambda: "2.30") monkeypatch.setattr( tags._manylinux, "_is_compatible", lambda _, glibc_version: glibc_version == _GLibCVersion(2, 17), ) - monkeypatch.setattr(sysconfig, "get_platform", lambda: "linux_armv7l") + monkeypatch.setattr(sysconfig, "get_platform", lambda: f"linux_{native_arch}") monkeypatch.setattr( sys, "executable", @@ -463,7 +469,7 @@ def test_linux_platforms_manylinux2014_armhf_abi(self, monkeypatch): ), ) platforms = list(tags._linux_platforms(is_32bit=True)) - expected = ["manylinux_2_17_armv7l", "manylinux2014_armv7l", "linux_armv7l"] + expected = ["manylinux_2_17_armv7l", "manylinux2014_armv7l", f"linux_{cross_arch}"] assert platforms == expected def test_linux_platforms_manylinux2014_i386_abi(self, monkeypatch): @@ -525,7 +531,8 @@ def test_linux_platforms_manylinux_glibc3(self, monkeypatch): @pytest.mark.parametrize( "native_arch, cross32_arch, musl_version", [ - ("aarch64", "armv7l", _MuslVersion(1, 1)), + ("armv7l", "armv7l", _MuslVersion(1, 1)), + ("aarch64", "armv8l", _MuslVersion(1, 1)), ("i386", "i386", _MuslVersion(1, 2)), ("x86_64", "i686", _MuslVersion(1, 2)), ], @@ -548,8 +555,9 @@ def test_linux_platforms_musllinux( platforms = list(tags._linux_platforms(is_32bit=cross32)) target_arch = cross32_arch if cross32 else native_arch + target_arch_musl = "armv7l" if target_arch == "armv8l" else target_arch expected = [ - f"musllinux_{musl_version[0]}_{minor}_{target_arch}" + f"musllinux_{musl_version[0]}_{minor}_{target_arch_musl}" for minor in range(musl_version[1], -1, -1) ] + [f"linux_{target_arch}"] assert platforms == expected From f2bdb76620a89a3da0edbb49c4f1ae11706e2e72 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 18 Jun 2023 13:08:39 +0200 Subject: [PATCH 2/3] generate both armv8l and armv7l tags for 32-bit python on aarch64 --- src/packaging/_manylinux.py | 63 ++++++++++++++++++++----------------- src/packaging/_musllinux.py | 21 ++++++------- src/packaging/tags.py | 12 +++++-- tests/test_tags.py | 38 +++++++++++++++------- 4 files changed, 81 insertions(+), 53 deletions(-) diff --git a/src/packaging/_manylinux.py b/src/packaging/_manylinux.py index d904b7c5..19f2de43 100644 --- a/src/packaging/_manylinux.py +++ b/src/packaging/_manylinux.py @@ -5,7 +5,7 @@ import re import sys import warnings -from typing import Dict, Generator, Iterator, NamedTuple, Optional, Tuple +from typing import Dict, Generator, Iterator, NamedTuple, Optional, Sequence, Tuple from ._elffile import EIClass, EIData, ELFFile, EMachine @@ -50,12 +50,13 @@ def _is_linux_i686(executable: str) -> bool: ) -def _have_compatible_abi(executable: str, arch: str) -> bool: - if arch == "armv7l": +def _have_compatible_abi(executable: str, archs: Sequence[str]) -> bool: + if "armv7l" in archs: return _is_linux_armhf(executable) - if arch == "i686": + if "i686" in archs: return _is_linux_i686(executable) - return arch in {"x86_64", "aarch64", "ppc64", "ppc64le", "s390x", "loongarch64"} + other_archs = {"x86_64", "aarch64", "ppc64", "ppc64le", "s390x", "loongarch64"} + return len(set(archs) & other_archs) > 0 # If glibc ever changes its major version, we need to know what the last @@ -203,17 +204,22 @@ def _is_compatible(arch: str, version: _GLibCVersion) -> bool: } -def platform_tags(linux: str, arch: str) -> Iterator[str]: - if arch == "armv8l": - # armv8l wheels are not accepted on PyPI - # As long as we pass the the ABI check below, - # the armv7l wheels can be installed. - arch = "armv7l" - if not _have_compatible_abi(sys.executable, arch): +def platform_tags(archs: Sequence[str]) -> Iterator[str]: + """Generate manylinux tags compatible to the current platform. + + :param archs: Sequence of compatible architectures. + The first one shall be the closest to the actual architecture and be the part of + platform tag after the ``linux_`` prefix, e.g. ``x86_64``. + The ``linux_`` prefix is assumed as a prerequisite for the current platform to + be manylinux-compatible. + + :returns: An iterator of compatible manylinux tags. + """ + if not _have_compatible_abi(sys.executable, archs): return # Oldest glibc to be supported regardless of architecture is (2, 17). too_old_glibc2 = _GLibCVersion(2, 16) - if arch in {"x86_64", "i686"}: + if set(archs) & {"x86_64", "i686"}: # On x86/i686 also oldest glibc to be supported is (2, 5). too_old_glibc2 = _GLibCVersion(2, 4) current_glibc = _GLibCVersion(*_get_glibc_version()) @@ -227,19 +233,20 @@ def platform_tags(linux: str, arch: str) -> Iterator[str]: for glibc_major in range(current_glibc.major - 1, 1, -1): glibc_minor = _LAST_GLIBC_MINOR[glibc_major] glibc_max_list.append(_GLibCVersion(glibc_major, glibc_minor)) - for glibc_max in glibc_max_list: - if glibc_max.major == too_old_glibc2.major: - min_minor = too_old_glibc2.minor - else: - # For other glibc major versions oldest supported is (x, 0). - min_minor = -1 - for glibc_minor in range(glibc_max.minor, min_minor, -1): - glibc_version = _GLibCVersion(glibc_max.major, glibc_minor) - tag = "manylinux_{}_{}".format(*glibc_version) - if _is_compatible(arch, glibc_version): - yield f"{tag}_{arch}" - # Handle the legacy manylinux1, manylinux2010, manylinux2014 tags. - if glibc_version in _LEGACY_MANYLINUX_MAP: - legacy_tag = _LEGACY_MANYLINUX_MAP[glibc_version] + for arch in archs: + for glibc_max in glibc_max_list: + if glibc_max.major == too_old_glibc2.major: + min_minor = too_old_glibc2.minor + else: + # For other glibc major versions oldest supported is (x, 0). + min_minor = -1 + for glibc_minor in range(glibc_max.minor, min_minor, -1): + glibc_version = _GLibCVersion(glibc_max.major, glibc_minor) + tag = "manylinux_{}_{}".format(*glibc_version) if _is_compatible(arch, glibc_version): - yield f"{legacy_tag}_{arch}" + yield f"{tag}_{arch}" + # Handle the legacy manylinux1, manylinux2010, manylinux2014 tags. + if glibc_version in _LEGACY_MANYLINUX_MAP: + legacy_tag = _LEGACY_MANYLINUX_MAP[glibc_version] + if _is_compatible(arch, glibc_version): + yield f"{legacy_tag}_{arch}" diff --git a/src/packaging/_musllinux.py b/src/packaging/_musllinux.py index b9e72f8f..a71f85ee 100644 --- a/src/packaging/_musllinux.py +++ b/src/packaging/_musllinux.py @@ -8,7 +8,7 @@ import re import subprocess import sys -from typing import Iterator, NamedTuple, Optional +from typing import Iterator, NamedTuple, Optional, Sequence from ._elffile import ELFFile @@ -51,24 +51,23 @@ def _get_musl_version(executable: str) -> Optional[_MuslVersion]: return _parse_musl_version(proc.stderr) -def platform_tags(arch: str) -> Iterator[str]: +def platform_tags(archs: Sequence[str]) -> Iterator[str]: """Generate musllinux tags compatible to the current platform. - :param arch: Should be the part of platform tag after the ``linux_`` - prefix, e.g. ``x86_64``. The ``linux_`` prefix is assumed as a - prerequisite for the current platform to be musllinux-compatible. + :param archs: Sequence of compatible architectures. + The first one shall be the closest to the actual architecture and be the part of + platform tag after the ``linux_`` prefix, e.g. ``x86_64``. + The ``linux_`` prefix is assumed as a prerequisite for the current platform to + be musllinux-compatible. :returns: An iterator of compatible musllinux tags. """ sys_musl = _get_musl_version(sys.executable) if sys_musl is None: # Python not dynamically linked against musl. return - if arch == "armv8l": - # armv8l wheels are not accepted on PyPI - # The armv7l wheels can be installed. - arch = "armv7l" - for minor in range(sys_musl.minor, -1, -1): - yield f"musllinux_{sys_musl.major}_{minor}_{arch}" + for arch in archs: + for minor in range(sys_musl.minor, -1, -1): + yield f"musllinux_{sys_musl.major}_{minor}_{arch}" if __name__ == "__main__": # pragma: no cover diff --git a/src/packaging/tags.py b/src/packaging/tags.py index a9f4f01d..a389a7b7 100644 --- a/src/packaging/tags.py +++ b/src/packaging/tags.py @@ -469,15 +469,21 @@ def mac_platforms( def _linux_platforms(is_32bit: bool = _32_BIT_INTERPRETER) -> Iterator[str]: linux = _normalize_string(sysconfig.get_platform()) + if not linux.startswith("linux_"): + # we should never be here, just yield the sysconfig one and return + yield linux + return if is_32bit: if linux == "linux_x86_64": linux = "linux_i686" elif linux == "linux_aarch64": linux = "linux_armv8l" _, arch = linux.split("_", 1) - yield from _manylinux.platform_tags(linux, arch) - yield from _musllinux.platform_tags(arch) - yield linux + archs = {"armv8l": ["armv8l", "armv7l"]}.get(arch, [arch]) + yield from _manylinux.platform_tags(archs) + yield from _musllinux.platform_tags(archs) + for arch in archs: + yield f"linux_{arch}" def _generic_platforms() -> Iterator[str]: diff --git a/tests/test_tags.py b/tests/test_tags.py index 892f153f..870ef4ec 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -360,10 +360,10 @@ def test_get_config_var_does_log(self, monkeypatch): @pytest.mark.parametrize( "arch,is_32bit,expected", [ - ("linux-x86_64", False, "linux_x86_64"), - ("linux-x86_64", True, "linux_i686"), - ("linux-aarch64", False, "linux_aarch64"), - ("linux-aarch64", True, "linux_armv8l"), + ("linux-x86_64", False, ["linux_x86_64"]), + ("linux-x86_64", True, ["linux_i686"]), + ("linux-aarch64", False, ["linux_aarch64"]), + ("linux-aarch64", True, ["linux_armv8l", "linux_armv7l"]), ], ) def test_linux_platforms_32_64bit_on_64bit_os( @@ -372,7 +372,9 @@ def test_linux_platforms_32_64bit_on_64bit_os( monkeypatch.setattr(sysconfig, "get_platform", lambda: arch) monkeypatch.setattr(os, "confstr", lambda x: "glibc 2.20", raising=False) monkeypatch.setattr(tags._manylinux, "_is_compatible", lambda *args: False) - linux_platform = list(tags._linux_platforms(is_32bit=is_32bit))[-1] + linux_platform = list(tags._linux_platforms(is_32bit=is_32bit))[ + -len(expected) : + ] assert linux_platform == expected def test_linux_platforms_manylinux_unsupported(self, monkeypatch): @@ -469,7 +471,11 @@ def test_linux_platforms_manylinux2014_armhf_abi( ), ) platforms = list(tags._linux_platforms(is_32bit=True)) - expected = ["manylinux_2_17_armv7l", "manylinux2014_armv7l", f"linux_{cross_arch}"] + archs = {"armv8l": ["armv8l", "armv7l"]}.get(cross_arch, [cross_arch]) + expected = [] + for arch in archs: + expected.extend([f"manylinux_2_17_{arch}", f"manylinux2014_{arch}"]) + expected.extend(f"linux_{arch}" for arch in archs) assert platforms == expected def test_linux_platforms_manylinux2014_i386_abi(self, monkeypatch): @@ -555,11 +561,14 @@ def test_linux_platforms_musllinux( platforms = list(tags._linux_platforms(is_32bit=cross32)) target_arch = cross32_arch if cross32 else native_arch - target_arch_musl = "armv7l" if target_arch == "armv8l" else target_arch - expected = [ - f"musllinux_{musl_version[0]}_{minor}_{target_arch_musl}" - for minor in range(musl_version[1], -1, -1) - ] + [f"linux_{target_arch}"] + archs = {"armv8l": ["armv8l", "armv7l"]}.get(target_arch, [target_arch]) + expected = [] + for arch in archs: + expected.extend( + f"musllinux_{musl_version[0]}_{minor}_{arch}" + for minor in range(musl_version[1], -1, -1) + ) + expected.extend(f"linux_{arch}" for arch in archs) assert platforms == expected assert recorder.calls == [pretend.call(fake_executable)] @@ -598,6 +607,13 @@ def test_linux_platforms_not_manylinux_abi( expected = [f"linux_{alt_machine}"] assert platforms == expected + def test_linux_not_linux(self, monkeypatch): + monkeypatch.setattr(sysconfig, "get_platform", lambda: "not_linux_x86_64") + monkeypatch.setattr(platform, "machine", lambda: "x86_64") + monkeypatch.setattr(os, "confstr", lambda x: "glibc 2.17", raising=False) + platforms = list(tags._linux_platforms(is_32bit=False)) + assert platforms == ["not_linux_x86_64"] + @pytest.mark.parametrize( "platform_name,dispatch_func", From 06df33b12a2c9d701e49364dc1daf12a57b1f857 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 8 Jul 2023 09:45:41 +0200 Subject: [PATCH 3/3] Apply review suggestion --- src/packaging/_manylinux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/packaging/_manylinux.py b/src/packaging/_manylinux.py index 19f2de43..3705d50d 100644 --- a/src/packaging/_manylinux.py +++ b/src/packaging/_manylinux.py @@ -55,8 +55,8 @@ def _have_compatible_abi(executable: str, archs: Sequence[str]) -> bool: return _is_linux_armhf(executable) if "i686" in archs: return _is_linux_i686(executable) - other_archs = {"x86_64", "aarch64", "ppc64", "ppc64le", "s390x", "loongarch64"} - return len(set(archs) & other_archs) > 0 + allowed_archs = {"x86_64", "aarch64", "ppc64", "ppc64le", "s390x", "loongarch64"} + return any(arch in allowed_archs for arch in archs) # If glibc ever changes its major version, we need to know what the last