From 4eef80b4f2cd29a128c34ca08ed6462d35293722 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Wed, 31 Jan 2024 14:37:42 -0800 Subject: [PATCH] Breaking Change: Removed obsolete/duplicate `setup.py` from Python. This copy of `setup.py` is obsolete, as it builds the old C++ backend for Protobuf, which has been deprecated and unused since 4.21.0 when [the backend was switch to upb](https://protobuf.dev/news/2022-05-06/#python-updates). The `setup.py` that we actually distribute in our source packages is located in [`python/dist/setup.py`](https://github.com/protocolbuffers/protobuf/blob/main/python/dist/setup.py). It is not possible to build this `setup.py` directly from the GitHub repo or GitHub release tarball, because it depends on the file layout of our Python source package ([as distributed on PyPI](https://pypi.org/project/protobuf/#files)). The Python source package uses a layout that pulls together all of the things Python needs: |Python Source Package Path|GitHub Repo Path|Description| |-----|-----|-----| |`setup.py`|`python/dist/setup.py`| |`google/protobuf/*`|`python/google/protobuf/*`|pure Python sources| |`python/*`|`python/*`|C extension sources| |`utf8_range/*`|`third_party/utf8_range`|C UTF-8 Validation Library| |`upb/*`|`upb/upb/*`|C Protobuf Library| Users who want to build their own Python packages should build from our source package on PyPI, not from our GitHub repo or our GitHub release tarball. It is also possible to build our source package from GitHub using the following command (this requires Bazel): ``` $ bazel build //python/dist:source_wheel ``` PiperOrigin-RevId: 603162788 --- python/README.md | 19 +- python/build_targets.bzl | 3 - python/setup.cfg | 2 - python/setup.py | 435 --------------------------------------- 4 files changed, 15 insertions(+), 444 deletions(-) delete mode 100644 python/setup.cfg delete mode 100755 python/setup.py diff --git a/python/README.md b/python/README.md index ab3935025436..febdf16d453c 100644 --- a/python/README.md +++ b/python/README.md @@ -2,6 +2,11 @@ This directory contains the Protobuf library for Python. +For user documentation about how to use Protobuf Python, see +https://protobuf.dev/getting-started/pythontutorial/ + +# Installation + In most cases you should install the library using `pip` or another package manager: @@ -12,10 +17,7 @@ $ pip install protobuf The packages released on https://pypi.org/project/protobuf/#files include both a source distribution and binary wheels. -For user documentation about how to use Protobuf Python, see -https://protobuf.dev/getting-started/pythontutorial/ - -# Building packages from this repo +## Building packages from this repo If for some reason you wish to build the packages directly from this repo, you can use the following Bazel commands: @@ -29,6 +31,15 @@ The binary wheel will build against whatever version of Python is installed on your system. The source package is always the same and does not depend on a local version of Python. +## Building from `setup.py` + +We support building from `setup.py`, but only from a Python source package. +You cannot build from `setup.py` using the GitHub repo or the GitHub source +tarball. + +To build a source package from this repo, see the instructions in the previous +section. + # Implementation backends There are three separate implementations of Python Protobuf. All of them offer diff --git a/python/build_targets.bzl b/python/build_targets.bzl index a1f1767fefb7..0a20662c3cdc 100644 --- a/python/build_targets.bzl +++ b/python/build_targets.bzl @@ -461,7 +461,6 @@ def build_targets(name): ":python_src_files", "README.md", "google/__init__.py", - "setup.cfg", ], strip_prefix = "", visibility = ["//python/dist:__pkg__"], @@ -485,8 +484,6 @@ def build_targets(name): "google/protobuf/python_protobuf.h", "internal.bzl", "python_version_test.py", - "setup.cfg", - "setup.py", ], strip_prefix = strip_prefix.from_root(""), visibility = ["//pkg:__pkg__"], diff --git a/python/setup.cfg b/python/setup.cfg deleted file mode 100644 index 2a9acf13daa9..000000000000 --- a/python/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal = 1 diff --git a/python/setup.py b/python/setup.py deleted file mode 100755 index 329dece30655..000000000000 --- a/python/setup.py +++ /dev/null @@ -1,435 +0,0 @@ -#! /usr/bin/env python -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file or at -# https://developers.google.com/open-source/licenses/bsd -# -# See README for usage instructions. - -# pylint:disable=missing-module-docstring -# pylint:disable=g-bad-import-order -import fnmatch -import glob -import os -import pkg_resources -import re -import shutil -import subprocess -import sys -import sysconfig - -# pylint:disable=g-importing-member -# pylint:disable=g-multiple-import - -from setuptools import setup, Extension, find_packages - -from setuptools.command.build_ext import build_ext as _build_ext -from setuptools.command.build_py import build_py as _build_py - -# Find the Protocol Compiler. -if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']): - protoc = os.environ['PROTOC'] -elif os.path.exists('../bazel-bin/protoc'): - protoc = '../bazel-bin/protoc' -elif os.path.exists('../bazel-bin/protoc.exe'): - protoc = '../bazel-bin/protoc.exe' -elif os.path.exists('../protoc'): - protoc = '../protoc' -elif os.path.exists('../protoc.exe'): - protoc = '../protoc.exe' -elif os.path.exists('../vsprojects/Debug/protoc.exe'): - protoc = '../vsprojects/Debug/protoc.exe' -elif os.path.exists('../vsprojects/Release/protoc.exe'): - protoc = '../vsprojects/Release/protoc.exe' -else: - protoc = shutil.which('protoc') - - -def GetVersion(): - """Reads and returns the version from google/protobuf/__init__.py. - - Do not import google.protobuf.__init__ directly, because an installed - protobuf library may be loaded instead. - - Returns: - The version. - """ - - with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file: - exec(version_file.read(), globals()) # pylint:disable=exec-used - return __version__ # pylint:disable=undefined-variable - - -def GenProto(source, require=True): - """Generates a _pb2.py from the given .proto file. - - Does nothing if the output already exists and is newer than the input. - - Args: - source: the .proto file path. - require: if True, exit immediately when a path is not found. - """ - - if not require and not os.path.exists(source): - return - - output = source.replace('.proto', '_pb2.py').replace('../src/', '') - - if (not os.path.exists(output) or - (os.path.exists(source) and - os.path.getmtime(source) > os.path.getmtime(output))): - print('Generating %s...' % output) - - if not os.path.exists(source): - sys.stderr.write("Can't find required file: %s\n" % source) - sys.exit(-1) - - if protoc is None: - sys.stderr.write( - 'protoc is not installed nor found in ../src. Please compile it ' - 'or install the binary package.\n') - sys.exit(-1) - - protoc_command = [protoc, '-I../src', '-I.', '--python_out=.', source] - if subprocess.call(protoc_command) != 0: - sys.exit(-1) - - -def GenerateUnittestProtos(): - """Generates protobuf code for unittests.""" - GenProto('../src/google/protobuf/any_test.proto', False) - GenProto('../src/google/protobuf/map_proto2_unittest.proto', False) - GenProto('../src/google/protobuf/map_unittest.proto', False) - GenProto('../src/google/protobuf/test_messages_proto3.proto', False) - GenProto('../src/google/protobuf/test_messages_proto2.proto', False) - GenProto('../src/google/protobuf/unittest.proto', False) - GenProto('../src/google/protobuf/unittest_custom_options.proto', False) - GenProto('../src/google/protobuf/unittest_import.proto', False) - GenProto('../src/google/protobuf/unittest_import_public.proto', False) - GenProto('../src/google/protobuf/unittest_mset.proto', False) - GenProto('../src/google/protobuf/unittest_mset_wire_format.proto', False) - GenProto('../src/google/protobuf/unittest_no_generic_services.proto', False) - GenProto('../src/google/protobuf/unittest_proto3_arena.proto', False) - GenProto('../src/google/protobuf/unittest_retention.proto', False) - GenProto('../src/google/protobuf/util/json_format.proto', False) - GenProto('../src/google/protobuf/util/json_format_proto3.proto', False) - GenProto('google/protobuf/internal/any_test.proto', False) - GenProto('google/protobuf/internal/descriptor_pool_test1.proto', False) - GenProto('google/protobuf/internal/descriptor_pool_test2.proto', False) - GenProto('google/protobuf/internal/factory_test1.proto', False) - GenProto('google/protobuf/internal/factory_test2.proto', False) - GenProto('google/protobuf/internal/file_options_test.proto', False) - GenProto('google/protobuf/internal/import_test_package/import_public.proto', - False) - GenProto( - 'google/protobuf/internal/import_test_package/import_public_nested.proto', - False) - GenProto('google/protobuf/internal/import_test_package/inner.proto', False) - GenProto('google/protobuf/internal/import_test_package/outer.proto', False) - GenProto('google/protobuf/internal/missing_enum_values.proto', False) - GenProto('google/protobuf/internal/message_set_extensions.proto', False) - GenProto('google/protobuf/internal/more_extensions.proto', False) - GenProto('google/protobuf/internal/more_extensions_dynamic.proto', False) - GenProto('google/protobuf/internal/more_messages.proto', False) - GenProto('google/protobuf/internal/no_package.proto', False) - GenProto('google/protobuf/internal/packed_field_test.proto', False) - GenProto('google/protobuf/internal/test_bad_identifiers.proto', False) - GenProto('google/protobuf/internal/test_proto3_optional.proto', False) - GenProto('google/protobuf/pyext/python.proto', False) - - -class BuildPyCmd(_build_py): - """Custom build_py command for building the protobuf runtime.""" - - def run(self): - # Generate necessary .proto file if it doesn't exist. - GenProto('../src/google/protobuf/descriptor.proto') - GenProto('../src/google/protobuf/compiler/plugin.proto') - GenProto('../src/google/protobuf/any.proto') - GenProto('../src/google/protobuf/api.proto') - GenProto('../src/google/protobuf/duration.proto') - GenProto('../src/google/protobuf/empty.proto') - GenProto('../src/google/protobuf/field_mask.proto') - GenProto('../src/google/protobuf/source_context.proto') - GenProto('../src/google/protobuf/struct.proto') - GenProto('../src/google/protobuf/timestamp.proto') - GenProto('../src/google/protobuf/type.proto') - GenProto('../src/google/protobuf/wrappers.proto') - GenerateUnittestProtos() - - # _build_py is an old-style class, so super() doesn't work. - _build_py.run(self) - - def find_package_modules(self, package, package_dir): - exclude = ( - '*test*', - 'google/protobuf/internal/*_pb2.py', - 'google/protobuf/internal/_parameterized.py', - 'google/protobuf/pyext/python_pb2.py', - ) - modules = _build_py.find_package_modules(self, package, package_dir) - return [(pkg, mod, fil) for (pkg, mod, fil) in modules - if not any(fnmatch.fnmatchcase(fil, pat=pat) for pat in exclude)] - - -class BuildExtCmd(_build_ext): - """Command class for building the protobuf Python extension.""" - - def get_ext_filename(self, ext_name): - # since python3.5, python extensions' shared libraries use a suffix that - # corresponds to the value of sysconfig.get_config_var('EXT_SUFFIX') and - # contains info about the architecture the library targets. E.g. on x64 - # linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so" When - # crosscompiling python wheels, we need to be able to override this - # suffix so that the resulting file name matches the target architecture - # and we end up with a well-formed wheel. - filename = _build_ext.get_ext_filename(self, ext_name) - orig_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') - new_ext_suffix = os.getenv('PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX') - if new_ext_suffix and filename.endswith(orig_ext_suffix): - filename = filename[:-len(orig_ext_suffix)] + new_ext_suffix - return filename - - -class TestConformanceCmd(_build_py): - target = '//python:conformance_test' - - def run(self): - # Python 2.6 dodges these extra failures. - os.environ['CONFORMANCE_PYTHON_EXTRA_FAILURES'] = ( - '--failure_list failure_list_python-post26.txt') - cmd = 'bazel test %s' % (TestConformanceCmd.target,) - subprocess.check_call(cmd, shell=True) - - -def GetOptionFromArgv(option_str): - if option_str in sys.argv: - sys.argv.remove(option_str) - return True - return False - - -def _GetFlagValues(flag_long, flag_short): - """Searches sys.argv for setuptools-style flags and yields values.""" - - expect_value = flag_long.endswith('=') - flag_res = [re.compile(r'--?%s(=(.*))?' % - (flag_long[:-1] if expect_value else flag_long))] - if flag_short: - flag_res.append(re.compile(r'-%s(.*)?' % (flag_short,))) - - flag_match = None - for arg in sys.argv: - # If the last arg was like '-O', check if this is the library we want. - if flag_match is not None: - yield arg - flag_match = None - continue - - for flag_re in flag_res: - m = flag_re.match(arg) - if m is None: - continue - if not expect_value: - yield arg - continue - groups = m.groups() - # Check for matches: - # --long-name=foo => ('=foo', 'foo') - # -Xfoo => ('foo') - # N.B.: if the flag is like '--long-name=', then there is a value - # (the empty string). - if groups[0] or groups[-1]: - yield groups[-1] - continue - flag_match = m - - return False - - -def HasStaticLibprotobufOpt(): - """Returns true if there is a --link-objects arg for libprotobuf.""" - - lib_re = re.compile(r'(.*[/\\])?(lib)?protobuf([.]pic)?[.](a|lib)') - for value in _GetFlagValues('link-objects=', 'O'): - if lib_re.match(value): - return True - return False - - -def HasLibraryDirsOpt(): - """Returns true if there is a --library-dirs arg.""" - return any(_GetFlagValues('library-dirs=', 'L')) - - -if __name__ == '__main__': - ext_module_list = [] - warnings_as_errors = '--warnings_as_errors' - if GetOptionFromArgv('--cpp_implementation'): - # Link libprotobuf.a and libprotobuf-lite.a statically with the - # extension. Note that those libraries have to be compiled with - # -fPIC for this to work. - compile_static_ext = HasStaticLibprotobufOpt() - if GetOptionFromArgv('--compile_static_extension'): - # FUTURE: add a warning and deprecate --compile_static_extension. - compile_static_ext = True - extra_objects = None - if compile_static_ext: - libraries = None - library_dirs = None - if not HasStaticLibprotobufOpt(): - if os.path.exists('../bazel-bin/src/google/protobuf/libprotobuf.a'): - extra_objects = ['../bazel-bin/src/google/protobuf/libprotobuf.a'] - else: - extra_objects = ['../libprotobuf.a'] - extra_objects += list( - glob.iglob('../third_party/utf8_range/*.a')) - # Repeat all of these enough times to eliminate order-dependence. - extra_objects += list( - glob.iglob('../third_party/abseil-cpp/absl/**/*.a')) - extra_objects += list( - glob.iglob('../third_party/abseil-cpp/absl/**/*.a')) - extra_objects += list( - glob.iglob('../third_party/abseil-cpp/absl/**/*.a')) - else: - libraries = ['protobuf'] - if HasLibraryDirsOpt(): - library_dirs = None - elif os.path.exists('../bazel-bin/src/google/protobuf/libprotobuf.a'): - library_dirs = ['../bazel-bin/src/google/protobuf'] - else: - library_dirs = ['..'] - - TestConformanceCmd.target = ('//python:conformance_test_cpp ' - '--define=use_fast_cpp_protos=true') - - extra_compile_args = [] - - message_extra_link_args = None - api_implementation_link_args = None - if 'darwin' in sys.platform: - if sys.version_info[0] == 2: - message_init_symbol = 'init_message' - api_implementation_init_symbol = 'init_api_implementation' - else: - message_init_symbol = 'PyInit__message' - api_implementation_init_symbol = 'PyInit__api_implementation' - message_extra_link_args = [ - '-Wl,-exported_symbol,_%s' % message_init_symbol - ] - api_implementation_link_args = [ - '-Wl,-exported_symbol,_%s' % api_implementation_init_symbol - ] - - if sys.platform != 'win32': - extra_compile_args.append('-Wno-write-strings') - extra_compile_args.append('-Wno-invalid-offsetof') - extra_compile_args.append('-Wno-sign-compare') - extra_compile_args.append('-Wno-unused-variable') - extra_compile_args.append('-std=c++14') - - if sys.platform == 'darwin': - extra_compile_args.append('-Wno-shorten-64-to-32') - extra_compile_args.append('-Wno-deprecated-register') - - # https://developer.apple.com/documentation/xcode_release_notes/xcode_10_release_notes - # C++ projects must now migrate to libc++ and are recommended to set a - # deployment target of macOS 10.9 or later, or iOS 7 or later. - if sys.platform == 'darwin': - mac_target = str(sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')) - if mac_target and (pkg_resources.parse_version(mac_target) < - pkg_resources.parse_version('10.9.0')): - os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9' - os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( - r'macosx-[0-9]+\.[0-9]+-(.+)', - r'macosx-10.9-\1', - sysconfig.get_platform(), - ) - - # https://github.com/Theano/Theano/issues/4926 - if sys.platform == 'win32': - extra_compile_args.append('-D_hypot=hypot') - - # https://github.com/tpaviot/pythonocc-core/issues/48 - if sys.platform == 'win32' and '64 bit' in sys.version: - extra_compile_args.append('-DMS_WIN64') - - # MSVS default is dymanic - if sys.platform == 'win32': - extra_compile_args.append('/MT') - - if 'clang' in os.popen('$CC --version 2> /dev/null').read(): - extra_compile_args.append('-Wno-shorten-64-to-32') - - if warnings_as_errors in sys.argv: - extra_compile_args.append('-Werror') - sys.argv.remove(warnings_as_errors) - - # C++ implementation extension - ext_module_list.extend([ - Extension( - 'google.protobuf.pyext._message', - glob.glob('google/protobuf/pyext/*.cc'), - include_dirs=['.', '../src', '../third_party/abseil-cpp'], - libraries=libraries, - extra_objects=extra_objects, - extra_link_args=message_extra_link_args, - library_dirs=library_dirs, - extra_compile_args=extra_compile_args, - ), - Extension( - 'google.protobuf.internal._api_implementation', - glob.glob('google/protobuf/internal/api_implementation.cc'), - extra_compile_args=(extra_compile_args + - ['-DPYTHON_PROTO2_CPP_IMPL_V2']), - extra_link_args=api_implementation_link_args, - ), - ]) - os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' - - install_requires = [] - - setup( - name='protobuf', - version=GetVersion(), - description='Protocol Buffers', - download_url='https://github.com/protocolbuffers/protobuf/releases', - long_description="Protocol Buffers are Google's data interchange format", - url='https://developers.google.com/protocol-buffers/', - project_urls={ - 'Source': 'https://github.com/protocolbuffers/protobuf', - }, - maintainer='protobuf@googlegroups.com', - maintainer_email='protobuf@googlegroups.com', - license='BSD-3-Clause', - classifiers=[ - 'Programming Language :: Python', - 'Programming Language :: Python :: 3', - # LINT.IfChange - # Remove importlib fallback path when we drop Python 3.8 support. - 'Programming Language :: Python :: 3.8', - # LINT.ThenChange(//depot/google3/google/protobuf/internal/test_util.py) - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - ], - namespace_packages=['google'], - packages=find_packages( - exclude=[ - 'import_test_package', - 'protobuf_distutils', - ], - ), - test_suite='google.protobuf.internal', - cmdclass={ - 'build_py': BuildPyCmd, - 'build_ext': BuildExtCmd, - 'test_conformance': TestConformanceCmd, - }, - install_requires=install_requires, - ext_modules=ext_module_list, - python_requires='>=3.8', - )