setup.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358
  1. #! /usr/bin/env python
  2. #
  3. # See README for usage instructions.
  4. # pylint:disable=missing-module-docstring
  5. # pylint:disable=g-bad-import-order
  6. from distutils import util
  7. import fnmatch
  8. import glob
  9. import os
  10. import pkg_resources
  11. import re
  12. import subprocess
  13. import sys
  14. import sysconfig
  15. # pylint:disable=g-importing-member
  16. # pylint:disable=g-multiple-import
  17. # We must use setuptools, not distutils, because we need to use the
  18. # namespace_packages option for the "google" package.
  19. from setuptools import setup, Extension, find_packages
  20. from distutils.command.build_ext import build_ext as _build_ext
  21. from distutils.command.build_py import build_py as _build_py
  22. from distutils.command.clean import clean as _clean
  23. from distutils.spawn import find_executable
  24. # Find the Protocol Compiler.
  25. if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']):
  26. protoc = os.environ['PROTOC']
  27. elif os.path.exists('../src/protoc'):
  28. protoc = '../src/protoc'
  29. elif os.path.exists('../src/protoc.exe'):
  30. protoc = '../src/protoc.exe'
  31. elif os.path.exists('../vsprojects/Debug/protoc.exe'):
  32. protoc = '../vsprojects/Debug/protoc.exe'
  33. elif os.path.exists('../vsprojects/Release/protoc.exe'):
  34. protoc = '../vsprojects/Release/protoc.exe'
  35. else:
  36. protoc = find_executable('protoc')
  37. def GetVersion():
  38. """Reads and returns the version from google/protobuf/__init__.py.
  39. Do not import google.protobuf.__init__ directly, because an installed
  40. protobuf library may be loaded instead.
  41. Returns:
  42. The version.
  43. """
  44. with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file:
  45. exec(version_file.read(), globals()) # pylint:disable=exec-used
  46. return __version__ # pylint:disable=undefined-variable
  47. def GenProto(source, require=True):
  48. """Generates a _pb2.py from the given .proto file.
  49. Does nothing if the output already exists and is newer than the input.
  50. Args:
  51. source: the .proto file path.
  52. require: if True, exit immediately when a path is not found.
  53. """
  54. if not require and not os.path.exists(source):
  55. return
  56. output = source.replace('.proto', '_pb2.py').replace('../src/', '')
  57. if (not os.path.exists(output) or
  58. (os.path.exists(source) and
  59. os.path.getmtime(source) > os.path.getmtime(output))):
  60. print('Generating %s...' % output)
  61. if not os.path.exists(source):
  62. sys.stderr.write("Can't find required file: %s\n" % source)
  63. sys.exit(-1)
  64. if protoc is None:
  65. sys.stderr.write(
  66. 'protoc is not installed nor found in ../src. Please compile it '
  67. 'or install the binary package.\n')
  68. sys.exit(-1)
  69. protoc_command = [protoc, '-I../src', '-I.', '--python_out=.', source]
  70. if subprocess.call(protoc_command) != 0:
  71. sys.exit(-1)
  72. def GenerateUnittestProtos():
  73. """Generates protobuf code for unittests."""
  74. GenProto('../src/google/protobuf/any_test.proto', False)
  75. GenProto('../src/google/protobuf/map_proto2_unittest.proto', False)
  76. GenProto('../src/google/protobuf/map_unittest.proto', False)
  77. GenProto('../src/google/protobuf/test_messages_proto3.proto', False)
  78. GenProto('../src/google/protobuf/test_messages_proto2.proto', False)
  79. GenProto('../src/google/protobuf/unittest_arena.proto', False)
  80. GenProto('../src/google/protobuf/unittest.proto', False)
  81. GenProto('../src/google/protobuf/unittest_custom_options.proto', False)
  82. GenProto('../src/google/protobuf/unittest_import.proto', False)
  83. GenProto('../src/google/protobuf/unittest_import_public.proto', False)
  84. GenProto('../src/google/protobuf/unittest_mset.proto', False)
  85. GenProto('../src/google/protobuf/unittest_mset_wire_format.proto', False)
  86. GenProto('../src/google/protobuf/unittest_no_generic_services.proto', False)
  87. GenProto('../src/google/protobuf/unittest_proto3_arena.proto', False)
  88. GenProto('../src/google/protobuf/util/json_format.proto', False)
  89. GenProto('../src/google/protobuf/util/json_format_proto3.proto', False)
  90. GenProto('google/protobuf/internal/any_test.proto', False)
  91. GenProto('google/protobuf/internal/descriptor_pool_test1.proto', False)
  92. GenProto('google/protobuf/internal/descriptor_pool_test2.proto', False)
  93. GenProto('google/protobuf/internal/factory_test1.proto', False)
  94. GenProto('google/protobuf/internal/factory_test2.proto', False)
  95. GenProto('google/protobuf/internal/file_options_test.proto', False)
  96. GenProto('google/protobuf/internal/import_test_package/import_public.proto',
  97. False)
  98. GenProto(
  99. 'google/protobuf/internal/import_test_package/import_public_nested.proto',
  100. False)
  101. GenProto('google/protobuf/internal/import_test_package/inner.proto', False)
  102. GenProto('google/protobuf/internal/import_test_package/outer.proto', False)
  103. GenProto('google/protobuf/internal/missing_enum_values.proto', False)
  104. GenProto('google/protobuf/internal/message_set_extensions.proto', False)
  105. GenProto('google/protobuf/internal/more_extensions.proto', False)
  106. GenProto('google/protobuf/internal/more_extensions_dynamic.proto', False)
  107. GenProto('google/protobuf/internal/more_messages.proto', False)
  108. GenProto('google/protobuf/internal/no_package.proto', False)
  109. GenProto('google/protobuf/internal/packed_field_test.proto', False)
  110. GenProto('google/protobuf/internal/test_bad_identifiers.proto', False)
  111. GenProto('google/protobuf/internal/test_proto3_optional.proto', False)
  112. GenProto('google/protobuf/pyext/python.proto', False)
  113. class CleanCmd(_clean):
  114. """Custom clean command for building the protobuf extension."""
  115. def run(self):
  116. # Delete generated files in the code tree.
  117. for (dirpath, unused_dirnames, filenames) in os.walk('.'):
  118. for filename in filenames:
  119. filepath = os.path.join(dirpath, filename)
  120. if (filepath.endswith('_pb2.py') or filepath.endswith('.pyc') or
  121. filepath.endswith('.so') or filepath.endswith('.o')):
  122. os.remove(filepath)
  123. # _clean is an old-style class, so super() doesn't work.
  124. _clean.run(self)
  125. class BuildPyCmd(_build_py):
  126. """Custom build_py command for building the protobuf runtime."""
  127. def run(self):
  128. # Generate necessary .proto file if it doesn't exist.
  129. GenProto('../src/google/protobuf/descriptor.proto')
  130. GenProto('../src/google/protobuf/compiler/plugin.proto')
  131. GenProto('../src/google/protobuf/any.proto')
  132. GenProto('../src/google/protobuf/api.proto')
  133. GenProto('../src/google/protobuf/duration.proto')
  134. GenProto('../src/google/protobuf/empty.proto')
  135. GenProto('../src/google/protobuf/field_mask.proto')
  136. GenProto('../src/google/protobuf/source_context.proto')
  137. GenProto('../src/google/protobuf/struct.proto')
  138. GenProto('../src/google/protobuf/timestamp.proto')
  139. GenProto('../src/google/protobuf/type.proto')
  140. GenProto('../src/google/protobuf/wrappers.proto')
  141. GenerateUnittestProtos()
  142. # _build_py is an old-style class, so super() doesn't work.
  143. _build_py.run(self)
  144. def find_package_modules(self, package, package_dir):
  145. exclude = (
  146. '*test*',
  147. 'google/protobuf/internal/*_pb2.py',
  148. 'google/protobuf/internal/_parameterized.py',
  149. 'google/protobuf/pyext/python_pb2.py',
  150. )
  151. modules = _build_py.find_package_modules(self, package, package_dir)
  152. return [(pkg, mod, fil) for (pkg, mod, fil) in modules
  153. if not any(fnmatch.fnmatchcase(fil, pat=pat) for pat in exclude)]
  154. class BuildExtCmd(_build_ext):
  155. """Command class for building the protobuf Python extension."""
  156. def get_ext_filename(self, ext_name):
  157. # since python3.5, python extensions' shared libraries use a suffix that
  158. # corresponds to the value of sysconfig.get_config_var('EXT_SUFFIX') and
  159. # contains info about the architecture the library targets. E.g. on x64
  160. # linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so" When
  161. # crosscompiling python wheels, we need to be able to override this
  162. # suffix so that the resulting file name matches the target architecture
  163. # and we end up with a well-formed wheel.
  164. filename = _build_ext.get_ext_filename(self, ext_name)
  165. orig_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
  166. new_ext_suffix = os.getenv('PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX')
  167. if new_ext_suffix and filename.endswith(orig_ext_suffix):
  168. filename = filename[:-len(orig_ext_suffix)] + new_ext_suffix
  169. return filename
  170. class TestConformanceCmd(_build_py):
  171. target = 'test_python'
  172. def run(self):
  173. # Python 2.6 dodges these extra failures.
  174. os.environ['CONFORMANCE_PYTHON_EXTRA_FAILURES'] = (
  175. '--failure_list failure_list_python-post26.txt')
  176. cmd = 'cd ../conformance && make %s' % (TestConformanceCmd.target)
  177. subprocess.check_call(cmd, shell=True)
  178. def GetOptionFromArgv(option_str):
  179. if option_str in sys.argv:
  180. sys.argv.remove(option_str)
  181. return True
  182. return False
  183. if __name__ == '__main__':
  184. ext_module_list = []
  185. warnings_as_errors = '--warnings_as_errors'
  186. if GetOptionFromArgv('--cpp_implementation'):
  187. # Link libprotobuf.a and libprotobuf-lite.a statically with the
  188. # extension. Note that those libraries have to be compiled with
  189. # -fPIC for this to work.
  190. compile_static_ext = GetOptionFromArgv('--compile_static_extension')
  191. libraries = ['protobuf']
  192. extra_objects = None
  193. if compile_static_ext:
  194. libraries = None
  195. extra_objects = ['../src/.libs/libprotobuf.a',
  196. '../src/.libs/libprotobuf-lite.a']
  197. TestConformanceCmd.target = 'test_python_cpp'
  198. extra_compile_args = []
  199. message_extra_link_args = None
  200. api_implementation_link_args = None
  201. if 'darwin' in sys.platform:
  202. if sys.version_info[0] == 2:
  203. message_init_symbol = 'init_message'
  204. api_implementation_init_symbol = 'init_api_implementation'
  205. else:
  206. message_init_symbol = 'PyInit__message'
  207. api_implementation_init_symbol = 'PyInit__api_implementation'
  208. message_extra_link_args = [
  209. '-Wl,-exported_symbol,_%s' % message_init_symbol
  210. ]
  211. api_implementation_link_args = [
  212. '-Wl,-exported_symbol,_%s' % api_implementation_init_symbol
  213. ]
  214. if sys.platform != 'win32':
  215. extra_compile_args.append('-Wno-write-strings')
  216. extra_compile_args.append('-Wno-invalid-offsetof')
  217. extra_compile_args.append('-Wno-sign-compare')
  218. extra_compile_args.append('-Wno-unused-variable')
  219. extra_compile_args.append('-std=c++11')
  220. if sys.platform == 'darwin':
  221. extra_compile_args.append('-Wno-shorten-64-to-32')
  222. extra_compile_args.append('-Wno-deprecated-register')
  223. # https://developer.apple.com/documentation/xcode_release_notes/xcode_10_release_notes
  224. # C++ projects must now migrate to libc++ and are recommended to set a
  225. # deployment target of macOS 10.9 or later, or iOS 7 or later.
  226. if sys.platform == 'darwin':
  227. mac_target = str(sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET'))
  228. if mac_target and (pkg_resources.parse_version(mac_target) <
  229. pkg_resources.parse_version('10.9.0')):
  230. os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
  231. os.environ['_PYTHON_HOST_PLATFORM'] = re.sub(
  232. r'macosx-[0-9]+\.[0-9]+-(.+)', r'macosx-10.9-\1',
  233. util.get_platform())
  234. # https://github.com/Theano/Theano/issues/4926
  235. if sys.platform == 'win32':
  236. extra_compile_args.append('-D_hypot=hypot')
  237. # https://github.com/tpaviot/pythonocc-core/issues/48
  238. if sys.platform == 'win32' and '64 bit' in sys.version:
  239. extra_compile_args.append('-DMS_WIN64')
  240. # MSVS default is dymanic
  241. if sys.platform == 'win32':
  242. extra_compile_args.append('/MT')
  243. if 'clang' in os.popen('$CC --version 2> /dev/null').read():
  244. extra_compile_args.append('-Wno-shorten-64-to-32')
  245. if warnings_as_errors in sys.argv:
  246. extra_compile_args.append('-Werror')
  247. sys.argv.remove(warnings_as_errors)
  248. # C++ implementation extension
  249. ext_module_list.extend([
  250. Extension(
  251. 'google.protobuf.pyext._message',
  252. glob.glob('google/protobuf/pyext/*.cc'),
  253. include_dirs=['.', '../src'],
  254. libraries=libraries,
  255. extra_objects=extra_objects,
  256. extra_link_args=message_extra_link_args,
  257. library_dirs=['../src/.libs'],
  258. extra_compile_args=extra_compile_args,
  259. ),
  260. Extension(
  261. 'google.protobuf.internal._api_implementation',
  262. glob.glob('google/protobuf/internal/api_implementation.cc'),
  263. extra_compile_args=(extra_compile_args +
  264. ['-DPYTHON_PROTO2_CPP_IMPL_V2']),
  265. extra_link_args=api_implementation_link_args,
  266. ),
  267. ])
  268. os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
  269. # Keep this list of dependencies in sync with tox.ini.
  270. install_requires = []
  271. setup(
  272. name='protobuf',
  273. version=GetVersion(),
  274. description='Protocol Buffers',
  275. download_url='https://github.com/protocolbuffers/protobuf/releases',
  276. long_description="Protocol Buffers are Google's data interchange format",
  277. url='https://developers.google.com/protocol-buffers/',
  278. maintainer='[email protected]',
  279. maintainer_email='[email protected]',
  280. license='BSD-3-Clause',
  281. classifiers=[
  282. 'Programming Language :: Python',
  283. 'Programming Language :: Python :: 3',
  284. 'Programming Language :: Python :: 3.7',
  285. 'Programming Language :: Python :: 3.8',
  286. 'Programming Language :: Python :: 3.9',
  287. 'Programming Language :: Python :: 3.10',
  288. ],
  289. namespace_packages=['google'],
  290. packages=find_packages(
  291. exclude=[
  292. 'import_test_package',
  293. 'protobuf_distutils',
  294. ],),
  295. test_suite='google.protobuf.internal',
  296. cmdclass={
  297. 'clean': CleanCmd,
  298. 'build_py': BuildPyCmd,
  299. 'build_ext': BuildExtCmd,
  300. 'test_conformance': TestConformanceCmd,
  301. },
  302. install_requires=install_requires,
  303. ext_modules=ext_module_list,
  304. python_requires='>=3.7',
  305. )