Add a setuptools extension for generating Python protobufs

python/protobuf_distutils/README.md

@@ -0,0 +1,99 @@
+# Python setuptools extension
+
+This is an extension for Python setuptools which uses an installed protobuf
+compiler (`protoc`) to generate Python sources.
+
+## Installing
+
+To use this extension, it needs to be installed so it can be imported by other
+projects' setup.py.
+
+```shell
+$ python setup.py build
+$ python setup.py install
+```
+
+(If you want to test changes to the extension, you can use `python setup.py
+develop`.)
+
+## Usage
+
+### Example setup.py configuration
+
+```python
+from setuptools import setup
+setup(
+    # ...
+    name='example_project',
+
+    # Require this package, but only for setup (not installation):
+    setup_requires=['protobuf_distutils'],
+
+    options={
+        # See below for details.
+        'generate_py_protobufs': {
+            'source_dir':        'path/to/protos',
+            'extra_proto_paths': ['path/to/other/project/protos'],
+            'output_dir':        'path/to/project/sources',  # default '.'
+            'proto_files':       ['relative/path/to/just_this_file.proto'],
+            'protoc':            'path/to/protoc.exe',
+        },
+    },
+)
+```
+
+### Example build invocation
+
+These steps will generate protobuf sources so they are included when building
+and installing `example_project` (see above):
+
+```shell
+$ python setup.py generate_py_protobufs
+$ python setup.py build
+$ python setup.py install
+```
+
+## Options
+
+- `source_dir`:
+
+  Sets the .proto file root path. This will be used to find proto files, and
+  to resolve imports.
+
+  The default behavior is to generate sources for all .proto files found
+  under `source_dirs`, recursively. This can be controlled with options
+  below.
+
+- `extra_proto_paths`:
+
+  Specifies additional paths that should be used to find imports, in
+  addition to `source_dir`.
+
+  This option can be used to specify the path to other protobuf sources,
+  which are separately converted into Python sources.  No Python code will
+  be generated for .proto files under these paths.
+
+- `output_dir`:
+
+  Specifies where generated code should be placed.
+
+  Typically, this should be the project root package. The generated files
+  will be placed under `output_dir` according to the relative source paths
+  under `source_dir`.
+
+  For example, the source file `${source_dir}/subdir/message.proto`
+  will be generated as the Python file `${output_dir}/subdir/message_pb2.py`.
+
+- `proto_files`:
+
+  Specific .proto files can be specified for generating code, instead of
+  searching for all .proto files under `source_path`.
+
+  These paths are relative to `source_dir`. For example, to generate code
+  for just ${source_dir}/subdir/message.proto:
+
+- `protoc`:
+
+  By default, the protoc binary (the Protobuf compiler) is found by
+  searching the environment path. To use a specific protoc binary, its
+  path can be specified.

python/protobuf_distutils/protobuf_distutils/generate_py_protobufs.py

@@ -0,0 +1,100 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Implements the generate_py_protobufs command."""
+
+__author__ = 'dlj@google.com (David L. Jones)'
+
+import glob
+import sys
+import os
+import distutils.spawn as spawn
+from distutils.cmd import Command
+from distutils.errors import DistutilsOptionError, DistutilsExecError
+
+class generate_py_protobufs(Command):
+    """Generates Python sources for .proto files."""
+
+    description = 'Generate Python sources for .proto files'
+    user_options = [
+        ('extra-proto-paths=', None,
+         'Additional paths to resolve imports in .proto files.'),
+
+        ('protoc', None,
+         'Path to a specific `protoc` command to use.'),
+    ]
+    boolean_options = ['recurse']
+
+    def initialize_options(self):
+        """Sets the defaults for the command options."""
+        self.source_dir = None
+        self.extra_proto_paths = []
+        self.output_dir = '.'
+        self.proto_files = None
+        self.recurse = True
+        self.protoc = None
+
+    def finalize_options(self):
+        """Sets the final values for the command options.
+
+        Defaults were set in `initialize_options`, but could have been changed
+        by command-line options or by other commands.
+        """
+        self.ensure_dirname('source_dir')
+        self.ensure_string_list('extra_proto_paths')
+
+        if self.output_dir is None:
+            self.output_dir = '.'
+        self.ensure_dirname('output_dir')
+
+        if self.proto_files is None:
+            files = glob.glob(os.path.join(self.source_dir, '*.proto'))
+            if self.recurse:
+                files.extend(glob.glob(os.path.join(self.source_dir, '**', '*.proto')))
+            self.proto_files = [f.partition(self.source_dir + os.path.sep)[-1] for f in files]
+            if not self.proto_files:
+                raise DistutilsOptionError('no .proto files were found under ' + self.source_dir)
+
+        self.ensure_string_list('proto_files')
+
+        if self.protoc is None:
+            self.protoc = spawn.find_executable('protoc')
+
+    def run(self):
+        # Run protoc. It was already resolved, so don't try to resolve
+        # through PATH.
+        proto_paths = ['--proto_path=' + self.source_dir]
+        for extra in self.extra_proto_paths:
+            proto_paths.append('--proto_path=' + extra)
+        spawn.spawn(
+            [self.protoc,
+             '--python_out=' + self.output_dir,
+            ] + proto_paths + self.proto_files,
+            search_path=0)

python/protobuf_distutils/setup.py

@@ -0,0 +1,127 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Setuptools/distutils extension for generating Python protobuf code.
+
+This extension uses a prebuilt 'protoc' binary to generate Python types for
+protobuf sources. By default, it will use a system-installed protoc binary, but
+a custom protoc can be specified by flag.
+
+This command should usually be run before the 'build' command, so that the
+generated sources are treated the same way as the rest of the Python
+sources.
+
+Options:
+
+    source_dir:
+        Sets the .proto file root path. This will be used to find proto files,
+        and to resolve imports.
+
+        The default behavior is to generate sources for all .proto files found
+        under `source_dirs`, recursively. This can be controlled with options
+        below.
+
+    extra_proto_paths:
+        Specifies additional paths that should be used to find imports, in
+        addition to `source_dir`.
+
+        This option can be used to specify the path to other protobuf sources,
+        which are separately converted into Python sources.  No Python code will
+        be generated for .proto files under these paths.
+
+    output_dir:
+        Specifies where generated code should be placed.
+
+        Typically, this should be the project root package. The generated files
+        will be placed under `output_dir` according to the relative source paths
+        under `source_dir`.
+
+        For example, the source file:
+            ${source_dir}/subdir/message.proto
+        will be generated as this Python file:
+            ${output_dir}/subdir/message_pb2.py
+
+    proto_files:
+        Specific .proto files can be specified for generating code, instead of
+        searching for all .proto files under `source_path`.
+
+        These paths are relative to `source_dir`. For example, to generate code
+        for just ${source_dir}/subdir/message.proto:
+
+    protoc:
+        By default, the protoc binary (the Protobuf compiler) is found by
+        searching the environment path. To use a specific protoc binary, its
+        path can be specified.
+
+    recurse:
+        If `proto_files` are not specified, then the default behavior is to
+        search `source_dir` recursively. This option controls the recursive
+        search; if it is False, only .proto files immediately under `source_dir`
+        will be used to generate sources.
+
+"""
+
+__author__ = 'dlj@google.com (David L. Jones)'
+
+from setuptools import setup, find_packages
+
+setup(
+    name='protobuf_distutils',
+    version='1.0',
+    packages=find_packages(),
+    maintainer='protobuf@googlegroups.com',
+    maintainer_email='protobuf@googlegroups.com',
+    license='3-Clause BSD License',
+    classifiers=[
+        "Framework :: Setuptools Plugin",
+        "Operating System :: OS Independent",
+        # These Python versions should match the protobuf package:
+        "Programming Language :: Python",
+        "Programming Language :: Python :: 2",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.3",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Topic :: Software Development :: Code Generators",
+    ],
+    description=('This is a distutils extension to generate Python code for '
+                 '.proto files using an installed protoc binary.'),
+    url='https://github.com/protocolbuffers/protobuf/',
+    entry_points={
+        'distutils.commands': [
+            ('generate_py_protobufs = '
+             'protobuf_distutils.generate_py_protobufs:generate_py_protobufs'),
+        ],
+    },
+)

python/setup.py

@@ -271,6 +271,7 @@ def get_option_from_sys_argv(option_str):
       packages=find_packages(
           exclude=[
               'import_test_package',
+              'protobuf_distutils',
           ],
       ),
       test_suite='google.protobuf.internal',

tests.sh

@@ -64,7 +64,7 @@ build_cpp_distcheck() {
   git ls-files | grep "^\(java\|python\|objectivec\|csharp\|js\|ruby\|php\|cmake\|examples\|src/google/protobuf/.*\.proto\)" |\
     grep -v ".gitignore" | grep -v "java/compatibility_tests" | grep -v "java/lite/proguard.pgcfg" |\
     grep -v "python/compatibility_tests" | grep -v "python/docs" | grep -v "python/.repo-metadata.json" |\
-    grep -v "csharp/compatibility_tests" > dist.lst
+    grep -v "python/protobuf_distutils" | grep -v "csharp/compatibility_tests" > dist.lst
   # Unzip the dist tar file.
   DIST=`ls *.tar.gz`
   tar -xf $DIST