From 0a3bcf1318ed9e00e73130e1850f1e2c5b0ed237 Mon Sep 17 00:00:00 2001 From: Alexander Shadchin Date: Wed, 24 Feb 2021 04:03:15 +0300 Subject: [PATCH 1/4] Switch on "new" buffer API. "Old" buffer API will removed in Python 3.10. This is also fix #7930. --- python/google/protobuf/pyext/message.cc | 12 ++++++------ python/tox.ini | 5 +---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index e8e1a17b9760..d3fc477d0fb6 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -1927,9 +1927,8 @@ PyObject* SetAllowOversizeProtos(PyObject* m, PyObject* arg) { } static PyObject* MergeFromString(CMessage* self, PyObject* arg) { - const void* data; - Py_ssize_t data_length; - if (PyObject_AsReadBuffer(arg, &data, &data_length) < 0) { + Py_buffer data; + if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) < 0) { return NULL; } @@ -1942,7 +1941,8 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) { const char* ptr; internal::ParseContext ctx( depth, false, &ptr, - StringPiece(static_cast(data), data_length)); + StringPiece(static_cast(data.buf), data.len)); + PyBuffer_Release(&data); ctx.data().pool = factory->pool->pool; ctx.data().factory = factory->message_factory; @@ -1968,9 +1968,9 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) { // TODO(jieluo): Raise error and return NULL instead. // b/27494216 PyErr_Warn(nullptr, "Unexpected end-group tag: Not all data was converted"); - return PyInt_FromLong(data_length - ctx.BytesUntilLimit(ptr)); + return PyInt_FromLong(data.len - ctx.BytesUntilLimit(ptr)); } - return PyInt_FromLong(data_length); + return PyInt_FromLong(data.len); } static PyObject* ParseFromString(CMessage* self, PyObject* arg) { diff --git a/python/tox.ini b/python/tox.ini index 9fabb6ddbb25..c3f24e2e2131 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -14,10 +14,7 @@ setenv = commands = python setup.py -q build_py python: python setup.py -q build - # --warnings_as_errors disabled until we update the Python C extension. See: - # https://github.com/protocolbuffers/protobuf/issues/7930 - # cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension - cpp: python setup.py -q build --cpp_implementation --compile_static_extension + cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension python: python setup.py -q test -q cpp: python setup.py -q test -q --cpp_implementation python: python setup.py -q test_conformance From 01cdbd48a88337f9a02e5c87379c85aad81ca6a0 Mon Sep 17 00:00:00 2001 From: Alexander Shadchin Date: Wed, 3 Mar 2021 01:06:49 +0300 Subject: [PATCH 2/4] Switch on PyObject_Call --- python/google/protobuf/pyext/descriptor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 75f1760ccf45..ab482da32f71 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -248,7 +248,7 @@ static PyObject* GetOrBuildOptions(const DescriptorClass *descriptor) { return NULL; } ScopedPyObjectPtr value( - PyEval_CallObject(message_class->AsPyObject(), NULL)); + PyObject_Call(message_class->AsPyObject(), NULL, NULL)); Py_DECREF(message_class); if (value == NULL) { return NULL; From 04b50d1eeccd236e91f8817b4e87464d7b632778 Mon Sep 17 00:00:00 2001 From: Alexander Shadchin Date: Wed, 3 Mar 2021 01:49:48 +0300 Subject: [PATCH 3/4] Fix tests for Python 2.7 --- python/google/protobuf/internal/message_test.py | 9 +-------- python/google/protobuf/internal/python_message.py | 6 ------ 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index feff228fb62c..77122a2875f9 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -841,8 +841,7 @@ def testMergeFromString(self, message_module): m1.MergeFromString(m2.SerializeToString()) self.assertEqual(1, m1.optional_nested_message.bb) - @unittest.skipIf(six.PY2, 'memoryview objects are not supported on py2') - def testMergeFromStringUsingMemoryViewWorksInPy3(self, message_module): + def testMergeFromStringUsingMemoryView(self, message_module): m2 = message_module.TestAllTypes() m2.optional_string = 'scalar string' m2.repeated_string.append('repeated string') @@ -864,12 +863,6 @@ def testMergeFromStringUsingMemoryViewWorksInPy3(self, message_module): self.assertIsInstance(m1.optional_string, six.text_type) self.assertIsInstance(m1.repeated_string[0], six.text_type) - @unittest.skipIf(six.PY3, 'memoryview is supported by py3') - def testMergeFromStringUsingMemoryViewIsPy2Error(self, message_module): - memview = memoryview(b'') - with self.assertRaises(TypeError): - message_module.TestAllTypes.FromString(memview) - def testMergeFromEmpty(self, message_module): m1 = message_module.TestAllTypes() # Cpp extension will lazily create a sub message which is immutable. diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index d1f4dcde8e5e..99d2f078de26 100644 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py @@ -1133,12 +1133,6 @@ def InternalSerialize(self, write_bytes, deterministic=None): def _AddMergeFromStringMethod(message_descriptor, cls): """Helper for _AddMessageMethods().""" def MergeFromString(self, serialized): - if isinstance(serialized, memoryview) and six.PY2: - raise TypeError( - 'memoryview not supported in Python 2 with the pure Python proto ' - 'implementation: this is to maintain compatibility with the C++ ' - 'implementation') - serialized = memoryview(serialized) length = len(serialized) try: From a3bc1f1ffb65bb2fb5f0e3789867e1adec2b7f08 Mon Sep 17 00:00:00 2001 From: Alexander Shadchin Date: Thu, 4 Mar 2021 03:10:10 +0300 Subject: [PATCH 4/4] Returned --warnings_as_errors for Python 2.7 --- python/tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/tox.ini b/python/tox.ini index c3f24e2e2131..f9eee920fdc7 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -14,7 +14,9 @@ setenv = commands = python setup.py -q build_py python: python setup.py -q build - cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension + # --warnings_as_errors disabled for Python 2.7 because _POSIX_C_SOURCE and _XOPEN_SOURCE are redefined + py27-cpp: python setup.py -q build --cpp_implementation --compile_static_extension + py{33,34,35,36,37,38,39}-cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension python: python setup.py -q test -q cpp: python setup.py -q test -q --cpp_implementation python: python setup.py -q test_conformance