CHANGES.txt

@@ -2,6 +2,22 @@
 lxml changelog
 ==============
 
+4.5.1 (2020-05-19)
+==================
+
+Bugs fixed
+----------
+
+* LP#1570388: Fix failures when serialising documents larger than 2GB in some cases.
+
+* LP#1865141, GH#298: ``QName`` values were not accepted by the ``el.iter()`` method.
+  Patch by xmo-odoo.
+
+* LP#1863413, GH#297: The build failed to detect libraries on Linux that are only
+  configured via pkg-config.
+  Patch by Hugh McMaster.
+
+
 4.5.0 (2020-01-29)
 ==================
 

doc/main.txt

@@ -159,8 +159,8 @@ Index <http://pypi.python.org/pypi/lxml/>`_ (PyPI).  It has the source
 that compiles on various platforms.  The source distribution is signed
 with `this key <pubkey.asc>`_.
 
-The latest version is `lxml 4.5.0`_, released 2020-01-29
-(`changes for 4.5.0`_).  `Older versions <#old-versions>`_
+The latest version is `lxml 4.5.1`_, released 2020-05-19
+(`changes for 4.5.1`_).  `Older versions <#old-versions>`_
 are listed below.
 
 Please take a look at the
@@ -255,7 +255,9 @@ See the websites of lxml
 ..
    and the `latest in-development version <http://lxml.de/dev/>`_.
 
-.. _`PDF documentation`: lxmldoc-4.5.0.pdf
+.. _`PDF documentation`: lxmldoc-4.5.1.pdf
+
+* `lxml 4.5.1`_, released 2020-05-19 (`changes for 4.5.1`_)
 
 * `lxml 4.5.0`_, released 2020-01-29 (`changes for 4.5.0`_)
 
@@ -269,12 +271,14 @@ See the websites of lxml
 
 * `older releases <http://lxml.de/4.3/#old-versions>`_
 
+.. _`lxml 4.5.1`: /files/lxml-4.5.1.tgz
 .. _`lxml 4.5.0`: /files/lxml-4.5.0.tgz
 .. _`lxml 4.4.3`: /files/lxml-4.4.3.tgz
 .. _`lxml 4.4.2`: /files/lxml-4.4.2.tgz
 .. _`lxml 4.4.1`: /files/lxml-4.4.1.tgz
 .. _`lxml 4.4.0`: /files/lxml-4.4.0.tgz
 
+.. _`changes for 4.5.1`: /changes-4.5.1.html
 .. _`changes for 4.5.0`: /changes-4.5.0.html
 .. _`changes for 4.4.3`: /changes-4.4.3.html
 .. _`changes for 4.4.2`: /changes-4.4.2.html

setupinfo.py

@@ -2,6 +2,7 @@
 import io
 import os
 import os.path
+import subprocess
 from distutils.core import Extension
 from distutils.errors import CompileError, DistutilsOptionError
 from distutils.command.build_ext import build_ext as _build_ext
@@ -109,17 +110,7 @@ def ext_modules(static_include_dirs, static_library_dirs,
         use_cython = False
         print("Building without Cython.")
 
-    lib_versions = get_library_versions()
-    versions_ok = True
-    if lib_versions[0]:
-        print("Using build configuration of libxml2 %s and libxslt %s" %
-              lib_versions)
-        versions_ok = check_min_version(lib_versions[0], (2, 7, 0), 'libxml2')
-    else:
-        print("Using build configuration of libxslt %s" %
-              lib_versions[1])
-    versions_ok |= check_min_version(lib_versions[1], (1, 1, 23), 'libxslt')
-    if not versions_ok:
+    if not check_build_dependencies():
         raise RuntimeError("Dependency missing")
 
     base_dir = get_base_dir()
@@ -360,53 +351,118 @@ def define_macros():
     macros.append(('CYTHON_CLINE_IN_TRACEBACK', '1' if OPTION_WITH_CLINES else '0'))
     return macros
 
-_ERROR_PRINTED = False
 
 def run_command(cmd, *args):
     if not cmd:
         return ''
     if args:
         cmd = ' '.join((cmd,) + args)
-    import subprocess
+
     p = subprocess.Popen(cmd, shell=True,
                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout_data, errors = p.communicate()
-    global _ERROR_PRINTED
-    if errors and not _ERROR_PRINTED:
-        _ERROR_PRINTED = True
-        print("ERROR: %s" % errors)
-        print("** make sure the development packages of libxml2 and libxslt are installed **\n")
+
+    if errors:
+        return ''
     return decode_input(stdout_data).strip()
 
 
-def check_min_version(version, min_version, error_name):
+def check_min_version(version, min_version, libname):
     if not version:
         # this is ok for targets like sdist etc.
         return True
-    version = tuple(map(int, version.split('.')[:3]))
-    min_version = tuple(min_version)
-    if version < min_version:
-        print("Minimum required version of %s is %s, found %s" % (
-            error_name, '.'.join(map(str, version)), '.'.join(map(str, min_version))))
+    lib_version = tuple(map(int, version.split('.')[:3]))
+    req_version = tuple(map(int, min_version.split('.')[:3]))
+    if lib_version < req_version:
+        print("Minimum required version of %s is %s. Your system has version %s." % (
+            libname, min_version, version))
         return False
     return True
 
 
-def get_library_version(config_tool):
-    is_pkgconfig = "pkg-config" in config_tool
-    return run_command(config_tool,
-                       "--modversion" if is_pkgconfig else "--version")
+def get_library_version(prog, libname=None):
+    if libname:
+        return run_command(prog, '--modversion %s' % libname)
+    else:
+        return run_command(prog, '--version')
+
 
+PKG_CONFIG = None
+XML2_CONFIG = None
+XSLT_CONFIG = None
 
 def get_library_versions():
-    xml2_version = get_library_version(find_xml2_config())
-    xslt_version = get_library_version(find_xslt_config())
-    return xml2_version, xslt_version
+    global XML2_CONFIG, XSLT_CONFIG
+
+    # Pre-built libraries
+    if XML2_CONFIG and XSLT_CONFIG:
+        xml2_version = get_library_version(XML2_CONFIG)
+        xslt_version = get_library_version(XSLT_CONFIG)
+        return xml2_version, xslt_version
+
+    # Path to xml2-config and xslt-config specified on the command line
+    if OPTION_WITH_XML2_CONFIG:
+        xml2_version = get_library_version(OPTION_WITH_XML2_CONFIG)
+        if xml2_version and OPTION_WITH_XSLT_CONFIG:
+            xslt_version = get_library_version(OPTION_WITH_XSLT_CONFIG)
+            if xslt_version:
+                XML2_CONFIG = OPTION_WITH_XML2_CONFIG
+                XSLT_CONFIG = OPTION_WITH_XSLT_CONFIG
+                return xml2_version, xslt_version
+
+    # Try pkg-config
+    global PKG_CONFIG
+    PKG_CONFIG = os.getenv('PKG_CONFIG', 'pkg-config')
+    xml2_version = get_library_version(PKG_CONFIG, 'libxml-2.0')
+    if xml2_version:
+        xslt_version = get_library_version(PKG_CONFIG, 'libxslt')
+        if xml2_version and xslt_version:
+            return xml2_version, xslt_version
+
+    # Try xml2-config and xslt-config
+    XML2_CONFIG = os.getenv('XML2_CONFIG', 'xml2-config')
+    xml2_version = get_library_version(XML2_CONFIG)
+    if xml2_version:
+        XSLT_CONFIG = os.getenv('XSLT_CONFIG', 'xslt-config')
+        xslt_version = get_library_version(XSLT_CONFIG)
+        if xml2_version and xslt_version:
+            return xml2_version, xslt_version
+
+    # One or both build dependencies not found. Fail on Linux platforms only.
+    if sys.platform.startswith('win'):
+        return '', ''
+    print("Error: Please make sure the libxml2 and libxslt development packages are installed.")
+    sys.exit(1)
+
+
+def check_build_dependencies():
+    xml2_version, xslt_version = get_library_versions()
+
+    xml2_ok = check_min_version(xml2_version, '2.7.0', 'libxml2')
+    xslt_ok = check_min_version(xslt_version, '1.1.23', 'libxslt')
+
+    if xml2_version and xslt_version:
+        print("Building against libxml2 %s and libxslt %s" % (xml2_version, xslt_version))
+    else:
+        print("Building against pre-built libxml2 andl libxslt libraries")
+
+    return (xml2_ok and xslt_ok)
+
+
+def get_flags(prog, option, libname=None):
+    if libname:
+        return run_command(prog, '--%s %s' % (option, libname))
+    else:
+        return run_command(prog, '--%s' % option)
 
 
 def flags(option):
-    xml2_flags = run_command(find_xml2_config(), "--%s" % option)
-    xslt_flags = run_command(find_xslt_config(), "--%s" % option)
+    if XML2_CONFIG:
+        xml2_flags = get_flags(XML2_CONFIG, option)
+        xslt_flags = get_flags(XSLT_CONFIG, option)
+    else:
+        xml2_flags = get_flags(PKG_CONFIG, option, 'libxml-2.0')
+        xslt_flags = get_flags(PKG_CONFIG, option, 'libxslt')
 
     flag_list = xml2_flags.split()
     for flag in xslt_flags.split():
@@ -418,37 +474,6 @@ def flags(option):
 def get_xcode_isysroot():
     return run_command('xcrun', '--show-sdk-path')
 
-XSLT_CONFIG = None
-XML2_CONFIG = None
-
-def find_xml2_config():
-    global XML2_CONFIG
-    if XML2_CONFIG:
-        return XML2_CONFIG
-    option = '--with-xml2-config='
-    for arg in sys.argv:
-        if arg.startswith(option):
-            sys.argv.remove(arg)
-            XML2_CONFIG = arg[len(option):]
-            return XML2_CONFIG
-    else:
-        # default: do nothing, rely only on xslt-config
-        XML2_CONFIG = os.getenv('XML2_CONFIG', '')
-    return XML2_CONFIG
-
-def find_xslt_config():
-    global XSLT_CONFIG
-    if XSLT_CONFIG:
-        return XSLT_CONFIG
-    option = '--with-xslt-config='
-    for arg in sys.argv:
-        if arg.startswith(option):
-            sys.argv.remove(arg)
-            XSLT_CONFIG = arg[len(option):]
-            return XSLT_CONFIG
-    else:
-        XSLT_CONFIG = os.getenv('XSLT_CONFIG', 'xslt-config')
-    return XSLT_CONFIG
 
 ## Option handling:
 
@@ -501,6 +526,8 @@ def option_value(name):
 OPTION_BUILD_LIBXML2XSLT = staticbuild or has_option('static-deps')
 if OPTION_BUILD_LIBXML2XSLT:
     OPTION_STATIC = True
+OPTION_WITH_XML2_CONFIG = option_value('xml2-config')
+OPTION_WITH_XSLT_CONFIG = option_value('xslt-config')
 OPTION_LIBXML2_VERSION = option_value('libxml2-version')
 OPTION_LIBXSLT_VERSION = option_value('libxslt-version')
 OPTION_LIBICONV_VERSION = option_value('libiconv-version')

src/lxml/__init__.py

@@ -1,6 +1,6 @@
 # this is a package
 
-__version__ = "4.5.0"
+__version__ = "4.5.1"
 
 
 def get_include():

src/lxml/etree.pyx

@@ -2741,6 +2741,8 @@ cdef class _MultiTagMatcher:
                 elif href == b'*':
                     href = None  # wildcard: any namespace, including none
                 self._py_tags.append((href, name))
+        elif isinstance(tag, QName):
+            self._storeTags(tag.text, seen)
         else:
             # support a sequence of tags
             for item in tag:

src/lxml/serializer.pxi

@@ -147,7 +147,7 @@ cdef _tostring(_Element element, encoding, doctype, method,
                 c_result_buffer))[:tree.xmlBufUse(c_result_buffer)]
     finally:
         error_result = tree.xmlOutputBufferClose(c_buffer)
-    if error_result < 0:
+    if error_result == -1:
         _raiseSerialisationError(error_result)
     return result
 
@@ -770,7 +770,7 @@ cdef int _serialise_node(tree.xmlOutputBuffer* c_buffer, const_xmlChar* c_doctyp
     error_result = c_buffer.error
     if error_result == xmlerror.XML_ERR_OK:
         error_result = tree.xmlOutputBufferClose(c_buffer)
-        if error_result > 0:
+        if error_result != -1:
             error_result = xmlerror.XML_ERR_OK
     else:
         tree.xmlOutputBufferClose(c_buffer)
@@ -870,6 +870,8 @@ cdef _tofilelikeC14N(f, _Element element, bint exclusive, bint with_comments,
                 error = tree.xmlOutputBufferClose(c_buffer)
                 if bytes_count < 0:
                     error = bytes_count
+                elif error != -1:
+                    error = xmlerror.XML_ERR_OK
         else:
             raise TypeError(f"File or filename expected, got '{python._fqtypename(f).decode('UTF-8')}'")
     finally:
@@ -1674,7 +1676,7 @@ cdef class _IncrementalFileWriter:
         error_result = self._c_out.error
         if error_result == xmlerror.XML_ERR_OK:
             error_result = tree.xmlOutputBufferClose(self._c_out)
-            if error_result > 0:
+            if error_result != -1:
                 error_result = xmlerror.XML_ERR_OK
         else:
             tree.xmlOutputBufferClose(self._c_out)

src/lxml/tests/test_etree.py

@@ -3266,6 +3266,30 @@ def test_elementtree_getelementpath_ns(self):
         self.assertRaises(ValueError, tree.getelementpath, d1)
         self.assertRaises(ValueError, tree.getelementpath, d2)
 
+    def test_elementtree_iter_qname(self):
+        XML = self.etree.XML
+        ElementTree = self.etree.ElementTree
+        QName = self.etree.QName
+        tree = ElementTree(XML(
+                _bytes('<a xmlns:x="X" xmlns:y="Y"><x:b><c/></x:b><b/><c><x:b/><b/></c><b/></a>')))
+        self.assertEqual(
+            list(tree.iter(QName("b"))),
+            list(tree.iter("b")),
+        )
+        self.assertEqual(
+            list(tree.iter(QName("X", "b"))),
+            list(tree.iter("{X}b")),
+        )
+
+        self.assertEqual(
+            [e.tag for e in tree.iter(QName("X", "b"), QName("b"))],
+            ['{X}b', 'b', '{X}b', 'b', 'b']
+        )
+        self.assertEqual(
+            list(tree.iter(QName("X", "b"), QName("b"))),
+            list(tree.iter("{X}b", "b"))
+        )
+
     def test_elementtree_find_qname(self):
         XML = self.etree.XML
         ElementTree = self.etree.ElementTree

src/lxml/xslt.pxi

@@ -744,7 +744,7 @@ cdef class _XSLTResultTree(_ElementTree):
             rclose = tree.xmlOutputBufferClose(c_buffer)
         if writer is not None:
             writer._exc_context._raise_if_stored()
-        if r < 0 or rclose < 0:
+        if r < 0 or rclose == -1:
             python.PyErr_SetFromErrno(IOError)  # raises IOError
 
     cdef _saveToStringAndSize(self, xmlChar** s, int* l):

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27, py34, py35, py36, py37
+envlist = py27, py35, py36, py37, py38
 
 [testenv]
 setenv =