.gitmodules

@@ -1,4 +1,4 @@
 [submodule "vendor/llhttp"]
     path = vendor/llhttp
     url = https://github.com/nodejs/llhttp.git
-    branch = v8.x
+    branch = main

.readthedocs.yml

@@ -6,14 +6,16 @@
 version: 2
 
 submodules:
-  include: all  # []
+  include: all
   exclude: []
   recursive: true
 
 build:
-  image: latest
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
 python:
-  version: 3.8
   install:
   - method: pip
     path: .

CHANGES.rst

@@ -10,6 +10,78 @@
 
 .. towncrier release notes start
 
+3.8.6 (2023-10-07)
+==================
+
+Security bugfixes
+-----------------
+
+- Upgraded the vendored copy of llhttp_ to v9.1.3 -- by :user:`Dreamsorcerer`
+
+  Thanks to :user:`kenballus` for reporting this, see
+  https://github.com/aio-libs/aiohttp/security/advisories/GHSA-pjjw-qhg8-p2p9.
+
+  .. _llhttp: https://llhttp.org
+
+  `#7647 <https://github.com/aio-libs/aiohttp/issues/7647>`_
+
+- Updated Python parser to comply with RFCs 9110/9112 -- by :user:`Dreamorcerer`
+
+  Thanks to :user:`kenballus` for reporting this, see
+  https://github.com/aio-libs/aiohttp/security/advisories/GHSA-gfw2-4jvh-wgfg.
+
+  `#7663 <https://github.com/aio-libs/aiohttp/issues/7663>`_
+
+
+Deprecation
+-----------
+
+- Added ``fallback_charset_resolver`` parameter in ``ClientSession`` to allow a user-supplied
+  character set detection function.
+
+  Character set detection will no longer be included in 3.9 as a default. If this feature is needed,
+  please use `fallback_charset_resolver <https://docs.aiohttp.org/en/stable/client_advanced.html#character-set-detection>`_.
+
+  `#7561 <https://github.com/aio-libs/aiohttp/issues/7561>`_
+
+
+Features
+--------
+
+- Enabled lenient response parsing for more flexible parsing in the client
+  (this should resolve some regressions when dealing with badly formatted HTTP responses). -- by :user:`Dreamsorcerer`
+
+  `#7490 <https://github.com/aio-libs/aiohttp/issues/7490>`_
+
+
+
+Bugfixes
+--------
+
+- Fixed ``PermissionError`` when ``.netrc`` is unreadable due to permissions.
+
+  `#7237 <https://github.com/aio-libs/aiohttp/issues/7237>`_
+
+- Fixed output of parsing errors pointing to a ``\n``. -- by :user:`Dreamsorcerer`
+
+  `#7468 <https://github.com/aio-libs/aiohttp/issues/7468>`_
+
+- Fixed ``GunicornWebWorker`` max_requests_jitter not working.
+
+  `#7518 <https://github.com/aio-libs/aiohttp/issues/7518>`_
+
+- Fixed sorting in ``filter_cookies`` to use cookie with longest path. -- by :user:`marq24`.
+
+  `#7577 <https://github.com/aio-libs/aiohttp/issues/7577>`_
+
+- Fixed display of ``BadStatusLine`` messages from llhttp_. -- by :user:`Dreamsorcerer`
+
+  `#7651 <https://github.com/aio-libs/aiohttp/issues/7651>`_
+
+
+----
+
+
 3.8.5 (2023-07-19)
 ==================
 
@@ -45,7 +117,6 @@ Bugfixes
   `#3355 <https://github.com/aio-libs/aiohttp/issues/3355>`_
 
 
-
 ----
 
 

CONTRIBUTORS.txt

@@ -153,6 +153,7 @@ Jake Davis
 Jakob Ackermann
 Jakub Wilk
 Jan Buchar
+Jan Gosmann
 Jashandeep Sohi
 Jens Steinhauser
 Jeonghun Lee
@@ -162,6 +163,7 @@ Jesus Cea
 Jian Zeng
 Jinkyu Yi
 Joel Watts
+John Parton
 Jon Nabozny
 Jonas Krüger Svensson
 Jonas Obrist
@@ -210,6 +212,7 @@ Martin Melka
 Martin Richard
 Mathias Fröjdman
 Mathieu Dugré
+Matthias Marquardt
 Matthieu Hauglustaine
 Matthieu Rigal
 Meet Mangukiya

Makefile

@@ -58,7 +58,7 @@ aiohttp/_find_header.c: $(call to-hash,aiohttp/hdrs.py ./tools/gen.py)
 
 # _find_headers generator creates _headers.pyi as well
 aiohttp/%.c: aiohttp/%.pyx $(call to-hash,$(CYS)) aiohttp/_find_header.c
-	cython -3 -o $@ $< -I aiohttp
+	cython -3 -o $@ $< -I aiohttp -Werror
 
 vendor/llhttp/node_modules: vendor/llhttp/package.json
 	cd vendor/llhttp; npm install

aiohttp/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.8.5"
+__version__ = "3.8.6"
 
 from typing import Tuple
 

aiohttp/_cparser.pxd

@@ -1,13 +1,4 @@
-from libc.stdint cimport (
-    int8_t,
-    int16_t,
-    int32_t,
-    int64_t,
-    uint8_t,
-    uint16_t,
-    uint32_t,
-    uint64_t,
-)
+from libc.stdint cimport int32_t, uint8_t, uint16_t, uint64_t
 
 
 cdef extern from "../vendor/llhttp/build/llhttp.h":
@@ -88,30 +79,14 @@ cdef extern from "../vendor/llhttp/build/llhttp.h":
     ctypedef llhttp_errno llhttp_errno_t
 
     enum llhttp_flags:
-        F_CONNECTION_KEEP_ALIVE,
-        F_CONNECTION_CLOSE,
-        F_CONNECTION_UPGRADE,
         F_CHUNKED,
-        F_UPGRADE,
-        F_CONTENT_LENGTH,
-        F_SKIPBODY,
-        F_TRAILING,
-        F_TRANSFER_ENCODING
-
-    enum llhttp_lenient_flags:
-        LENIENT_HEADERS,
-        LENIENT_CHUNKED_LENGTH
+        F_CONTENT_LENGTH
 
     enum llhttp_type:
         HTTP_REQUEST,
         HTTP_RESPONSE,
         HTTP_BOTH
 
-    enum llhttp_finish_t:
-        HTTP_FINISH_SAFE,
-        HTTP_FINISH_SAFE_WITH_CB,
-        HTTP_FINISH_UNSAFE
-
     enum llhttp_method:
         HTTP_DELETE,
         HTTP_GET,
@@ -167,24 +142,17 @@ cdef extern from "../vendor/llhttp/build/llhttp.h":
                  const llhttp_settings_t* settings)
 
     llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len)
-    llhttp_errno_t llhttp_finish(llhttp_t* parser)
-
-    int llhttp_message_needs_eof(const llhttp_t* parser)
 
     int llhttp_should_keep_alive(const llhttp_t* parser)
 
-    void llhttp_pause(llhttp_t* parser)
-    void llhttp_resume(llhttp_t* parser)
-
     void llhttp_resume_after_upgrade(llhttp_t* parser)
 
     llhttp_errno_t llhttp_get_errno(const llhttp_t* parser)
     const char* llhttp_get_error_reason(const llhttp_t* parser)
-    void llhttp_set_error_reason(llhttp_t* parser, const char* reason)
     const char* llhttp_get_error_pos(const llhttp_t* parser)
-    const char* llhttp_errno_name(llhttp_errno_t err)
 
     const char* llhttp_method_name(llhttp_method_t method)
 
     void llhttp_set_lenient_headers(llhttp_t* parser, int enabled)
-    void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled)
+    void llhttp_set_lenient_optional_cr_before_lf(llhttp_t* parser, int enabled)
+    void llhttp_set_lenient_spaces_after_chunk_size(llhttp_t* parser, int enabled)

aiohttp/_http_parser.pyx

@@ -2,7 +2,6 @@
 #
 # Based on https://github.com/MagicStack/httptools
 #
-from __future__ import absolute_import, print_function
 
 from cpython cimport (
     Py_buffer,
@@ -20,6 +19,7 @@ from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiD
 from yarl import URL as _URL
 
 from aiohttp import hdrs
+from aiohttp.helpers import DEBUG
 
 from .http_exceptions import (
     BadHttpMessage,
@@ -548,8 +548,8 @@ cdef class HttpParser:
                 else:
                     after = cparser.llhttp_get_error_pos(self._cparser)
                     before = data[:after - <char*>self.py_buf.buf]
-                    after_b = after.split(b"\n", 1)[0]
-                    before = before.rsplit(b"\n", 1)[-1]
+                    after_b = after.split(b"\r\n", 1)[0]
+                    before = before.rsplit(b"\r\n", 1)[-1]
                     data = before + after_b
                     pointer = " " * (len(repr(before))-1) + "^"
                     ex = parser_error_from_errno(self._cparser, data, pointer)
@@ -648,6 +648,11 @@ cdef class HttpResponseParser(HttpParser):
                    max_line_size, max_headers, max_field_size,
                    payload_exception, response_with_body, read_until_eof,
                    auto_decompress)
+        # Use strict parsing on dev mode, so users are warned about broken servers.
+        if not DEBUG:
+            cparser.llhttp_set_lenient_headers(self._cparser, 1)
+            cparser.llhttp_set_lenient_optional_cr_before_lf(self._cparser, 1)
+            cparser.llhttp_set_lenient_spaces_after_chunk_size(self._cparser, 1)
 
     cdef object _on_status_complete(self):
         if self._buf:
@@ -807,7 +812,9 @@ cdef parser_error_from_errno(cparser.llhttp_t* parser, data, pointer):
     cdef cparser.llhttp_errno_t errno = cparser.llhttp_get_errno(parser)
     cdef bytes desc = cparser.llhttp_get_error_reason(parser)
 
-    if errno in (cparser.HPE_CB_MESSAGE_BEGIN,
+    err_msg = "{}:\n\n  {!r}\n  {}".format(desc.decode("latin-1"), data, pointer)
+
+    if errno in {cparser.HPE_CB_MESSAGE_BEGIN,
                  cparser.HPE_CB_HEADERS_COMPLETE,
                  cparser.HPE_CB_MESSAGE_COMPLETE,
                  cparser.HPE_CB_CHUNK_HEADER,
@@ -817,22 +824,13 @@ cdef parser_error_from_errno(cparser.llhttp_t* parser, data, pointer):
                  cparser.HPE_INVALID_CONTENT_LENGTH,
                  cparser.HPE_INVALID_CHUNK_SIZE,
                  cparser.HPE_INVALID_EOF_STATE,
-                 cparser.HPE_INVALID_TRANSFER_ENCODING):
-        cls = BadHttpMessage
-
-    elif errno == cparser.HPE_INVALID_STATUS:
-        cls = BadStatusLine
-
-    elif errno == cparser.HPE_INVALID_METHOD:
-        cls = BadStatusLine
-
-    elif errno == cparser.HPE_INVALID_VERSION:
-        cls = BadStatusLine
-
+                 cparser.HPE_INVALID_TRANSFER_ENCODING}:
+        return BadHttpMessage(err_msg)
+    elif errno in {cparser.HPE_INVALID_STATUS,
+                   cparser.HPE_INVALID_METHOD,
+                   cparser.HPE_INVALID_VERSION}:
+        return BadStatusLine(error=err_msg)
     elif errno == cparser.HPE_INVALID_URL:
-        cls = InvalidURLError
-
-    else:
-        cls = BadHttpMessage
+        return InvalidURLError(err_msg)
 
-    return cls("{}:\n\n  {!r}\n  {}".format(desc.decode("latin-1"), data, pointer))
+    return BadHttpMessage(err_msg)

aiohttp/client.py

@@ -88,6 +88,11 @@
 from .tracing import Trace, TraceConfig
 from .typedefs import Final, JSONEncoder, LooseCookies, LooseHeaders, StrOrURL
 
+try:
+    import cchardet as chardet
+except ImportError:  # pragma: no cover
+    import charset_normalizer as chardet  # type: ignore[no-redef]
+
 __all__ = (
     # client_exceptions
     "ClientConnectionError",
@@ -159,6 +164,22 @@ class ClientTimeout:
 DEFAULT_TIMEOUT: Final[ClientTimeout] = ClientTimeout(total=5 * 60)
 
 _RetType = TypeVar("_RetType")
+_CharsetResolver = Callable[[ClientResponse, bytes], str]
+
+
+def _default_fallback_charset_resolver(response: ClientResponse, body: bytes) -> str:
+
+    ret: str = chardet.detect(body)["encoding"] or "utf-8"
+
+    if ret != "utf-8":
+        warnings.warn(
+            "Automatic charset detection will be removed in 3.9, see: "
+            "https://docs.aiohttp.org/en/stable/client_advanced.html#character-set-detection",  # noqa: E501
+            DeprecationWarning,
+            stacklevel=3,
+        )
+
+    return ret
 
 
 class ClientSession:
@@ -220,6 +241,9 @@ def __init__(
         requote_redirect_url: bool = True,
         trace_configs: Optional[List[TraceConfig]] = None,
         read_bufsize: int = 2**16,
+        fallback_charset_resolver: _CharsetResolver = (
+            _default_fallback_charset_resolver
+        ),
     ) -> None:
         if loop is None:
             if connector is not None:
@@ -313,6 +337,8 @@ def __init__(
         for trace_config in self._trace_configs:
             trace_config.freeze()
 
+        self._resolve_charset = fallback_charset_resolver
+
     def __init_subclass__(cls: Type["ClientSession"]) -> None:
         warnings.warn(
             "Inheritance class {} from ClientSession "