Merge branch 'main' into idle_timeout

.github/workflows/changelog.yml

@@ -13,7 +13,7 @@ jobs:
 
     steps:
       - name: "Checkout repository"
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0
         with:
           # `towncrier check` runs `git diff --name-only origin/main...`, which
           # needs a non-shallow clone.

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
 
     steps:
       - name: "Checkout repository"
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0
 
       - name: "Setup Python"
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
@@ -34,7 +34,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
         os:
           - macos-12
           - windows-latest
@@ -89,6 +89,8 @@ jobs:
             os: ubuntu-latest
             nox-session: emscripten
             experimental: true
+          - python-version: "3.13"
+            experimental: true
         exclude:
           # Ubuntu 22.04 comes with OpenSSL 3.0, so only CPython 3.9+ is compatible with it
           # https://github.com/python/cpython/issues/83001
@@ -101,7 +103,7 @@ jobs:
     timeout-minutes: 30
     steps:
       - name: "Checkout repository"
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0
 
       - name: "Setup Python ${{ matrix.python-version }}"
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
@@ -113,10 +115,10 @@ jobs:
         run: python -m pip install --upgrade pip setuptools nox
 
       - name: "Install Chrome"
-        uses: browser-actions/setup-chrome@52f10de5479c69bcbbab2eab094c9d373148005e # v1.4.0
+        uses: browser-actions/setup-chrome@97349de5c98094d4fc9412f31c524d7697115ad8 # v1.5.0
         if: ${{ matrix.nox-session == 'emscripten' }}
       - name: "Install Firefox"
-        uses: browser-actions/setup-firefox@29a706787c6fb2196f091563261e1273bf379ead # v1.4.0
+        uses: browser-actions/setup-firefox@233224b712fc07910ded8c15fb95a555c86da76f # v1.5.0
         if: ${{ matrix.nox-session == 'emscripten' }}
       - name: "Run tests"
         # If no explicit NOX_SESSION is set, run the default tests for the chosen Python version
@@ -139,7 +141,7 @@ jobs:
     needs: test
     steps:
       - name: "Checkout repository"
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0
 
       - name: "Setup Python"
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0

.github/workflows/codeql.yml

@@ -21,7 +21,7 @@ jobs:
       security-events: write
     steps:
     - name: "Checkout repository"
-      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+      uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0
 
     - name: "Run CodeQL init"
       uses: github/codeql-action/init@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4

.github/workflows/downstream.yml

@@ -15,7 +15,7 @@ jobs:
 
     steps:
       - name: "Checkout repository"
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0
 
       - name: "Setup Python"
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0

.github/workflows/lint.yml

@@ -11,7 +11,7 @@ jobs:
 
     steps:
       - name: "Checkout repository"
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0
 
       - name: "Setup Python"
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0

.github/workflows/publish.yml

@@ -19,7 +19,7 @@ jobs:
 
     steps:
       - name: "Checkout repository"
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0
 
       - name: "Setup Python"
         uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0

.github/workflows/scorecards.yml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: "Checkout repository"
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@b32f140b0c872d58512e0a66172253c302617b90 # v4.0.0
         with:
           persist-credentials: false
 

CHANGES.rst

@@ -1,19 +1,38 @@
-2.1.0 (2023-11-13)
+2.2.1 (2024-02-16)
 ==================
 
-Read the `v2 migration guide <https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html>`__ for help upgrading to the latest version of urllib3.
+- Fixed issue where ``InsecureRequestWarning`` was emitted for HTTPS connections when using Emscripten. (`#3331 <https://github.com/urllib3/urllib3/issues/3331>`__)
+- Fixed ``HTTPConnectionPool.urlopen`` to stop automatically casting non-proxy headers to ``HTTPHeaderDict``. This change was premature as it did not apply to proxy headers and ``HTTPHeaderDict`` does not handle byte header values correctly yet. (`#3343 <https://github.com/urllib3/urllib3/issues/3343>`__)
+- Changed ``InvalidChunkLength`` to ``ProtocolError`` when response terminates before the chunk length is sent. (`#2860 <https://github.com/urllib3/urllib3/issues/2860>`__)
+- Changed ``ProtocolError`` to be more verbose on incomplete reads with excess content. (`#3261 <https://github.com/urllib3/urllib3/issues/3261>`__)
 
-Removals
---------
 
-- Removed support for the deprecated urllib3[secure] extra. (`#2680 <https://github.com/urllib3/urllib3/issues/2680>`__)
-- Removed support for the deprecated SecureTransport TLS implementation. (`#2681 <https://github.com/urllib3/urllib3/issues/2681>`__)
-- Removed support for the end-of-life Python 3.7. (`#3143 <https://github.com/urllib3/urllib3/issues/3143>`__)
+2.2.0 (2024-01-30)
+==================
 
+- Added support for `Emscripten and Pyodide <https://urllib3.readthedocs.io/en/latest/reference/contrib/emscripten.html>`__, including streaming support in cross-origin isolated browser environments where threading is enabled. (`#2951 <https://github.com/urllib3/urllib3/issues/2951>`__)
+- Added support for ``HTTPResponse.read1()`` method. (`#3186 <https://github.com/urllib3/urllib3/issues/3186>`__)
+- Added rudimentary support for HTTP/2. (`#3284 <https://github.com/urllib3/urllib3/issues/3284>`__)
+- Fixed issue where requests against urls with trailing dots were failing due to SSL errors
+  when using proxy. (`#2244 <https://github.com/urllib3/urllib3/issues/2244>`__)
+- Fixed ``HTTPConnection.proxy_is_verified`` and ``HTTPSConnection.proxy_is_verified``
+  to be always set to a boolean after connecting to a proxy. It could be
+  ``None`` in some cases previously. (`#3130 <https://github.com/urllib3/urllib3/issues/3130>`__)
+- Fixed an issue where ``headers`` passed in a request with ``json=`` would be mutated (`#3203 <https://github.com/urllib3/urllib3/issues/3203>`__)
+- Fixed ``HTTPSConnection.is_verified`` to be set to ``False`` when connecting
+  from a HTTPS proxy to an HTTP target. It was set to ``True`` previously. (`#3267 <https://github.com/urllib3/urllib3/issues/3267>`__)
+- Fixed handling of new error message from OpenSSL 3.2.0 when configuring an HTTP proxy as HTTPS (`#3268 <https://github.com/urllib3/urllib3/issues/3268>`__)
+- Fixed TLS 1.3 post-handshake auth when the server certificate validation is disabled (`#3325 <https://github.com/urllib3/urllib3/issues/3325>`__)
+- Note for downstream distributors: To run integration tests, you now need to run the tests a second
+  time with the ``--integration`` pytest flag. (`#3181 <https://github.com/urllib3/urllib3/issues/3181>`__)
 
-Bugfixes
---------
 
+2.1.0 (2023-11-13)
+==================
+
+- Removed support for the deprecated urllib3[secure] extra. (`#2680 <https://github.com/urllib3/urllib3/issues/2680>`__)
+- Removed support for the deprecated SecureTransport TLS implementation. (`#2681 <https://github.com/urllib3/urllib3/issues/2681>`__)
+- Removed support for the end-of-life Python 3.7. (`#3143 <https://github.com/urllib3/urllib3/issues/3143>`__)
 - Allowed loading CA certificates from memory for proxies. (`#3065 <https://github.com/urllib3/urllib3/issues/3065>`__)
 - Fixed decoding Gzip-encoded responses which specified ``x-gzip`` content-encoding. (`#3174 <https://github.com/urllib3/urllib3/issues/3174>`__)
 
@@ -23,11 +42,13 @@ Bugfixes
 
 * Made body stripped from HTTP requests changing the request method to GET after HTTP 303 "See Other" redirect responses.
 
+
 2.0.6 (2023-10-02)
 ==================
 
 * Added the ``Cookie`` header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via ``Retry.remove_headers_on_redirect``.
 
+
 2.0.5 (2023-09-20)
 ==================
 

changelog/3122.bugfix.rst

@@ -0,0 +1,2 @@
+Allowed passing negative integers as ``amt`` to read methods of
+:class:`http.client.HTTPResponse` as an alternative to ``None``.

changelog/3342.doc.rst

@@ -0,0 +1 @@
+Updated docs for ``.json()``

dev-requirements.txt

@@ -1,20 +1,29 @@
 h2==4.1.0
-coverage==7.4.0
+coverage==7.4.1
 PySocks==1.7.1
-pytest==7.4.2
+pytest==7.4.4
 pytest-timeout==2.1.0
-pyOpenSSL==23.2.0
+pyOpenSSL==24.0.0
 idna==3.4
 trustme==1.1.0
-cryptography==41.0.6
+cryptography==42.0.4
 backports.zoneinfo==0.2.1;python_version<"3.9"
 towncrier==23.6.0
-pytest-memray==1.5.0;sys_platform!="win32" and implementation_name=="cpython"
-trio==0.23.1
+pytest-memray==1.5.0;python_version<"3.13" and sys_platform!="win32" and implementation_name=="cpython"
+trio==0.23.1;python_version<"3.13"
+# We need a release of Trio newer than 0.24.0 to support CPython 3.13.
+# https://github.com/python-trio/trio/issues/2903
+trio @ git+https://github.com/python-trio/trio@e4c8eb2d7ef59eeea1441656e392fe1b0870a374; python_version == "3.13"
 Quart==0.19.4
 quart-trio==0.11.1
 # https://github.com/pgjones/hypercorn/issues/62
 # https://github.com/pgjones/hypercorn/issues/168
 # https://github.com/pgjones/hypercorn/issues/169
 hypercorn @ git+https://github.com/urllib3/hypercorn@urllib3-changes
 httpx==0.25.2
+pytest-socket==0.7.0
+# CFFI is not going to support CPython 3.13 in an actual release until
+# there is a release candidate for 3.13.
+# https://github.com/python-cffi/cffi/issues/23#issuecomment-1845861410
+cffi @ git+https://github.com/python-cffi/cffi@14723b0bbd127790c450945099db31018d80fa83; python_version == "3.13"
+

docs/contributing.rst

@@ -71,6 +71,7 @@ suite::
   nox > Session test-3.10 was successful.
   nox > Session test-3.11 was successful.
   nox > Session test-3.12 was successful.
+  nox > Session test-3.13 was successful.
   nox > Session test-pypy was successful.
 
 Our test suite `runs continuously on GitHub Actions

docs/reference/contrib/emscripten.rst

@@ -0,0 +1,87 @@
+Pyodide, Emscripten, and PyScript
+=================================
+
+From the Pyodide documentation, `Pyodide <https://pyodide.org>`_ is a Python distribution for the browser and Node.js based on WebAssembly and `Emscripten <https://emscripten.org/>`_.
+This technology also underpins the `PyScript framework <https://pyscript.net/>`_ and `Jupyterlite <https://jupyterlite.readthedocs.io/>`_, so should work in those environments too.
+
+Starting in version 2.2.0 urllib3 supports being used in a Pyodide runtime utilizing
+the `JavaScript fetch API <https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API>`_
+or falling back on `XMLHttpRequest <https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest>`_
+if the fetch API isn't available (such as when cross-origin isolation
+isn't active). This means you can use Python libraries to make HTTP requests from your browser!
+
+Because urllib3's Emscripten support is API-compatible, this means that
+libraries that depend on urllib3 may now be usable from Emscripten and Pyodide environments, too.
+
+ .. warning::
+
+    **Support for Emscripten and Pyodide is experimental**. Report all bugs to the `urllib3 issue tracker <https://github.com/urllib3/urllib3/issues>`_.
+    Currently only supports browsers, does not yet support running in Node.js.
+
+It's recommended to `run Pyodide in a Web Worker <https://pyodide.org/en/stable/usage/webworker.html#using-from-webworker>`_
+in order to take full advantage of features like the fetch API which enables streaming of HTTP response bodies.
+
+Getting started
+---------------
+
+Using urllib3 with Pyodide means you need to `get started with Pyodide first <https://pyodide.org/en/stable/usage/quickstart.html>`_.
+The Pyodide project provides a `useful online REPL <https://pyodide.org/en/stable/console.html>`_ to try in your browser without
+any setup or installation to test out the code examples below.
+
+urllib3's Emscripten support is automatically enabled if ``sys.platform`` is ``"emscripten"``, so no setup is required beyond installation and importing the module.
+
+You can install urllib3 in a Pyodide environment using micropip.
+Try using the following code in a Pyodide console or ``<script>`` tag:
+
+ .. code-block:: python
+
+    import micropip
+    await micropip.install("urllib3")
+
+    import urllib3
+    resp = urllib3.request("GET", "https://httpbin.org/anything")
+
+    print(resp.status)  # 200
+    print(resp.headers) # HTTPHeaderDict(...)
+    print(resp.json())  # {"headers": {"Accept": "*/*", ...}, ...}
+
+Because `Requests <https://requests.readthedocs.io/en/latest/>`_ is built on urllib3, Requests also works out of the box:
+
+ .. code-block:: python
+
+    import micropip
+    await micropip.install("requests")
+
+    import requests
+    resp = requests.request("GET", "https://httpbin.org/anything")
+
+    print(resp.status_code)  # 200
+    print(resp.headers)
+
+Features
+--------
+
+Because we use JavaScript APIs under the hood, it's not possible to use all of urllib3 features.
+Features which are usable with Emscripten support are:
+
+* Requests over HTTP and HTTPS
+* Timeouts
+* Retries
+* Streaming (with Web Workers and Cross-Origin Isolation)
+* Redirects
+* Decompressing response bodies
+
+Features which don't work with Emscripten:
+
+* Proxies, both forwarding and tunneling
+* Customizing TLS and certificates (uses browsers' configuration)
+* Configuring low-level socket options or source address
+
+Streaming with Web Workers
+--------------------------
+
+To access the fetch API and do HTTP response streaming with urllib3
+you must be running the code within a Web Worker and set specific HTTP headers
+for the serving website to enable `Cross-Origin Isolation <https://developer.mozilla.org/en-US/docs/Web/API/crossOriginIsolated>`_.
+
+You can verify whether a given environment is cross-origin isolated by evaluating the global ``crossOriginIsolated`` JavaScript property.

docs/reference/contrib/index.rst

@@ -6,5 +6,6 @@ prime time or that require optional third-party dependencies.
 
 .. toctree::
 
+   emscripten
    pyopenssl
    socks

docs/user-guide.rst

@@ -99,6 +99,8 @@ The :class:`~response.HTTPResponse` object provides
     print(resp.headers)
     # HTTPHeaderDict({"Content-Length": "32", ...})
 
+.. _json_content:
+
 JSON Content
 ~~~~~~~~~~~~
 JSON content can be loaded by :meth:`~response.HTTPResponse.json` 

dummyserver/hypercornserver.py

@@ -52,7 +52,7 @@ def run_hypercorn_in_thread(
     ) as executor:
         future = executor.submit(
             trio.run,
-            _start_server,  # type: ignore[arg-type]
+            _start_server,
             config,
             app,
             ready_event,

mypy-requirements.txt

@@ -1,4 +1,4 @@
-mypy==1.5.1
+mypy==1.8.0
 idna>=2.0.0
 cryptography>=1.3.4
 pytest>=6.2
@@ -11,3 +11,4 @@ httpx==0.25.2
 types-backports
 types-requests
 nox
+zstandard