v2.30.0

* pin cryptography<40 on pypy 3.7

.github/workflows/lint.yml

@@ -1,8 +1,6 @@
 name: Lint code
 
-on:
-  push:
-  pull_request:
+on: [push, pull_request]
 
 permissions:
   contents: read
@@ -10,10 +8,13 @@ permissions:
 jobs:
   lint:
     runs-on: ubuntu-20.04
+    timeout-minutes: 10
 
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
     - name: Run pre-commit
       uses: pre-commit/action@v3.0.0

.github/workflows/run-tests.yml

@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
-        os: [ubuntu-18.04, macOS-latest, windows-latest]
+        os: [ubuntu-22.04, macOS-latest, windows-latest]
         include:
           # pypy-3.7 on Mac OS currently fails trying to compile
           # brotlipy. Moving pypy3 to only test linux.

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
   - id: end-of-file-fixer
   - id: trailing-whitespace
 - repo: https://github.com/PyCQA/isort
-  rev: 5.10.1
+  rev: 5.12.0
   hooks:
     - id: isort
 - repo: https://github.com/psf/black

HISTORY.md

@@ -6,6 +6,26 @@ dev
 
 - \[Short description of non-trivial change.\]
 
+2.30.0 (2023-05-03)
+-------------------
+
+**Dependencies**
+- ⚠️ Added support for urllib3 2.0. ⚠️
+
+  This may contain minor breaking changes so we advise careful testing and
+  reviewing https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html
+  prior to upgrading.
+
+  Users who wish to stay on urllib3 1.x can pin to `urllib3<2`.
+
+2.29.0 (2023-04-26)
+-------------------
+
+**Improvements**
+
+- Requests now defers chunked requests to the urllib3 implementation to improve
+  standardization. (#6226)
+- Requests relaxes header component requirements to support bytes/str subclasses. (#6356)
 
 2.28.2 (2023-01-12)
 -------------------

docs/user/quickstart.rst

@@ -177,7 +177,7 @@ server, you can access ``r.raw``. If you want to do this, make sure you set
     <urllib3.response.HTTPResponse object at 0x101194810>
 
     >>> r.raw.read(10)
-    '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03'
+    b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03'
 
 In general, however, you should use a pattern like this to save what is being
 streamed to a file::
@@ -237,7 +237,7 @@ dictionary of data will automatically be form-encoded when the request is made::
 
     >>> payload = {'key1': 'value1', 'key2': 'value2'}
 
-    >>> r = requests.post("https://httpbin.org/post", data=payload)
+    >>> r = requests.post('https://httpbin.org/post', data=payload)
     >>> print(r.text)
     {
       ...

pyproject.toml

@@ -4,7 +4,7 @@ src_paths = ["requests", "test"]
 honor_noqa = true
 
 [tool.pytest.ini_options]
-addopts = "-p no:warnings --doctest-modules"
+addopts = "--doctest-modules"
 doctest_optionflags = "NORMALIZE_WHITESPACE ELLIPSIS"
 minversion = "6.2"
 testpaths = [

requests/__init__.py

@@ -66,10 +66,10 @@ def check_compatibility(urllib3_version, chardet_version, charset_normalizer_ver
     # Check urllib3 for compatibility.
     major, minor, patch = urllib3_version  # noqa: F811
     major, minor, patch = int(major), int(minor), int(patch)
-    # urllib3 >= 1.21.1, <= 1.26
-    assert major == 1
-    assert minor >= 21
-    assert minor <= 26
+    # urllib3 >= 1.21.1
+    assert major >= 1
+    if major == 1:
+        assert minor >= 21
 
     # Check charset_normalizer for compatibility.
     if chardet_version:

requests/__version__.py

@@ -5,8 +5,8 @@
 __title__ = "requests"
 __description__ = "Python HTTP for Humans."
 __url__ = "https://requests.readthedocs.io"
-__version__ = "2.28.2"
-__build__ = 0x022802
+__version__ = "2.30.0"
+__build__ = 0x023000
 __author__ = "Kenneth Reitz"
 __author_email__ = "me@kennethreitz.org"
 __license__ = "Apache 2.0"

requests/_internal_utils.py

@@ -14,9 +14,11 @@ _VALID_HEADER_NAME_RE_STR = re.compile(r"^[^:\s][^:\r\n]*$")
 _VALID_HEADER_VALUE_RE_BYTE = re.compile(rb"^\S[^\r\n]*$|^$")
 _VALID_HEADER_VALUE_RE_STR = re.compile(r"^\S[^\r\n]*$|^$")
 
+_HEADER_VALIDATORS_STR = (_VALID_HEADER_NAME_RE_STR, _VALID_HEADER_VALUE_RE_STR)
+_HEADER_VALIDATORS_BYTE = (_VALID_HEADER_NAME_RE_BYTE, _VALID_HEADER_VALUE_RE_BYTE)
 HEADER_VALIDATORS = {
-    bytes: (_VALID_HEADER_NAME_RE_BYTE, _VALID_HEADER_VALUE_RE_BYTE),
-    str: (_VALID_HEADER_NAME_RE_STR, _VALID_HEADER_VALUE_RE_STR),
+    bytes: _HEADER_VALIDATORS_BYTE,
+    str: _HEADER_VALIDATORS_STR,
 }
 
 

requests/adapters.py

@@ -22,7 +22,6 @@ from urllib3.exceptions import ProxyError as _ProxyError
 from urllib3.exceptions import ReadTimeoutError, ResponseError
 from urllib3.exceptions import SSLError as _SSLError
 from urllib3.poolmanager import PoolManager, proxy_from_url
-from urllib3.response import HTTPResponse
 from urllib3.util import Timeout as TimeoutSauce
 from urllib3.util import parse_url
 from urllib3.util.retry import Retry
@@ -194,7 +193,6 @@ class HTTPAdapter(BaseAdapter):
             num_pools=connections,
             maxsize=maxsize,
             block=block,
-            strict=True,
             **pool_kwargs,
         )
 
@@ -485,63 +483,19 @@ class HTTPAdapter(BaseAdapter):
             timeout = TimeoutSauce(connect=timeout, read=timeout)
 
         try:
-            if not chunked:
-                resp = conn.urlopen(
-                    method=request.method,
-                    url=url,
-                    body=request.body,
-                    headers=request.headers,
-                    redirect=False,
-                    assert_same_host=False,
-                    preload_content=False,
-                    decode_content=False,
-                    retries=self.max_retries,
-                    timeout=timeout,
-                )
-
-            # Send the request.
-            else:
-                if hasattr(conn, "proxy_pool"):
-                    conn = conn.proxy_pool
-
-                low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)
-
-                try:
-                    skip_host = "Host" in request.headers
-                    low_conn.putrequest(
-                        request.method,
-                        url,
-                        skip_accept_encoding=True,
-                        skip_host=skip_host,
-                    )
-
-                    for header, value in request.headers.items():
-                        low_conn.putheader(header, value)
-
-                    low_conn.endheaders()
-
-                    for i in request.body:
-                        low_conn.send(hex(len(i))[2:].encode("utf-8"))
-                        low_conn.send(b"\r\n")
-                        low_conn.send(i)
-                        low_conn.send(b"\r\n")
-                    low_conn.send(b"0\r\n\r\n")
-
-                    # Receive the response from the server
-                    r = low_conn.getresponse()
-
-                    resp = HTTPResponse.from_httplib(
-                        r,
-                        pool=conn,
-                        connection=low_conn,
-                        preload_content=False,
-                        decode_content=False,
-                    )
-                except Exception:
-                    # If we hit any problems here, clean up the connection.
-                    # Then, raise so that we can handle the actual exception.
-                    low_conn.close()
-                    raise
+            resp = conn.urlopen(
+                method=request.method,
+                url=url,
+                body=request.body,
+                headers=request.headers,
+                redirect=False,
+                assert_same_host=False,
+                preload_content=False,
+                decode_content=False,
+                retries=self.max_retries,
+                timeout=timeout,
+                chunked=chunked,
+            )
 
         except (ProtocolError, OSError) as err:
             raise ConnectionError(err, request=request)

requests/api.py

@@ -106,7 +106,7 @@ def post(url, data=None, json=None, **kwargs):
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, list of tuples, bytes, or file-like
         object to send in the body of the :class:`Request`.
-    :param json: (optional) json data to send in the body of the :class:`Request`.
+    :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
     :return: :class:`Response <Response>` object
     :rtype: requests.Response
@@ -121,7 +121,7 @@ def put(url, data=None, **kwargs):
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, list of tuples, bytes, or file-like
         object to send in the body of the :class:`Request`.
-    :param json: (optional) json data to send in the body of the :class:`Request`.
+    :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
     :return: :class:`Response <Response>` object
     :rtype: requests.Response
@@ -136,7 +136,7 @@ def patch(url, data=None, **kwargs):
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, list of tuples, bytes, or file-like
         object to send in the body of the :class:`Request`.
-    :param json: (optional) json data to send in the body of the :class:`Request`.
+    :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
     :return: :class:`Response <Response>` object
     :rtype: requests.Response

requests/utils.py

@@ -25,7 +25,12 @@ from . import certs
 from .__version__ import __version__
 
 # to_native_string is unused here, but imported here for backwards compatibility
-from ._internal_utils import HEADER_VALIDATORS, to_native_string  # noqa: F401
+from ._internal_utils import (  # noqa: F401
+    _HEADER_VALIDATORS_BYTE,
+    _HEADER_VALIDATORS_STR,
+    HEADER_VALIDATORS,
+    to_native_string,
+)
 from .compat import (
     Mapping,
     basestring,
@@ -1031,20 +1036,23 @@ def check_header_validity(header):
     :param header: tuple, in the format (name, value).
     """
     name, value = header
-
-    for part in header:
-        if type(part) not in HEADER_VALIDATORS:
-            raise InvalidHeader(
-                f"Header part ({part!r}) from {{{name!r}: {value!r}}} must be "
-                f"of type str or bytes, not {type(part)}"
-            )
-
-    _validate_header_part(name, "name", HEADER_VALIDATORS[type(name)][0])
-    _validate_header_part(value, "value", HEADER_VALIDATORS[type(value)][1])
+    _validate_header_part(header, name, 0)
+    _validate_header_part(header, value, 1)
 
 
-def _validate_header_part(header_part, header_kind, validator):
+def _validate_header_part(header, header_part, header_validator_index):
+    if isinstance(header_part, str):
+        validator = _HEADER_VALIDATORS_STR[header_validator_index]
+    elif isinstance(header_part, bytes):
+        validator = _HEADER_VALIDATORS_BYTE[header_validator_index]
+    else:
+        raise InvalidHeader(
+            f"Header part ({header_part!r}) from {header} "
+            f"must be of type str or bytes, not {type(header_part)}"
+        )
+
     if not validator.match(header_part):
+        header_kind = "name" if header_validator_index == 0 else "value"
         raise InvalidHeader(
             f"Invalid leading whitespace, reserved character(s), or return"
             f"character(s) in header {header_kind}: {header_part!r}"

requirements-dev.txt

@@ -6,6 +6,7 @@ pytest-mock==2.0.0
 httpbin==0.7.0
 trustme
 wheel
+cryptography<40.0.0; python_version <= '3.7' and platform_python_implementation == 'PyPy'
 
 # Flask Stack
 Flask>1.0,<2.0

setup.py

@@ -61,7 +61,7 @@ if sys.argv[-1] == "publish":
 requires = [
     "charset_normalizer>=2,<4",
     "idna>=2.5,<4",
-    "urllib3>=1.21.1,<1.27",
+    "urllib3>=1.21.1,<3",
     "certifi>=2017.4.17",
 ]
 test_requirements = [
@@ -94,7 +94,7 @@ setup(
     package_data={"": ["LICENSE", "NOTICE"]},
     package_dir={"requests": "requests"},
     include_package_data=True,
-    python_requires=">=3.7, <4",
+    python_requires=">=3.7",
     install_requires=requires,
     license=about["__license__"],
     zip_safe=False,

tests/__init__.py

@@ -2,9 +2,13 @@
 
 import warnings
 
-from urllib3.exceptions import SNIMissingWarning
+try:
+    from urllib3.exceptions import SNIMissingWarning
 
-# urllib3 sets SNIMissingWarning to only go off once,
-# while this test suite requires it to always fire
-# so that it occurs during test_requests.test_https_warnings
-warnings.simplefilter("always", SNIMissingWarning)
+    # urllib3 1.x sets SNIMissingWarning to only go off once,
+    # while this test suite requires it to always fire
+    # so that it occurs during test_requests.test_https_warnings
+    warnings.simplefilter("always", SNIMissingWarning)
+except ImportError:
+    # urllib3 2.0 removed that warning and errors out instead
+    SNIMissingWarning = None

tests/test_requests.py

@@ -48,6 +48,7 @@ from requests.models import PreparedRequest, urlencode
 from requests.sessions import SessionRedirectMixin
 from requests.structures import CaseInsensitiveDict
 
+from . import SNIMissingWarning
 from .compat import StringIO
 from .utils import override_environ
 
@@ -974,6 +975,10 @@ class TestRequests:
         r = requests.get(httpbin(), cert=".")
         assert r.status_code == 200
 
+    @pytest.mark.skipif(
+        SNIMissingWarning is None,
+        reason="urllib3 2.0 removed that warning and errors out instead",
+    )
     def test_https_warnings(self, nosan_server):
         """warnings are emitted with requests.get"""
         host, port, ca_bundle = nosan_server
@@ -1747,6 +1752,31 @@ class TestRequests:
         with pytest.raises(InvalidHeader):
             requests.get(httpbin("get"), headers=invalid_header)
 
+    def test_header_with_subclass_types(self, httpbin):
+        """If the subclasses does not behave *exactly* like
+        the base bytes/str classes, this is not supported.
+        This test is for backwards compatibility.
+        """
+
+        class MyString(str):
+            pass
+
+        class MyBytes(bytes):
+            pass
+
+        r_str = requests.get(httpbin("get"), headers={MyString("x-custom"): "myheader"})
+        assert r_str.request.headers["x-custom"] == "myheader"
+
+        r_bytes = requests.get(
+            httpbin("get"), headers={MyBytes(b"x-custom"): b"myheader"}
+        )
+        assert r_bytes.request.headers["x-custom"] == b"myheader"
+
+        r_mixed = requests.get(
+            httpbin("get"), headers={MyString("x-custom"): MyBytes(b"myheader")}
+        )
+        assert r_mixed.request.headers["x-custom"] == b"myheader"
+
     @pytest.mark.parametrize("files", ("foo", b"foo", bytearray(b"foo")))
     def test_can_send_objects_with_files(self, httpbin, files):
         data = {"a": "this is a string"}