v2.29.0

* pin cryptography<40 on pypy 3.7

.github/workflows/codeql-analysis.yml

@@ -14,8 +14,15 @@ on:
   schedule:
     - cron: '0 23 * * 0'
 
+permissions:
+  contents: read
+
 jobs:
   analyze:
+    permissions:
+      actions: read  # for github/codeql-action/init to get workflow details
+      contents: read  # for actions/checkout to fetch code
+      security-events: write  # for github/codeql-action/autobuild to send a status report
     name: Analyze
     runs-on: ubuntu-latest
 

.github/workflows/lint.yml

@@ -1,16 +1,20 @@
 name: Lint code
 
-on:
-  push:
-  pull_request:
+on: [push, pull_request]
+
+permissions:
+  contents: read
 
 jobs:
   lint:
     runs-on: ubuntu-20.04
+    timeout-minutes: 10
 
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
     - name: Run pre-commit
       uses: pre-commit/action@v3.0.0

.github/workflows/run-tests.yml

@@ -2,6 +2,9 @@ name: Tests
 
 on: [push, pull_request]
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ${{ matrix.os }}
@@ -9,8 +12,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev"]
-        os: [ubuntu-18.04, macOS-latest, windows-latest]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        os: [ubuntu-22.04, macOS-latest, windows-latest]
         include:
           # pypy-3.7 on Mac OS currently fails trying to compile
           # brotlipy. Moving pypy3 to only test linux.

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
   - id: end-of-file-fixer
   - id: trailing-whitespace
 - repo: https://github.com/PyCQA/isort
-  rev: 5.10.1
+  rev: 5.12.0
   hooks:
     - id: isort
 - repo: https://github.com/psf/black
@@ -22,7 +22,7 @@ repos:
   hooks:
     - id: pyupgrade
       args: [--py37-plus]
-- repo: https://gitlab.com/pycqa/flake8
-  rev: 4.0.1
+- repo: https://github.com/PyCQA/flake8
+  rev: 6.0.0
   hooks:
     - id: flake8

HISTORY.md

@@ -6,6 +6,37 @@ dev
 
 - \[Short description of non-trivial change.\]
 
+2.29.0 (2023-04-26)
+-------------------
+
+**Improvements**
+
+- Requests now defers chunked requests to the urllib3 implementation to improve
+  standardization. (#6226)
+- Requests relaxes header component requirements to support bytes/str subclasses. (#6356)
+
+2.28.2 (2023-01-12)
+-------------------
+
+**Dependencies**
+
+- Requests now supports charset\_normalizer 3.x. (#6261)
+
+**Bugfixes**
+
+- Updated MissingSchema exception to suggest https scheme rather than http. (#6188)
+
+2.28.1 (2022-06-29)
+-------------------
+
+**Improvements**
+
+- Speed optimization in `iter_content` with transition to `yield from`. (#6170)
+
+**Dependencies**
+
+- Added support for chardet 5.0.0 (#6179)
+- Added support for charset-normalizer 2.1.0 (#6169)
 
 2.28.0 (2022-06-09)
 -------------------

Makefile

@@ -2,7 +2,7 @@
 init:
 	pip install -r requirements-dev.txt
 test:
-	# This runs all of the tests, on both Python 2 and Python 3.
+	# This runs all of the tests on all supported Python versions.
 	tox -p
 ci:
 	pytest tests --junitxml=report.xml

docs/api.rst

@@ -36,6 +36,7 @@ Exceptions
 .. autoexception:: requests.ConnectTimeout
 .. autoexception:: requests.ReadTimeout
 .. autoexception:: requests.Timeout
+.. autoexception:: requests.JSONDecodeError
 
 
 Request Sessions

docs/user/advanced.rst

@@ -656,10 +656,10 @@ certificates trusted by Requests can be found with::
     from requests.utils import DEFAULT_CA_BUNDLE_PATH
     print(DEFAULT_CA_BUNDLE_PATH)
 
-You override this default certificate bundle by setting the standard
-``curl_ca_bundle`` environment variable to another file path::
+You override this default certificate bundle by setting the ``REQUESTS_CA_BUNDLE``
+(or ``CURL_CA_BUNDLE``) environment variable to another file path::
 
-    $ export curl_ca_bundle="/usr/local/myproxy_info/cacert.pem"
+    $ export REQUESTS_CA_BUNDLE="/usr/local/myproxy_info/cacert.pem"
     $ export https_proxy="http://10.10.1.10:1080"
 
     $ python
@@ -717,10 +717,9 @@ If ``chardet`` is installed, ``requests`` uses it, however for python3
 library is an LGPL-licenced dependency and some users of requests
 cannot depend on mandatory LGPL-licensed dependencies.
 
-When you install ``request`` without specifying ``[use_chardet_on_py3]]`` extra,
+When you install ``requests`` without specifying ``[use_chardet_on_py3]`` extra,
 and ``chardet`` is not already installed, ``requests`` uses ``charset-normalizer``
-(MIT-licensed) to guess the encoding. For Python 2, ``requests`` uses only
-``chardet`` and is a mandatory dependency there.
+(MIT-licensed) to guess the encoding.
 
 The only time Requests will not guess the encoding is if no explicit charset
 is present in the HTTP headers **and** the ``Content-Type``

docs/user/install.rst

@@ -22,7 +22,7 @@ Requests is actively developed on GitHub, where the code is
 
 You can either clone the public repository::
 
-    $ git clone git://github.com/psf/requests.git
+    $ git clone https://github.com/psf/requests.git
 
 Or, download the `tarball <https://github.com/psf/requests/tarball/main>`_::
 

docs/user/quickstart.rst

@@ -177,7 +177,7 @@ server, you can access ``r.raw``. If you want to do this, make sure you set
     <urllib3.response.HTTPResponse object at 0x101194810>
 
     >>> r.raw.read(10)
-    '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03'
+    b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03'
 
 In general, however, you should use a pattern like this to save what is being
 streamed to a file::
@@ -237,7 +237,7 @@ dictionary of data will automatically be form-encoded when the request is made::
 
     >>> payload = {'key1': 'value1', 'key2': 'value2'}
 
-    >>> r = requests.post("https://httpbin.org/post", data=payload)
+    >>> r = requests.post('https://httpbin.org/post', data=payload)
     >>> print(r.text)
     {
       ...

pyproject.toml

@@ -4,7 +4,7 @@ src_paths = ["requests", "test"]
 honor_noqa = true
 
 [tool.pytest.ini_options]
-addopts = "-p no:warnings --doctest-modules"
+addopts = "--doctest-modules"
 doctest_optionflags = "NORMALIZE_WHITESPACE ELLIPSIS"
 minversion = "6.2"
 testpaths = [

requests/__init__.py

@@ -75,13 +75,13 @@ def check_compatibility(urllib3_version, chardet_version, charset_normalizer_ver
     if chardet_version:
         major, minor, patch = chardet_version.split(".")[:3]
         major, minor, patch = int(major), int(minor), int(patch)
-        # chardet_version >= 3.0.2, < 5.0.0
-        assert (3, 0, 2) <= (major, minor, patch) < (5, 0, 0)
+        # chardet_version >= 3.0.2, < 6.0.0
+        assert (3, 0, 2) <= (major, minor, patch) < (6, 0, 0)
     elif charset_normalizer_version:
         major, minor, patch = charset_normalizer_version.split(".")[:3]
         major, minor, patch = int(major), int(minor), int(patch)
-        # charset_normalizer >= 2.0.0 < 3.0.0
-        assert (2, 0, 0) <= (major, minor, patch) < (3, 0, 0)
+        # charset_normalizer >= 2.0.0 < 4.0.0
+        assert (2, 0, 0) <= (major, minor, patch) < (4, 0, 0)
     else:
         raise Exception("You need either charset_normalizer or chardet installed")
 

requests/__version__.py

@@ -5,10 +5,10 @@
 __title__ = "requests"
 __description__ = "Python HTTP for Humans."
 __url__ = "https://requests.readthedocs.io"
-__version__ = "2.28.0"
-__build__ = 0x022800
+__version__ = "2.29.0"
+__build__ = 0x022900
 __author__ = "Kenneth Reitz"
 __author_email__ = "me@kennethreitz.org"
 __license__ = "Apache 2.0"
-__copyright__ = "Copyright 2022 Kenneth Reitz"
+__copyright__ = "Copyright Kenneth Reitz"
 __cake__ = "\u2728 \U0001f370 \u2728"

requests/_internal_utils.py

@@ -14,9 +14,11 @@ _VALID_HEADER_NAME_RE_STR = re.compile(r"^[^:\s][^:\r\n]*$")
 _VALID_HEADER_VALUE_RE_BYTE = re.compile(rb"^\S[^\r\n]*$|^$")
 _VALID_HEADER_VALUE_RE_STR = re.compile(r"^\S[^\r\n]*$|^$")
 
+_HEADER_VALIDATORS_STR = (_VALID_HEADER_NAME_RE_STR, _VALID_HEADER_VALUE_RE_STR)
+_HEADER_VALIDATORS_BYTE = (_VALID_HEADER_NAME_RE_BYTE, _VALID_HEADER_VALUE_RE_BYTE)
 HEADER_VALIDATORS = {
-    bytes: (_VALID_HEADER_NAME_RE_BYTE, _VALID_HEADER_VALUE_RE_BYTE),
-    str: (_VALID_HEADER_NAME_RE_STR, _VALID_HEADER_VALUE_RE_STR),
+    bytes: _HEADER_VALIDATORS_BYTE,
+    str: _HEADER_VALIDATORS_STR,
 }
 
 

requests/adapters.py

@@ -22,7 +22,6 @@ from urllib3.exceptions import ProxyError as _ProxyError
 from urllib3.exceptions import ReadTimeoutError, ResponseError
 from urllib3.exceptions import SSLError as _SSLError
 from urllib3.poolmanager import PoolManager, proxy_from_url
-from urllib3.response import HTTPResponse
 from urllib3.util import Timeout as TimeoutSauce
 from urllib3.util import parse_url
 from urllib3.util.retry import Retry
@@ -485,63 +484,19 @@ class HTTPAdapter(BaseAdapter):
             timeout = TimeoutSauce(connect=timeout, read=timeout)
 
         try:
-            if not chunked:
-                resp = conn.urlopen(
-                    method=request.method,
-                    url=url,
-                    body=request.body,
-                    headers=request.headers,
-                    redirect=False,
-                    assert_same_host=False,
-                    preload_content=False,
-                    decode_content=False,
-                    retries=self.max_retries,
-                    timeout=timeout,
-                )
-
-            # Send the request.
-            else:
-                if hasattr(conn, "proxy_pool"):
-                    conn = conn.proxy_pool
-
-                low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT)
-
-                try:
-                    skip_host = "Host" in request.headers
-                    low_conn.putrequest(
-                        request.method,
-                        url,
-                        skip_accept_encoding=True,
-                        skip_host=skip_host,
-                    )
-
-                    for header, value in request.headers.items():
-                        low_conn.putheader(header, value)
-
-                    low_conn.endheaders()
-
-                    for i in request.body:
-                        low_conn.send(hex(len(i))[2:].encode("utf-8"))
-                        low_conn.send(b"\r\n")
-                        low_conn.send(i)
-                        low_conn.send(b"\r\n")
-                    low_conn.send(b"0\r\n\r\n")
-
-                    # Receive the response from the server
-                    r = low_conn.getresponse()
-
-                    resp = HTTPResponse.from_httplib(
-                        r,
-                        pool=conn,
-                        connection=low_conn,
-                        preload_content=False,
-                        decode_content=False,
-                    )
-                except Exception:
-                    # If we hit any problems here, clean up the connection.
-                    # Then, raise so that we can handle the actual exception.
-                    low_conn.close()
-                    raise
+            resp = conn.urlopen(
+                method=request.method,
+                url=url,
+                body=request.body,
+                headers=request.headers,
+                redirect=False,
+                assert_same_host=False,
+                preload_content=False,
+                decode_content=False,
+                retries=self.max_retries,
+                timeout=timeout,
+                chunked=chunked,
+            )
 
         except (ProtocolError, OSError) as err:
             raise ConnectionError(err, request=request)

requests/api.py

@@ -106,7 +106,7 @@ def post(url, data=None, json=None, **kwargs):
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, list of tuples, bytes, or file-like
         object to send in the body of the :class:`Request`.
-    :param json: (optional) json data to send in the body of the :class:`Request`.
+    :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
     :return: :class:`Response <Response>` object
     :rtype: requests.Response
@@ -121,7 +121,7 @@ def put(url, data=None, **kwargs):
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, list of tuples, bytes, or file-like
         object to send in the body of the :class:`Request`.
-    :param json: (optional) json data to send in the body of the :class:`Request`.
+    :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
     :return: :class:`Response <Response>` object
     :rtype: requests.Response
@@ -136,7 +136,7 @@ def patch(url, data=None, **kwargs):
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, list of tuples, bytes, or file-like
         object to send in the body of the :class:`Request`.
-    :param json: (optional) json data to send in the body of the :class:`Request`.
+    :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
     :return: :class:`Response <Response>` object
     :rtype: requests.Response

requests/models.py

@@ -438,7 +438,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         if not scheme:
             raise MissingSchema(
                 f"Invalid URL {url!r}: No scheme supplied. "
-                f"Perhaps you meant http://{url}?"
+                f"Perhaps you meant https://{url}?"
             )
 
         if not host:
@@ -813,8 +813,7 @@ class Response:
             # Special case for urllib3.
             if hasattr(self.raw, "stream"):
                 try:
-                    for chunk in self.raw.stream(chunk_size, decode_content=True):
-                        yield chunk
+                    yield from self.raw.stream(chunk_size, decode_content=True)
                 except ProtocolError as e:
                     raise ChunkedEncodingError(e)
                 except DecodeError as e:

requests/utils.py

@@ -25,7 +25,12 @@ from . import certs
 from .__version__ import __version__
 
 # to_native_string is unused here, but imported here for backwards compatibility
-from ._internal_utils import HEADER_VALIDATORS, to_native_string  # noqa: F401
+from ._internal_utils import (  # noqa: F401
+    _HEADER_VALIDATORS_BYTE,
+    _HEADER_VALIDATORS_STR,
+    HEADER_VALIDATORS,
+    to_native_string,
+)
 from .compat import (
     Mapping,
     basestring,
@@ -1031,20 +1036,23 @@ def check_header_validity(header):
     :param header: tuple, in the format (name, value).
     """
     name, value = header
-
-    for part in header:
-        if type(part) not in HEADER_VALIDATORS:
-            raise InvalidHeader(
-                f"Header part ({part!r}) from {{{name!r}: {value!r}}} must be "
-                f"of type str or bytes, not {type(part)}"
-            )
-
-    _validate_header_part(name, "name", HEADER_VALIDATORS[type(name)][0])
-    _validate_header_part(value, "value", HEADER_VALIDATORS[type(value)][1])
+    _validate_header_part(header, name, 0)
+    _validate_header_part(header, value, 1)
 
 
-def _validate_header_part(header_part, header_kind, validator):
+def _validate_header_part(header, header_part, header_validator_index):
+    if isinstance(header_part, str):
+        validator = _HEADER_VALIDATORS_STR[header_validator_index]
+    elif isinstance(header_part, bytes):
+        validator = _HEADER_VALIDATORS_BYTE[header_validator_index]
+    else:
+        raise InvalidHeader(
+            f"Header part ({header_part!r}) from {header} "
+            f"must be of type str or bytes, not {type(header_part)}"
+        )
+
     if not validator.match(header_part):
+        header_kind = "name" if header_validator_index == 0 else "value"
         raise InvalidHeader(
             f"Invalid leading whitespace, reserved character(s), or return"
             f"character(s) in header {header_kind}: {header_part!r}"

requirements-dev.txt

@@ -6,6 +6,7 @@ pytest-mock==2.0.0
 httpbin==0.7.0
 trustme
 wheel
+cryptography<40.0.0; python_version <= '3.7' and platform_python_implementation == 'PyPy'
 
 # Flask Stack
 Flask>1.0,<2.0

setup.cfg

@@ -5,7 +5,7 @@ provides-extra =
     use_chardet_on_py3
 requires-dist =
     certifi>=2017.4.17
-    charset_normalizer~=2.0.0
+    charset_normalizer>=2,<4
     idna>=2.5,<4
     urllib3>=1.21.1,<1.27
 

setup.py

@@ -59,7 +59,7 @@ if sys.argv[-1] == "publish":
     sys.exit()
 
 requires = [
-    "charset_normalizer~=2.0.0",
+    "charset_normalizer>=2,<4",
     "idna>=2.5,<4",
     "urllib3>=1.21.1,<1.27",
     "certifi>=2017.4.17",
@@ -94,7 +94,7 @@ setup(
     package_data={"": ["LICENSE", "NOTICE"]},
     package_dir={"requests": "requests"},
     include_package_data=True,
-    python_requires=">=3.7, <4",
+    python_requires=">=3.7",
     install_requires=requires,
     license=about["__license__"],
     zip_safe=False,
@@ -123,7 +123,7 @@ setup(
     extras_require={
         "security": [],
         "socks": ["PySocks>=1.5.6, !=1.5.7"],
-        "use_chardet_on_py3": ["chardet>=3.0.2,<5"],
+        "use_chardet_on_py3": ["chardet>=3.0.2,<6"],
     },
     project_urls={
         "Documentation": "https://requests.readthedocs.io",

tests/__init__.py

@@ -2,9 +2,13 @@
 
 import warnings
 
-from urllib3.exceptions import SNIMissingWarning
+try:
+    from urllib3.exceptions import SNIMissingWarning
 
-# urllib3 sets SNIMissingWarning to only go off once,
-# while this test suite requires it to always fire
-# so that it occurs during test_requests.test_https_warnings
-warnings.simplefilter("always", SNIMissingWarning)
+    # urllib3 1.x sets SNIMissingWarning to only go off once,
+    # while this test suite requires it to always fire
+    # so that it occurs during test_requests.test_https_warnings
+    warnings.simplefilter("always", SNIMissingWarning)
+except ImportError:
+    # urllib3 2.0 removed that warning and errors out instead
+    SNIMissingWarning = None

tests/test_requests.py

@@ -48,6 +48,7 @@ from requests.models import PreparedRequest, urlencode
 from requests.sessions import SessionRedirectMixin
 from requests.structures import CaseInsensitiveDict
 
+from . import SNIMissingWarning
 from .compat import StringIO
 from .utils import override_environ
 
@@ -974,6 +975,10 @@ class TestRequests:
         r = requests.get(httpbin(), cert=".")
         assert r.status_code == 200
 
+    @pytest.mark.skipif(
+        SNIMissingWarning is None,
+        reason="urllib3 2.0 removed that warning and errors out instead",
+    )
     def test_https_warnings(self, nosan_server):
         """warnings are emitted with requests.get"""
         host, port, ca_bundle = nosan_server
@@ -1747,6 +1752,31 @@ class TestRequests:
         with pytest.raises(InvalidHeader):
             requests.get(httpbin("get"), headers=invalid_header)
 
+    def test_header_with_subclass_types(self, httpbin):
+        """If the subclasses does not behave *exactly* like
+        the base bytes/str classes, this is not supported.
+        This test is for backwards compatibility.
+        """
+
+        class MyString(str):
+            pass
+
+        class MyBytes(bytes):
+            pass
+
+        r_str = requests.get(httpbin("get"), headers={MyString("x-custom"): "myheader"})
+        assert r_str.request.headers["x-custom"] == "myheader"
+
+        r_bytes = requests.get(
+            httpbin("get"), headers={MyBytes(b"x-custom"): b"myheader"}
+        )
+        assert r_bytes.request.headers["x-custom"] == b"myheader"
+
+        r_mixed = requests.get(
+            httpbin("get"), headers={MyString("x-custom"): MyBytes(b"myheader")}
+        )
+        assert r_mixed.request.headers["x-custom"] == b"myheader"
+
     @pytest.mark.parametrize("files", ("foo", b"foo", bytearray(b"foo")))
     def test_can_send_objects_with_files(self, httpbin, files):
         data = {"a": "this is a string"}

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{37,38,39,310}-{default, use_chardet_on_py3}
+envlist = py{37,38,39,310,311}-{default, use_chardet_on_py3}
 
 [testenv]
 deps = -rrequirements-dev.txt