Pass urllib3.SKIP_HEADER when headers should be unset

urllib3 introduced some default headers and a way to skip them if
desired. Let's use that sentinel value to pass along information about
Requests' users desire to skip those headers as well.

Closes gh-5671

HISTORY.md

@@ -6,17 +6,6 @@ dev
 
 -   \[Short description of non-trivial change.\]
 
-2.25.1 (2020-12-16)
--------------------
-
-**Bugfixes**
-
-- Requests now treats `application/json` as `utf8` by default. Resolving
-  inconsistencies between `r.text` and `r.json` output. (#5673)
-
-**Dependencies**
-
-- Requests now supports chardet v4.x.
 
 2.25.0 (2020-11-11)
 ------------------

requests/__init__.py

@@ -65,8 +65,10 @@ def check_compatibility(urllib3_version, chardet_version):
     # Check chardet for compatibility.
     major, minor, patch = chardet_version.split('.')[:3]
     major, minor, patch = int(major), int(minor), int(patch)
-    # chardet >= 3.0.2, < 5.0.0
-    assert (3, 0, 2) <= (major, minor, patch) < (5, 0, 0)
+    # chardet >= 3.0.2, < 3.1.0
+    assert major == 3
+    assert minor < 1
+    assert patch >= 2
 
 
 def _check_cryptography(cryptography_version):

requests/__version__.py

@@ -5,8 +5,8 @@
 __title__ = 'requests'
 __description__ = 'Python HTTP for Humans.'
 __url__ = 'https://requests.readthedocs.io'
-__version__ = '2.25.1'
-__build__ = 0x022501
+__version__ = '2.25.0'
+__build__ = 0x022500
 __author__ = 'Kenneth Reitz'
 __author_email__ = 'me@kennethreitz.org'
 __license__ = 'Apache 2.0'

requests/compat.py

@@ -30,6 +30,16 @@ try:
 except ImportError:
     import json
 
+
+import urllib3
+
+try:
+    SKIP_HEADER = urllib3.util.SKIP_HEADER
+    SKIPPABLE_HEADERS = urllib3.util.SKIPPABLE_HEADERS
+except AttributeError:
+    SKIP_HEADER = None
+    SKIPPABLE_HEADERS = frozenset([])
+
 # ---------
 # Specifics
 # ---------

requests/models.py

@@ -15,6 +15,7 @@ import sys
 # such as in Embedded Python. See https://github.com/psf/requests/issues/3578.
 import encodings.idna
 
+import urllib3
 from urllib3.fields import RequestField
 from urllib3.filepost import encode_multipart_formdata
 from urllib3.util import parse_url
@@ -36,9 +37,21 @@ from .utils import (
     stream_decode_response_unicode, to_key_val_list, parse_header_links,
     iter_slices, guess_json_utf, super_len, check_header_validity)
 from .compat import (
-    Callable, Mapping,
-    cookielib, urlunparse, urlsplit, urlencode, str, bytes,
-    is_py2, chardet, builtin_str, basestring)
+    SKIP_HEADER,
+    SKIPPABLE_HEADERS,
+    Callable,
+    Mapping,
+    cookielib,
+    urlunparse,
+    urlsplit,
+    urlencode,
+    str,
+    bytes,
+    is_py2,
+    chardet,
+    builtin_str,
+    basestring,
+)
 from .compat import json as complexjson
 from .status_codes import codes
 
@@ -447,9 +460,14 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         self.headers = CaseInsensitiveDict()
         if headers:
             for header in headers.items():
+                name, value = header
+                if value is None:
+                    if name.lower() in SKIPPABLE_HEADERS:
+                        value = SKIP_HEADER
+                    else:
+                        continue
                 # Raise exception on invalid header value.
                 check_header_validity(header)
-                name, value = header
                 self.headers[to_native_string(name)] = value
 
     def prepare_body(self, data, files, json=None):

requests/sessions.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 
 """
-requests.sessions
-~~~~~~~~~~~~~~~~~
+requests.session
+~~~~~~~~~~~~~~~~
 
 This module provides a Session object to manage and persist settings across
 requests (cookies, auth, proxies).
@@ -47,7 +47,9 @@ else:
     preferred_clock = time.time
 
 
-def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
+def merge_setting(
+    request_setting, session_setting, dict_class=OrderedDict, delete_none=True
+):
     """Determines appropriate setting for a given request, taking into account
     the explicit setting on that request, and the setting in the session. If a
     setting is a dictionary, they will be merged together using `dict_class`
@@ -69,11 +71,12 @@ def merge_setting(request_setting, session_setting, dict_class=OrderedDict):
     merged_setting = dict_class(to_key_val_list(session_setting))
     merged_setting.update(to_key_val_list(request_setting))
 
-    # Remove keys that are set to None. Extract keys first to avoid altering
-    # the dictionary during iteration.
-    none_keys = [k for (k, v) in merged_setting.items() if v is None]
-    for key in none_keys:
-        del merged_setting[key]
+    if delete_none:
+        # Remove keys that are set to None. Extract keys first to avoid altering
+        # the dictionary during iteration.
+        none_keys = [k for (k, v) in merged_setting.items() if v is None]
+        for key in none_keys:
+            del merged_setting[key]
 
     return merged_setting
 
@@ -459,7 +462,12 @@ class Session(SessionRedirectMixin):
             files=request.files,
             data=request.data,
             json=request.json,
-            headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict),
+            headers=merge_setting(
+                request.headers,
+                self.headers,
+                dict_class=CaseInsensitiveDict,
+                delete_none=False,
+            ),
             params=merge_setting(request.params, self.params),
             auth=merge_setting(auth, self.auth),
             cookies=merged_cookies,

requests/utils.py

@@ -947,6 +947,8 @@ def check_header_validity(header):
     :param header: tuple, in the format (name, value).
     """
     name, value = header
+    if value is None:
+        return
 
     if isinstance(value, bytes):
         pat = _CLEAN_HEADER_REGEX_BYTE

setup.py

@@ -42,7 +42,7 @@ if sys.argv[-1] == 'publish':
 packages = ['requests']
 
 requires = [
-    'chardet>=3.0.2,<5',
+    'chardet>=3.0.2,<4',
     'idna>=2.5,<3',
     'urllib3>=1.21.1,<1.27',
     'certifi>=2017.4.17'

tests/test_requests.py

@@ -17,10 +17,15 @@ import pytest
 from requests.adapters import HTTPAdapter
 from requests.auth import HTTPDigestAuth, _basic_auth_str
 from requests.compat import (
-    Morsel, cookielib, getproxies, str, urlparse,
-    builtin_str)
-from requests.cookies import (
-    cookiejar_from_dict, morsel_to_cookie)
+    Morsel,
+    cookielib,
+    getproxies,
+    str,
+    urlparse,
+    builtin_str,
+    SKIP_HEADER,
+)
+from requests.cookies import cookiejar_from_dict, morsel_to_cookie
 from requests.exceptions import (
     ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL,
     MissingSchema, ReadTimeout, Timeout, RetryError, TooManyRedirects,
@@ -438,10 +443,13 @@ class TestRequests:
     def test_headers_on_session_with_None_are_not_sent(self, httpbin):
         """Do not send headers in Session.headers with None values."""
         ses = requests.Session()
-        ses.headers['Accept-Encoding'] = None
-        req = requests.Request('GET', httpbin('get'))
+        ses.headers["Accept-Encoding"] = None
+        req = requests.Request("GET", httpbin("get"))
         prep = ses.prepare_request(req)
-        assert 'Accept-Encoding' not in prep.headers
+        if not SKIP_HEADER:
+            assert "Accept-Encoding" not in prep.headers
+        else:
+            assert SKIP_HEADER == prep.headers["Accept-Encoding"]
 
     def test_headers_preserve_order(self, httpbin):
         """Preserve order when headers provided as OrderedDict."""