Bump version for 2.13.0

Add release notes for 2.13.0

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+omit = requests/packages/*

.travis.yml

@@ -0,0 +1,18 @@
+language: python
+python:
+  - "2.6"
+  - "2.7"
+  - "3.3"
+  - "3.4"
+  - "3.5"
+  - "3.6"
+  - "3.7-dev"
+  # - "pypy"  -- appears to hang
+  # - "pypy3"
+# command to install dependencies
+install: "make"
+# command to run tests
+script:
+  - make coverage
+after_success:
+  - codecov

AUTHORS.rst

@@ -173,3 +173,6 @@ Patches and Suggestions
 - Om Prakash Kumar <omprakash070@gmail.com> (`@iamprakashom <https://github.com/iamprakashom>`_)
 - Philipp Konrad <gardiac2002@gmail.com> (`@gardiac2002 <https://github.com/gardiac2002>`_)
 - Hussain Tamboli <hussaintamboli18@gmail.com> (`@hussaintamboli <https://github.com/hussaintamboli>`_)
+- Casey Davidson (`@davidsoncasey <https://github.com/davidsoncasey>`_)
+- Andrii Soldatenko (`@a_soldatenko <https://github.com/andriisoldatenko>`_)
+- Moinuddin Quadri <moin18@gmail.com> (`@moin18 <https://github.com/moin18>`_)

HISTORY.rst

@@ -3,6 +3,48 @@
 Release History
 ---------------
 
+2.13.0 (2017-01-24)
++++++++++++++++++++
+
+**Features**
+
+- Only load the ``idna`` library when we've determined we need it. This will
+  save some memory for users.
+
+**Miscellaneous**
+
+- Updated bundled urllib3 to 1.20.
+- Updated bundled idna to 2.2.
+
+2.12.5 (2017-01-18)
++++++++++++++++++++
+
+**Bugfixes**
+
+- Fixed an issue with JSON encoding detection, specifically detecting
+  big-endian UTF-32 with BOM.
+
+2.12.4 (2016-12-14)
++++++++++++++++++++
+
+**Bugfixes**
+
+- Fixed regression from 2.12.2 where non-string types were rejected in the
+  basic auth parameters. While support for this behaviour has been readded,
+  the behaviour is deprecated and will be removed in the future.
+
+2.12.3 (2016-12-01)
++++++++++++++++++++
+
+**Bugfixes**
+
+- Fixed regression from v2.12.1 for URLs with schemes that begin with "http".
+  These URLs have historically been processed as though they were HTTP-schemed
+  URLs, and so have had parameters added. This was removed in v2.12.2 in an
+  overzealous attempt to resolve problems with IDNA-encoding those URLs. This
+  change was reverted: the other fixes for IDNA-encoding have been judged to
+  be sufficient to return to the behaviour Requests had before v2.12.0.
+
 2.12.2 (2016-11-30)
 +++++++++++++++++++
 

Makefile

@@ -1,18 +1,16 @@
 .PHONY: docs
 
 init:
-	pip install -r requirements.txt
+	pip install 'pipenv>=0.1.6'
+	pipenv install --dev
 
 test:
 	# This runs all of the tests. To run an individual test, run py.test with
 	# the -k flag, like "py.test -k test_path_is_not_double_encoded"
-	py.test tests
+	pipenv run py.test tests
 
 coverage:
-	py.test --verbose --cov-report term --cov=requests tests
-
-ci: init
-	py.test --junitxml=junit.xml
+	pipenv run py.test --cov-config .coveragerc --verbose --cov-report term --cov-report xml --cov=requests tests
 
 certs:
 	curl http://ci.kennethreitz.org/job/ca-bundle/lastSuccessfulBuild/artifact/cacerts.pem -o requests/cacert.pem
@@ -20,7 +18,7 @@ certs:
 deps: urllib3 chardet idna
 
 urllib3:
-	git clone https://github.com/shazow/urllib3.git && \
+	git clone -b release https://github.com/shazow/urllib3.git && \
 	    rm -fr requests/packages/urllib3 && \
 	    cd urllib3 && \
 	    git checkout `git describe --abbrev=0 --tags` && \
@@ -44,13 +42,13 @@ idna:
 	    git checkout `git describe --abbrev=0 --tags` && \
 	    cd .. && \
 	    mv idna/idna requests/packages/ && \
-	    find requests/packages/idna -type f -exec sed -i "" "s/^from idna/from /" {} \; && \
 	    rm -fr idna
 
 publish:
-	python setup.py register
-	python setup.py sdist upload
-	python setup.py bdist_wheel --universal upload
+	pip install 'twine>=1.5.0'
+	python setup.py sdist
+	python setup.py bdist_wheel --universal
+	twine upload dist/*
 	rm -fr build dist .egg requests.egg-info
 
 

Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+url = "https://pypi.org/"
+verify_ssl = true
+
+[dev-packages]
+sphinx = "*"
+pytest-mock = "*"
+pytest-cov = "*"
+pytest = "*"
+codecov = "*"
+pytest-httpbin = "*"
+PySocks = "*"

Pipfile.lock

@@ -0,0 +1,45 @@
+{
+    "default": {},
+    "develop": {
+        "Werkzeug": "==0.11.15",
+        "six": "==1.10.0",
+        "httpbin": "==0.5.0",
+        "codecov": "==2.0.5",
+        "Flask": "==0.12",
+        "Babel": "==2.3.4",
+        "alabaster": "==0.7.9",
+        "pytest-mock": "==1.5.0",
+        "MarkupSafe": "==0.23",
+        "pytz": "==2016.10",
+        "coverage": "==4.3.4",
+        "pytest-httpbin": "==0.2.3",
+        "funcsigs": "==1.0.2",
+        "click": "==6.7",
+        "decorator": "==4.0.11",
+        "imagesize": "==0.7.1",
+        "Pygments": "==2.1.3",
+        "sphinx": "*",
+        "pbr": "==1.10.0",
+        "snowballstemmer": "==1.2.1",
+        "py": "==1.4.32",
+        "pytest-cov": "==2.4.0",
+        "docutils": "==0.13.1",
+        "pytest": "==3.0.5",
+        "Jinja2": "==2.9.4",
+        "Sphinx": "==1.5.1",
+        "requests": "==2.12.5",
+        "itsdangerous": "==0.24",
+        "PySocks": "==1.6.5",
+        "mock": "==2.0.0"
+    },
+    "_meta": {
+        "sources": [
+            {
+                "url": "https://pypi.org/",
+                "verify_ssl": true
+            }
+        ],
+        "requires": {},
+        "Pipfile-sha256": "f306fe105cc76e259ab1e5d2933ad2d0fe7a8b2d28686f64f7a94f767c0ce5d4"
+    }
+}

README.rst

@@ -3,7 +3,28 @@ Requests: HTTP for Humans
 
 .. image:: https://img.shields.io/pypi/v/requests.svg
     :target: https://pypi.python.org/pypi/requests
+
+.. image:: https://img.shields.io/pypi/l/requests.svg
+    :target: https://pypi.python.org/pypi/requests
     
+.. image:: https://img.shields.io/pypi/wheel/requests.svg
+    :target: https://pypi.python.org/pypi/requests
+ 
+.. image:: https://img.shields.io/pypi/pyversions/requests.svg
+    :target: https://pypi.python.org/pypi/requests
+
+.. image:: https://travis-ci.org/kennethreitz/requests.svg?branch=master
+    :target: https://travis-ci.org/kennethreitz/requests
+
+.. image:: https://codecov.io/github/kennethreitz/requests/coverage.svg?branch=master
+    :target: https://codecov.io/github/kennethreitz/requests
+    :alt: codecov.io
+    
+.. image:: https://img.shields.io/badge/SayThanks.io-☼-1EAEDB.svg
+    :target: https://saythanks.io/to/kennethreitz
+
+
+
 Requests is the only *Non-GMO* HTTP library for Python, safe for human
 consumption.
 
@@ -65,7 +86,7 @@ Requests is ready for today's web.
 - Chunked Requests
 - Thread-safety
 
-Requests officially supports Python 2.6–2.7 & 3.3–3.5, and runs great on PyPy.
+Requests officially supports Python 2.6–2.7 & 3.3–3.7, and runs great on PyPy.
 
 Installation
 ------------

docs/community/faq.rst

@@ -59,6 +59,7 @@ supported:
 * Python 3.3
 * Python 3.4
 * Python 3.5
+* Python 3.6
 * PyPy
 
 What are "hostname doesn't match" errors?

docs/dev/todo.rst

@@ -41,6 +41,7 @@ Requests currently supports the following versions of Python:
 - Python 3.3
 - Python 3.4
 - Python 3.5
+- Python 3.6
 - PyPy
 
 Google AppEngine is not officially supported although support is available

docs/index.rst

@@ -43,7 +43,7 @@ User Testimonials
 
 The NSA, Her Majesty's Government, Amazon, Google, Twilio, Runscope, Mozilla, Heroku,
 PayPal, NPR, Obama for America, Transifex, Native Instruments, The Washington
-Post, Twitter, SoundCloud, Kippt, Readability, Sony, and Federal U.S.
+Post, Twitter, SoundCloud, Kippt, Sony, and Federal U.S.
 Institutions that prefer to be unnamed claim to use Requests internally.
 
 **Armin Ronacher**
@@ -56,7 +56,7 @@ Institutions that prefer to be unnamed claim to use Requests internally.
 
 **Daniel Greenfeld**
     Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to
-    @kennethreitz's request library. Today has been AWESOME.
+    `@kennethreitz <https://twitter.com/kennethreitz>`_'s request library. Today has been AWESOME.
 
 **Kenny Meyers**
     Python HTTP: When in doubt, or when not in doubt, use Requests. Beautiful,
@@ -87,7 +87,7 @@ Requests is ready for today's web.
 - Chunked Requests
 - Thread-safety
 
-Requests officially supports Python 2.6–2.7 & 3.3–3.5, and runs great on PyPy.
+Requests officially supports Python 2.6–2.7 & 3.3–3.7, and runs great on PyPy.
 
 
 The User Guide

docs/user/advanced.rst

@@ -220,7 +220,7 @@ This list of trusted CAs can also be specified through the ``REQUESTS_CA_BUNDLE`
 
 Requests can also ignore verifying the SSL certificate if you set ``verify`` to False::
 
-    >>> requests.get('https://kennethreitz.com', verify=False)
+    >>> requests.get('https://kennethreitz.org', verify=False)
     <Response [200]>
 
 By default, ``verify`` is set to True. Option ``verify`` only applies to host certs.
@@ -229,7 +229,7 @@ You can also specify a local cert to use as client side certificate, as a single
 file (containing the private key and the certificate) or as a tuple of both
 file's path::
 
-    >>> requests.get('https://kennethreitz.com', cert=('/path/client.cert', '/path/client.key'))
+    >>> requests.get('https://kennethreitz.org', cert=('/path/client.cert', '/path/client.key'))
     <Response [200]>
 
 or persistent::
@@ -239,7 +239,7 @@ or persistent::
 
 If you specify a wrong path or an invalid cert, you'll get a SSLError::
 
-    >>> requests.get('https://kennethreitz.com', cert='/wrong_path/client.pem')
+    >>> requests.get('https://kennethreitz.org', cert='/wrong_path/client.pem')
     SSLError: [Errno 336265225] _ssl.c:347: error:140B0009:SSL routines:SSL_CTX_use_PrivateKey_file:PEM lib
 
 .. warning:: The private key to your local certificate *must* be unencrypted.

docs/user/authentication.rst

@@ -60,7 +60,7 @@ OAuth 1 Authentication
 ----------------------
 
 A common form of authentication for several web APIs is OAuth. The ``requests-oauthlib``
-library allows Requests users to easily make OAuth authenticated requests::
+library allows Requests users to easily make OAuth 1 authenticated requests::
 
     >>> import requests
     >>> from requests_oauthlib import OAuth1
@@ -76,6 +76,17 @@ For more information on how to OAuth flow works, please see the official `OAuth`
 For examples and documentation on requests-oauthlib, please see the `requests_oauthlib`_
 repository on GitHub
 
+OAuth 2 and OpenID Connect Authentication
+-----------------------------------------
+
+The ``requests-oauthlib`` library also handles OAuth 2, the authentication mechanism
+underpinning OpenID Connect. See the `requests-oauthlib OAuth2 documentation`_ for
+details of the various OAuth 2 credential management flows:
+
+* `Web Application Flow`_
+* `Mobile Application Flow`_
+* `Legacy Application Flow`_
+* `Backend Application Flow`_
 
 Other Authentication
 --------------------
@@ -123,6 +134,11 @@ Further examples can be found under the `Requests organization`_ and in the
 
 .. _OAuth: http://oauth.net/
 .. _requests_oauthlib: https://github.com/requests/requests-oauthlib
+.. _requests-oauthlib OAuth2 documentation: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html
+.. _Web Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#web-application-flow
+.. _Mobile Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#mobile-application-flow
+.. _Legacy Application Flow:  http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#legacy-application-flow
+.. _Backend Application Flow:  http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#backend-application-flow
 .. _Kerberos: https://github.com/requests/requests-kerberos
 .. _NTLM: https://github.com/requests/requests-ntlm
 .. _Requests organization: https://github.com/requests

requests/__init__.py

@@ -41,8 +41,8 @@ is at <http://python-requests.org>.
 """
 
 __title__ = 'requests'
-__version__ = '2.12.2'
-__build__ = 0x021202
+__version__ = '2.13.0'
+__build__ = 0x021300
 __author__ = 'Kenneth Reitz'
 __license__ = 'Apache 2.0'
 __copyright__ = 'Copyright 2016 Kenneth Reitz'

requests/api.py

@@ -33,7 +33,7 @@ def request(method, url, **kwargs):
         before giving up, as a float, or a :ref:`(connect timeout, read
         timeout) <timeouts>` tuple.
     :type timeout: float or tuple
-    :param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection.
+    :param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``True``.
     :type allow_redirects: bool
     :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
     :param verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``.

requests/auth.py

@@ -12,10 +12,11 @@ import re
 import time
 import hashlib
 import threading
+import warnings
 
 from base64 import b64encode
 
-from .compat import urlparse, str
+from .compat import urlparse, str, basestring
 from .cookies import extract_cookies_to_jar
 from ._internal_utils import to_native_string
 from .utils import parse_dict_header
@@ -27,7 +28,35 @@ CONTENT_TYPE_MULTI_PART = 'multipart/form-data'
 
 def _basic_auth_str(username, password):
     """Returns a Basic Auth string."""
-    
+
+    # "I want us to put a big-ol' comment on top of it that
+    # says that this behaviour is dumb but we need to preserve
+    # it because people are relying on it."
+    #    - Lukasa
+    #
+    # These are here solely to maintain backwards compatibility
+    # for things like ints. This will be removed in 3.0.0.
+    if not isinstance(username, basestring):
+        warnings.warn(
+            "Non-string usernames will no longer be supported in Requests "
+            "3.0.0. Please convert the object you've passed in ({0!r}) to "
+            "a string or bytes object in the near future to avoid "
+            "problems.".format(username),
+            category=DeprecationWarning,
+        )
+        username = str(username)
+
+    if not isinstance(password, basestring):
+        warnings.warn(
+            "Non-string passwords will no longer be supported in Requests "
+            "3.0.0. Please convert the object you've passed in ({0!r}) to "
+            "a string or bytes object in the near future to avoid "
+            "problems.".format(password),
+            category=DeprecationWarning,
+        )
+        password = str(password)
+    # -- End Removal --
+
     if isinstance(username, str):
         username = username.encode('latin1')
 

requests/models.py

@@ -9,6 +9,7 @@ This module contains the primary objects that power Requests.
 
 import collections
 import datetime
+import sys
 
 # Import encoding now, to avoid implicit import later.
 # Implicit import within threads may cause LookupError when standard library is in a ZIP,
@@ -21,7 +22,6 @@ from .structures import CaseInsensitiveDict
 
 from .auth import HTTPBasicAuth
 from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar
-from .packages import idna
 from .packages.urllib3.fields import RequestField
 from .packages.urllib3.filepost import encode_multipart_formdata
 from .packages.urllib3.util import parse_url
@@ -331,6 +331,22 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         if self.method is not None:
             self.method = to_native_string(self.method.upper())
 
+    @staticmethod
+    def _get_idna_encoded_host(host):
+        try:
+            from .packages import idna
+        except ImportError:
+            # tolerate the possibility of downstream repackagers unvendoring `requests`
+            # For more information, read: packages/__init__.py
+            import idna
+            sys.modules['requests.packages.idna'] = idna
+
+        try:
+            host = idna.encode(host, uts46=True).decode('utf-8')
+        except idna.IDNAError:
+            raise UnicodeError
+        return host
+
     def prepare_url(self, url, params):
         """Prepares the given HTTP URL."""
         #: Accept objects that have string representations.
@@ -347,9 +363,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         url = url.lstrip()
 
         # Don't do any URL preparation for non-HTTP schemes like `mailto`,
-        # `data`, `http+unix` etc to work around exceptions from `url_parse`,
-        # which handles RFC 3986 only.
-        if ':' in url and not url.lower().startswith(('http://', 'https://')):
+        # `data` etc to work around exceptions from `url_parse`, which
+        # handles RFC 3986 only.
+        if ':' in url and not url.lower().startswith('http'):
             self.url = url
             return
 
@@ -368,17 +384,17 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         if not host:
             raise InvalidURL("Invalid URL %r: No host supplied" % url)
 
-        # In general, we want to try IDNA encoding every hostname, as that
-        # allows users to automatically get the correct behaviour. However,
-        # we’re quite strict about IDNA encoding, so certain valid hostnames
-        # may fail to encode. On failure, we verify the hostname meets a
-        # minimum standard of only containing ASCII characters, and not starting
-        # with a wildcard (*), before allowing the unencoded hostname through.
-        try:
-            host = idna.encode(host, uts46=True).decode('utf-8')
-        except (UnicodeError, idna.IDNAError):
-            if not unicode_is_ascii(host) or host.startswith(u'*'):
+        # In general, we want to try IDNA encoding the hostname if the string contains
+        # non-ASCII characters. This allows users to automatically get the correct IDNA
+        # behaviour. For strings containing only ASCII characters, we need to also verify
+        # it doesn't start with a wildcard (*), before allowing the unencoded hostname.
+        if not unicode_is_ascii(host):
+            try:
+                host = self._get_idna_encoded_host(host)
+            except UnicodeError:
                 raise InvalidURL('URL has an invalid label.')
+        elif host.startswith(u'*'):
+            raise InvalidURL('URL has an invalid label.')
 
         # Carefully reconstruct the network location
         netloc = auth or ''

requests/packages/__init__.py

@@ -34,9 +34,3 @@ try:
 except ImportError:
     import chardet
     sys.modules['%s.chardet' % __name__] = chardet
-
-try:
-    from . import idna
-except ImportError:
-    import idna
-    sys.modules['%s.idna' % __name__] = idna

requests/packages/idna/uts46data.py

@@ -3,7 +3,9 @@
 
 """IDNA Mapping Table from UTS46."""
 
-uts46data = (
+
+def _seg_0():
+    return [
     (0x0, '3'),
     (0x1, '3'),
     (0x2, '3'),
@@ -104,6 +106,10 @@ uts46data = (
     (0x61, 'V'),
     (0x62, 'V'),
     (0x63, 'V'),
+    ]
+
+def _seg_1():
+    return [
     (0x64, 'V'),
     (0x65, 'V'),
     (0x66, 'V'),
@@ -204,6 +210,10 @@ uts46data = (
     (0xC5, 'M', u'å'),
     (0xC6, 'M', u'æ'),
     (0xC7, 'M', u'ç'),
+    ]
+
+def _seg_2():
+    return [
     (0xC8, 'M', u'è'),
     (0xC9, 'M', u'é'),
     (0xCA, 'M', u'ê'),
@@ -304,6 +314,10 @@ uts46data = (
     (0x129, 'V'),
     (0x12A, 'M', u'ī'),
     (0x12B, 'V'),
+    ]
+
+def _seg_3():
+    return [
     (0x12C, 'M', u'ĭ'),
     (0x12D, 'V'),
     (0x12E, 'M', u'į'),
@@ -404,6 +418,10 @@ uts46data = (
     (0x191, 'M', u'ƒ'),
     (0x192, 'V'),
     (0x193, 'M', u'ɠ'),
+    ]
+
+def _seg_4():
+    return [
     (0x194, 'M', u'ɣ'),
     (0x195, 'V'),
     (0x196, 'M', u'ɩ'),
@@ -504,6 +522,10 @@ uts46data = (
     (0x20A, 'M', u'ȋ'),
     (0x20B, 'V'),
     (0x20C, 'M', u'ȍ'),
+    ]
+
+def _seg_5():
+    return [
     (0x20D, 'V'),
     (0x20E, 'M', u'ȏ'),
     (0x20F, 'V'),
@@ -604,6 +626,10 @@ uts46data = (
     (0x375, 'V'),
     (0x376, 'M', u'ͷ'),
     (0x377, 'V'),
+    ]
+
+def _seg_6():
+    return [
     (0x378, 'X'),
     (0x37A, '3', u' ι'),
     (0x37B, 'V'),
@@ -704,6 +730,10 @@ uts46data = (
     (0x401, 'M', u'ё'),
     (0x402, 'M', u'ђ'),
     (0x403, 'M', u'ѓ'),
+    ]
+
+def _seg_7():
+    return [
     (0x404, 'M', u'є'),
     (0x405, 'M', u'ѕ'),
     (0x406, 'M', u'і'),
@@ -804,6 +834,10 @@ uts46data = (
     (0x49C, 'M', u'ҝ'),
     (0x49D, 'V'),
     (0x49E, 'M', u'ҟ'),
+    ]
+
+def _seg_8():
+    return [
     (0x49F, 'V'),
     (0x4A0, 'M', u'ҡ'),
     (0x4A1, 'V'),
@@ -904,6 +938,10 @@ uts46data = (
     (0x501, 'V'),
     (0x502, 'M', u'ԃ'),
     (0x503, 'V'),
+    ]
+
+def _seg_9():
+    return [
     (0x504, 'M', u'ԅ'),
     (0x505, 'V'),
     (0x506, 'M', u'ԇ'),
@@ -1004,6 +1042,10 @@ uts46data = (
     (0x678, 'M', u'يٴ'),
     (0x679, 'V'),
     (0x6DD, 'X'),
+    ]
+
+def _seg_10():
+    return [
     (0x6DE, 'V'),
     (0x70E, 'X'),
     (0x710, 'V'),
@@ -1104,6 +1146,10 @@ uts46data = (
     (0xA5D, 'X'),
     (0xA5E, 'M', u'ਫ਼'),
     (0xA5F, 'X'),
+    ]
+
+def _seg_11():
+    return [
     (0xA66, 'V'),
     (0xA76, 'X'),
     (0xA81, 'V'),
@@ -1204,6 +1250,10 @@ uts46data = (
     (0xC2A, 'V'),
     (0xC34, 'X'),
     (0xC35, 'V'),
+    ]
+
+def _seg_12():
+    return [
     (0xC3A, 'X'),
     (0xC3D, 'V'),
     (0xC45, 'X'),
@@ -1304,6 +1354,10 @@ uts46data = (
     (0xE84, 'V'),
     (0xE85, 'X'),
     (0xE87, 'V'),
+    ]
+
+def _seg_13():
+    return [
     (0xE89, 'X'),
     (0xE8A, 'V'),
     (0xE8B, 'X'),
@@ -1404,6 +1458,10 @@ uts46data = (
     (0x1250, 'V'),
     (0x1257, 'X'),
     (0x1258, 'V'),
+    ]
+
+def _seg_14():
+    return [
     (0x1259, 'X'),
     (0x125A, 'V'),
     (0x125E, 'X'),
@@ -1504,6 +1562,10 @@ uts46data = (
     (0x1A8A, 'X'),
     (0x1A90, 'V'),
     (0x1A9A, 'X'),
+    ]
+
+def _seg_15():
+    return [
     (0x1AA0, 'V'),
     (0x1AAE, 'X'),
     (0x1B00, 'V'),
@@ -1604,6 +1666,10 @@ uts46data = (
     (0x1DA7, 'M', u'ᵻ'),
     (0x1DA8, 'M', u'ʝ'),
     (0x1DA9, 'M', u'ɭ'),
+    ]
+
+def _seg_16():
+    return [
     (0x1DAA, 'M', u'ᶅ'),
     (0x1DAB, 'M', u'ʟ'),
     (0x1DAC, 'M', u'ɱ'),
@@ -1704,6 +1770,10 @@ uts46data = (
     (0x1E48, 'M', u'ṉ'),
     (0x1E49, 'V'),
     (0x1E4A, 'M', u'ṋ'),
+    ]
+
+def _seg_17():
+    return [
     (0x1E4B, 'V'),
     (0x1E4C, 'M', u'ṍ'),
     (0x1E4D, 'V'),
@@ -1804,6 +1874,10 @@ uts46data = (
     (0x1EB1, 'V'),
     (0x1EB2, 'M', u'ẳ'),
     (0x1EB3, 'V'),
+    ]
+
+def _seg_18():
+    return [
     (0x1EB4, 'M', u'ẵ'),
     (0x1EB5, 'V'),
     (0x1EB6, 'M', u'ặ'),
@@ -1904,6 +1978,10 @@ uts46data = (
     (0x1F2B, 'M', u'ἣ'),
     (0x1F2C, 'M', u'ἤ'),
     (0x1F2D, 'M', u'ἥ'),
+    ]
+
+def _seg_19():
+    return [
     (0x1F2E, 'M', u'ἦ'),
     (0x1F2F, 'M', u'ἧ'),
     (0x1F30, 'V'),
@@ -2004,6 +2082,10 @@ uts46data = (
     (0x1FAC, 'M', u'ὤι'),
     (0x1FAD, 'M', u'ὥι'),
     (0x1FAE, 'M', u'ὦι'),
+    ]
+
+def _seg_20():
+    return [
     (0x1FAF, 'M', u'ὧι'),
     (0x1FB0, 'V'),
     (0x1FB2, 'M', u'ὰι'),
@@ -2104,6 +2186,10 @@ uts46data = (
     (0x204A, 'V'),
     (0x2057, 'M', u'′′′′'),
     (0x2058, 'V'),
+    ]
+
+def _seg_21():
+    return [
     (0x205F, '3', u' '),
     (0x2060, 'I'),
     (0x2061, 'X'),
@@ -2204,6 +2290,10 @@ uts46data = (
     (0x2133, 'M', u'm'),
     (0x2134, 'M', u'o'),
     (0x2135, 'M', u'א'),
+    ]
+
+def _seg_22():
+    return [
     (0x2136, 'M', u'ב'),
     (0x2137, 'M', u'ג'),
     (0x2138, 'M', u'ד'),
@@ -2304,6 +2394,10 @@ uts46data = (
     (0x2469, 'M', u'10'),
     (0x246A, 'M', u'11'),
     (0x246B, 'M', u'12'),
+    ]
+
+def _seg_23():
+    return [
     (0x246C, 'M', u'13'),
     (0x246D, 'M', u'14'),
     (0x246E, 'M', u'15'),
@@ -2404,6 +2498,10 @@ uts46data = (
     (0x24E0, 'M', u'q'),
     (0x24E1, 'M', u'r'),
     (0x24E2, 'M', u's'),
+    ]
+
+def _seg_24():
+    return [
     (0x24E3, 'M', u't'),
     (0x24E4, 'M', u'u'),
     (0x24E5, 'M', u'v'),
@@ -2504,6 +2602,10 @@ uts46data = (
     (0x2C80, 'M', u'ⲁ'),
     (0x2C81, 'V'),
     (0x2C82, 'M', u'ⲃ'),
+    ]
+
+def _seg_25():
+    return [
     (0x2C83, 'V'),
     (0x2C84, 'M', u'ⲅ'),
     (0x2C85, 'V'),
@@ -2604,6 +2706,10 @@ uts46data = (
     (0x2CEB, 'M', u'ⳬ'),
     (0x2CEC, 'V'),
     (0x2CED, 'M', u'ⳮ'),
+    ]
+
+def _seg_26():
+    return [
     (0x2CEE, 'V'),
     (0x2CF2, 'M', u'ⳳ'),
     (0x2CF3, 'V'),
@@ -2704,6 +2810,10 @@ uts46data = (
     (0x2F37, 'M', u'弋'),
     (0x2F38, 'M', u'弓'),
     (0x2F39, 'M', u'彐'),
+    ]
+
+def _seg_27():
+    return [
     (0x2F3A, 'M', u'彡'),
     (0x2F3B, 'M', u'彳'),
     (0x2F3C, 'M', u'心'),
@@ -2804,6 +2914,10 @@ uts46data = (
     (0x2F9B, 'M', u'走'),
     (0x2F9C, 'M', u'足'),
     (0x2F9D, 'M', u'身'),
+    ]
+
+def _seg_28():
+    return [
     (0x2F9E, 'M', u'車'),
     (0x2F9F, 'M', u'辛'),
     (0x2FA0, 'M', u'辰'),
@@ -2904,6 +3018,10 @@ uts46data = (
     (0x3142, 'M', u'ᄇ'),
     (0x3143, 'M', u'ᄈ'),
     (0x3144, 'M', u'ᄡ'),
+    ]
+
+def _seg_29():
+    return [
     (0x3145, 'M', u'ᄉ'),
     (0x3146, 'M', u'ᄊ'),
     (0x3147, 'M', u'ᄋ'),
@@ -3004,6 +3122,10 @@ uts46data = (
     (0x3202, '3', u'(ᄃ)'),
     (0x3203, '3', u'(ᄅ)'),
     (0x3204, '3', u'(ᄆ)'),
+    ]
+
+def _seg_30():
+    return [
     (0x3205, '3', u'(ᄇ)'),
     (0x3206, '3', u'(ᄉ)'),
     (0x3207, '3', u'(ᄋ)'),
@@ -3104,6 +3226,10 @@ uts46data = (
     (0x326D, 'M', u'ᄒ'),
     (0x326E, 'M', u'가'),
     (0x326F, 'M', u'나'),
+    ]
+
+def _seg_31():
+    return [
     (0x3270, 'M', u'다'),
     (0x3271, 'M', u'라'),
     (0x3272, 'M', u'마'),
@@ -3204,6 +3330,10 @@ uts46data = (
     (0x32D1, 'M', u'イ'),
     (0x32D2, 'M', u'ウ'),
     (0x32D3, 'M', u'エ'),
+    ]
+
+def _seg_32():
+    return [
     (0x32D4, 'M', u'オ'),
     (0x32D5, 'M', u'カ'),
     (0x32D6, 'M', u'キ'),
@@ -3304,6 +3434,10 @@ uts46data = (
     (0x3335, 'M', u'フラン'),
     (0x3336, 'M', u'ヘクタール'),
     (0x3337, 'M', u'ペソ'),
+    ]
+
+def _seg_33():
+    return [
     (0x3338, 'M', u'ペニヒ'),
     (0x3339, 'M', u'ヘルツ'),
     (0x333A, 'M', u'ペンス'),
@@ -3404,6 +3538,10 @@ uts46data = (
     (0x3399, 'M', u'fm'),
     (0x339A, 'M', u'nm'),
     (0x339B, 'M', u'μm'),
+    ]
+
+def _seg_34():
+    return [
     (0x339C, 'M', u'mm'),
     (0x339D, 'M', u'cm'),
     (0x339E, 'M', u'km'),
@@ -3504,6 +3642,10 @@ uts46data = (
     (0x33FD, 'M', u'30日'),
     (0x33FE, 'M', u'31日'),
     (0x33FF, 'M', u'gal'),
+    ]
+
+def _seg_35():
+    return [
     (0x3400, 'V'),
     (0x4DB6, 'X'),
     (0x4DC0, 'V'),
@@ -3604,6 +3746,10 @@ uts46data = (
     (0xA72F, 'V'),
     (0xA732, 'M', u'ꜳ'),
     (0xA733, 'V'),
+    ]
+
+def _seg_36():
+    return [
     (0xA734, 'M', u'ꜵ'),
     (0xA735, 'V'),
     (0xA736, 'M', u'ꜷ'),
@@ -3704,6 +3850,10 @@ uts46data = (
     (0xA7AA, 'M', u'ɦ'),
     (0xA7AB, 'X'),
     (0xA7F8, 'M', u'ħ'),
+    ]
+
+def _seg_37():
+    return [
     (0xA7F9, 'M', u'œ'),
     (0xA7FA, 'V'),
     (0xA82C, 'X'),
@@ -3804,6 +3954,10 @@ uts46data = (
     (0xF92B, 'M', u'狼'),
     (0xF92C, 'M', u'郎'),
     (0xF92D, 'M', u'來'),
+    ]
+
+def _seg_38():
+    return [
     (0xF92E, 'M', u'冷'),
     (0xF92F, 'M', u'勞'),
     (0xF930, 'M', u'擄'),
@@ -3904,6 +4058,10 @@ uts46data = (
     (0xF98F, 'M', u'憐'),
     (0xF990, 'M', u'戀'),
     (0xF991, 'M', u'撚'),
+    ]
+
+def _seg_39():
+    return [
     (0xF992, 'M', u'漣'),
     (0xF993, 'M', u'煉'),
     (0xF994, 'M', u'璉'),
@@ -4004,6 +4162,10 @@ uts46data = (
     (0xF9F3, 'M', u'麟'),
     (0xF9F4, 'M', u'林'),
     (0xF9F5, 'M', u'淋'),
+    ]
+
+def _seg_40():
+    return [
     (0xF9F6, 'M', u'臨'),
     (0xF9F7, 'M', u'立'),
     (0xF9F8, 'M', u'笠'),
@@ -4104,6 +4266,10 @@ uts46data = (
     (0xFA5C, 'M', u'臭'),
     (0xFA5D, 'M', u'艹'),
     (0xFA5F, 'M', u'著'),
+    ]
+
+def _seg_41():
+    return [
     (0xFA60, 'M', u'褐'),
     (0xFA61, 'M', u'視'),
     (0xFA62, 'M', u'謁'),
@@ -4204,6 +4370,10 @@ uts46data = (
     (0xFAC2, 'M', u'輸'),
     (0xFAC3, 'M', u'遲'),
     (0xFAC4, 'M', u'醙'),
+    ]
+
+def _seg_42():
+    return [
     (0xFAC5, 'M', u'鉶'),
     (0xFAC6, 'M', u'陼'),
     (0xFAC7, 'M', u'難'),
@@ -4304,6 +4474,10 @@ uts46data = (
     (0xFB7A, 'M', u'چ'),
     (0xFB7E, 'M', u'ڇ'),
     (0xFB82, 'M', u'ڍ'),
+    ]
+
+def _seg_43():
+    return [
     (0xFB84, 'M', u'ڌ'),
     (0xFB86, 'M', u'ڎ'),
     (0xFB88, 'M', u'ڈ'),
@@ -4404,6 +4578,10 @@ uts46data = (
     (0xFC3C, 'M', u'كم'),
     (0xFC3D, 'M', u'كى'),
     (0xFC3E, 'M', u'كي'),
+    ]
+
+def _seg_44():
+    return [
     (0xFC3F, 'M', u'لج'),
     (0xFC40, 'M', u'لح'),
     (0xFC41, 'M', u'لخ'),
@@ -4504,6 +4682,10 @@ uts46data = (
     (0xFCA0, 'M', u'به'),
     (0xFCA1, 'M', u'تج'),
     (0xFCA2, 'M', u'تح'),
+    ]
+
+def _seg_45():
+    return [
     (0xFCA3, 'M', u'تخ'),
     (0xFCA4, 'M', u'تم'),
     (0xFCA5, 'M', u'ته'),
@@ -4604,6 +4786,10 @@ uts46data = (
     (0xFD04, 'M', u'خي'),
     (0xFD05, 'M', u'صى'),
     (0xFD06, 'M', u'صي'),
+    ]
+
+def _seg_46():
+    return [
     (0xFD07, 'M', u'ضى'),
     (0xFD08, 'M', u'ضي'),
     (0xFD09, 'M', u'شج'),
@@ -4704,6 +4890,10 @@ uts46data = (
     (0xFD87, 'M', u'لمح'),
     (0xFD89, 'M', u'محج'),
     (0xFD8A, 'M', u'محم'),
+    ]
+
+def _seg_47():
+    return [
     (0xFD8B, 'M', u'محي'),
     (0xFD8C, 'M', u'مجح'),
     (0xFD8D, 'M', u'مجم'),
@@ -4804,6 +4994,10 @@ uts46data = (
     (0xFE3C, 'M', u'】'),
     (0xFE3D, 'M', u'《'),
     (0xFE3E, 'M', u'》'),
+    ]
+
+def _seg_48():
+    return [
     (0xFE3F, 'M', u'〈'),
     (0xFE40, 'M', u'〉'),
     (0xFE41, 'M', u'「'),
@@ -4904,6 +5098,10 @@ uts46data = (
     (0xFF00, 'X'),
     (0xFF01, '3', u'!'),
     (0xFF02, '3', u'"'),
+    ]
+
+def _seg_49():
+    return [
     (0xFF03, '3', u'#'),
     (0xFF04, '3', u'$'),
     (0xFF05, '3', u'%'),
@@ -5004,6 +5202,10 @@ uts46data = (
     (0xFF64, 'M', u'、'),
     (0xFF65, 'M', u'・'),
     (0xFF66, 'M', u'ヲ'),
+    ]
+
+def _seg_50():
+    return [
     (0xFF67, 'M', u'ァ'),
     (0xFF68, 'M', u'ィ'),
     (0xFF69, 'M', u'ゥ'),
@@ -5104,6 +5306,10 @@ uts46data = (
     (0xFFCB, 'M', u'ᅨ'),
     (0xFFCC, 'M', u'ᅩ'),
     (0xFFCD, 'M', u'ᅪ'),
+    ]
+
+def _seg_51():
+    return [
     (0xFFCE, 'M', u'ᅫ'),
     (0xFFCF, 'M', u'ᅬ'),
     (0xFFD0, 'X'),
@@ -5204,6 +5410,10 @@ uts46data = (
     (0x1041B, 'M', u'𐑃'),
     (0x1041C, 'M', u'𐑄'),
     (0x1041D, 'M', u'𐑅'),
+    ]
+
+def _seg_52():
+    return [
     (0x1041E, 'M', u'𐑆'),
     (0x1041F, 'M', u'𐑇'),
     (0x10420, 'M', u'𐑈'),
@@ -5304,6 +5514,10 @@ uts46data = (
     (0x12474, 'X'),
     (0x13000, 'V'),
     (0x1342F, 'X'),
+    ]
+
+def _seg_53():
+    return [
     (0x16800, 'V'),
     (0x16A39, 'X'),
     (0x16F00, 'V'),
@@ -5404,6 +5618,10 @@ uts46data = (
     (0x1D43A, 'M', u'g'),
     (0x1D43B, 'M', u'h'),
     (0x1D43C, 'M', u'i'),
+    ]
+
+def _seg_54():
+    return [
     (0x1D43D, 'M', u'j'),
     (0x1D43E, 'M', u'k'),
     (0x1D43F, 'M', u'l'),
@@ -5504,6 +5722,10 @@ uts46data = (
     (0x1D49E, 'M', u'c'),
     (0x1D49F, 'M', u'd'),
     (0x1D4A0, 'X'),
+    ]
+
+def _seg_55():
+    return [
     (0x1D4A2, 'M', u'g'),
     (0x1D4A3, 'X'),
     (0x1D4A5, 'M', u'j'),
@@ -5604,6 +5826,10 @@ uts46data = (
     (0x1D505, 'M', u'b'),
     (0x1D506, 'X'),
     (0x1D507, 'M', u'd'),
+    ]
+
+def _seg_56():
+    return [
     (0x1D508, 'M', u'e'),
     (0x1D509, 'M', u'f'),
     (0x1D50A, 'M', u'g'),
@@ -5704,6 +5930,10 @@ uts46data = (
     (0x1D56C, 'M', u'a'),
     (0x1D56D, 'M', u'b'),
     (0x1D56E, 'M', u'c'),
+    ]
+
+def _seg_57():
+    return [
     (0x1D56F, 'M', u'd'),
     (0x1D570, 'M', u'e'),
     (0x1D571, 'M', u'f'),
@@ -5804,6 +6034,10 @@ uts46data = (
     (0x1D5D0, 'M', u'w'),
     (0x1D5D1, 'M', u'x'),
     (0x1D5D2, 'M', u'y'),
+    ]
+
+def _seg_58():
+    return [
     (0x1D5D3, 'M', u'z'),
     (0x1D5D4, 'M', u'a'),
     (0x1D5D5, 'M', u'b'),
@@ -5904,6 +6138,10 @@ uts46data = (
     (0x1D634, 'M', u's'),
     (0x1D635, 'M', u't'),
     (0x1D636, 'M', u'u'),
+    ]
+
+def _seg_59():
+    return [
     (0x1D637, 'M', u'v'),
     (0x1D638, 'M', u'w'),
     (0x1D639, 'M', u'x'),
@@ -6004,6 +6242,10 @@ uts46data = (
     (0x1D698, 'M', u'o'),
     (0x1D699, 'M', u'p'),
     (0x1D69A, 'M', u'q'),
+    ]
+
+def _seg_60():
+    return [
     (0x1D69B, 'M', u'r'),
     (0x1D69C, 'M', u's'),
     (0x1D69D, 'M', u't'),
@@ -6104,6 +6346,10 @@ uts46data = (
     (0x1D6FE, 'M', u'γ'),
     (0x1D6FF, 'M', u'δ'),
     (0x1D700, 'M', u'ε'),
+    ]
+
+def _seg_61():
+    return [
     (0x1D701, 'M', u'ζ'),
     (0x1D702, 'M', u'η'),
     (0x1D703, 'M', u'θ'),
@@ -6204,6 +6450,10 @@ uts46data = (
     (0x1D764, 'M', u'ο'),
     (0x1D765, 'M', u'π'),
     (0x1D766, 'M', u'ρ'),
+    ]
+
+def _seg_62():
+    return [
     (0x1D767, 'M', u'θ'),
     (0x1D768, 'M', u'σ'),
     (0x1D769, 'M', u'τ'),
@@ -6304,6 +6554,10 @@ uts46data = (
     (0x1D7CA, 'M', u'ϝ'),
     (0x1D7CC, 'X'),
     (0x1D7CE, 'M', u'0'),
+    ]
+
+def _seg_63():
+    return [
     (0x1D7CF, 'M', u'1'),
     (0x1D7D0, 'M', u'2'),
     (0x1D7D1, 'M', u'3'),
@@ -6404,6 +6658,10 @@ uts46data = (
     (0x1EE30, 'M', u'ف'),
     (0x1EE31, 'M', u'ص'),
     (0x1EE32, 'M', u'ق'),
+    ]
+
+def _seg_64():
+    return [
     (0x1EE33, 'X'),
     (0x1EE34, 'M', u'ش'),
     (0x1EE35, 'M', u'ت'),
@@ -6504,6 +6762,10 @@ uts46data = (
     (0x1EEA2, 'M', u'ج'),
     (0x1EEA3, 'M', u'د'),
     (0x1EEA4, 'X'),
+    ]
+
+def _seg_65():
+    return [
     (0x1EEA5, 'M', u'و'),
     (0x1EEA6, 'M', u'ز'),
     (0x1EEA7, 'M', u'ح'),
@@ -6604,6 +6866,10 @@ uts46data = (
     (0x1F140, 'M', u'q'),
     (0x1F141, 'M', u'r'),
     (0x1F142, 'M', u's'),
+    ]
+
+def _seg_66():
+    return [
     (0x1F143, 'M', u't'),
     (0x1F144, 'M', u'u'),
     (0x1F145, 'M', u'v'),
@@ -6704,6 +6970,10 @@ uts46data = (
     (0x1F400, 'V'),
     (0x1F43F, 'X'),
     (0x1F440, 'V'),
+    ]
+
+def _seg_67():
+    return [
     (0x1F441, 'X'),
     (0x1F442, 'V'),
     (0x1F4F8, 'X'),
@@ -6804,6 +7074,10 @@ uts46data = (
     (0x2F84B, 'M', u'圖'),
     (0x2F84C, 'M', u'嘆'),
     (0x2F84D, 'M', u'圗'),
+    ]
+
+def _seg_68():
+    return [
     (0x2F84E, 'M', u'噑'),
     (0x2F84F, 'M', u'噴'),
     (0x2F850, 'M', u'切'),
@@ -6904,6 +7178,10 @@ uts46data = (
     (0x2F8B2, 'M', u'成'),
     (0x2F8B3, 'M', u'戛'),
     (0x2F8B4, 'M', u'扝'),
+    ]
+
+def _seg_69():
+    return [
     (0x2F8B5, 'M', u'抱'),
     (0x2F8B6, 'M', u'拔'),
     (0x2F8B7, 'M', u'捐'),
@@ -7004,6 +7282,10 @@ uts46data = (
     (0x2F916, 'M', u'㶖'),
     (0x2F917, 'M', u'灊'),
     (0x2F918, 'M', u'災'),
+    ]
+
+def _seg_70():
+    return [
     (0x2F919, 'M', u'灷'),
     (0x2F91A, 'M', u'炭'),
     (0x2F91B, 'M', u'𠔥'),
@@ -7104,6 +7386,10 @@ uts46data = (
     (0x2F97D, 'M', u'聠'),
     (0x2F97E, 'M', u'𦖨'),
     (0x2F97F, 'M', u'聰'),
+    ]
+
+def _seg_71():
+    return [
     (0x2F980, 'M', u'𣍟'),
     (0x2F981, 'M', u'䏕'),
     (0x2F982, 'M', u'育'),
@@ -7204,6 +7490,10 @@ uts46data = (
     (0x2F9E1, 'M', u'𨗭'),
     (0x2F9E2, 'M', u'邔'),
     (0x2F9E3, 'M', u'郱'),
+    ]
+
+def _seg_72():
+    return [
     (0x2F9E4, 'M', u'鄑'),
     (0x2F9E5, 'M', u'𨜮'),
     (0x2F9E6, 'M', u'鄛'),
@@ -7264,4 +7554,80 @@ uts46data = (
     (0x2FA1E, 'X'),
     (0xE0100, 'I'),
     (0xE01F0, 'X'),
+    ]
+
+uts46data = tuple(
+    _seg_0()
+    + _seg_1()
+    + _seg_2()
+    + _seg_3()
+    + _seg_4()
+    + _seg_5()
+    + _seg_6()
+    + _seg_7()
+    + _seg_8()
+    + _seg_9()
+    + _seg_10()
+    + _seg_11()
+    + _seg_12()
+    + _seg_13()
+    + _seg_14()
+    + _seg_15()
+    + _seg_16()
+    + _seg_17()
+    + _seg_18()
+    + _seg_19()
+    + _seg_20()
+    + _seg_21()
+    + _seg_22()
+    + _seg_23()
+    + _seg_24()
+    + _seg_25()
+    + _seg_26()
+    + _seg_27()
+    + _seg_28()
+    + _seg_29()
+    + _seg_30()
+    + _seg_31()
+    + _seg_32()
+    + _seg_33()
+    + _seg_34()
+    + _seg_35()
+    + _seg_36()
+    + _seg_37()
+    + _seg_38()
+    + _seg_39()
+    + _seg_40()
+    + _seg_41()
+    + _seg_42()
+    + _seg_43()
+    + _seg_44()
+    + _seg_45()
+    + _seg_46()
+    + _seg_47()
+    + _seg_48()
+    + _seg_49()
+    + _seg_50()
+    + _seg_51()
+    + _seg_52()
+    + _seg_53()
+    + _seg_54()
+    + _seg_55()
+    + _seg_56()
+    + _seg_57()
+    + _seg_58()
+    + _seg_59()
+    + _seg_60()
+    + _seg_61()
+    + _seg_62()
+    + _seg_63()
+    + _seg_64()
+    + _seg_65()
+    + _seg_66()
+    + _seg_67()
+    + _seg_68()
+    + _seg_69()
+    + _seg_70()
+    + _seg_71()
+    + _seg_72()
 )

requests/packages/urllib3/__init__.py

@@ -32,7 +32,7 @@ except ImportError:
 
 __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
 __license__ = 'MIT'
-__version__ = '1.19.1'
+__version__ = '1.20'
 
 __all__ = (
     'HTTPConnectionPool',

requests/packages/urllib3/connection.py

@@ -56,7 +56,10 @@ port_by_scheme = {
     'https': 443,
 }
 
-RECENT_DATE = datetime.date(2014, 1, 1)
+# When updating RECENT_DATE, move it to
+# within two years of the current date, and no
+# earlier than 6 months ago.
+RECENT_DATE = datetime.date(2016, 1, 1)
 
 
 class DummyConnection(object):

requests/packages/urllib3/connectionpool.py

@@ -25,7 +25,7 @@ from .exceptions import (
 )
 from .packages.ssl_match_hostname import CertificateError
 from .packages import six
-from .packages.six.moves.queue import LifoQueue, Empty, Full
+from .packages.six.moves import queue
 from .connection import (
     port_by_scheme,
     DummyConnection,
@@ -36,6 +36,7 @@ from .request import RequestMethods
 from .response import HTTPResponse
 
 from .util.connection import is_connection_dropped
+from .util.request import set_file_position
 from .util.response import assert_header_parsing
 from .util.retry import Retry
 from .util.timeout import Timeout
@@ -61,19 +62,13 @@ class ConnectionPool(object):
     """
 
     scheme = None
-    QueueCls = LifoQueue
+    QueueCls = queue.LifoQueue
 
     def __init__(self, host, port=None):
         if not host:
             raise LocationValueError("No host specified.")
 
-        # httplib doesn't like it when we include brackets in ipv6 addresses
-        # Specifically, if we include brackets but also pass the port then
-        # httplib crazily doubles up the square brackets on the Host header.
-        # Instead, we need to make sure we never pass ``None`` as the port.
-        # However, for backward compatibility reasons we can't actually
-        # *assert* that.
-        self.host = host.strip('[]')
+        self.host = _ipv6_host(host).lower()
         self.port = port
 
     def __str__(self):
@@ -154,7 +149,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         A dictionary with proxy headers, should not be used directly,
         instead, see :class:`urllib3.connectionpool.ProxyManager`"
 
-    :param \**conn_kw:
+    :param \\**conn_kw:
         Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
         :class:`urllib3.connection.HTTPSConnection` instances.
     """
@@ -235,7 +230,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         except AttributeError:  # self.pool is None
             raise ClosedPoolError(self, "Pool is closed.")
 
-        except Empty:
+        except queue.Empty:
             if self.block:
                 raise EmptyPoolError(self,
                                      "Pool reached maximum size and no more "
@@ -274,7 +269,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         except AttributeError:
             # self.pool is None.
             pass
-        except Full:
+        except queue.Full:
             # This should never happen if self.block == True
             log.warning(
                 "Connection pool is full, discarding connection: %s",
@@ -424,7 +419,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                 if conn:
                     conn.close()
 
-        except Empty:
+        except queue.Empty:
             pass  # Done.
 
     def is_same_host(self, url):
@@ -438,6 +433,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         # TODO: Add optional support for socket.gethostbyname checking.
         scheme, host, port = get_host(url)
 
+        host = _ipv6_host(host).lower()
+
         # Use explicit default port for comparison when none is given
         if self.port and not port:
             port = port_by_scheme.get(scheme)
@@ -449,7 +446,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
     def urlopen(self, method, url, body=None, headers=None, retries=None,
                 redirect=True, assert_same_host=True, timeout=_Default,
                 pool_timeout=None, release_conn=None, chunked=False,
-                **response_kw):
+                body_pos=None, **response_kw):
         """
         Get a connection from the pool and perform an HTTP request. This is the
         lowest level call for making a request, so you'll need to specify all
@@ -531,7 +528,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             encoding. Otherwise, urllib3 will send the body using the standard
             content-length form. Defaults to False.
 
-        :param \**response_kw:
+        :param int body_pos:
+            Position to seek to in file-like body in the event of a retry or
+            redirect. Typically this won't need to be set because urllib3 will
+            auto-populate the value when needed.
+
+        :param \\**response_kw:
             Additional parameters are passed to
             :meth:`urllib3.response.HTTPResponse.from_httplib`
         """
@@ -576,6 +578,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         # ensures we do proper cleanup in finally.
         clean_exit = False
 
+        # Rewind body position, if needed. Record current position
+        # for future rewinds in the event of a redirect/retry.
+        body_pos = set_file_position(body, body_pos)
+
         try:
             # Request a connection from the queue.
             timeout_obj = self._get_timeout(timeout)
@@ -612,7 +618,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # Everything went great!
             clean_exit = True
 
-        except Empty:
+        except queue.Empty:
             # Timed out by queue.
             raise EmptyPoolError(self, "No pool connections are available.")
 
@@ -668,7 +674,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             return self.urlopen(method, url, body, headers, retries,
                                 redirect, assert_same_host,
                                 timeout=timeout, pool_timeout=pool_timeout,
-                                release_conn=release_conn, **response_kw)
+                                release_conn=release_conn, body_pos=body_pos,
+                                **response_kw)
 
         # Handle redirect?
         redirect_location = redirect and response.get_redirect_location()
@@ -693,7 +700,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                 retries=retries, redirect=redirect,
                 assert_same_host=assert_same_host,
                 timeout=timeout, pool_timeout=pool_timeout,
-                release_conn=release_conn, **response_kw)
+                release_conn=release_conn, body_pos=body_pos,
+                **response_kw)
 
         # Check if we should retry the HTTP response.
         has_retry_after = bool(response.getheader('Retry-After'))
@@ -714,7 +722,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                 retries=retries, redirect=redirect,
                 assert_same_host=assert_same_host,
                 timeout=timeout, pool_timeout=pool_timeout,
-                release_conn=release_conn, **response_kw)
+                release_conn=release_conn,
+                body_pos=body_pos, **response_kw)
 
         return response
 
@@ -853,7 +862,7 @@ def connection_from_url(url, **kw):
     :param url:
         Absolute URL string that must include the scheme. Port is optional.
 
-    :param \**kw:
+    :param \\**kw:
         Passes additional parameters to the constructor of the appropriate
         :class:`.ConnectionPool`. Useful for specifying things like
         timeout, maxsize, headers, etc.
@@ -869,3 +878,22 @@ def connection_from_url(url, **kw):
         return HTTPSConnectionPool(host, port=port, **kw)
     else:
         return HTTPConnectionPool(host, port=port, **kw)
+
+
+def _ipv6_host(host):
+    """
+    Process IPv6 address literals
+    """
+
+    # httplib doesn't like it when we include brackets in IPv6 addresses
+    # Specifically, if we include brackets but also pass the port then
+    # httplib crazily doubles up the square brackets on the Host header.
+    # Instead, we need to make sure we never pass ``None`` as the port.
+    # However, for backward compatibility reasons we can't actually
+    # *assert* that.  See http://bugs.python.org/issue28539
+    #
+    # Also if an IPv6 address literal has a zone identifier, the
+    # percent sign might be URIencoded, convert it back into ASCII
+    if host.startswith('[') and host.endswith(']'):
+        host = host.replace('%25', '%').strip('[]')
+    return host

requests/packages/urllib3/contrib/appengine.py

@@ -111,7 +111,7 @@ class AppEngineManager(RequestMethods):
         warnings.warn(
             "urllib3 is using URLFetch on Google App Engine sandbox instead "
             "of sockets. To use sockets directly instead of URLFetch see "
-            "https://urllib3.readthedocs.io/en/latest/contrib.html.",
+            "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.",
             AppEnginePlatformWarning)
 
         RequestMethods.__init__(self, headers)

requests/packages/urllib3/contrib/pyopenssl.py

@@ -43,7 +43,6 @@ set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable.
 """
 from __future__ import absolute_import
 
-import idna
 import OpenSSL.SSL
 from cryptography import x509
 from cryptography.hazmat.backends.openssl import backend as openssl_backend
@@ -60,7 +59,6 @@ except ImportError:  # Platform-specific: Python 3
 
 import logging
 import ssl
-import select
 import six
 import sys
 
@@ -111,6 +109,8 @@ log = logging.getLogger(__name__)
 def inject_into_urllib3():
     'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.'
 
+    _validate_dependencies_met()
+
     util.ssl_.SSLContext = PyOpenSSLContext
     util.HAS_SNI = HAS_SNI
     util.ssl_.HAS_SNI = HAS_SNI
@@ -128,6 +128,26 @@ def extract_from_urllib3():
     util.ssl_.IS_PYOPENSSL = False
 
 
+def _validate_dependencies_met():
+    """
+    Verifies that PyOpenSSL's package-level dependencies have been met.
+    Throws `ImportError` if they are not met.
+    """
+    # Method added in `cryptography==1.1`; not available in older versions
+    from cryptography.x509.extensions import Extensions
+    if getattr(Extensions, "get_extension_for_class", None) is None:
+        raise ImportError("'cryptography' module missing required functionality.  "
+                          "Try upgrading to v1.3.4 or newer.")
+
+    # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509
+    # attribute is only present on those versions.
+    from OpenSSL.crypto import X509
+    x509 = X509()
+    if getattr(x509, "_x509", None) is None:
+        raise ImportError("'pyOpenSSL' module missing required functionality. "
+                          "Try upgrading to v0.14 or newer.")
+
+
 def _dnsname_to_stdlib(name):
     """
     Converts a dNSName SubjectAlternativeName field to the form used by the
@@ -144,6 +164,8 @@ def _dnsname_to_stdlib(name):
         that we can't just safely call `idna.encode`: it can explode for
         wildcard names. This avoids that problem.
         """
+        import idna
+
         for prefix in [u'*.', u'.']:
             if name.startswith(prefix):
                 name = name[len(prefix):]
@@ -242,8 +264,7 @@ class WrappedSocket(object):
             else:
                 raise
         except OpenSSL.SSL.WantReadError:
-            rd, wd, ed = select.select(
-                [self.socket], [], [], self.socket.gettimeout())
+            rd = util.wait_for_read(self.socket, self.socket.gettimeout())
             if not rd:
                 raise timeout('The read operation timed out')
             else:
@@ -265,8 +286,7 @@ class WrappedSocket(object):
             else:
                 raise
         except OpenSSL.SSL.WantReadError:
-            rd, wd, ed = select.select(
-                [self.socket], [], [], self.socket.gettimeout())
+            rd = util.wait_for_read(self.socket, self.socket.gettimeout())
             if not rd:
                 raise timeout('The read operation timed out')
             else:
@@ -280,9 +300,8 @@ class WrappedSocket(object):
             try:
                 return self.connection.send(data)
             except OpenSSL.SSL.WantWriteError:
-                _, wlist, _ = select.select([], [self.socket], [],
-                                            self.socket.gettimeout())
-                if not wlist:
+                wr = util.wait_for_write(self.socket, self.socket.gettimeout())
+                if not wr:
                     raise timeout()
                 continue
 
@@ -416,7 +435,7 @@ class PyOpenSSLContext(object):
             try:
                 cnx.do_handshake()
             except OpenSSL.SSL.WantReadError:
-                rd, _, _ = select.select([sock], [], [], sock.gettimeout())
+                rd = util.wait_for_read(sock, sock.gettimeout())
                 if not rd:
                     raise timeout('select timed out')
                 continue

requests/packages/urllib3/contrib/socks.py

@@ -83,6 +83,7 @@ class SOCKSConnection(HTTPConnection):
                 proxy_port=self._socks_options['proxy_port'],
                 proxy_username=self._socks_options['username'],
                 proxy_password=self._socks_options['password'],
+                proxy_rdns=self._socks_options['rdns'],
                 timeout=self.timeout,
                 **extra_kw
             )
@@ -153,8 +154,16 @@ class SOCKSProxyManager(PoolManager):
 
         if parsed.scheme == 'socks5':
             socks_version = socks.PROXY_TYPE_SOCKS5
+            rdns = False
+        elif parsed.scheme == 'socks5h':
+            socks_version = socks.PROXY_TYPE_SOCKS5
+            rdns = True
         elif parsed.scheme == 'socks4':
             socks_version = socks.PROXY_TYPE_SOCKS4
+            rdns = False
+        elif parsed.scheme == 'socks4a':
+            socks_version = socks.PROXY_TYPE_SOCKS4
+            rdns = True
         else:
             raise ValueError(
                 "Unable to determine SOCKS version from %s" % proxy_url
@@ -168,6 +177,7 @@ class SOCKSProxyManager(PoolManager):
             'proxy_port': parsed.port,
             'username': username,
             'password': password,
+            'rdns': rdns
         }
         connection_pool_kw['_socks_options'] = socks_options
 

requests/packages/urllib3/exceptions.py

@@ -239,3 +239,8 @@ class HeaderParsingError(HTTPError):
     def __init__(self, defects, unparsed_data):
         message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data)
         super(HeaderParsingError, self).__init__(message)
+
+
+class UnrewindableBodyError(HTTPError):
+    "urllib3 encountered an error when trying to rewind a body"
+    pass

requests/packages/urllib3/poolmanager.py

@@ -93,7 +93,7 @@ class PoolManager(RequestMethods):
         Headers to include with all requests, unless other headers are given
         explicitly.
 
-    :param \**connection_pool_kw:
+    :param \\**connection_pool_kw:
         Additional parameters are used to create fresh
         :class:`urllib3.connectionpool.ConnectionPool` instances.
 

requests/packages/urllib3/util/__init__.py

@@ -24,6 +24,10 @@ from .url import (
     split_first,
     Url,
 )
+from .wait import (
+    wait_for_read,
+    wait_for_write
+)
 
 __all__ = (
     'HAS_SNI',
@@ -43,4 +47,6 @@ __all__ = (
     'resolve_ssl_version',
     'split_first',
     'ssl_wrap_socket',
+    'wait_for_read',
+    'wait_for_write'
 )

requests/packages/urllib3/util/connection.py

@@ -1,13 +1,7 @@
 from __future__ import absolute_import
 import socket
-try:
-    from select import poll, POLLIN
-except ImportError:  # `poll` doesn't exist on OSX and other platforms
-    poll = False
-    try:
-        from select import select
-    except ImportError:  # `select` doesn't exist on AppEngine.
-        select = False
+from .wait import wait_for_read
+from .selectors import HAS_SELECT, SelectorError
 
 
 def is_connection_dropped(conn):  # Platform-specific
@@ -26,22 +20,13 @@ def is_connection_dropped(conn):  # Platform-specific
     if sock is None:  # Connection already closed (such as by httplib).
         return True
 
-    if not poll:
-        if not select:  # Platform-specific: AppEngine
-            return False
+    if not HAS_SELECT:
+        return False
 
-        try:
-            return select([sock], [], [], 0.0)[0]
-        except socket.error:
-            return True
-
-    # This version is better on platforms that support it.
-    p = poll()
-    p.register(sock, POLLIN)
-    for (fno, ev) in p.poll(0.0):
-        if fno == sock.fileno():
-            # Either data is buffered (bad), or the connection is dropped.
-            return True
+    try:
+        return bool(wait_for_read(sock, timeout=0.0))
+    except SelectorError:
+        return True
 
 
 # This function is copied from socket.py in the Python 2.7 standard

requests/packages/urllib3/util/request.py

@@ -1,9 +1,11 @@
 from __future__ import absolute_import
 from base64 import b64encode
 
-from ..packages.six import b
+from ..packages.six import b, integer_types
+from ..exceptions import UnrewindableBodyError
 
 ACCEPT_ENCODING = 'gzip,deflate'
+_FAILEDTELL = object()
 
 
 def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
@@ -70,3 +72,47 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
         headers['cache-control'] = 'no-cache'
 
     return headers
+
+
+def set_file_position(body, pos):
+    """
+    If a position is provided, move file to that point.
+    Otherwise, we'll attempt to record a position for future use.
+    """
+    if pos is not None:
+        rewind_body(body, pos)
+    elif getattr(body, 'tell', None) is not None:
+        try:
+            pos = body.tell()
+        except (IOError, OSError):
+            # This differentiates from None, allowing us to catch
+            # a failed `tell()` later when trying to rewind the body.
+            pos = _FAILEDTELL
+
+    return pos
+
+
+def rewind_body(body, body_pos):
+    """
+    Attempt to rewind body to a certain position.
+    Primarily used for request redirects and retries.
+
+    :param body:
+        File-like object that supports seek.
+
+    :param int pos:
+        Position to seek to in file.
+    """
+    body_seek = getattr(body, 'seek', None)
+    if body_seek is not None and isinstance(body_pos, integer_types):
+        try:
+            body_seek(body_pos)
+        except (IOError, OSError):
+            raise UnrewindableBodyError("An error occured when rewinding request "
+                                        "body for redirect/retry.")
+    elif body_pos is _FAILEDTELL:
+        raise UnrewindableBodyError("Unable to record file position for rewinding "
+                                    "request body during a redirect/retry.")
+    else:
+        raise ValueError("body_pos must be of type integer, "
+                         "instead it was %s." % type(body_pos))

requests/packages/urllib3/util/retry.py

@@ -273,12 +273,25 @@ class Retry(object):
         """
         return isinstance(err, (ReadTimeoutError, ProtocolError))
 
-    def is_retry(self, method, status_code, has_retry_after=False):
-        """ Is this method/status code retryable? (Based on method/codes whitelists)
+    def _is_method_retryable(self, method):
+        """ Checks if a given HTTP method should be retried upon, depending if
+        it is included on the method whitelist.
         """
         if self.method_whitelist and method.upper() not in self.method_whitelist:
             return False
 
+        return True
+
+    def is_retry(self, method, status_code, has_retry_after=False):
+        """ Is this method/status code retryable? (Based on whitelists and control
+        variables such as the number of total retries to allow, whether to
+        respect the Retry-After header, whether this header is present, and
+        whether the returned status code is on the list of status codes to
+        be retried upon on the presence of the aforementioned header)
+        """
+        if not self._is_method_retryable(method):
+            return False
+
         if self.status_forcelist and status_code in self.status_forcelist:
             return True
 
@@ -330,7 +343,7 @@ class Retry(object):
 
         elif error and self._is_read_error(error):
             # Read retry?
-            if read is False:
+            if read is False or not self._is_method_retryable(method):
                 raise six.reraise(type(error), error, _stacktrace)
             elif read is not None:
                 read -= 1

requests/packages/urllib3/util/selectors.py

@@ -0,0 +1,524 @@
+# Backport of selectors.py from Python 3.5+ to support Python < 3.4
+# Also has the behavior specified in PEP 475 which is to retry syscalls
+# in the case of an EINTR error. This module is required because selectors34
+# does not follow this behavior and instead returns that no dile descriptor
+# events have occurred rather than retry the syscall. The decision to drop
+# support for select.devpoll is made to maintain 100% test coverage.
+
+import errno
+import math
+import select
+from collections import namedtuple, Mapping
+
+import time
+try:
+    monotonic = time.monotonic
+except (AttributeError, ImportError):  # Python 3.3<
+    monotonic = time.time
+
+EVENT_READ = (1 << 0)
+EVENT_WRITE = (1 << 1)
+
+HAS_SELECT = True  # Variable that shows whether the platform has a selector.
+_SYSCALL_SENTINEL = object()  # Sentinel in case a system call returns None.
+
+
+class SelectorError(Exception):
+    def __init__(self, errcode):
+        super(SelectorError, self).__init__()
+        self.errno = errcode
+
+    def __repr__(self):
+        return "<SelectorError errno={0}>".format(self.errno)
+
+    def __str__(self):
+        return self.__repr__()
+
+
+def _fileobj_to_fd(fileobj):
+    """ Return a file descriptor from a file object. If
+    given an integer will simply return that integer back. """
+    if isinstance(fileobj, int):
+        fd = fileobj
+    else:
+        try:
+            fd = int(fileobj.fileno())
+        except (AttributeError, TypeError, ValueError):
+            raise ValueError("Invalid file object: {0!r}".format(fileobj))
+    if fd < 0:
+        raise ValueError("Invalid file descriptor: {0}".format(fd))
+    return fd
+
+
+def _syscall_wrapper(func, recalc_timeout, *args, **kwargs):
+    """ Wrapper function for syscalls that could fail due to EINTR.
+    All functions should be retried if there is time left in the timeout
+    in accordance with PEP 475. """
+    timeout = kwargs.get("timeout", None)
+    if timeout is None:
+        expires = None
+        recalc_timeout = False
+    else:
+        timeout = float(timeout)
+        if timeout < 0.0:  # Timeout less than 0 treated as no timeout.
+            expires = None
+        else:
+            expires = monotonic() + timeout
+
+    args = list(args)
+    if recalc_timeout and "timeout" not in kwargs:
+        raise ValueError(
+            "Timeout must be in args or kwargs to be recalculated")
+
+    result = _SYSCALL_SENTINEL
+    while result is _SYSCALL_SENTINEL:
+        try:
+            result = func(*args, **kwargs)
+        # OSError is thrown by select.select
+        # IOError is thrown by select.epoll.poll
+        # select.error is thrown by select.poll.poll
+        # Aren't we thankful for Python 3.x rework for exceptions?
+        except (OSError, IOError, select.error) as e:
+            # select.error wasn't a subclass of OSError in the past.
+            errcode = None
+            if hasattr(e, "errno"):
+                errcode = e.errno
+            elif hasattr(e, "args"):
+                errcode = e.args[0]
+
+            # Also test for the Windows equivalent of EINTR.
+            is_interrupt = (errcode == errno.EINTR or (hasattr(errno, "WSAEINTR") and
+                                                       errcode == errno.WSAEINTR))
+
+            if is_interrupt:
+                if expires is not None:
+                    current_time = monotonic()
+                    if current_time > expires:
+                        raise OSError(errno=errno.ETIMEDOUT)
+                    if recalc_timeout:
+                        if "timeout" in kwargs:
+                            kwargs["timeout"] = expires - current_time
+                continue
+            if errcode:
+                raise SelectorError(errcode)
+            else:
+                raise
+    return result
+
+
+SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data'])
+
+
+class _SelectorMapping(Mapping):
+    """ Mapping of file objects to selector keys """
+
+    def __init__(self, selector):
+        self._selector = selector
+
+    def __len__(self):
+        return len(self._selector._fd_to_key)
+
+    def __getitem__(self, fileobj):
+        try:
+            fd = self._selector._fileobj_lookup(fileobj)
+            return self._selector._fd_to_key[fd]
+        except KeyError:
+            raise KeyError("{0!r} is not registered.".format(fileobj))
+
+    def __iter__(self):
+        return iter(self._selector._fd_to_key)
+
+
+class BaseSelector(object):
+    """ Abstract Selector class
+
+    A selector supports registering file objects to be monitored
+    for specific I/O events.
+
+    A file object is a file descriptor or any object with a
+    `fileno()` method. An arbitrary object can be attached to the
+    file object which can be used for example to store context info,
+    a callback, etc.
+
+    A selector can use various implementations (select(), poll(), epoll(),
+    and kqueue()) depending on the platform. The 'DefaultSelector' class uses
+    the most efficient implementation for the current platform.
+    """
+    def __init__(self):
+        # Maps file descriptors to keys.
+        self._fd_to_key = {}
+
+        # Read-only mapping returned by get_map()
+        self._map = _SelectorMapping(self)
+
+    def _fileobj_lookup(self, fileobj):
+        """ Return a file descriptor from a file object.
+        This wraps _fileobj_to_fd() to do an exhaustive
+        search in case the object is invalid but we still
+        have it in our map. Used by unregister() so we can
+        unregister an object that was previously registered
+        even if it is closed. It is also used by _SelectorMapping
+        """
+        try:
+            return _fileobj_to_fd(fileobj)
+        except ValueError:
+
+            # Search through all our mapped keys.
+            for key in self._fd_to_key.values():
+                if key.fileobj is fileobj:
+                    return key.fd
+
+            # Raise ValueError after all.
+            raise
+
+    def register(self, fileobj, events, data=None):
+        """ Register a file object for a set of events to monitor. """
+        if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)):
+            raise ValueError("Invalid events: {0!r}".format(events))
+
+        key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data)
+
+        if key.fd in self._fd_to_key:
+            raise KeyError("{0!r} (FD {1}) is already registered"
+                           .format(fileobj, key.fd))
+
+        self._fd_to_key[key.fd] = key
+        return key
+
+    def unregister(self, fileobj):
+        """ Unregister a file object from being monitored. """
+        try:
+            key = self._fd_to_key.pop(self._fileobj_lookup(fileobj))
+        except KeyError:
+            raise KeyError("{0!r} is not registered".format(fileobj))
+        return key
+
+    def modify(self, fileobj, events, data=None):
+        """ Change a registered file object monitored events and data. """
+        # NOTE: Some subclasses optimize this operation even further.
+        try:
+            key = self._fd_to_key[self._fileobj_lookup(fileobj)]
+        except KeyError:
+            raise KeyError("{0!r} is not registered".format(fileobj))
+
+        if events != key.events:
+            self.unregister(fileobj)
+            key = self.register(fileobj, events, data)
+
+        elif data != key.data:
+            # Use a shortcut to update the data.
+            key = key._replace(data=data)
+            self._fd_to_key[key.fd] = key
+
+        return key
+
+    def select(self, timeout=None):
+        """ Perform the actual selection until some monitored file objects
+        are ready or the timeout expires. """
+        raise NotImplementedError()
+
+    def close(self):
+        """ Close the selector. This must be called to ensure that all
+        underlying resources are freed. """
+        self._fd_to_key.clear()
+        self._map = None
+
+    def get_key(self, fileobj):
+        """ Return the key associated with a registered file object. """
+        mapping = self.get_map()
+        if mapping is None:
+            raise RuntimeError("Selector is closed")
+        try:
+            return mapping[fileobj]
+        except KeyError:
+            raise KeyError("{0!r} is not registered".format(fileobj))
+
+    def get_map(self):
+        """ Return a mapping of file objects to selector keys """
+        return self._map
+
+    def _key_from_fd(self, fd):
+        """ Return the key associated to a given file descriptor
+         Return None if it is not found. """
+        try:
+            return self._fd_to_key[fd]
+        except KeyError:
+            return None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+
+# Almost all platforms have select.select()
+if hasattr(select, "select"):
+    class SelectSelector(BaseSelector):
+        """ Select-based selector. """
+        def __init__(self):
+            super(SelectSelector, self).__init__()
+            self._readers = set()
+            self._writers = set()
+
+        def register(self, fileobj, events, data=None):
+            key = super(SelectSelector, self).register(fileobj, events, data)
+            if events & EVENT_READ:
+                self._readers.add(key.fd)
+            if events & EVENT_WRITE:
+                self._writers.add(key.fd)
+            return key
+
+        def unregister(self, fileobj):
+            key = super(SelectSelector, self).unregister(fileobj)
+            self._readers.discard(key.fd)
+            self._writers.discard(key.fd)
+            return key
+
+        def _select(self, r, w, timeout=None):
+            """ Wrapper for select.select because timeout is a positional arg """
+            return select.select(r, w, [], timeout)
+
+        def select(self, timeout=None):
+            # Selecting on empty lists on Windows errors out.
+            if not len(self._readers) and not len(self._writers):
+                return []
+
+            timeout = None if timeout is None else max(timeout, 0.0)
+            ready = []
+            r, w, _ = _syscall_wrapper(self._select, True, self._readers,
+                                       self._writers, timeout)
+            r = set(r)
+            w = set(w)
+            for fd in r | w:
+                events = 0
+                if fd in r:
+                    events |= EVENT_READ
+                if fd in w:
+                    events |= EVENT_WRITE
+
+                key = self._key_from_fd(fd)
+                if key:
+                    ready.append((key, events & key.events))
+            return ready
+
+
+if hasattr(select, "poll"):
+    class PollSelector(BaseSelector):
+        """ Poll-based selector """
+        def __init__(self):
+            super(PollSelector, self).__init__()
+            self._poll = select.poll()
+
+        def register(self, fileobj, events, data=None):
+            key = super(PollSelector, self).register(fileobj, events, data)
+            event_mask = 0
+            if events & EVENT_READ:
+                event_mask |= select.POLLIN
+            if events & EVENT_WRITE:
+                event_mask |= select.POLLOUT
+            self._poll.register(key.fd, event_mask)
+            return key
+
+        def unregister(self, fileobj):
+            key = super(PollSelector, self).unregister(fileobj)
+            self._poll.unregister(key.fd)
+            return key
+
+        def _wrap_poll(self, timeout=None):
+            """ Wrapper function for select.poll.poll() so that
+            _syscall_wrapper can work with only seconds. """
+            if timeout is not None:
+                if timeout <= 0:
+                    timeout = 0
+                else:
+                    # select.poll.poll() has a resolution of 1 millisecond,
+                    # round away from zero to wait *at least* timeout seconds.
+                    timeout = math.ceil(timeout * 1e3)
+
+            result = self._poll.poll(timeout)
+            return result
+
+        def select(self, timeout=None):
+            ready = []
+            fd_events = _syscall_wrapper(self._wrap_poll, True, timeout=timeout)
+            for fd, event_mask in fd_events:
+                events = 0
+                if event_mask & ~select.POLLIN:
+                    events |= EVENT_WRITE
+                if event_mask & ~select.POLLOUT:
+                    events |= EVENT_READ
+
+                key = self._key_from_fd(fd)
+                if key:
+                    ready.append((key, events & key.events))
+
+            return ready
+
+
+if hasattr(select, "epoll"):
+    class EpollSelector(BaseSelector):
+        """ Epoll-based selector """
+        def __init__(self):
+            super(EpollSelector, self).__init__()
+            self._epoll = select.epoll()
+
+        def fileno(self):
+            return self._epoll.fileno()
+
+        def register(self, fileobj, events, data=None):
+            key = super(EpollSelector, self).register(fileobj, events, data)
+            events_mask = 0
+            if events & EVENT_READ:
+                events_mask |= select.EPOLLIN
+            if events & EVENT_WRITE:
+                events_mask |= select.EPOLLOUT
+            _syscall_wrapper(self._epoll.register, False, key.fd, events_mask)
+            return key
+
+        def unregister(self, fileobj):
+            key = super(EpollSelector, self).unregister(fileobj)
+            try:
+                _syscall_wrapper(self._epoll.unregister, False, key.fd)
+            except SelectorError:
+                # This can occur when the fd was closed since registry.
+                pass
+            return key
+
+        def select(self, timeout=None):
+            if timeout is not None:
+                if timeout <= 0:
+                    timeout = 0.0
+                else:
+                    # select.epoll.poll() has a resolution of 1 millisecond
+                    # but luckily takes seconds so we don't need a wrapper
+                    # like PollSelector. Just for better rounding.
+                    timeout = math.ceil(timeout * 1e3) * 1e-3
+                timeout = float(timeout)
+            else:
+                timeout = -1.0  # epoll.poll() must have a float.
+
+            # We always want at least 1 to ensure that select can be called
+            # with no file descriptors registered. Otherwise will fail.
+            max_events = max(len(self._fd_to_key), 1)
+
+            ready = []
+            fd_events = _syscall_wrapper(self._epoll.poll, True,
+                                         timeout=timeout,
+                                         maxevents=max_events)
+            for fd, event_mask in fd_events:
+                events = 0
+                if event_mask & ~select.EPOLLIN:
+                    events |= EVENT_WRITE
+                if event_mask & ~select.EPOLLOUT:
+                    events |= EVENT_READ
+
+                key = self._key_from_fd(fd)
+                if key:
+                    ready.append((key, events & key.events))
+            return ready
+
+        def close(self):
+            self._epoll.close()
+            super(EpollSelector, self).close()
+
+
+if hasattr(select, "kqueue"):
+    class KqueueSelector(BaseSelector):
+        """ Kqueue / Kevent-based selector """
+        def __init__(self):
+            super(KqueueSelector, self).__init__()
+            self._kqueue = select.kqueue()
+
+        def fileno(self):
+            return self._kqueue.fileno()
+
+        def register(self, fileobj, events, data=None):
+            key = super(KqueueSelector, self).register(fileobj, events, data)
+            if events & EVENT_READ:
+                kevent = select.kevent(key.fd,
+                                       select.KQ_FILTER_READ,
+                                       select.KQ_EV_ADD)
+
+                _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+
+            if events & EVENT_WRITE:
+                kevent = select.kevent(key.fd,
+                                       select.KQ_FILTER_WRITE,
+                                       select.KQ_EV_ADD)
+
+                _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+
+            return key
+
+        def unregister(self, fileobj):
+            key = super(KqueueSelector, self).unregister(fileobj)
+            if key.events & EVENT_READ:
+                kevent = select.kevent(key.fd,
+                                       select.KQ_FILTER_READ,
+                                       select.KQ_EV_DELETE)
+                try:
+                    _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+                except SelectorError:
+                    pass
+            if key.events & EVENT_WRITE:
+                kevent = select.kevent(key.fd,
+                                       select.KQ_FILTER_WRITE,
+                                       select.KQ_EV_DELETE)
+                try:
+                    _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0)
+                except SelectorError:
+                    pass
+
+            return key
+
+        def select(self, timeout=None):
+            if timeout is not None:
+                timeout = max(timeout, 0)
+
+            max_events = len(self._fd_to_key) * 2
+            ready_fds = {}
+
+            kevent_list = _syscall_wrapper(self._kqueue.control, True,
+                                           None, max_events, timeout)
+
+            for kevent in kevent_list:
+                fd = kevent.ident
+                event_mask = kevent.filter
+                events = 0
+                if event_mask == select.KQ_FILTER_READ:
+                    events |= EVENT_READ
+                if event_mask == select.KQ_FILTER_WRITE:
+                    events |= EVENT_WRITE
+
+                key = self._key_from_fd(fd)
+                if key:
+                    if key.fd not in ready_fds:
+                        ready_fds[key.fd] = (key, events & key.events)
+                    else:
+                        old_events = ready_fds[key.fd][1]
+                        ready_fds[key.fd] = (key, (events | old_events) & key.events)
+
+            return list(ready_fds.values())
+
+        def close(self):
+            self._kqueue.close()
+            super(KqueueSelector, self).close()
+
+
+# Choose the best implementation, roughly:
+# kqueue == epoll > poll > select. Devpoll not supported. (See above)
+# select() also can't accept a FD > FD_SETSIZE (usually around 1024)
+if 'KqueueSelector' in globals():  # Platform-specific: Mac OS and BSD
+    DefaultSelector = KqueueSelector
+elif 'EpollSelector' in globals():  # Platform-specific: Linux
+    DefaultSelector = EpollSelector
+elif 'PollSelector' in globals():  # Platform-specific: Linux
+    DefaultSelector = PollSelector
+elif 'SelectSelector' in globals():  # Platform-specific: Windows
+    DefaultSelector = SelectSelector
+else:  # Platform-specific: AppEngine
+    def no_selector(_):
+        raise ValueError("Platform does not have a selector")
+    DefaultSelector = no_selector
+    HAS_SELECT = False

requests/packages/urllib3/util/timeout.py

@@ -11,11 +11,8 @@ from ..exceptions import TimeoutStateError
 _Default = object()
 
 
-def current_time():
-    """
-    Retrieve the current time. This function is mocked out in unit testing.
-    """
-    return time.time()
+# Use time.monotonic if available.
+current_time = getattr(time, "monotonic", time.time)
 
 
 class Timeout(object):

requests/packages/urllib3/util/wait.py

@@ -0,0 +1,40 @@
+from .selectors import (
+    HAS_SELECT,
+    DefaultSelector,
+    EVENT_READ,
+    EVENT_WRITE
+)
+
+
+def _wait_for_io_events(socks, events, timeout=None):
+    """ Waits for IO events to be available from a list of sockets
+    or optionally a single socket if passed in. Returns a list of
+    sockets that can be interacted with immediately. """
+    if not HAS_SELECT:
+        raise ValueError('Platform does not have a selector')
+    if not isinstance(socks, list):
+        # Probably just a single socket.
+        if hasattr(socks, "fileno"):
+            socks = [socks]
+        # Otherwise it might be a non-list iterable.
+        else:
+            socks = list(socks)
+    with DefaultSelector() as selector:
+        for sock in socks:
+            selector.register(sock, events)
+        return [key[0].fileobj for key in
+                selector.select(timeout) if key[1] & events]
+
+
+def wait_for_read(socks, timeout=None):
+    """ Waits for reading to be available from a list of sockets
+    or optionally a single socket if passed in. Returns a list of
+    sockets that can be read from immediately. """
+    return _wait_for_io_events(socks, EVENT_READ, timeout)
+
+
+def wait_for_write(socks, timeout=None):
+    """ Waits for writing to be available from a list of sockets
+    or optionally a single socket if passed in. Returns a list of
+    sockets that can be written to immediately. """
+    return _wait_for_io_events(socks, EVENT_WRITE, timeout)

requests/utils.py

@@ -714,7 +714,7 @@ def guess_json_utf(data):
     # easy as counting the nulls and from their location and count
     # determine the encoding. Also detect a BOM, if present.
     sample = data[:4]
-    if sample in (codecs.BOM_UTF32_LE, codecs.BOM32_BE):
+    if sample in (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE):
         return 'utf-32'     # BOM included
     if sample[:3] == codecs.BOM_UTF8:
         return 'utf-8-sig'  # BOM included, MS style (discouraged)

requirements-to-freeze.txt

@@ -1,4 +0,0 @@
-pytest
-pytest-cov
-pytest-httpbin
-sphinx

requirements.txt

@@ -1,24 +0,0 @@
-alabaster==0.7.7
-Babel==2.2.0
-coverage==4.0.3
-decorator==4.0.9
-docutils==0.12
-Flask==0.10.1
-httpbin==0.5.0
-itsdangerous==0.24
-Jinja2==2.8
-MarkupSafe==0.23
-py==1.4.31
-Pygments==2.1.1
-PySocks==1.5.6
-pytest==2.8.7
-pytest-cov==2.2.1
-pytest-httpbin==0.2.0
-pytest-mock==0.11.0
-pytz==2015.7
-six==1.10.0
-snowballstemmer==1.2.1
-Sphinx==1.3.5
-sphinx-rtd-theme==0.1.9
-Werkzeug==0.11.4
-wheel==0.29.0

setup.py

@@ -47,7 +47,7 @@ packages = [
 ]
 
 requires = []
-test_requirements = ['pytest>=2.8.0', 'pytest-httpbin==0.0.7', 'pytest-cov']
+test_requirements = ['pytest>=2.8.0', 'pytest-httpbin==0.0.7', 'pytest-cov', 'pytest-mock']
 
 with open('requests/__init__.py', 'r') as fd:
     version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
@@ -88,6 +88,7 @@ setup(
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy'
     ),

tests/test_lowlevel.py

@@ -5,7 +5,7 @@ import pytest
 import threading
 import requests
 
-from tests.testserver.server import Server
+from tests.testserver.server import Server, consume_socket_content
 
 from .utils import override_environ
 
@@ -25,6 +25,117 @@ def test_chunked_upload():
     assert r.request.headers['Transfer-Encoding'] == 'chunked'
 
 
+def test_digestauth_401_count_reset_on_redirect():
+    """Ensure we correctly reset num_401_calls after a successful digest auth,
+    followed by a 302 redirect to another digest auth prompt.
+
+    See https://github.com/kennethreitz/requests/issues/1979.
+    """
+    text_401 = (b'HTTP/1.1 401 UNAUTHORIZED\r\n'
+                b'Content-Length: 0\r\n'
+                b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"'
+                b', opaque="372825293d1c26955496c80ed6426e9e", '
+                b'realm="me@kennethreitz.com", qop=auth\r\n\r\n')
+
+    text_302 = (b'HTTP/1.1 302 FOUND\r\n'
+                b'Content-Length: 0\r\n'
+                b'Location: /\r\n\r\n')
+
+    text_200 = (b'HTTP/1.1 200 OK\r\n'
+                b'Content-Length: 0\r\n\r\n')
+
+    expected_digest = (b'Authorization: Digest username="user", '
+                       b'realm="me@kennethreitz.com", '
+                       b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"')
+
+    auth = requests.auth.HTTPDigestAuth('user', 'pass')
+
+    def digest_response_handler(sock):
+        # Respond to initial GET with a challenge.
+        request_content = consume_socket_content(sock, timeout=0.5)
+        assert request_content.startswith(b"GET / HTTP/1.1")
+        sock.send(text_401)
+
+        # Verify we receive an Authorization header in response, then redirect.
+        request_content = consume_socket_content(sock, timeout=0.5)
+        assert expected_digest in request_content
+        sock.send(text_302)
+
+        # Verify Authorization isn't sent to the redirected host,
+        # then send another challenge.
+        request_content = consume_socket_content(sock, timeout=0.5)
+        assert b'Authorization:' not in request_content
+        sock.send(text_401)
+
+        # Verify Authorization is sent correctly again, and return 200 OK.
+        request_content = consume_socket_content(sock, timeout=0.5)
+        assert expected_digest in request_content
+        sock.send(text_200)
+
+        return request_content
+
+    close_server = threading.Event()
+    server = Server(digest_response_handler, wait_to_close_event=close_server)
+
+    with server as (host, port):
+        url = 'http://{0}:{1}/'.format(host, port)
+        r = requests.get(url, auth=auth)
+        # Verify server succeeded in authenticating.
+        assert r.status_code == 200
+        # Verify Authorization was sent in final request.
+        assert 'Authorization' in r.request.headers
+        assert r.request.headers['Authorization'].startswith('Digest ')
+        # Verify redirect happened as we expected.
+        assert r.history[0].status_code == 302
+        close_server.set()
+
+
+def test_digestauth_401_only_sent_once():
+    """Ensure we correctly respond to a 401 challenge once, and then
+    stop responding if challenged again.
+    """
+    text_401 = (b'HTTP/1.1 401 UNAUTHORIZED\r\n'
+                b'Content-Length: 0\r\n'
+                b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"'
+                b', opaque="372825293d1c26955496c80ed6426e9e", '
+                b'realm="me@kennethreitz.com", qop=auth\r\n\r\n')
+
+    expected_digest = (b'Authorization: Digest username="user", '
+                       b'realm="me@kennethreitz.com", '
+                       b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"')
+
+    auth = requests.auth.HTTPDigestAuth('user', 'pass')
+
+    def digest_failed_response_handler(sock):
+        # Respond to initial GET with a challenge.
+        request_content = consume_socket_content(sock, timeout=0.5)
+        assert request_content.startswith(b"GET / HTTP/1.1")
+        sock.send(text_401)
+
+        # Verify we receive an Authorization header in response, then
+        # challenge again.
+        request_content = consume_socket_content(sock, timeout=0.5)
+        assert expected_digest in request_content
+        sock.send(text_401)
+
+        # Verify the client didn't respond to second challenge.
+        request_content = consume_socket_content(sock, timeout=0.5)
+        assert request_content == b''
+
+        return request_content
+
+    close_server = threading.Event()
+    server = Server(digest_failed_response_handler, wait_to_close_event=close_server)
+
+    with server as (host, port):
+        url = 'http://{0}:{1}/'.format(host, port)
+        r = requests.get(url, auth=auth)
+        # Verify server didn't authenticate us.
+        assert r.status_code == 401
+        assert r.history[0].status_code == 401
+        close_server.set()
+
+
 _schemes_by_var_prefix = [
     ('http', ['http']),
     ('https', ['https']),

tests/test_requests.py

@@ -20,7 +20,7 @@ from requests.compat import (
     Morsel, cookielib, getproxies, str, urlparse,
     builtin_str, OrderedDict)
 from requests.cookies import (
-    cookiejar_from_dict, morsel_to_cookie, merge_cookies)
+    cookiejar_from_dict, morsel_to_cookie)
 from requests.exceptions import (
     ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL,
     MissingSchema, ReadTimeout, Timeout, RetryError, TooManyRedirects,
@@ -484,6 +484,8 @@ class TestRequests:
         'username, password', (
             ('user', 'pass'),
             (u'имя'.encode('utf-8'), u'пароль'.encode('utf-8')),
+            (42, 42),
+            (None, None),
         ))
     def test_set_basicauth(self, httpbin, username, password):
         auth = (username, password)
@@ -494,6 +496,16 @@ class TestRequests:
 
         assert p.headers['Authorization'] == _basic_auth_str(username, password)
 
+    def test_basicauth_encodes_byte_strings(self):
+        """Ensure b'test' formats as the byte string "test" rather
+        than the unicode string "b'test'" in Python 3.
+        """
+        auth = (b'\xc5\xafsername', b'test\xc6\xb6')
+        r = requests.Request('GET', 'http://localhost', auth=auth)
+        p = r.prepare()
+
+        assert p.headers['Authorization'] == 'Basic xa9zZXJuYW1lOnRlc3TGtg=='
+
     @pytest.mark.parametrize(
         'url, exception', (
             # Connecting to an unknown domain should raise a ConnectionError
@@ -620,7 +632,7 @@ class TestRequests:
         post1 = requests.post(url, data={'some': 'data'})
         assert post1.status_code == 200
 
-        with open('requirements.txt') as f:
+        with open('Pipfile') as f:
             post2 = requests.post(url, files={'some': f})
         assert post2.status_code == 200
 
@@ -680,7 +692,7 @@ class TestRequests:
         post1 = requests.post(url, data={'some': 'data'})
         assert post1.status_code == 200
 
-        with open('requirements.txt') as f:
+        with open('Pipfile') as f:
             post2 = requests.post(url, data={'some': 'data'}, files={'some': f})
         assert post2.status_code == 200
 
@@ -717,7 +729,7 @@ class TestRequests:
 
     def test_conflicting_post_params(self, httpbin):
         url = httpbin('post')
-        with open('requirements.txt') as f:
+        with open('Pipfile') as f:
             pytest.raises(ValueError, "requests.post(url, data='[{\"some\": \"data\"}]', files={'some': f})")
             pytest.raises(ValueError, "requests.post(url, data=u('[{\"some\": \"data\"}]'), files={'some': f})")
 
@@ -1693,6 +1705,42 @@ class TestRequests:
         resp.close()
         assert resp.raw.closed
 
+    def test_empty_stream_with_auth_does_not_set_content_length_header(self, httpbin):
+        """Ensure that a byte stream with size 0 will not set both a Content-Length
+        and Transfer-Encoding header.
+        """
+        auth = ('user', 'pass')
+        url = httpbin('post')
+        file_obj = io.BytesIO(b'')
+        r = requests.Request('POST', url, auth=auth, data=file_obj)
+        prepared_request = r.prepare()
+        assert 'Transfer-Encoding' in prepared_request.headers
+        assert 'Content-Length' not in prepared_request.headers
+
+    def test_stream_with_auth_does_not_set_transfer_encoding_header(self, httpbin):
+        """Ensure that a byte stream with size > 0 will not set both a Content-Length
+        and Transfer-Encoding header.
+        """
+        auth = ('user', 'pass')
+        url = httpbin('post')
+        file_obj = io.BytesIO(b'test data')
+        r = requests.Request('POST', url, auth=auth, data=file_obj)
+        prepared_request = r.prepare()
+        assert 'Transfer-Encoding' not in prepared_request.headers
+        assert 'Content-Length' in prepared_request.headers
+
+    def test_chunked_upload_does_not_set_content_length_header(self, httpbin):
+        """Ensure that requests with a generator body stream using
+        Transfer-Encoding: chunked, not a Content-Length header.
+        """
+        data = (i for i in [b'a', b'b', b'c'])
+        url = httpbin('post')
+        r = requests.Request('POST', url, data=data)
+        prepared_request = r.prepare()
+        assert 'Transfer-Encoding' in prepared_request.headers
+        assert 'Content-Length' not in prepared_request.headers
+
+
 class TestCaseInsensitiveDict:
 
     @pytest.mark.parametrize(
@@ -2177,18 +2225,71 @@ class TestPreparingURLs(object):
             r.prepare()
 
     @pytest.mark.parametrize(
-        'protocol, url',
+        'input, expected',
         (
-            ("http+unix://", b"http+unix://%2Fvar%2Frun%2Fsocket/path"),
-            ("http+unix://", u"http+unix://%2Fvar%2Frun%2Fsocket/path"),
-            ("mailto", b"mailto:user@example.org"),
-            ("mailto", u"mailto:user@example.org"),
-            ("data", b"data:SSDimaUgUHl0aG9uIQ=="),
+            (
+                b"http+unix://%2Fvar%2Frun%2Fsocket/path",
+                u"http+unix://%2fvar%2frun%2fsocket/path",
+            ),
+            (
+                u"http+unix://%2Fvar%2Frun%2Fsocket/path",
+                u"http+unix://%2fvar%2frun%2fsocket/path",
+            ),
+            (
+                b"mailto:user@example.org",
+                u"mailto:user@example.org",
+            ),
+            (
+                u"mailto:user@example.org",
+                u"mailto:user@example.org",
+            ),
+            (
+                b"data:SSDimaUgUHl0aG9uIQ==",
+                u"data:SSDimaUgUHl0aG9uIQ==",
+            )
         )
     )
-    def test_url_passthrough(self, protocol, url):
-        session = requests.Session()
-        session.mount(protocol, HTTPAdapter())
-        p = requests.Request('GET', url=url)
-        p.prepare()
-        assert p.url == url
+    def test_url_mutation(self, input, expected):
+        """
+        This test validates that we correctly exclude some URLs from
+        preparation, and that we handle others. Specifically, it tests that
+        any URL whose scheme doesn't begin with "http" is left alone, and
+        those whose scheme *does* begin with "http" are mutated.
+        """
+        r = requests.Request('GET', url=input)
+        p = r.prepare()
+        assert p.url == expected
+
+    @pytest.mark.parametrize(
+        'input, params, expected',
+        (
+            (
+                b"http+unix://%2Fvar%2Frun%2Fsocket/path",
+                {"key": "value"},
+                u"http+unix://%2fvar%2frun%2fsocket/path?key=value",
+            ),
+            (
+                u"http+unix://%2Fvar%2Frun%2Fsocket/path",
+                {"key": "value"},
+                u"http+unix://%2fvar%2frun%2fsocket/path?key=value",
+            ),
+            (
+                b"mailto:user@example.org",
+                {"key": "value"},
+                u"mailto:user@example.org",
+            ),
+            (
+                u"mailto:user@example.org",
+                {"key": "value"},
+                u"mailto:user@example.org",
+            ),
+        )
+    )
+    def test_parameters_for_nonstandard_schemes(self, input, params, expected):
+        """
+        Setting paramters for nonstandard schemes is allowed if those schemes
+        begin with "http", and is forbidden otherwise.
+        """
+        r = requests.Request('GET', url=input, params=params)
+        p = r.prepare()
+        assert p.url == expected

tests/test_utils.py

@@ -4,7 +4,7 @@ from io import BytesIO
 
 import pytest
 from requests import compat
-from requests.cookies import RequestsCookieJar, cookiejar_from_dict
+from requests.cookies import RequestsCookieJar
 from requests.structures import CaseInsensitiveDict
 from requests.utils import (
     address_in_network, dotted_netmask,
@@ -274,6 +274,17 @@ class TestGuessJSONUTF:
     def test_bad_utf_like_encoding(self):
         assert guess_json_utf(b'\x00\x00\x00\x00') is None
 
+    @pytest.mark.parametrize(
+        ('encoding', 'expected'), (
+            ('utf-16-be', 'utf-16'),
+            ('utf-16-le', 'utf-16'),
+            ('utf-32-be', 'utf-32'),
+            ('utf-32-le', 'utf-32')
+        ))
+    def test_guess_by_bom(self, encoding, expected):
+        data = u'\ufeff{}'.encode(encoding)
+        assert guess_json_utf(data) == expected
+
 
 USER = PASSWORD = "%!*'();:@&=+$,/?#[] "
 ENCODED_USER = compat.quote(USER, '')