v2.12.3

Revert "Restrict URL preparation to HTTP/HTTPS"

AUTHORS.rst

@@ -172,3 +172,4 @@ Patches and Suggestions
 - Brian Bamsch <bbamsch32@gmail.com> (`@bbamsch <https://github.com/bbamsch>`_)
 - Om Prakash Kumar <omprakash070@gmail.com> (`@iamprakashom <https://github.com/iamprakashom>`_)
 - Philipp Konrad <gardiac2002@gmail.com> (`@gardiac2002 <https://github.com/gardiac2002>`_)
+- Hussain Tamboli <hussaintamboli18@gmail.com> (`@hussaintamboli <https://github.com/hussaintamboli>`_)

HISTORY.rst

@@ -3,6 +3,38 @@
 Release History
 ---------------
 
+2.12.3 (2016-12-01)
++++++++++++++++++++
+
+**Bugfixes**
+
+- Fixed regression from v2.12.1 for URLs with schemes that begin with "http".
+  These URLs have historically been processed as though they were HTTP-schemed
+  URLs, and so have had parameters added. This was removed in v2.12.2 in an
+  overzealous attempt to resolve problems with IDNA-encoding those URLs. This
+  change was reverted: the other fixes for IDNA-encoding have been judged to
+  be sufficient to return to the behaviour Requests had before v2.12.0.
+
+2.12.2 (2016-11-30)
++++++++++++++++++++
+
+**Bugfixes**
+
+- Fixed several issues with IDNA-encoding URLs that are technically invalid but
+  which are widely accepted. Requests will now attempt to IDNA-encode a URL if
+  it can but, if it fails, and the host contains only ASCII characters, it will
+  be passed through optimistically. This will allow users to opt-in to using
+  IDNA2003 themselves if they want to, and will also allow technically invalid
+  but still common hostnames.
+- Fixed an issue where URLs with leading whitespace would raise
+  ``InvalidSchema`` errors.
+- Fixed an issue where some URLs without the HTTP or HTTPS schemes would still
+  have HTTP URL preparation applied to them.
+- Fixed an issue where Unicode strings could not be used in basic auth.
+- Fixed an issue encountered by some Requests plugins where constructing a
+  Response object would cause ``Response.content`` to raise an
+  ``AttributeError``.
+
 2.12.1 (2016-11-16)
 +++++++++++++++++++
 

README.rst

@@ -3,7 +3,7 @@ Requests: HTTP for Humans
 
 .. image:: https://img.shields.io/pypi/v/requests.svg
     :target: https://pypi.python.org/pypi/requests
-
+    
 Requests is the only *Non-GMO* HTTP library for Python, safe for human
 consumption.
 
@@ -29,6 +29,10 @@ Behold, the power of Requests:
 
 See `the similar code, sans Requests <https://gist.github.com/973705>`_.
 
+.. image:: http://docs.python-requests.org/en/master/_static/requests-sidebar.png
+    :target: http://docs.python-requests.org/
+
+
 Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, without the
 need for manual labor. There's no need to manually add query strings to your
 URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling

docs/_templates/sidebarintro.html

@@ -1,6 +1,6 @@
 <p class="logo">
   <a href="{{ pathto(master_doc) }}">
-    <img class="logo" src="{{ pathto('_static/requests-sidebar.png', 1) }}" title="Rezzy the Requests Sea Turtle"/>
+    <img class="logo" src="{{ pathto('_static/requests-sidebar.png', 1) }}" title="https://kennethreitz.org/tattoos"/>
   </a>
 </p>
 
@@ -21,7 +21,7 @@
   allowtransparency="true" frameborder="0" scrolling="0" width="200" height="20"></iframe></p>
 
 <p><a href="https://twitter.com/kennethreitz" class="twitter-follow-button" data-show-count="false">Follow @kennethreitz</a> <script>!function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],p=/^http:/.test(d.location)?'http':'https';if(!d.getElementById(id)){js=d.createElement(s);js.id=id;js.src=p+'://platform.twitter.com/widgets.js';fjs.parentNode.insertBefore(js,fjs);}}(document, 'script', 'twitter-wjs');</script></p>
-
+<p><a href="https://saythanks.io/to/kennethreitz">Say Thanks!</a></p>
 <p><a href="http://tinyletter.com/kennethreitz">Join Mailing List</a>.</p>
 
 <h3>Other Projects</h3>

docs/_templates/sidebarlogo.html

@@ -1,6 +1,6 @@
 <p class="logo">
   <a href="{{ pathto(master_doc) }}">
-    <img class="logo" src="{{ pathto('_static/requests-sidebar.png', 1) }}" title="Rezzy the Requests Sea Turtle"/>
+    <img class="logo" src="{{ pathto('_static/requests-sidebar.png', 1) }}" title="https://kennethreitz.org/tattoos"/>
   </a>
 </p>
 <p>
@@ -19,6 +19,10 @@
 
 <p><a href="http://tinyletter.com/kennethreitz">Join Mailing List</a>.</p>
 
+<hr/>
+
+<p>If you enjoy using this project, <a href="https://saythanks.io/to/kennethreitz">Say Thanks!</a></p>
+
 <p><iframe src="http://ghbtns.com/github-btn.html?user=kennethreitz&type=follow&count=false"
   allowtransparency="true" frameborder="0" scrolling="0" width="200" height="20"></iframe></p>
 

docs/user/quickstart.rst

@@ -178,13 +178,14 @@ In general, however, you should use a pattern like this to save what is being
 streamed to a file::
 
     with open(filename, 'wb') as fd:
-        for chunk in r.iter_content(chunk_size):
+        for chunk in r.iter_content(chunk_size=128):
             fd.write(chunk)
 
 Using ``Response.iter_content`` will handle a lot of what you would otherwise
 have to handle when using ``Response.raw`` directly. When streaming a
 download, the above is the preferred and recommended way to retrieve the
-content.
+content. Note that ``chunk_size`` can be freely adjusted to a number that
+may better fit your use cases.
 
 
 Custom Headers

requests/__init__.py

@@ -41,8 +41,8 @@ is at <http://python-requests.org>.
 """
 
 __title__ = 'requests'
-__version__ = '2.12.1'
-__build__ = 0x021201
+__version__ = '2.12.3'
+__build__ = 0x021203
 __author__ = 'Kenneth Reitz'
 __license__ = 'Apache 2.0'
 __copyright__ = 'Copyright 2016 Kenneth Reitz'

requests/_internal_utils.py

@@ -8,7 +8,7 @@ Provides utility functions that are consumed internally by Requests
 which depend on extremely few external helpers (such as compat)
 """
 
-from .compat import is_py2, builtin_str
+from .compat import is_py2, builtin_str, str
 
 
 def to_native_string(string, encoding='ascii'):
@@ -25,3 +25,18 @@ def to_native_string(string, encoding='ascii'):
             out = string.decode(encoding)
 
     return out
+
+
+def unicode_is_ascii(u_string):
+    """Determine if unicode string only contains ASCII characters.
+
+    :param str u_string: unicode string to check. Must be unicode
+        and not Python 2 `str`.
+    :rtype: bool
+    """
+    assert isinstance(u_string, str)
+    try:
+        u_string.encode('ascii')
+        return True
+    except UnicodeEncodeError:
+        return False

requests/auth.py

@@ -27,9 +27,15 @@ CONTENT_TYPE_MULTI_PART = 'multipart/form-data'
 
 def _basic_auth_str(username, password):
     """Returns a Basic Auth string."""
+    
+    if isinstance(username, str):
+        username = username.encode('latin1')
+
+    if isinstance(password, str):
+        password = password.encode('latin1')
 
     authstr = 'Basic ' + to_native_string(
-        b64encode(('%s:%s' % (username, password)).encode('latin1')).strip()
+        b64encode(b':'.join((username, password))).strip()
     )
 
     return authstr

requests/models.py

@@ -30,7 +30,7 @@ from .packages.urllib3.exceptions import (
 from .exceptions import (
     HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError,
     ContentDecodingError, ConnectionError, StreamConsumedError)
-from ._internal_utils import to_native_string
+from ._internal_utils import to_native_string, unicode_is_ascii
 from .utils import (
     guess_filename, get_auth_from_url, requote_uri,
     stream_decode_response_unicode, to_key_val_list, parse_header_links,
@@ -343,6 +343,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         else:
             url = unicode(url) if is_py2 else str(url)
 
+        # Remove leading whitespaces from url
+        url = url.lstrip()
+
         # Don't do any URL preparation for non-HTTP schemes like `mailto`,
         # `data` etc to work around exceptions from `url_parse`, which
         # handles RFC 3986 only.
@@ -365,11 +368,17 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         if not host:
             raise InvalidURL("Invalid URL %r: No host supplied" % url)
 
-        # Only want to apply IDNA to the hostname
+        # In general, we want to try IDNA encoding every hostname, as that
+        # allows users to automatically get the correct behaviour. However,
+        # we’re quite strict about IDNA encoding, so certain valid hostnames
+        # may fail to encode. On failure, we verify the hostname meets a
+        # minimum standard of only containing ASCII characters, and not starting
+        # with a wildcard (*), before allowing the unencoded hostname through.
         try:
             host = idna.encode(host, uts46=True).decode('utf-8')
         except (UnicodeError, idna.IDNAError):
-            raise InvalidURL('URL has an invalid label.')
+            if not unicode_is_ascii(host) or host.startswith(u'*'):
+                raise InvalidURL('URL has an invalid label.')
 
         # Carefully reconstruct the network location
         netloc = auth or ''
@@ -766,7 +775,7 @@ class Response(object):
                 raise RuntimeError(
                     'The content for this response was already consumed')
 
-            if self.status_code == 0:
+            if self.status_code == 0 or self.raw is None:
                 self._content = None
             else:
                 self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()

tests/test_requests.py

@@ -140,6 +140,11 @@ class TestRequests:
                                    data=u"ööö".encode("utf-8")).prepare()
         assert isinstance(request.body, bytes)
 
+    def test_whitespaces_are_removed_from_url(self):
+        # Test for issue #3696
+        request = requests.Request('GET', ' http://example.com').prepare()
+        assert request.url == 'http://example.com/'
+
     @pytest.mark.parametrize('scheme', ('http://', 'HTTP://', 'hTTp://', 'HttP://'))
     def test_mixed_case_scheme_acceptable(self, httpbin, scheme):
         s = requests.Session()
@@ -475,6 +480,20 @@ class TestRequests:
         r = s.get(url)
         assert r.status_code == 200
 
+    @pytest.mark.parametrize(
+        'username, password', (
+            ('user', 'pass'),
+            (u'имя'.encode('utf-8'), u'пароль'.encode('utf-8')),
+        ))
+    def test_set_basicauth(self, httpbin, username, password):
+        auth = (username, password)
+        url = httpbin('get')
+
+        r = requests.Request('GET', url, auth=auth)
+        p = r.prepare()
+
+        assert p.headers['Authorization'] == _basic_auth_str(username, password)
+
     @pytest.mark.parametrize(
         'url, exception', (
             # Connecting to an unknown domain should raise a ConnectionError
@@ -1075,6 +1094,10 @@ class TestRequests:
         total_seconds = ((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6)
         assert total_seconds > 0.0
 
+    def test_empty_response_has_content_none(self):
+        r = requests.Response()
+        assert r.content is None
+
     def test_response_is_iterable(self):
         r = requests.Response()
         io = StringIO.StringIO('abc')
@@ -1568,10 +1591,15 @@ class TestRequests:
         self._patch_adapter_gzipped_redirect(s, url)
         s.get(url)
 
-    def test_basic_auth_str_is_always_native(self):
-        s = _basic_auth_str("test", "test")
+    @pytest.mark.parametrize(
+        'username, password, auth_str', (
+            ('test', 'test', 'Basic dGVzdDp0ZXN0'),
+            (u'имя'.encode('utf-8'), u'пароль'.encode('utf-8'), 'Basic 0LjQvNGPOtC/0LDRgNC+0LvRjA=='),
+        ))
+    def test_basic_auth_str_is_always_native(self, username, password, auth_str):
+        s = _basic_auth_str(username, password)
         assert isinstance(s, builtin_str)
-        assert s == "Basic dGVzdDp0ZXN0"
+        assert s == auth_str
 
     def test_requests_history_is_saved(self, httpbin):
         r = requests.get(httpbin('redirect/5'))
@@ -2093,6 +2121,7 @@ class TestPreparingURLs(object):
         (
             ('http://google.com', 'http://google.com/'),
             (u'http://ジェーピーニック.jp', u'http://xn--hckqz9bzb1cyrb.jp/'),
+            (u'http://xn--n3h.net/', u'http://xn--n3h.net/'),
             (
                 u'http://ジェーピーニック.jp'.encode('utf-8'),
                 u'http://xn--hckqz9bzb1cyrb.jp/'
@@ -2113,6 +2142,18 @@ class TestPreparingURLs(object):
                 u'http://Königsgäßchen.de/straße'.encode('utf-8'),
                 u'http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe'
             ),
+            (
+                b'http://xn--n3h.net/',
+                u'http://xn--n3h.net/'
+            ),
+            (
+                b'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/',
+                u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/'
+            ),
+            (
+                u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/',
+                u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/'
+            )
         )
     )
     def test_preparing_url(self, url, expected):
@@ -2127,9 +2168,81 @@ class TestPreparingURLs(object):
             b"http://*",
             u"http://*.google.com",
             u"http://*",
+            u"http://☃.net/"
         )
     )
     def test_preparing_bad_url(self, url):
         r = requests.Request('GET', url=url)
         with pytest.raises(requests.exceptions.InvalidURL):
             r.prepare()
+
+    @pytest.mark.parametrize(
+        'input, expected',
+        (
+            (
+                b"http+unix://%2Fvar%2Frun%2Fsocket/path",
+                u"http+unix://%2fvar%2frun%2fsocket/path",
+            ),
+            (
+                u"http+unix://%2Fvar%2Frun%2Fsocket/path",
+                u"http+unix://%2fvar%2frun%2fsocket/path",
+            ),
+            (
+                b"mailto:user@example.org",
+                u"mailto:user@example.org",
+            ),
+            (
+                u"mailto:user@example.org",
+                u"mailto:user@example.org",
+            ),
+            (
+                b"data:SSDimaUgUHl0aG9uIQ==",
+                u"data:SSDimaUgUHl0aG9uIQ==",
+            )
+        )
+    )
+    def test_url_mutation(self, input, expected):
+        """
+        This test validates that we correctly exclude some URLs from
+        preparation, and that we handle others. Specifically, it tests that
+        any URL whose scheme doesn't begin with "http" is left alone, and
+        those whose scheme *does* begin with "http" are mutated.
+        """
+        r = requests.Request('GET', url=input)
+        p = r.prepare()
+        assert p.url == expected
+
+    @pytest.mark.parametrize(
+        'input, params, expected',
+        (
+            (
+                b"http+unix://%2Fvar%2Frun%2Fsocket/path",
+                {"key": "value"},
+                u"http+unix://%2fvar%2frun%2fsocket/path?key=value",
+            ),
+            (
+                u"http+unix://%2Fvar%2Frun%2Fsocket/path",
+                {"key": "value"},
+                u"http+unix://%2fvar%2frun%2fsocket/path?key=value",
+            ),
+            (
+                b"mailto:user@example.org",
+                {"key": "value"},
+                u"mailto:user@example.org",
+            ),
+            (
+                u"mailto:user@example.org",
+                {"key": "value"},
+                u"mailto:user@example.org",
+            ),
+        )
+    )
+    def test_parameters_for_nonstandard_schemes(self, input, params, expected):
+        """
+        Setting paramters for nonstandard schemes is allowed if those schemes
+        begin with "http", and is forbidden otherwise.
+        """
+        r = requests.Request('GET', url=input, params=params)
+        p = r.prepare()
+        assert p.url == expected
+

tests/test_utils.py

@@ -17,6 +17,7 @@ from requests.utils import (
     to_key_val_list, to_native_string,
     unquote_header_value, unquote_unreserved,
     urldefragauth, add_dict_to_cookiejar)
+from requests._internal_utils import unicode_is_ascii
 
 from .compat import StringIO, cStringIO
 
@@ -515,6 +516,7 @@ def test_should_bypass_proxies(url, expected, monkeypatch):
     monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1')
     assert should_bypass_proxies(url) == expected
 
+
 @pytest.mark.parametrize(
     'cookiejar', (
         compat.cookielib.CookieJar(),
@@ -529,3 +531,14 @@ def test_add_dict_to_cookiejar(cookiejar):
     cj = add_dict_to_cookiejar(cookiejar, cookiedict)
     cookies = dict((cookie.name, cookie.value) for cookie in cj)
     assert cookiedict == cookies
+
+
+@pytest.mark.parametrize(
+    'value, expected', (
+                (u'test', True),
+                (u'æíöû', False),
+                (u'ジェーピーニック', False),
+    )
+)
+def test_unicode_is_ascii(value, expected):
+    assert unicode_is_ascii(value) is expected