Merge pull request #2492 from sigmavirus24/release/v2.6.0

Bump version and add release notes for 2.6.0

AUTHORS.rst

@@ -159,3 +159,4 @@ Patches and Suggestions
 - Syed Suhail Ahmed <ssuhail.ahmed93@gmail.com> (`@syedsuhail <https://github.com/syedsuhail>`_)
 - Scott Sadler (`@ssadler <https://github.com/ssadler>`_)
 - Arthur Darcet (`@arthurdarcet <https://github.com/arthurdarcet>`_)
+- Ulrich Petri (`@ulope <https://github.com/ulope>`_)

HISTORY.rst

@@ -3,6 +3,36 @@
 Release History
 ---------------
 
+2.6.0 (2015-03-14)
+++++++++++++++++++
+
+**Bugfixes**
+
+- Fix handling of cookies on redirect. Previously a cookie without a host
+  value set would use the hostname for the redirected URL exposing requests
+  users to session fixation attacks and potentially cookie stealing. This was
+  disclosed privately by Matthew Daley of `BugFuzz <https://bugfuzz.com>`_.
+  An CVE identifier has not yet been assigned for this. This affects all
+  versions of requests from v2.1.0 to v2.5.3 (inclusive on both ends).
+
+- Fix error when requests is an ``install_requires`` dependency and ``python
+  setup.py test`` is run. (#2462)
+
+- Fix error when urllib3 is unbundled and requests continues to use the
+  vendored import location.
+
+- Include fixes to ``urllib3``'s header handling.
+
+- Requests' handling of unvendored dependencies is now more restrictive.
+
+**Features and Improvements**
+
+- Support bytearrays when passed as parameters in the ``files`` argument.
+  (#2468)
+
+- Avoid data duplication when creating a request with ``str``, ``bytes``, or
+  ``bytearray`` input to the ``files`` argument.
+
 2.5.3 (2015-02-24)
 ++++++++++++++++++
 

docs/community/support.rst

@@ -5,12 +5,57 @@ Support
 
 If you have questions or issues about Requests, there are several options:
 
+StackOverflow
+-------------
+
+If your question does not contain sensitive (possibly proprietary)
+information or can be properly anonymized, please ask a question on
+`StackOverflow <https://stackoverflow.com/questions/tagged/python-requests>`_
+and use the tag ``python-requests``.
+
 Send a Tweet
 ------------
 
 If your question is less than 140 characters, feel free to send a tweet to
-`@kennethreitz <http://twitter.com/kennethreitz>`_.
+`@kennethreitz <https://twitter.com/kennethreitz>`_,
+`@sigmavirus24 <https://twitter.com/sigmavirus24>`_, or
+`@lukasaoz <https://twitter.com/lukasaoz>`_.
 
+Vulnerability Disclosure
+------------------------
+
+If you think you have found a potential security vulnerability in requests,
+please email `sigmavirus24 <mailto:graffatcolmingov@gmail.com>`_ and
+`Lukasa <mailto:cory@lukasa.co.uk>`_ directly. **Do not file a public issue.**
+
+Our PGP Key fingerprints are:
+
+- 0161 BB7E B208 B5E0 4FDC  9F81 D9DA 0A04 9113 F853 (@sigmavirus24)
+
+- 90DC AE40 FEA7 4B14 9B70  662D F25F 2144 EEC1 373D (@lukasa)
+
+If English is not your first language, please try to describe the problem and
+its impact to the best of your ability. For greater detail, please use your native
+language and we will try our best to translate it using online services.
+
+Please also include the code you used to find the problem and the shortest amount
+of code necessary to reproduce it.
+
+Please do not disclose this to anyone else. We will retrieve a CVE identifier if
+necessary and give you full credit under whatever name or alias you provide.
+We will only request an identifier when we have a fix and can publish it in a release.
+
+We will respect your privacy and will only publicize your involvement if you grant
+us permission.
+
+Previous CVEs
+~~~~~~~~~~~~~
+
+- Fixed in 2.3.0
+
+  - `CVE 2014-1829 <http://www.cve.mitre.org/cgi-bin/cvename.cgi?name=2014-1829>`_
+
+  - `CVE 2014-1830 <http://www.cve.mitre.org/cgi-bin/cvename.cgi?name=2014-1830>`_
 
 File an Issue
 -------------
@@ -34,4 +79,9 @@ IRC
 The official Freenode channel for Requests is
 `#python-requests <irc://irc.freenode.net/python-requests>`_
 
-I'm also available as **kennethreitz** on Freenode.
+The core developers of requests are on IRC throughout the day.
+You can find them in ``#python-requests`` as:
+
+- kennethreitz
+- lukasa
+- sigmavirus24

docs/user/advanced.rst

@@ -399,6 +399,19 @@ set ``stream`` to ``True`` and iterate over the response with
         if line:
             print(json.loads(line))
 
+.. warning::
+
+    :class:`~requests.Response.iter_lines()` is not reentrant safe.
+    Calling this method multiple times causes some of the received data
+    being lost. In case you need to call it from multiple places, use
+    the resulting iterator object instead::
+
+        lines = r.iter_lines()
+        # Save the first line for later or just skip it
+        first_line = next(lines)
+        for line in lines:
+            print(line)
+
 .. _proxies:
 
 Proxies

requests/__init__.py

@@ -42,7 +42,7 @@ is at <http://python-requests.org>.
 """
 
 __title__ = 'requests'
-__version__ = '2.5.3'
+__version__ = '2.6.0'
 __build__ = 0x020503
 __author__ = 'Kenneth Reitz'
 __license__ = 'Apache 2.0'

requests/adapters.py

@@ -11,10 +11,10 @@ and maintain connections.
 import socket
 
 from .models import Response
-from .packages.urllib3 import Retry
 from .packages.urllib3.poolmanager import PoolManager, proxy_from_url
 from .packages.urllib3.response import HTTPResponse
 from .packages.urllib3.util import Timeout as TimeoutSauce
+from .packages.urllib3.util.retry import Retry
 from .compat import urlparse, basestring
 from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
                     prepend_scheme_if_needed, get_auth_from_url, urldefragauth)

requests/api.py

@@ -16,7 +16,6 @@ from . import sessions
 
 def request(method, url, **kwargs):
     """Constructs and sends a :class:`Request <Request>`.
-    Returns :class:`Response <Response>` object.
 
     :param method: method for the new :class:`Request` object.
     :param url: URL for the new :class:`Request` object.
@@ -37,6 +36,8 @@ def request(method, url, **kwargs):
     :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
     :param stream: (optional) if ``False``, the response content will be immediately downloaded.
     :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
 
     Usage::
 
@@ -55,10 +56,12 @@ def request(method, url, **kwargs):
 
 
 def get(url, **kwargs):
-    """Sends a GET request. Returns :class:`Response` object.
+    """Sends a GET request.
 
     :param url: URL for the new :class:`Request` object.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     kwargs.setdefault('allow_redirects', True)
@@ -66,10 +69,12 @@ def get(url, **kwargs):
 
 
 def options(url, **kwargs):
-    """Sends a OPTIONS request. Returns :class:`Response` object.
+    """Sends a OPTIONS request.
 
     :param url: URL for the new :class:`Request` object.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     kwargs.setdefault('allow_redirects', True)
@@ -77,10 +82,12 @@ def options(url, **kwargs):
 
 
 def head(url, **kwargs):
-    """Sends a HEAD request. Returns :class:`Response` object.
+    """Sends a HEAD request.
 
     :param url: URL for the new :class:`Request` object.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     kwargs.setdefault('allow_redirects', False)
@@ -88,44 +95,52 @@ def head(url, **kwargs):
 
 
 def post(url, data=None, json=None, **kwargs):
-    """Sends a POST request. Returns :class:`Response` object.
+    """Sends a POST request.
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
     :param json: (optional) json data to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     return request('post', url, data=data, json=json, **kwargs)
 
 
 def put(url, data=None, **kwargs):
-    """Sends a PUT request. Returns :class:`Response` object.
+    """Sends a PUT request.
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     return request('put', url, data=data, **kwargs)
 
 
 def patch(url, data=None, **kwargs):
-    """Sends a PATCH request. Returns :class:`Response` object.
+    """Sends a PATCH request.
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     return request('patch', url,  data=data, **kwargs)
 
 
 def delete(url, **kwargs):
-    """Sends a DELETE request. Returns :class:`Response` object.
+    """Sends a DELETE request.
 
     :param url: URL for the new :class:`Request` object.
     :param \*\*kwargs: Optional arguments that ``request`` takes.
+    :return: :class:`Response <Response>` object
+    :rtype: requests.Response
     """
 
     return request('delete', url, **kwargs)

requests/models.py

@@ -143,12 +143,13 @@ class RequestEncodingMixin(object):
             else:
                 fn = guess_filename(v) or k
                 fp = v
-            if isinstance(fp, str):
-                fp = StringIO(fp)
-            if isinstance(fp, bytes):
-                fp = BytesIO(fp)
 
-            rf = RequestField(name=k, data=fp.read(),
+            if isinstance(fp, (str, bytes, bytearray)):
+                fdata = fp
+            else:
+                fdata = fp.read()
+
+            rf = RequestField(name=k, data=fdata,
                               filename=fn, headers=fh)
             rf.make_multipart(content_type=ft)
             new_fields.append(rf)
@@ -688,6 +689,8 @@ class Response(object):
         """Iterates over the response data, one line at a time.  When
         stream=True is set on the request, this avoids reading the
         content at once into memory for large responses.
+
+        .. note:: This method is not reentrant safe.
         """
 
         pending = None

requests/packages/__init__.py

@@ -27,9 +27,13 @@ import sys
 
 class VendorAlias(object):
 
-    def __init__(self):
+    def __init__(self, package_names):
+        self._package_names = package_names
         self._vendor_name = __name__
         self._vendor_pkg = self._vendor_name + "."
+        self._vendor_pkgs = [
+            self._vendor_pkg + name for name in self._package_names
+        ]
 
     def find_module(self, fullname, path=None):
         if fullname.startswith(self._vendor_pkg):
@@ -44,6 +48,14 @@ class VendorAlias(object):
                 )
             )
 
+        if not (name == self._vendor_name or
+                any(name.startswith(pkg) for pkg in self._vendor_pkgs)):
+            raise ImportError(
+                "Cannot import %s, must be one of %s." % (
+                    name, self._vendor_pkgs
+                )
+            )
+
         # Check to see if we already have this item in sys.modules, if we do
         # then simply return that.
         if name in sys.modules:
@@ -92,4 +104,4 @@ class VendorAlias(object):
         return module
 
 
-sys.meta_path.append(VendorAlias())
+sys.meta_path.append(VendorAlias(["urllib3", "chardet"]))

requests/packages/urllib3/__init__.py

@@ -4,7 +4,7 @@ urllib3 - Thread-safe connection pooling and re-using.
 
 __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
 __license__ = 'MIT'
-__version__ = 'dev'
+__version__ = '1.10.2'
 
 
 from .connectionpool import (

requests/packages/urllib3/_collections.py

@@ -20,8 +20,6 @@ from .packages.six import iterkeys, itervalues, PY3
 __all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict']
 
 
-MULTIPLE_HEADERS_ALLOWED = frozenset(['cookie', 'set-cookie', 'set-cookie2'])
-
 _Null = object()
 
 
@@ -143,7 +141,10 @@ class HTTPHeaderDict(dict):
     def __init__(self, headers=None, **kwargs):
         dict.__init__(self)
         if headers is not None:
-            self.extend(headers)
+            if isinstance(headers, HTTPHeaderDict):
+                self._copy_from(headers)
+            else:
+                self.extend(headers)
         if kwargs:
             self.extend(kwargs)
 
@@ -223,11 +224,8 @@ class HTTPHeaderDict(dict):
                 vals.append(val)
             else:
                 # vals should be a tuple then, i.e. only one item so far
-                if key_lower in MULTIPLE_HEADERS_ALLOWED:
-                    # Need to convert the tuple to list for further extension
-                    _dict_setitem(self, key_lower, [vals[0], vals[1], val])
-                else:
-                    _dict_setitem(self, key_lower, new_vals)
+                # Need to convert the tuple to list for further extension
+                _dict_setitem(self, key_lower, [vals[0], vals[1], val])
 
     def extend(*args, **kwargs):
         """Generic import function for any type of header-like object.
@@ -276,14 +274,17 @@ class HTTPHeaderDict(dict):
     def __repr__(self):
         return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))
 
-    def copy(self):
-        clone = type(self)()
-        for key in self:
-            val = _dict_getitem(self, key)
+    def _copy_from(self, other):
+        for key in other:
+            val = _dict_getitem(other, key)
             if isinstance(val, list):
                 # Don't need to convert tuples
                 val = list(val)
-            _dict_setitem(clone, key, val)
+            _dict_setitem(self, key, val)
+
+    def copy(self):
+        clone = type(self)()
+        clone._copy_from(self)
         return clone
 
     def iteritems(self):

requests/packages/urllib3/exceptions.py

@@ -157,3 +157,8 @@ class InsecureRequestWarning(SecurityWarning):
 class SystemTimeWarning(SecurityWarning):
     "Warned when system time is suspected to be wrong"
     pass
+
+
+class InsecurePlatformWarning(SecurityWarning):
+    "Warned when certain SSL configuration is not available on a platform."
+    pass

requests/packages/urllib3/util/ssl_.py

@@ -1,7 +1,7 @@
 from binascii import hexlify, unhexlify
 from hashlib import md5, sha1, sha256
 
-from ..exceptions import SSLError
+from ..exceptions import SSLError, InsecurePlatformWarning
 
 
 SSLContext = None
@@ -10,6 +10,7 @@ create_default_context = None
 
 import errno
 import ssl
+import warnings
 
 try:  # Test for SSL features
     from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
@@ -69,6 +70,14 @@ except ImportError:
             self.ciphers = cipher_suite
 
         def wrap_socket(self, socket, server_hostname=None):
+            warnings.warn(
+                'A true SSLContext object is not available. This prevents '
+                'urllib3 from configuring SSL appropriately and may cause '
+                'certain SSL connections to fail. For more information, see '
+                'https://urllib3.readthedocs.org/en/latest/security.html'
+                '#insecureplatformwarning.',
+                InsecurePlatformWarning
+            )
             kwargs = {
                 'keyfile': self.keyfile,
                 'certfile': self.certfile,

requests/sessions.py

@@ -171,7 +171,10 @@ class SessionRedirectMixin(object):
             except KeyError:
                 pass
 
-            extract_cookies_to_jar(prepared_request._cookies, prepared_request, resp.raw)
+            # Extract any cookies sent on the response to the cookiejar
+            # in the new request. Because we've mutated our copied prepared
+            # request, use the old one that we haven't yet touched.
+            extract_cookies_to_jar(prepared_request._cookies, req, resp.raw)
             prepared_request._cookies.update(self.cookies)
             prepared_request.prepare_cookies(prepared_request._cookies)
 

setup.py

@@ -1,10 +1,9 @@
 #!/usr/bin/env python
 
 import os
+import re
 import sys
 
-import requests
-
 from codecs import open
 
 try:
@@ -29,6 +28,18 @@ packages = [
 
 requires = []
 
+version = ''
+with open('requests/__init__.py', 'r') as fd:
+    reg = re.compile(r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]')
+    for line in fd:
+        m = reg.match(line)
+        if m:
+            version = m.group(1)
+            break
+
+if not version:
+    raise RuntimeError('Cannot find version information')
+
 with open('README.rst', 'r', 'utf-8') as f:
     readme = f.read()
 with open('HISTORY.rst', 'r', 'utf-8') as f:
@@ -36,7 +47,7 @@ with open('HISTORY.rst', 'r', 'utf-8') as f:
 
 setup(
     name='requests',
-    version=requests.__version__,
+    version=version,
     description='Python HTTP for Humans.',
     long_description=readme + '\n\n' + history,
     author='Kenneth Reitz',
@@ -60,7 +71,6 @@ setup(
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4'
-
     ),
     extras_require={
         'security': ['pyOpenSSL', 'ndg-httpsclient', 'pyasn1'],

test_requests.py

@@ -935,6 +935,19 @@ class RequestsTestCase(unittest.TestCase):
 
         assert 'multipart/form-data' in p.headers['Content-Type']
 
+    def test_can_send_bytes_bytearray_objects_with_files(self):
+        # Test bytes:
+        data = {'a': 'this is a string'}
+        files = {'b': b'foo'}
+        r = requests.Request('POST', httpbin('post'), data=data, files=files)
+        p = r.prepare()
+        assert 'multipart/form-data' in p.headers['Content-Type']
+        # Test bytearrays:
+        files = {'b': bytearray(b'foo')}
+        r = requests.Request('POST', httpbin('post'), data=data, files=files)
+        p = r.prepare()
+        assert 'multipart/form-data' in p.headers['Content-Type']
+
     def test_can_send_file_object_with_non_string_filename(self):
         f = io.BytesIO()
         f.name = 2
@@ -1039,6 +1052,23 @@ class RequestsTestCase(unittest.TestCase):
         assert 'application/json' in r.request.headers['Content-Type']
         assert {'life': 42} == r.json()['json']
 
+    def test_response_iter_lines(self):
+        r = requests.get(httpbin('stream/4'), stream=True)
+        assert r.status_code == 200
+
+        it = r.iter_lines()
+        next(it)
+        assert len(list(it)) == 3
+
+    @pytest.mark.xfail
+    def test_response_iter_lines_reentrant(self):
+        """Response.iter_lines() is not reentrant safe"""
+        r = requests.get(httpbin('stream/4'), stream=True)
+        assert r.status_code == 200
+
+        next(r.iter_lines())
+        assert len(list(r.iter_lines())) == 3
+
 
 class TestContentEncodingDetection(unittest.TestCase):
 
@@ -1598,5 +1628,12 @@ def test_urllib3_retries():
     with pytest.raises(RetryError):
         s.get(httpbin('status/500'))
 
+def test_vendor_aliases():
+    from requests.packages import urllib3
+    from requests.packages import chardet
+
+    with pytest.raises(ImportError):
+        from requests.packages import webbrowser
+
 if __name__ == '__main__':
     unittest.main()