authors update

Make requests.async.send asynchronous

AUTHORS.rst

@@ -34,7 +34,7 @@ Patches and Suggestions
 - Alberto Paro
 - Jérémy Bethmont
 - 潘旭 (Xu Pan)
-- Tamás Gulácsi
+- Gulácsi Tamás
 - Rubén Abad
 - Peter Manser
 - Jeremy Selier
@@ -69,3 +69,30 @@ Patches and Suggestions
 - Ori Livneh
 - Jason Emerick
 - Bryan Helmig
+- Jonas Obrist
+- Lucian Ursu
+- Tom Moertel
+- Frank Kumro Jr
+- Mike Waldner
+- Luca De Vitis
+- digitalxero
+- Ronny Pfannschmidt
+- Chase Sterling
+- Andrey Petrov
+- Michael Van Veen
+- Devin Sevilla
+- Douglas Morrison
+- jbergstroem
+- jbrendel
+- Mark Story
+- Nick Hatch
+- Pat Nakajima
+- Piotr Dobrogost
+- Raymond Penners
+- Bryan Berg
+- Shane McDonald
+- Simon Sapin
+- Clermont
+- Ben Edwards
+- verm666
+- Juarez Bochi

HISTORY.rst

@@ -1,6 +1,19 @@
 History
 -------
 
+0.9.2 (2012-01-18)
+++++++++++++++++++
+
+* Asyncronous async.send method.
+* SSL verify=False bugfix (apparent on windows machines).
+* Support for proper chunk streams with boundaries.
+* session argument for Session classes.
+* Print entire hook tracebacks, not just exception instance.
+* Fix response.iter_lines from pending next line.
+* Fix but in HTTP-digest auth w/ URI having query strings.
+* Fix in Event Hooks section.
+* Urllib3 update.
+
 
 0.9.1 (2012-01-06)
 ++++++++++++++++++
@@ -87,6 +100,7 @@ History
 * OPTION method
 * Async pool size throttling
 * File uploads send real names
+* Vendored in urllib3
 
 0.7.6 (2011-11-07)
 ++++++++++++++++++

README.rst

@@ -27,8 +27,9 @@ See `the same code, without Requests <https://gist.github.com/973705>`_.
 Requests allow you to send  **HEAD**, **GET**, **POST**, **PUT**,
 **PATCH**, and **DELETE** HTTP requests. You can add headers, form data,
 multipart files, and parameters with simple Python dictionaries, and access the
-response data in the same way. It's powered by httplib, but it does
-all the hard work and crazy hacks for you.
+response data in the same way. It's powered by httplib and `urllib3
+<https://github.com/shazow/urllib3>`_, but it does all the hard work and crazy
+hacks for you.
 
 
 Features

docs/_themes/kr/layout.html

@@ -25,6 +25,14 @@
         })();
     /* ]]> */
     </script>
+
+
+    <script type="text/javascript" src="//www.hellobar.com/hellobar.js"></script>
+    <script type="text/javascript">
+        new HelloBar(36402,48802);
+    </script>
+
+
     <script type="text/javascript">
 
       var _gaq = _gaq || [];
@@ -54,4 +62,6 @@
         s.parentNode.insertBefore(t, s);
       })();
     </script>
+
+
 {%- endblock %}

docs/index.rst

@@ -28,7 +28,7 @@ Things shouldn’t be this way. Not in Python.
 
 See `the same code, without Requests <https://gist.github.com/973705>`_.
 
-Requests takes all of the work out of Python HTTP — making your integration with web services seamless. There's no need to manually add query strings to your URLs, or to form-encode your POST data.
+Requests takes all of the work out of Python HTTP — making your integration with web services seamless. There's no need to manually add query strings to your URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling are 100%  automatic, powered by `urllib3 <https://github.com/shazow/urllib3>`_, which is embedded within Requests.
 
 
 Testimonials

docs/user/advanced.rst

@@ -53,9 +53,9 @@ Requests can verify SSL certificates for HTTPS requests, just like a web browser
     >>> requests.get('https://kennethreitz.com', verify=True)
     requests.exceptions.SSLError: hostname 'kennethreitz.com' doesn't match either of '*.herokuapp.com', 'herokuapp.com'
 
-I don't have SSL setup on this domain, so it fails. Excellent. I do have certs for httpbin.org though::
+I don't have SSL setup on this domain, so it fails. Excellent. Github does though::
 
-    >>> requests.get('https://httpbin.org', verify=True)
+    >>> requests.get('https://github.com', verify=True)
     <Response [200]>
 
 You can also pass ``verify`` the path to a CA_BUNDLE file for private certs. You can also set the ``REQUESTS_CA_BUNDLE`` environment variable.
@@ -194,7 +194,7 @@ Let's print some request method arguments at runtime::
 Let's hijack some arguments this time with a new callback::
 
     def hack_headers(args):
-        if not args[headers]:
+        if args.get('headers') is None:
             args['headers'] = dict()
 
         args['headers'].update({'X-Testing': 'True'})

requests/__init__.py

@@ -15,8 +15,8 @@ requests
 """
 
 __title__ = 'requests'
-__version__ = '0.9.1'
-__build__ = 0x000901
+__version__ = '0.9.2'
+__build__ = 0x000902
 __author__ = 'Kenneth Reitz'
 __license__ = 'ISC'
 __copyright__ = 'Copyright 2012 Kenneth Reitz'

requests/api.py

@@ -32,9 +32,10 @@ def request(method, url, **kwargs):
     :param session: (optional) A :class:`Session` object to be used for the request.
     :param config: (optional) A configuration dictionary.
     :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
+    :param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
     """
 
-    s = kwargs.get('session') or sessions.session()
+    s = kwargs.pop('session') if 'session' in kwargs else sessions.session()
     return s.request(method=method, url=url, **kwargs)
 
 

requests/async.py

@@ -46,15 +46,15 @@ def patched(f):
     return wrapped
 
 
-def send(r, pools=None):
-    """Sends a given Request object."""
+def send(r, pool=None):
+    """Sends the request object using the specified pool. If a pool isn't 
+    specified this method blocks. Pools are useful because you can specify size
+    and can hence limit concurrency."""
 
-    if pools:
-        r._pools = pools
+    if pool != None:
+        return pool.spawn(r.send)
 
-    r.send()
-
-    return r.response
+    return gevent.spawn(r.send)
 
 
 # Patched requests.api functions.
@@ -80,17 +80,13 @@ def map(requests, prefetch=True, size=None):
 
     if size:
         pool = Pool(size)
-        pool.map(send, requests)
-        pool.join()
+        jobs = [pool.spawn(r.send) for r in requests]
     else:
-        jobs = [gevent.spawn(send, r) for r in requests]
-        gevent.joinall(jobs)
+        jobs = [gevent.spawn(r.send) for r in requests]
+    
+    gevent.joinall(jobs)
 
     if prefetch:
         [r.response.content for r in requests]
 
-    return [r.response for r in requests]
-
-
-
-
+    return [r.response for r in requests]

requests/auth.py

@@ -86,7 +86,9 @@ class HTTPDigestAuth(AuthBase):
             # XXX not implemented yet
             entdig = None
             p_parsed = urlparse(r.request.url)
-            path = p_parsed.path + p_parsed.query
+            path = p_parsed.path
+            if p_parsed.query:
+                path += '?' + p_parsed.query
 
             A1 = '%s:%s:%s' % (self.username, realm, self.password)
             A2 = '%s:%s' % (r.request.method, path)

requests/hooks.py

@@ -22,7 +22,7 @@ Available hooks:
 
 """
 
-import warnings
+import traceback
 
 
 def dispatch_hook(key, hooks, hook_data):
@@ -34,7 +34,7 @@ def dispatch_hook(key, hooks, hook_data):
         try:
             return hooks.get(key).__call__(hook_data) or hook_data
 
-        except Exception, why:
-            warnings.warn(str(why))
+        except Exception:
+            traceback.print_exc()
 
     return hook_data

requests/models.py

@@ -82,7 +82,6 @@ class Request(object):
         #: Dictionary or byte of querystring data to attach to the
         #: :class:`Request <Request>`.
         self.params = None
-        self.params = dict(params or [])
 
         #: True if :class:`Request <Request>` is part of a redirect chain (disables history
         #: and HTTPError storage).
@@ -298,7 +297,7 @@ class Request(object):
         scheme, netloc, path, params, query, fragment = urlparse(self.url)
 
         if not scheme:
-            raise ValueError()
+            raise ValueError("Invalid URL %r: No schema supplied" %self.url)
 
         netloc = netloc.encode('idna')
 
@@ -612,8 +611,36 @@ class Response(object):
                     break
                 yield chunk
             self._content_consumed = True
+        
+        def generate_chunked():
+            resp = self.raw._original_response
+            fp = resp.fp
+            if resp.chunk_left is not None:
+                pending_bytes = resp.chunk_left
+                while pending_bytes:
+                    chunk = fp.read(min(chunk_size, pending_bytes))
+                    pending_bytes-=len(chunk)
+                    yield chunk
+                fp.read(2) # throw away crlf
+            while 1:
+                #XXX correct line size? (httplib has 64kb, seems insane)
+                pending_bytes = fp.readline(40).strip()
+                pending_bytes = int(pending_bytes, 16)
+                if pending_bytes == 0:
+                    break
+                while pending_bytes:
+                    chunk = fp.read(min(chunk_size, pending_bytes))
+                    pending_bytes-=len(chunk)
+                    yield chunk
+                fp.read(2) # throw away crlf
+            self._content_consumed = True
+            fp.close()
 
-        gen = generate()
+
+        if getattr(getattr(self.raw, '_original_response', None), 'chunked', False):
+            gen = generate_chunked()
+        else:
+            gen = generate()
 
         if 'gzip' in self.headers.get('content-encoding', ''):
             gen = stream_decompress(gen, mode='gzip')
@@ -634,7 +661,7 @@ class Response(object):
         avoids reading the content at once into memory for large
         responses.
         """
-
+        #XXX: why rstrip by default
         pending = None
         for chunk in self.iter_content(chunk_size, decode_unicode=decode_unicode):
             if pending is not None:
@@ -643,7 +670,15 @@ class Response(object):
             for line in lines[:-1]:
                 yield line.rstrip()
             # Save the last part of the chunk for next iteration, to keep full line together
-            pending = lines[-1]
+            # lines may be empty for the last chunk of a chunked response
+            if lines:
+                pending = lines[-1]
+                #if pending is a complete line, give it baack
+                if pending[-1] == '\n':
+                    yield pending.rstrip()
+                    pending = None
+            else:
+                pending = None
 
         # Yield the last line
         if pending is not None:

requests/packages/urllib3/connectionpool.py

@@ -306,7 +306,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             headers = self.headers
 
         if retries < 0:
-            raise MaxRetryError("Max retries exceeded for url: %s" % url)
+            raise MaxRetryError(url)
 
         if timeout is _Default:
             timeout = self.timeout
@@ -320,8 +320,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             if self.port:
                 host = "%s:%d" % (host, self.port)
 
-            raise HostChangedError("Connection pool with host '%s' tried to "
-                                   "open a foreign host: %s" % (host, url))
+            raise HostChangedError(host, url, retries - 1)
 
         conn = None
 
@@ -369,7 +368,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         except (CertificateError), e:
             # Name mismatch
             raise SSLError(e)
-            
+
         except (HTTPException, SocketError), e:
             # Connection broken, discard. It will be replaced next _get_conn().
             conn = None
@@ -385,15 +384,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             return self.urlopen(method, url, body, headers, retries - 1,
                                 redirect, assert_same_host)  # Try again
 
-        # Handle redirection
-        if (redirect and
-            response.status in [301, 302, 303, 307] and
-            'location' in response.headers):  # Redirect, retry
-            log.info("Redirecting %s -> %s" %
-                     (url, response.headers.get('location')))
-            return self.urlopen(method, response.headers.get('location'), body,
-                                headers, retries - 1, redirect,
-                                assert_same_host)
+        # Handle redirect?
+        redirect_location = redirect and response.get_redirect_location()
+        if redirect_location:
+            log.info("Redirecting %s -> %s" % (url, redirect_location))
+            return self.urlopen(method, redirect_location, body, headers,
+                                retries - 1, redirect, assert_same_host)
 
         return response
 

requests/packages/urllib3/exceptions.py

@@ -18,7 +18,9 @@ class SSLError(Exception):
 
 class MaxRetryError(HTTPError):
     "Raised when the maximum number of retries is exceeded."
-    pass
+    def __init__(self, url):
+        HTTPError.__init__(self, "Max retries exceeded for url: %s" % url)
+        self.url = url
 
 
 class TimeoutError(HTTPError):
@@ -28,7 +30,15 @@ class TimeoutError(HTTPError):
 
 class HostChangedError(HTTPError):
     "Raised when an existing pool gets a request for a foreign host."
-    pass
+    def __init__(self, original_host, new_url, retries=3):
+        HTTPError.__init__(self,
+            "Connection pool with host '%s' tried to open a foreign host: %s" %
+            (original_host, new_url))
+
+        self.original_host = original_host
+        self.new_url = new_url
+        self.retries = retries
+
 
 class EmptyPoolError(HTTPError):
     "Raised when a pool runs out of connections and no more are allowed."

requests/packages/urllib3/poolmanager.py

@@ -4,20 +4,18 @@
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
+import logging
+
 from ._collections import RecentlyUsedContainer
-from .connectionpool import (
-    HTTPConnectionPool, HTTPSConnectionPool,
-    get_host, connection_from_url,
-)
+from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
+from .connectionpool import get_host, connection_from_url
+from .exceptions import HostChangedError
+from .request import RequestMethods
 
 
 __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
 
 
-from .request import RequestMethods
-from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
-
-
 pool_classes_by_scheme = {
     'http': HTTPConnectionPool,
     'https': HTTPSConnectionPool,
@@ -28,6 +26,8 @@ port_by_scheme = {
     'https': 443,
 }
 
+log = logging.getLogger(__name__)
+
 
 class PoolManager(RequestMethods):
     """
@@ -105,7 +105,12 @@ class PoolManager(RequestMethods):
         :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
         """
         conn = self.connection_from_url(url)
-        return conn.urlopen(method, url, assert_same_host=False, **kw)
+        try:
+            return conn.urlopen(method, url, **kw)
+
+        except HostChangedError, e:
+            kw['retries'] = e.retries # Persist retries countdown
+            return self.urlopen(method, e.new_url, **kw)
 
 
 class ProxyManager(RequestMethods):

requests/packages/urllib3/response.py

@@ -84,6 +84,19 @@ class HTTPResponse(object):
         if preload_content:
             self._body = self.read(decode_content=decode_content)
 
+    def get_redirect_location(self):
+        """
+        Should we redirect and where to?
+
+        :returns: Truthy redirect location string if we got a redirect status
+            code and valid location. ``None`` if redirect status and no
+            location. ``False`` if not a redirect status code.
+        """
+        if self.status in [301, 302, 303, 307]:
+            return self.headers.get('location')
+
+        return False
+
     def release_conn(self):
         if not self._pool or not self._connection:
             return
@@ -98,10 +111,9 @@ class HTTPResponse(object):
             return self._body
 
         if self._fp:
-            return self.read(decode_content=self._decode_content,
-                             cache_content=True)
+            return self.read(cache_content=True)
 
-    def read(self, amt=None, decode_content=True, cache_content=False):
+    def read(self, amt=None, decode_content=None, cache_content=False):
         """
         Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
         parameters: ``decode_content`` and ``cache_content``.
@@ -124,6 +136,8 @@ class HTTPResponse(object):
         """
         content_encoding = self.headers.get('content-encoding')
         decoder = self.CONTENT_DECODERS.get(content_encoding)
+        if decode_content is None:
+            decode_content = self._decode_content
 
         data = self._fp and self._fp.read(amt)
 
@@ -154,8 +168,8 @@ class HTTPResponse(object):
             if self._original_response and self._original_response.isclosed():
                 self.release_conn()
 
-    @staticmethod
-    def from_httplib(r, **response_kw):
+    @classmethod
+    def from_httplib(ResponseCls, r, **response_kw):
         """
         Given an :class:`httplib.HTTPResponse` instance ``r``, return a
         corresponding :class:`urllib3.response.HTTPResponse` object.
@@ -164,14 +178,14 @@ class HTTPResponse(object):
         with ``original_response=r``.
         """
 
-        return HTTPResponse(body=r,
-                            headers=dict(r.getheaders()),
-                            status=r.status,
-                            version=r.version,
-                            reason=r.reason,
-                            strict=r.strict,
-                            original_response=r,
-                            **response_kw)
+        return ResponseCls(body=r,
+                           headers=dict(r.getheaders()),
+                           status=r.status,
+                           version=r.version,
+                           reason=r.reason,
+                           strict=r.strict,
+                           original_response=r,
+                           **response_kw)
 
     # Backwards-compatibility methods for httplib.HTTPResponse
     def getheaders(self):

test_requests.py

@@ -608,9 +608,9 @@ class RequestsTestSuite(unittest.TestCase):
         lines = (0, 2, 10, 100)
 
         for i in lines:
-
             r = requests.get(httpbin('stream', str(i)), prefetch=False)
-            len_lines = len([l for l in r.iter_lines()])
+            lines = list(r.iter_lines())
+            len_lines = len(lines)
 
             self.assertEqual(i, len_lines)