v0.6.5

Content Streaming

.gitignore

@@ -7,3 +7,4 @@ pylint.txt
 docs/_build
 toy.py
 .gitignore
+junit-report.xml

AUTHORS

@@ -40,3 +40,7 @@ Patches and Suggestions
 - Mikko Ohtamaa
 - Den Shabalin
 - Daniel Miller <danielm@vs-networks.com>
+- Alejandro Giacometti
+- Rick Mak
+- Johan Bergström
+- Josselin Jacquard

AUTHORS.orig

@@ -1,40 +0,0 @@
-Requests is written and maintained by Kenneth Reitz and
-various contributors:
-
-Development Lead
-````````````````
-
-- Kenneth Reitz <me@kennethreitz.com>
-
-
-Patches and Suggestions
-```````````````````````
-
-- Various Pocoo Members
-- Chris Adams
-- Flavio Percoco Premoli
-- Dj Gilcrease
-- Justin Murphy
-- Rob Madole
-- Aram Dulyan
-- Johannes Gorset
-- 村山めがね (Megane Murayama)
-- James Rowe
-- Daniel Schauenberg
-- Zbigniew Siciarz
-- Daniele Tricoli 'Eriol'
-- Richard Boulton
-- Miguel Olivares <miguel@moliware.com>
-- Alberto Paro
-- Jérémy Bethmont
-- 潘旭 (Xu Pan)
-- Tamás Gulácsi
-- Rubén Abad
-- Peter Manser
-- Jeremy Selie
-<<<<<<< HEAD
-- Jens Diemer
-- Alex <@alopatin>
-=======
-- Tom Hogans <tomhsx@gmail.com>
->>>>>>> 0ed641a26ec2200de00e4bbf3d170c767375351e

HACKING

@@ -1,15 +0,0 @@
-Where possible, please follow PEP8 with regard to coding style. Sometimes the
-line length restriction is too hard to follow, so don't bend over backwards
-there.
-
-Triple-quotes should always be """, single quotes are ' unless using " would
-result in less escaping within the string.
-
-All modules, functions, and methods should be well documented reStructuredText
-for Sphinx AutoDoc.
-
-All functionality should be available in pure Python. Optional C (via Cython)
-implementations may be written for performance reasons, but should never
-replace the Python implementation.
-
-Lastly, don't take yourself too seriously :)

HISTORY.rst

@@ -1,6 +1,32 @@
 History
 -------
 
+0.6.5 (2011-10-19)
+++++++++++++++++++
+
+* Offline (fast) test suite.
+* Session dictionary argument merging.
+
+
+0.6.4 (2011-10-13)
+++++++++++++++++++
+
+* Automatic decoding of unicode, based on HTTP Headers.
+* New ``decode_unicode`` setting
+* Removal of ``r.read/close`` methods
+* New ``r.faw`` interface for advanced response usage.*
+* Automatic expansion of parameterized headers
+
+0.6.3 (2011-10-13)
+++++++++++++++++++
+
+* Beautiful ``requests.async`` module, for making async requests w/ gevent.
+
+0.6.2 (2011-10-09)
+++++++++++++++++++
+
+* GET/HEAD obeys allow_redirects=False
+
 
 0.6.1 (2011-08-20)
 ++++++++++++++++++
@@ -28,6 +54,7 @@ History
 * Improved https testing
 * Bugfixes
 
+
 0.5.1 (2011-07-23)
 ++++++++++++++++++
 

MANIFEST.in

@@ -1 +1 @@
-include README.rst LICENSE HISTORY.rst 
+include README.rst LICENSE HISTORY.rst test_requests.py

Makefile

@@ -0,0 +1,16 @@
+init:
+	pip install -r reqs.txt
+
+test:
+	nosetests test_requests.py --with-color
+
+ci: init
+	nosetests test_requests.py --with-xunit --xunit-file=junit-report.xml
+
+stats:
+	pyflakes requests | awk -F\: '{printf "%s:%s: [E]%s\n", $1, $2, $3}' > violations.pyflakes.txt
+
+site:
+	cd docs; make dirhtml
+
+docs: site

debian/changelog

@@ -1,5 +0,0 @@
-python-requests (0.4.1-0) testing; urgency=low
-
-  * Initial Debian package
-
- -- Bruno Clermont <bruno.clermont@gmail.com>  Thu, 26 May 2011 16:25:00 -0500

debian/compat

@@ -1 +0,0 @@
-5

debian/control

@@ -1,13 +0,0 @@
-Source: python-requests
-Section: python
-Priority: optional
-Maintainer: Bruno Clermont <bruno.clermont@gmail.com>
-Homepage: https://github.com/bclermont/requests
-Bugs: https://github.com/bclermont/requests/issues
-Build-Depends: debhelper, python-support
-
-Package: python-requests
-Architecture: all
-Depends: ${python:Depends}, python-support
-Provides: ${python:Provides}
-Description: Python HTTP Requests for Humans.

debian/docs

@@ -1 +0,0 @@
-docs/user/*.rst

debian/pyversions

@@ -1 +0,0 @@
-2.4-

debian/rules

@@ -1,42 +0,0 @@
-#!/usr/bin/make -f
-
-# Verbose mode
-#export DH_VERBOSE=1
-
-clean:
-	dh_testdir
-	dh_testroot
-
-	rm -rf build requests.egg-info
-#	find django-sentry/ -name *.pyc | xargs rm -f
-
-	dh_clean
-
-build:
-	dh_testdir
-
-	python setup.py build
-
-install:
-	dh_testdir
-	dh_installdirs
-
-	python setup.py install --root $(CURDIR)/debian/python-requests
-
-binary-indep: install
-
-binary-arch: install
-	dh_install
-	dh_installdocs
-#	dh_installchangelogs
-	dh_compress
-	dh_fixperms
-	dh_pysupport
-	dh_gencontrol
-	dh_installdeb
-	dh_md5sums
-	dh_builddeb -- -Z lzma -z9
-
-binary: binary-indep binary-arch
-.PHONY: build clean binary-indep binary-arch binary
-

docs/api.rst

@@ -13,34 +13,64 @@ important right here and provide links to the canonical documentation.
 Main Interface
 --------------
 
-All of Request's functionality can be accessed by these 6 methods. They
-all return a :class:`Response <models.Response>` object.
+All of Request's functionality can be accessed by these 7 methods.
+They all return an instance of the :class:`Response <Response>` object.
 
+.. autofunction:: request
 .. autofunction:: head
 .. autofunction:: get
 .. autofunction:: post
 .. autofunction:: put
 .. autofunction:: patch
 .. autofunction:: delete
-.. autofunction:: request
+
 
 -----------
 
 
-.. autoclass:: requests.models.Response
+.. autoclass:: Response
    :inherited-members:
 
+Async
+-----
 
-Exceptions
-----------
+.. module:: requests.async
 
-.. autoexception:: HTTPError
 
-.. autoexception:: RequestException
+.. autofunction:: map
+.. autofunction:: request
+.. autofunction:: head
+.. autofunction:: get
+.. autofunction:: post
+.. autofunction:: put
+.. autofunction:: patch
+.. autofunction:: delete
 
-.. autoexception:: requests.models.AuthenticationError
-.. autoexception:: requests.models.URLRequired
-.. autoexception:: requests.models.InvalidMethod
+
+
+Utilities
+---------
+
+These functions are used internally, but may be useful outside of
+Requests.
+
+.. module:: requests.utils
+
+Cookies
+~~~~~~~
+
+.. autofunction:: dict_from_cookiejar
+.. autofunction:: cookiejar_from_dict
+.. autofunction:: add_dict_to_cookiejar
+
+
+Encodings
+~~~~~~~~~
+
+.. autofunction:: get_encodings_from_content
+.. autofunction:: get_encoding_from_headers
+.. autofunction:: get_unicode_from_response
+.. autofunction:: decode_gzip
 
 
 Internals
@@ -50,8 +80,26 @@ These items are an internal component to Requests, and should never be
 seen by the end user (developer). This part of the API documentation
 exists for those who are extending the functionality of Requests.
 
+Exceptions
+~~~~~~~~~~
+
+.. module:: requests
+
+.. autoexception:: HTTPError
+
+.. autoexception:: RequestException
+
+.. autoexception:: AuthenticationError
+.. autoexception:: URLRequired
+.. autoexception:: InvalidMethod
+.. autoexception:: TooManyRedirects
+
+
+
 Classes
 ~~~~~~~
 
-.. autoclass:: requests.models.Request
+.. autoclass:: requests.Request
    :inherited-members:
+
+

docs/community/faq.rst

@@ -3,7 +3,25 @@
 Frequently Asked Questions
 ==========================
 
-This part of the documentation covers common questions about Requests.
+This part of the documentation answers common questions about Requests.
+
+Encoded Data?
+-------------
+
+Requests automatically decompresses gzip-encoded responses, and does
+its best to decodes response content to unicode when possible.
+it's best to decodes response content to unicode when possible.
+
+You can get direct access to the raw response (and even the socket),
+if needed as well.
+
+
+Custom User-Agents?
+-------------------
+
+Requests allows you to easily override User-Agent strings, along with
+any other HTTP Header.
+
 
 Why not Httplib2?
 -----------------
@@ -34,10 +52,33 @@ Chris Adams gave an excellent summary on
 Python 3 Support?
 -----------------
 
-It's on the way. Here's a list of `supported interpreters <interpreters>`_.
+It's on the way. Here's a list of Python platforms that are officially
+supported:
+
+* cPython 2.5
+* cPython 2.5.5
+* cPython 2.5.6
+* cPython 2.6
+* cPython 2.6.6
+* cPython 2.6.7
+* cPython 2.7
+* cPython 2.7.1
+* cPython 2.7.2
+* PyPy-c 1.4
+* PyPy-c 1.5
+
+
+Support for Python 3.x is coming *very* soon.
 
 
 Keep-alive Support?
 -------------------
 
-It's on the way.
+It's on the way.
+
+
+Proxy Support?
+--------------
+
+You bet!
+

docs/community/out-there.rst

@@ -0,0 +1,63 @@
+Modules
+=======
+
+- `requests-oauth-hook <https://github.com/maraujop/requests-oauth-hook>`_, adds OAuth support to Requests.
+- `FacePy <https://github.com/jgorset/facepy>`_, a Python wrapper to the Facebook API.
+- `robotframework-requests <https://github.com/bulkan/robotframework-requests>`_, a Robot Framework API wrapper.
+- `fullerene <https://github.com/bitprophet/fullerene>`_, a Graphite Dashboard.
+- `urbanairship-python <https://github.com/benjaminws/urbanairship-python>`_, a fork of the Urban Airship API wrapper.
+
+
+Articles & Talks
+================
+- `Python for the Web <http://gun.io/blog/python-for-the-web/>`_ teaches how to use Python to interact with the web, using Requests.
+- `Daniel Greenfield's Review of Requests <http://pydanny.blogspot.com/2011/05/python-http-requests-for-humans.html>`_
+- `My 'Python for Humans' talk <http://python-for-humans.heroku.com>`_ ( `audio <http://codeconf.s3.amazonaws.com/2011/pycodeconf/talks/PyCodeConf2011%20-%20Kenneth%20Reitz.m4a>`_ )
+- `Issac Kelly's 'Consuming Web APIs' talk <http://issackelly.github.com/Consuming-Web-APIs-with-Python-Talk/slides/slides.html>`_
+- `Blog post about Requests via Yum <http://arunsag.wordpress.com/2011/08/17/new-package-python-requests-http-for-humans/>`_
+- `Russian blog post introducing Requests <http://habrahabr.ru/blogs/python/126262/>`_
+
+
+Integrations
+============
+
+ScraperWiki
+------------
+
+`ScraperWiki <https://scraperwiki.com/>`_ is an excellent service that allows
+you to run Python, Ruby, and PHP scraper scripts on the web. Now, Requests
+v0.6.1 is available to use in your scrapers!
+
+To give it a try, simply::
+
+    import requests
+
+
+Managed Packages
+================
+
+Requests is available in a number of popular package formats. Of course,
+the ideal way to install Requests is via The Cheeseshop.
+
+
+Ubuntu & Debian
+---------------
+
+Requests is available installed as a Debian package! Debian Etch Ubuntu, since Oneiric::
+
+    $ apt-get install python-requests
+
+Unfortunately, the most recent version available is  v0.5.0. If you're on the
+Debian Python Package team, I'd love an update of that :)
+
+
+Fedora and RedHat
+-----------------
+
+You can easily install Requests v0.6.1 with yum on rpm-based systems::
+
+    $ yum install python-requests
+
+
+
+

docs/community/updates.rst

@@ -20,7 +20,6 @@ I often tweet about new features and releases of Requests.
 Follow `@kennethreitz <https://twitter.com/kennethreitz>`_ for updates.
 
 
-
 Mailing List
 ------------
 

docs/index.rst

@@ -36,10 +36,18 @@ multipart files, and parameters with simple Python dictionaries, and access the
 response data in the same way. It's powered by :py:class:`urllib2`, but it does
 all the hard work and crazy hacks for you.
 
-Testimonals
------------
+Testimonials
+------------
 
-`Twitter, Inc <http://twitter.com>`_ uses Requests internally.
+`Twitter, Inc <http://twitter.com>`_,
+a U.S. Federal Institution,
+`Readability <http://readability.com>`_, and
+`Work for Pie <http://workforpie.com>`_
+use Requests internally.
+
+**Armin Ronacher**
+    Requests is the perfect example how beautiful an API can be with the
+    right level of abstraction.
 
 **Daniel Greenfeld**
     Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to
@@ -52,10 +60,6 @@ Testimonals
 **Rich Leland**
    Requests is awesome. That is all.
 
-**Steve Pike**
-  I can never remember how to do it the regular way.
-  ``import requests; requests.get()`` is just so easy!
-
 
 User Guide
 ----------
@@ -83,6 +87,7 @@ Requests ecosystem and community.
    :maxdepth: 2
 
    community/faq
+   community/out-there.rst
    community/support
    community/updates
 

docs/user/advanced.rst

@@ -43,6 +43,40 @@ Sessions can also be used to provide default data to the request methods::
     (e.g.. a global proxy, user agent header).
 
 
+Asynchronous Requests
+----------------------
+
+Requests has first-class support for non-blocking i/o requests, powered
+by gevent. This allows you to send a bunch of HTTP requests at the same
+
+First, let's import the async module. Heads up — if you don't have
+`gevent <gevent>`_ this will fail::
+
+    from requests import async
+
+The ``async`` module has the exact same api as ``requests``, except it
+doesn't send the request immediately. Instead, it returns the ``Request``
+object.
+
+We can build a list of ``Request`` objects easily::
+
+    urls = [
+        'http://python-requests.org',
+        'http://httpbin.org',
+        'http://python-guide.org',
+        'http://kennethreitz.com'
+    ]
+
+    rs = [async.get(u) for u in urls]
+
+Now we have a list of ``Request`` objects, ready to be sent. We could
+send them one at a time with ``Request.send()``, but that would take a while.
+Instead, we'll send them all at the same time with ``async.map()``::
+
+    >>> async.map(rs)
+    [<Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>]
+
+
 Event Hooks
 -----------
 
@@ -66,7 +100,7 @@ Available hooks:
 
 You can assign a hook function on a per-request basis by passing a
 ``{hook_name: callback_function}`` dictionary to the ``hooks`` request
-paramaeter::
+parameter::
 
     hooks=dict(args=print_url)
 
@@ -134,4 +168,4 @@ To do so, just configure Requests with a stream to write to::
     >>> requests.settings.verbose = sys.stderr
     >>> requests.get('http://httpbin.org/headers')
     2011-08-17T03:04:23.380175   GET   http://httpbin.org/headers
-    <Response [200]>
+    <Response [200]>

docs/user/install.rst

@@ -3,7 +3,8 @@
 Installation
 ============
 
-This part of the documentation covers the installation of Requests. The first step to using any software package is getting it properly installed.
+This part of the documentation covers the installation of Requests.
+The first step to using any software package is getting it properly installed.
 
 
 Distribute & Pip
@@ -24,7 +25,8 @@ But, you really `shouldn't do that <http://www.pip-installer.org/en/latest/index
 Cheeseshop Mirror
 -----------------
 
-If the Cheeseshop is down, you can also install Requests from Kenneth Reitz's personal `Cheeseshop mirror <http://pip.kreitz.co/>`_::
+If the Cheeseshop is down, you can also install Requests from Kenneth Reitz's
+personal `Cheeseshop mirror <http://pip.kreitz.co/>`_::
 
     $ pip install -i http://pip.kreitz.co/simple requests
 
@@ -48,6 +50,29 @@ Or, download the `zipball <https://github.com/kennethreitz/requests/zipball/mast
     $ curl -O https://github.com/kennethreitz/requests/zipball/master
 
 
-Once you have a copy of the source, you can embed it in your Python package, or install it into your site-packages easily::
+Once you have a copy of the source, you can embed it in your Python package,
+or install it into your site-packages easily::
 
     $ python setup.py install
+
+.. _gevent:
+
+Installing Gevent
+-----------------
+
+If you are using the ``requests.async`` module for making concurrent
+requests, you need to install gevent.
+
+To install gevent, you'll need ``libevent``.
+
+OSX::
+
+    $ brew install libevent
+
+Ubuntu::
+
+    $ apt-get install libevent-dev
+
+Once you have ``libevent``, you can install ``gevent`` with ``pip``::
+
+    $ pip install gevent

docs/user/intro.rst

@@ -21,10 +21,16 @@ All contributions to Requests should keep these important rules in mind.
 ISC License
 -----------
 
+A large number of open source projects you find today are `GPL Licensed`_.
+While the GPL has its time and place, it should most certainly not be your
+go-to license for your next open source project.
 
-A large number of open source projects you find today are `GPL Licensed`_. While the GPL has its time and place, it should most certainly not be your go-to license for your next open source project.
+A project that is released as GPL cannot be used in any commercial product
+without the product itself also being offered as open source.
 
-A project that is released as GPL cannot be used in any commercial product without the product itself also being offered as open source. The MIT, BSD, ISC, and Apache2 licenses are great alternatives to the GPL that allow your open-source software to be used freely in proprietary, closed-source software.
+The MIT, BSD, ISC, and Apache2 licenses are great alternatives to the GPL
+that allow your open-source software to be used freely in proprietary,
+closed-source software.
 
 Requests is released under terms of `The ISC License`_.
 
@@ -41,24 +47,3 @@ Requests License
 
     THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-
-.. _interpreters:
-Python Interpreters
--------------------
-
-At this time, the following Python platforms are officially supported:
-
-* cPython 2.5
-* cPython 2.5.5
-* cPython 2.5.6
-* cPython 2.6
-* cPython 2.6.6
-* cPython 2.6.7
-* cPython 2.7
-* cPython 2.7.1
-* cPython 2.7.2
-* PyPy-c 1.4
-* PyPy-c 1.5
-
-
-Support for Python 3.x is planned.

docs/user/quickstart.rst

@@ -5,7 +5,9 @@ Quickstart
 
 .. module:: requests.models
 
-Eager to get started? This page gives a good introduction in how to get started with Requests. This assumes you already have Requests installed. If you do not, head over to the :ref:`Installation <install>` section.
+Eager to get started? This page gives a good introduction in how to get started
+with Requests. This assumes you already have Requests installed. If you do not,
+head over to the :ref:`Installation <install>` section.
 
 First, make sure that:
 
@@ -25,7 +27,8 @@ Let's get GitHub's public timeline ::
 
     r = requests.get('https://github.com/timeline.json')
 
-Now, we have a :class:`Response` object called ``r``. We can get all the information we need from this.
+Now, we have a :class:`Response` object called ``r``. We can get all the
+information we need from this.
 
 
 Response Content
@@ -90,7 +93,8 @@ interface::
         'content-type': 'application/json; charset=utf-8'
     }
 
-The dictionary is special, though: it's made just for HTTP headers. According to `RFC 2616 <http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html>`_, HTTP
+The dictionary is special, though: it's made just for HTTP headers. According to
+`RFC 2616 <http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html>`_, HTTP
 Headers are case-insensitive.
 
 So, we can access the headers using any capitalization we want::
@@ -118,7 +122,7 @@ If a response contains some Cookies, you can get quick access to them::
     >>> print r.cookies
     {'requests-is': 'awesome'}
 
-The underlying CookieJar is also available for more advanced handing::
+The underlying CookieJar is also available for more advanced handling::
 
     >>> r.request.cookiejar
     <cookielib.CookieJar>

reqs.txt

@@ -0,0 +1,7 @@
+envoy==0.0.2
+httpbin==0.0.4
+gunicorn
+nose
+pyflakes
+omnijson
+rudolf

requests/api.py

@@ -15,18 +15,17 @@ import config
 from .models import Request, Response, AuthObject
 from .status_codes import codes
 from .hooks import dispatch_hook
-from .utils import cookiejar_from_dict
+from .utils import cookiejar_from_dict, header_expand
 
-from urlparse import urlparse
 
 __all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete')
 
 def request(method, url,
     params=None, data=None, headers=None, cookies=None, files=None, auth=None,
-    timeout=None, allow_redirects=False, proxies=None, hooks=None):
+    timeout=None, allow_redirects=False, proxies=None, hooks=None, return_response=True):
 
-    """Constructs and sends a :class:`Request <models.Request>`.
-    Returns :class:`Response <models.Response>` object.
+    """Constructs and sends a :class:`Request <Request>`.
+    Returns :class:`Response <Response>` object.
 
     :param method: method for the new :class:`Request` object.
     :param url: URL for the new :class:`Request` object.
@@ -39,13 +38,21 @@ def request(method, url,
     :param timeout: (optional) Float describing the timeout of the request.
     :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
     :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param return_response: (optional) If False, an un-sent Request object will returned.
     """
 
+    method = str(method).upper()
+
     if cookies is None:
         cookies = {}
 
     cookies = cookiejar_from_dict(cookies)
 
+    # Expand header values
+    if headers:
+        for k, v in headers.items() or {}:
+            headers[k] = header_expand(v)
+
     args = dict(
         method = method,
         url = url,
@@ -55,6 +62,7 @@ def request(method, url,
         cookiejar = cookies,
         files = files,
         auth = auth,
+        hooks = hooks,
         timeout = timeout or config.settings.timeout,
         allow_redirects = allow_redirects,
         proxies = proxies or config.settings.proxies,
@@ -68,6 +76,10 @@ def request(method, url,
     # Pre-request hook.
     r = dispatch_hook('pre_request', hooks, r)
 
+    # Don't send if asked nicely.
+    if not return_response:
+        return r
+
     # Send the HTTP Request.
     r.send()
 
@@ -85,31 +97,19 @@ def get(url, **kwargs):
     """Sends a GET request. Returns :class:`Response` object.
 
     :param url: URL for the new :class:`Request` object.
-    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
-    :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
-    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request.
-    :param allow_redirects: (optional) Boolean. Set to False to disable redirect following.
-    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
+
     kwargs.setdefault('allow_redirects', True)
     return request('GET', url, **kwargs)
 
 
 def head(url, **kwargs):
-
     """Sends a HEAD request. Returns :class:`Response` object.
 
     :param url: URL for the new :class:`Request` object.
-    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
-    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
-    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request.
-    :param allow_redirects: (optional) Boolean. Set to False to disable redirect following.
-    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
     kwargs.setdefault('allow_redirects', True)
@@ -117,22 +117,14 @@ def head(url, **kwargs):
 
 
 def post(url, data='', **kwargs):
-
     """Sends a POST request. Returns :class:`Response` object.
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
-    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
-    :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
-    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request.
-    :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
-    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
-    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
-    return request('POST', url, data=data, **kwargs)
+    return request('post', url, data=data, **kwargs)
 
 
 def put(url, data='', **kwargs):
@@ -140,17 +132,10 @@ def put(url, data='', **kwargs):
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
-    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
-    :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
-    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request.
-    :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
-    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
-    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
-    return request('PUT', url, data=data, **kwargs)
+    return request('put', url, data=data, **kwargs)
 
 
 def patch(url, data='', **kwargs):
@@ -158,31 +143,17 @@ def patch(url, data='', **kwargs):
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
-    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
-    :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
-    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request.
-    :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
-    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
-    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
-    return request('PATCH', url, **kwargs)
+    return request('patch', url, **kwargs)
 
 
 def delete(url, **kwargs):
-
     """Sends a DELETE request. Returns :class:`Response` object.
 
     :param url: URL for the new :class:`Request` object.
-    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
-    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
-    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request.
-    :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
-    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
-    return request('DELETE', url, **kwargs)
+    return request('delete', url, **kwargs)

requests/async.py

@@ -1,41 +1,84 @@
 # -*- coding: utf-8 -*-
 
 """
-    requests.async
-    ~~~~~~~~~~~~~~
+requests.async
+~~~~~~~~~~~~~~
 
-    This module implements the main Requests system, after monkey-patching
-    the urllib2 module with eventlet or gevent..
-
-    :copyright: (c) 2011 by Kenneth Reitz.
-    :license: ISC, see LICENSE for more details.
+This module contains an asynchronous replica of ``requests.api``, powered
+by gevent. All API methods return a ``Request`` instance (as opposed to
+``Response``). A list of requests can be sent with ``map()``.
 """
 
-
-from __future__ import absolute_import
-
-import urllib
-import urllib2
-
-from urllib2 import HTTPError
-
-
 try:
-    import eventlet
-    eventlet.monkey_patch()
+    import gevent
+    from gevent import monkey as curious_george
 except ImportError:
-    pass
+    raise RuntimeError('Gevent is required for requests.async.')
 
-if not 'eventlet' in locals():
-    try:
-        from gevent import monkey
-        monkey.patch_all()
-    except ImportError:
-        pass
+# Monkey-patch.
+curious_george.patch_all(thread=False)
+
+from . import api
+from .hooks import dispatch_hook
+
+
+__all__ = (
+    'map',
+    'get', 'head', 'post', 'put', 'patch', 'delete', 'request'
+)
+
+
+def _patched(f):
+    """Patches a given API function to not send."""
+
+    def wrapped(*args, **kwargs):
+        return f(*args, return_response=False, **kwargs)
+
+    return wrapped
+
+
+def _send(r, pools=None):
+    """Sends a given Request object."""
+
+    if pools:
+        r._pools = pools
+
+    r.send()
+
+    # Post-request hook.
+    r = dispatch_hook('post_request', r.hooks, r)
+
+    # Response manipulation hook.
+    r.response = dispatch_hook('response', r.hooks, r.response)
+
+    return r.response
+
+
+# Patched requests.api functions.
+get = _patched(api.get)
+head = _patched(api.head)
+post = _patched(api.post)
+put = _patched(api.put)
+patch = _patched(api.patch)
+delete = _patched(api.delete)
+request = _patched(api.request)
+
+
+def map(requests, prefetch=True):
+    """Concurrently converts a list of Requests to Responses.
+
+    :param requests: a collection of Request objects.
+    :param prefetch: If False, the content will not be downloaded immediately.
+    """
+
+    jobs = [gevent.spawn(_send, r) for r in requests]
+    gevent.joinall(jobs)
+
+    if prefetch:
+        [r.response.content for r in requests]
+
+    return [r.response for r in requests]
 
 
-if not 'eventlet' in locals():
-    raise ImportError('No Async adaptations of urllib2 found!')
 
 
-from .core import *

requests/config.py

@@ -62,6 +62,7 @@ settings.proxies = None
 settings.verbose = None
 settings.timeout = None
 settings.max_redirects = 30
+settings.decode_unicode = True
 
 #: Use socket.setdefaulttimeout() as fallback?
 settings.timeout_fallback = True

requests/core.py

@@ -12,16 +12,18 @@ This module implements the main Requests system.
 """
 
 __title__ = 'requests'
-__version__ = '0.6.1'
-__build__ = 0x000601
+__version__ = '0.6.5'
+__build__ = 0x000605
 __author__ = 'Kenneth Reitz'
 __license__ = 'ISC'
 __copyright__ = 'Copyright 2011 Kenneth Reitz'
 
 
-from models import HTTPError
+from models import HTTPError, Request, Response
 from api import *
 from exceptions import *
 from sessions import session
 from status_codes import codes
-from config import settings
+from config import settings
+
+import utils

requests/models.py

@@ -9,8 +9,10 @@ requests.models
 import urllib
 import urllib2
 import socket
+import codecs
 import zlib
 
+
 from urllib2 import HTTPError
 from urlparse import urlparse, urlunparse, urljoin
 from datetime import datetime
@@ -20,9 +22,9 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA
 from .structures import CaseInsensitiveDict
 from .packages.poster.encode import multipart_encode
 from .packages.poster.streaminghttp import register_openers, get_handlers
-from .utils import dict_from_cookiejar
-from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects
+from .utils import dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip
 from .status_codes import codes
+from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects
 
 
 REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
@@ -30,14 +32,14 @@ REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
 
 
 class Request(object):
-    """The :class:`Request <models.Request>` object. It carries out all functionality of
+    """The :class:`Request <Request>` object. It carries out all functionality of
     Requests. Recommended interface is with the Requests functions.
     """
 
     def __init__(self,
         url=None, headers=dict(), files=None, method=None, data=dict(),
         params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False,
-        allow_redirects=False, proxies=None):
+        allow_redirects=False, proxies=None, hooks=None):
 
         #: Float describ the timeout of the request.
         #  (Use socket.setdefaulttimeout() as fallback)
@@ -46,24 +48,24 @@ class Request(object):
         #: Request URL.
         self.url = url
 
-        #: Dictonary of HTTP Headers to attach to the :class:`Request <models.Request>`.
+        #: Dictonary of HTTP Headers to attach to the :class:`Request <Request>`.
         self.headers = headers
 
         #: Dictionary of files to multipart upload (``{filename: content}``).
         self.files = files
 
-        #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE.
+        #: HTTP Method to use.
         self.method = method
 
         #: Dictionary or byte of request body data to attach to the
-        #: :class:`Request <models.Request>`.
+        #: :class:`Request <Request>`.
         self.data = None
 
         #: Dictionary or byte of querystring data to attach to the
-        #: :class:`Request <models.Request>`.
+        #: :class:`Request <Request>`.
         self.params = None
 
-        #: True if :class:`Request <models.Request>` is part of a redirect chain (disables history
+        #: True if :class:`Request <Request>` is part of a redirect chain (disables history
         #: and HTTPError storage).
         self.redirect = redirect
 
@@ -76,7 +78,7 @@ class Request(object):
         self.data, self._enc_data = self._encode_params(data)
         self.params, self._enc_params = self._encode_params(params)
 
-        #: :class:`Response <models.Response>` instance, containing
+        #: :class:`Response <Response>` instance, containing
         #: content and metadata of HTTP Response, once :attr:`sent <send>`.
         self.response = Response()
 
@@ -85,15 +87,17 @@ class Request(object):
         if not auth:
             auth = auth_manager.get_auth(self.url)
 
-        #: :class:`AuthObject` to attach to :class:`Request <models.Request>`.
+        #: :class:`AuthObject` to attach to :class:`Request <Request>`.
         self.auth = auth
 
-        #: CookieJar to attach to :class:`Request <models.Request>`.
+        #: CookieJar to attach to :class:`Request <Request>`.
         self.cookiejar = cookiejar
 
         #: True if Request has been sent.
         self.sent = False
 
+        #: Event-handling hooks.
+        self.hooks = hooks
 
         # Header manipulation and defaults.
 
@@ -132,9 +136,16 @@ class Request(object):
             _handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar))
 
         if self.auth:
-            if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)):
+            if not isinstance(self.auth.handler,
+                (urllib2.AbstractBasicAuthHandler,
+                urllib2.AbstractDigestAuthHandler)):
+
                 # TODO: REMOVE THIS COMPLETELY
-                auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password)
+                auth_manager.add_password(
+                    self.auth.realm, self.url,
+                    self.auth.username,
+                    self.auth.password)
+
                 self.auth.handler = self.auth.handler(auth_manager)
                 auth_manager.add_auth(self.url, self.auth)
 
@@ -166,7 +177,10 @@ class Request(object):
 
 
     def _build_response(self, resp, is_error=False):
-        """Build internal :class:`Response <models.Response>` object from given response."""
+        """Build internal :class:`Response <Response>` object
+        from given response.
+        """
+
 
         def build(resp):
 
@@ -175,12 +189,9 @@ class Request(object):
 
             try:
                 response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None))
-                response.read = resp.read
-                response._resp = resp
-                response._close = resp.close
+                response.raw = resp
 
                 if self.cookiejar:
-
                     response.cookies = dict_from_cookiejar(self.cookiejar)
 
 
@@ -206,7 +217,7 @@ class Request(object):
                 ((r.status_code is codes.see_other) or (self.allow_redirects))
             ):
 
-                r.close()
+                r.raw.close()
 
                 if not len(history) < settings.max_redirects:
                     raise TooManyRedirects()
@@ -255,8 +266,8 @@ class Request(object):
 
         Otherwise, assumes the data is already encoded appropriately, and
         returns it twice.
-
         """
+
         if hasattr(data, 'items'):
             result = []
             for k, vs in data.items():
@@ -300,7 +311,6 @@ class Request(object):
         """
 
         self._checks()
-        success = False
 
         # Logging
         if settings.verbose:
@@ -361,10 +371,11 @@ class Request(object):
                 if hasattr(why, 'reason'):
                     if isinstance(why.reason, socket.timeout):
                         why = Timeout(why)
+                    elif isinstance(why.reason, socket.error):
+                        why = Timeout(why)
 
                 self._build_response(why, is_error=True)
 
-
             else:
                 self._build_response(resp)
                 self.response.ok = True
@@ -375,37 +386,46 @@ class Request(object):
         return self.sent
 
 
-
 class Response(object):
-    """The core :class:`Response <models.Response>` object. All
-    :class:`Request <models.Request>` objects contain a
-    :class:`response <models.Response>` attribute, which is an instance
+    """The core :class:`Response <Response>` object. All
+    :class:`Request <Request>` objects contain a
+    :class:`response <Response>` attribute, which is an instance
     of this class.
     """
 
     def __init__(self):
-        #: Raw content of the response, in bytes.
-        #: If ``content-encoding`` of response was set to ``gzip``, the
-        #: response data will be automatically deflated.
+
         self._content = None
+        self._content_consumed = False
+
         #: Integer Code of responded HTTP Status.
         self.status_code = None
+
         #: Case-insensitive Dictionary of Response Headers.
         #: For example, ``headers['content-encoding']`` will return the
         #: value of a ``'Content-Encoding'`` response header.
         self.headers = CaseInsensitiveDict()
+
+        #: File-like object representation of response (for advanced usage).
+        self.raw = None
+
         #: Final URL location of Response.
         self.url = None
+
         #: True if no :attr:`error` occured.
         self.ok = False
+
         #: Resulting :class:`HTTPError` of request, if one occured.
         self.error = None
-        #: A list of :class:`Response <models.Response>` objects from
+
+        #: A list of :class:`Response <Response>` objects from
         #: the history of the Request. Any redirect responses will end
         #: up here.
         self.history = []
-        #: The Request that created the Response.
+
+        #: The :class:`Request <Request>` that created the Response.
         self.request = None
+
         #: A dictionary of Cookies the server sent back.
         self.cookies = None
 
@@ -416,23 +436,65 @@ class Response(object):
 
     def __nonzero__(self):
         """Returns true if :attr:`status_code` is 'OK'."""
+
         return not self.error
 
+    def iter_content(self, chunk_size=10 * 1024, decode_unicode=None):
+        """Iterates over the response data.  This avoids reading the content
+        at once into memory for large responses.  The chunk size is the number
+        of bytes it should read into memory.  This is not necessarily the
+        length of each item returned as decoding can take place.
+        """
+        if self._content_consumed:
+            raise RuntimeError('The content for this response was '
+                               'already consumed')
 
-    def __getattr__(self, name):
-        """Read and returns the full stream when accessing to :attr: `content`"""
-        if name == 'content':
-            if self._content is not None:
-                return self._content
-            self._content = self.read()
-            if self.headers.get('content-encoding', '') == 'gzip':
-                try:
-                    self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS)
-                except zlib.error:
-                    pass
+        def generate():
+            while 1:
+                chunk = self.raw.read(chunk_size)
+                if not chunk:
+                    break
+                yield chunk
+            self._content_consumed = True
+        gen = generate()
+        if 'gzip' in self.headers.get('content-encoding', ''):
+            gen = stream_decode_gzip(gen)
+        if decode_unicode is None:
+            decode_unicode = settings.decode_unicode
+        if decode_unicode:
+            gen = stream_decode_response_unicode(gen, self)
+        return gen
+
+    @property
+    def content(self):
+        """Content of the response, in bytes or unicode
+        (if available).
+        """
+
+        if self._content is not None:
             return self._content
-        else:
-            raise AttributeError
+
+        if self._content_consumed:
+            raise RuntimeError('The content for this response was '
+                               'already consumed')
+
+        # Read the contents.
+        self._content = self.raw.read()
+
+        # Decode GZip'd content.
+        if 'gzip' in self.headers.get('content-encoding', ''):
+            try:
+                self._content = decode_gzip(self._content)
+            except zlib.error:
+                pass
+
+        # Decode unicode content.
+        if settings.decode_unicode:
+            self._content = get_unicode_from_response(self)
+
+        self._content_consumed = True
+        return self._content
+
 
     def raise_for_status(self):
         """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured."""
@@ -440,10 +502,6 @@ class Response(object):
             raise self.error
 
 
-    def close(self):
-        if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'):
-            self._resp.fp._sock.recv = None
-        self._close()
 
 class AuthManager(object):
     """Requests Authentication Manager."""

requests/patches.py

@@ -1,5 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""
-requests.monkeys
-"""

requests/sessions.py

@@ -16,21 +16,60 @@ from .utils import add_dict_to_cookiejar
 
 
 
+def merge_kwargs(local_kwarg, default_kwarg):
+    """Merges kwarg dictionaries.
+
+    If a local key in the dictionary is set to None, it will be removed.
+    """
+
+    if default_kwarg is None:
+        return local_kwarg
+
+    if local_kwarg is None:
+        return default_kwarg
+
+    # Bypass if not a dictionary (e.g. timeout)
+    if not hasattr(default_kwarg, 'items'):
+        return local_kwarg
+
+
+
+    # Update new values.
+    kwargs = default_kwarg.copy()
+    kwargs.update(local_kwarg)
+
+    # Remove keys that are set to None.
+    for (k,v) in local_kwarg.items():
+        if v is None:
+            del kwargs[k]
+
+    return kwargs
+
+
 class Session(object):
     """A Requests session."""
 
     __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks']
 
 
-    def __init__(self, **kwargs):
+    def __init__(self,
+        headers=None,
+        cookies=None,
+        auth=None,
+        timeout=None,
+        proxies=None,
+        hooks=None):
+
+        self.headers = headers or {}
+        self.cookies = cookies or {}
+        self.auth = auth
+        self.timeout = timeout
+        self.proxies = proxies or {}
+        self.hooks = hooks or {}
 
         # Set up a CookieJar to be used by default
         self.cookies = cookielib.FileCookieJar()
 
-        # Map args from kwargs to instance-local variables
-        map(lambda k, v: (k in self.__attrs__) and setattr(self, k, v),
-                kwargs.iterkeys(), kwargs.itervalues())
-
         # Map and wrap requests.api methods
         self._map_api_methods()
 
@@ -42,10 +81,8 @@ class Session(object):
         return self
 
     def __exit__(self, *args):
-        # print args
         pass
 
-
     def _map_api_methods(self):
         """Reads each available method from requests.api and decorates
         them with a wrapper, which inserts any instance-local attributes
@@ -54,23 +91,35 @@ class Session(object):
 
         def pass_args(func):
             def wrapper_func(*args, **kwargs):
-                inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems()
-                        if k in self.__attrs__)
-                # Combine instance-local values with kwargs values, with
-                # priority to values in kwargs
-                kwargs = dict(inst_attrs.items() + kwargs.items())
+
+                # Argument collector.
+                _kwargs = {}
 
                 # If a session request has a cookie_dict, inject the
                 # values into the existing CookieJar instead.
                 if isinstance(kwargs.get('cookies', None), dict):
                     kwargs['cookies'] = add_dict_to_cookiejar(
-                        inst_attrs['cookies'], kwargs['cookies']
+                        self.cookies, kwargs['cookies']
                     )
 
-                if kwargs.get('headers', None) and inst_attrs.get('headers', None):
-                    kwargs['headers'].update(inst_attrs['headers'])
+                for attr in self.__attrs__:
+                # for attr in ['headers',]:
+                    s_val = self.__dict__.get(attr)
+                    r_val = kwargs.get(attr)
+
+                    new_attr = merge_kwargs(r_val, s_val)
+
+                    # Skip attributes that were set to None.
+                    if new_attr is not None:
+                        _kwargs[attr] = new_attr
+
+                # Make sure we didn't miss anything.
+                for (k, v) in kwargs.items():
+                    if k not in _kwargs:
+                        _kwargs[k] = v
+
+                return func(*args, **_kwargs)
 
-                return func(*args, **kwargs)
             return wrapper_func
 
         # Map and decorate each function available in requests.api

requests/utils.py

@@ -9,15 +9,69 @@ that are also useful for external consumption.
 
 """
 
+import cgi
+import codecs
 import cookielib
+import re
+import zlib
 
 
-def dict_from_cookiejar(cookiejar):
-    """Returns a key/value dictionary from a CookieJar."""
+def header_expand(headers):
+    """Returns an HTTP Header value string from a dictionary.
+
+    Example expansion::
+
+        {'text/x-dvi': {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}, 'text/x-c': {}}
+        # Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
+
+        (('text/x-dvi', {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}), ('text/x-c', {}))
+        # Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
+    """
+
+    collector = []
+
+    if isinstance(headers, dict):
+        headers = headers.items()
+
+    elif isinstance(headers, basestring):
+        return headers
+
+    for i, (value, params) in enumerate(headers):
+
+        _params = []
+
+        for (p_k, p_v) in params.items():
+
+            _params.append('%s=%s' % (p_k, p_v))
+
+        collector.append(value)
+        collector.append('; ')
+
+        if len(params):
+
+            collector.append('; '.join(_params))
+
+            if not len(headers) == i+1:
+                collector.append(', ')
+
+
+    # Remove trailing seperators.
+    if collector[-1] in (', ', '; '):
+        del collector[-1]
+
+    return ''.join(collector)
+
+
+
+def dict_from_cookiejar(cj):
+    """Returns a key/value dictionary from a CookieJar.
+
+    :param cj: CookieJar object to extract cookies from.
+    """
 
     cookie_dict = {}
 
-    for _, cookies in cookiejar._cookies.items():
+    for _, cookies in cj._cookies.items():
         for _, cookies in cookies.items():
             for cookie in cookies.values():
                 # print cookie
@@ -27,7 +81,10 @@ def dict_from_cookiejar(cookiejar):
 
 
 def cookiejar_from_dict(cookie_dict):
-    """Returns a CookieJar from a key/value dictionary."""
+    """Returns a CookieJar from a key/value dictionary.
+
+    :param cookie_dict: Dict of key/values to insert into CookieJar.
+    """
 
     # return cookiejar if one was passed in
     if isinstance(cookie_dict, cookielib.CookieJar):
@@ -42,7 +99,11 @@ def cookiejar_from_dict(cookie_dict):
 
 
 def add_dict_to_cookiejar(cj, cookie_dict):
-    """Returns a CookieJar from a key/value dictionary."""
+    """Returns a CookieJar from a key/value dictionary.
+
+    :param cj: CookieJar to insert cookies into.
+    :param cookie_dict: Dict of key/values to insert into CookieJar.
+    """
 
     for k, v in cookie_dict.items():
 
@@ -70,3 +131,124 @@ def add_dict_to_cookiejar(cj, cookie_dict):
         cj.set_cookie(cookie)
 
     return cj
+
+
+def get_encodings_from_content(content):
+    """Returns encodings from given content string.
+
+    :param content: bytestring to extract encodings from.
+    """
+
+    charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
+
+    return charset_re.findall(content)
+
+
+def get_encoding_from_headers(headers):
+    """Returns encodings from given HTTP Header Dict.
+
+    :param headers: dictionary to extract encoding from.
+    """
+
+    content_type = headers.get('content-type')
+
+    if not content_type:
+        return None
+
+    content_type, params = cgi.parse_header(content_type)
+
+    if 'charset' in params:
+        return params['charset'].strip("'\"")
+
+
+def unicode_from_html(content):
+    """Attempts to decode an HTML string into unicode.
+    If unsuccessful, the original content is returned.
+    """
+
+    encodings = get_encodings_from_content(content)
+
+    for encoding in encodings:
+
+        try:
+            return unicode(content, encoding)
+        except (UnicodeError, TypeError):
+            pass
+
+        return content
+
+
+def stream_decode_response_unicode(iterator, r):
+    """Stream decodes a iterator."""
+    encoding = get_encoding_from_headers(r.headers)
+    if encoding is None:
+        for item in iterator:
+            yield item
+        return
+
+    decoder = codecs.getincrementaldecoder(encoding)(errors='replace')
+    for chunk in iterator:
+        rv = decoder.decode(chunk)
+        if rv:
+            yield rv
+    rv = decoder.decode('', final=True)
+    if rv:
+        yield rv
+
+
+def get_unicode_from_response(r):
+    """Returns the requested content back in unicode.
+
+    :param r: Reponse object to get unicode content from.
+
+    Tried:
+
+    1. charset from content-type
+
+    2. every encodings from ``<meta ... charset=XXX>``
+
+    3. fall back and replace all unicode characters
+
+    """
+
+    tried_encodings = []
+
+    # Try charset from content-type
+    encoding = get_encoding_from_headers(r.headers)
+
+    if encoding:
+        try:
+            return unicode(r.content, encoding)
+        except UnicodeError:
+            tried_encodings.append(encoding)
+
+    # Fall back:
+    try:
+        return unicode(r.content, encoding, errors='replace')
+    except TypeError:
+        return r.content
+
+
+def decode_gzip(content):
+    """Return gzip-decoded string.
+
+    :param content: bytestring to gzip-decode.
+    """
+
+    return zlib.decompress(content, 16 + zlib.MAX_WBITS)
+
+
+def stream_decode_gzip(iterator):
+    """Stream decodes a gzip-encoded iterator"""
+    try:
+        dec = zlib.decompressobj(16 + zlib.MAX_WBITS)
+        for chunk in iterator:
+            rv = dec.decompress(chunk)
+            if rv:
+                yield rv
+        buf = dec.decompress('')
+        rv = buf + dec.flush()
+        if rv:
+            yield rv
+    except zlib.error:
+        pass

test_requests.py

@@ -3,8 +3,12 @@
 
 from __future__ import with_statement
 
-import unittest
+import time
 import cookielib
+import os
+import unittest
+
+import envoy
 
 try:
     import omnijson as json
@@ -15,12 +19,10 @@ import requests
 
 from requests.sessions import Session
 
+PORT = os.environ.get('HTTPBIN_PORT', '7045')
 
-HTTPBIN_URL = 'http://httpbin.org/'
-HTTPSBIN_URL = 'https://httpbin.ep.io/'
-
-# HTTPBIN_URL = 'http://staging.httpbin.org/'
-# HTTPSBIN_URL = 'https://httpbin-staging.ep.io/'
+HTTPBIN_URL = 'http://0.0.0.0:%s/' % (PORT)
+# HTTPBIN_URL = 'http://127.0.0.1:8000/'
 
 
 def httpbin(*suffix):
@@ -29,27 +31,32 @@ def httpbin(*suffix):
     return HTTPBIN_URL + '/'.join(suffix)
 
 
-def httpsbin(*suffix):
-    """Returns url for HTTPSBIN resource."""
-
-    return HTTPSBIN_URL + '/'.join(suffix)
-
-
-SERVICES = (httpbin, httpsbin)
-
+SERVICES = (httpbin, )
 
+_httpbin = False
 
 class RequestsTestSuite(unittest.TestCase):
     """Requests test cases."""
 
+    # It goes to eleven.
+    _multiprocess_can_split_ = True
 
     def setUp(self):
-        pass
+
+        global _httpbin
+
+        if not _httpbin:
+
+            self.httpbin = envoy.connect('gunicorn httpbin:app --bind=0.0.0.0:%s' % (PORT))
+
+            _httpbin = True
+            time.sleep(1)
+
 
 
     def tearDown(self):
         """Teardown."""
-        pass
+        # self.httpbin.kill()
 
 
     def test_invalid_url(self):
@@ -57,13 +64,16 @@ class RequestsTestSuite(unittest.TestCase):
 
 
     def test_HTTP_200_OK_GET(self):
-        r = requests.get(httpbin('/'))
+        r = requests.get(httpbin('/get'))
         self.assertEqual(r.status_code, 200)
 
+    def test_HTTP_302_ALLOW_REDIRECT_GET(self):
+	r = requests.get(httpbin('redirect', '1'))
+	self.assertEqual(r.status_code, 200)
 
-    def test_HTTPS_200_OK_GET(self):
-        r = requests.get(httpsbin('/'))
-        self.assertEqual(r.status_code, 200)
+    def test_HTTP_302_GET(self):
+	r = requests.get(httpbin('redirect', '1'), allow_redirects=False)
+	self.assertEqual(r.status_code, 302)
 
 
     def test_HTTP_200_OK_GET_WITH_PARAMS(self):
@@ -103,12 +113,7 @@ class RequestsTestSuite(unittest.TestCase):
 
 
     def test_HTTP_200_OK_HEAD(self):
-        r = requests.head(httpbin('/'))
-        self.assertEqual(r.status_code, 200)
-
-
-    def test_HTTPS_200_OK_HEAD(self):
-        r = requests.head(httpsbin('/'))
+        r = requests.head(httpbin('/get'))
         self.assertEqual(r.status_code, 200)
 
 
@@ -117,21 +122,11 @@ class RequestsTestSuite(unittest.TestCase):
         self.assertEqual(r.status_code, 200)
 
 
-    def test_HTTPS_200_OK_PUT(self):
-        r = requests.put(httpsbin('put'))
-        self.assertEqual(r.status_code, 200)
-
-
     def test_HTTP_200_OK_PATCH(self):
         r = requests.patch(httpbin('patch'))
         self.assertEqual(r.status_code, 200)
 
 
-    def test_HTTPS_200_OK_PATCH(self):
-        r = requests.patch(httpsbin('patch'))
-        self.assertEqual(r.status_code, 200)
-
-
     def test_AUTH_HTTP_200_OK_GET(self):
 
         for service in SERVICES:
@@ -197,7 +192,7 @@ class RequestsTestSuite(unittest.TestCase):
             r = requests.get(service('status', '500'))
             self.assertEqual(bool(r), False)
 
-            r = requests.get(service('/'))
+            r = requests.get(service('/get'))
             self.assertEqual(bool(r), True)
 
 
@@ -248,7 +243,7 @@ class RequestsTestSuite(unittest.TestCase):
 
         for service in SERVICES:
 
-            url = service('/')
+            url = service('/get')
 
             requests.get(url, params={'foo': u'føø'})
             requests.get(url, params={u'føø': u'føø'})
@@ -266,19 +261,6 @@ class RequestsTestSuite(unittest.TestCase):
             self.assertEquals(r.status_code, 401)
 
 
-    def test_settings(self):
-
-        def test():
-            r = requests.get(httpbin(''))
-            r.raise_for_status()
-
-        with requests.settings(timeout=0.0000001):
-            self.assertRaises(requests.Timeout, test)
-
-        with requests.settings(timeout=100):
-            requests.get(httpbin(''))
-
-
     def test_urlencoded_post_data(self):
 
         for service in SERVICES:
@@ -377,9 +359,9 @@ class RequestsTestSuite(unittest.TestCase):
             self.assertEquals(rbody.get('data'), 'foobar')
 
 
-    def test_idna(self):
-        r = requests.get(u'http://➡.ws/httpbin')
-        assert 'httpbin' in r.url
+    # def test_idna(self):
+    #     r = requests.get(u'http://➡.ws/httpbin')
+    #     assert 'httpbin' in r.url
 
 
     def test_urlencoded_get_query_multivalued_param(self):
@@ -461,14 +443,7 @@ class RequestsTestSuite(unittest.TestCase):
     def test_session_HTTP_200_OK_GET(self):
 
         s = Session()
-        r = s.get(httpbin('/'))
-        self.assertEqual(r.status_code, 200)
-
-
-    def test_session_HTTPS_200_OK_GET(self):
-
-        s = Session()
-        r = s.get(httpsbin('/'))
+        r = s.get(httpbin('/get'))
         self.assertEqual(r.status_code, 200)
 
 
@@ -478,15 +453,19 @@ class RequestsTestSuite(unittest.TestCase):
 
         s = Session()
         s.headers = heads
+
         # Make 2 requests from Session object, should send header both times
         r1 = s.get(httpbin('user-agent'))
-
         assert heads['User-agent'] in r1.content
+
         r2 = s.get(httpbin('user-agent'))
-
         assert heads['User-agent'] in r2.content
-        self.assertEqual(r2.status_code, 200)
 
+        new_heads = {'User-agent': 'blah'}
+        r3 = s.get(httpbin('user-agent'), headers=new_heads)
+        assert new_heads['User-agent'] in r3.content
+
+        self.assertEqual(r2.status_code, 200)
 
 
 if __name__ == '__main__':

tox.ini

@@ -1,8 +0,0 @@
-[tox]
-envlist = py25,py26,py27
-
-[testenv]
-commands=py.test --junitxml=junit-{envname}.xml
-deps =
-    pytest
-    omnijson