v0.14.2

Use a JSON-specific encoding detection when no encoding has been specified

.gitignore

@@ -12,3 +12,5 @@ requests.egg-info/
 *.pyc
 *.swp
 env/
+
+.workon

.workon

@@ -1 +0,0 @@
-/Users/kreitz/.virtualenvs/r26/

AUTHORS.rst

@@ -105,3 +105,14 @@ Patches and Suggestions
 - Danilo Bargen (gwrtheyrn)
 - Torsten Landschoff
 - Michael Holler (apotheos)
+- Timnit Gebru
+- Sarah Gonzalez
+- Victoria Mo
+- Leila Muhtasib
+- Matthias Rahlf <matthias@webding.de>
+- Jakub Roztocil <jakub@roztocil.name>
+- Ian Cordasco <graffatcolmingov@gmail.com> @sigmavirus24
+- Rhys Elsmore
+- André Graf (dergraf)
+- Stephen Zhuang (everbird)
+- Martijn Pieters

HISTORY.rst

@@ -3,6 +3,79 @@
 History
 -------
 
+0.14.2 (2012-10-27)
++++++++++++++++++++
+
+- Improved mime-compatible JSON handling
+- Proxy fixes
+- Path hack fixes
+- Case-Insensistive Content-Encoding headers
+- Support for CJK parameters in form posts
+
+
+0.14.1 (2012-10-01)
++++++++++++++++++++
+
+- Python 3.3 Compatibility
+- Simply default accept-encoding
+- Bugfixes
+
+
+0.14.0 (2012-09-02)
+++++++++++++++++++++
+
+- No more iter_content errors if already downloaded.
+
+0.13.9 (2012-08-25)
++++++++++++++++++++
+
+- Fix for OAuth + POSTs
+- Remove exception eating from dispatch_hook
+- General bugfixes
+
+0.13.8 (2012-08-21)
++++++++++++++++++++
+
+- Incredible Link header support :)
+
+0.13.7 (2012-08-19)
++++++++++++++++++++
+
+- Support for (key, value) lists everywhere.
+- Digest Authentication improvements.
+- Ensure proxy exclusions work properly.
+- Clearer UnicodeError exceptions.
+- Automatic casting of URLs to tsrings (fURL and such)
+- Bugfixes.
+
+0.13.6 (2012-08-06)
++++++++++++++++++++
+
+- Long awaited fix for hanging connections!
+
+0.13.5 (2012-07-27)
++++++++++++++++++++
+
+- Packaging fix
+
+0.13.4 (2012-07-27)
++++++++++++++++++++
+
+- GSSAPI/Kerberos authentication!
+- App Engine 2.7 Fixes!
+- Fix leaking connections (from urllib3 update)
+- OAuthlib path hack fix
+- OAuthlib URL parameters fix.
+
+0.13.3 (2012-07-12)
++++++++++++++++++++
+
+- Use simplejson if available.
+- Do not hide SSLErrors behind Timeouts.
+- Fixed param handling with urls containing fragments.
+- Significantly improved information in User Agent.
+- client certificates are ignored when verify=False
+
 0.13.2 (2012-06-28)
 +++++++++++++++++++
 

NOTICE

@@ -1,4 +1,4 @@
-Request includes some vendorized python libraries to ease installation.
+Requests includes some vendorized python libraries to ease installation.
 
 Urllib3 License
 ===============
@@ -113,4 +113,4 @@ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.rst

@@ -3,6 +3,7 @@ Requests: HTTP for Humans
 
 
 .. image:: https://secure.travis-ci.org/kennethreitz/requests.png?branch=develop
+        :target: https://secure.travis-ci.org/kennethreitz/requests
 
 Requests is an ISC Licensed HTTP library, written in Python, for human
 beings.

docs/MANIFEST.in

@@ -1 +1 @@
-include HISTORY.rst README.rst LICENSE 
+include HISTORY.rst README.rst LICENSE

docs/_templates/sidebarintro.html

@@ -5,7 +5,7 @@
 </p>
 
 <p>
-  <iframe src="http://markdotto.github.com/github-buttons/github-btn.html?user=kennethreitz&repo=requests&type=watch&count=true&size=large"
+  <iframe src="http://ghbtns.com/github-btn.html?user=kennethreitz&repo=requests&type=watch&count=true&size=large"
     allowtransparency="true" frameborder="0" scrolling="0" width="200px" height="35px"></iframe>
 </p>
 
@@ -18,12 +18,20 @@
 
 <h3>Donate</h3>
 <p>
-    If you love Requests, consider making a small donation <a href="https://flattr.com/thing/442264/Requests">on Flattr</a>:
+    If you love Requests, consider supporting the author <a href="https://www.gittip.com/kennethreitz/">on Gittip</a>:
 </p>
 <p>
-    <a class="FlattrButton" style="display:none;" rev="flattr;button:compact;" href="http://docs.python-requests.org/"></a>
-    <noscript><a href="http://flattr.com/thing/442264/Requests" target="_blank">
-    <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></noscript>
+  <iframe style="border: 0; margin: 0; padding: 0;"
+      src="https://www.gittip.com/kennethreitz/widget.html"
+      width="48pt" height="20pt"></iframe>
+</p>
+
+<p>
+    If your organization uses Requests, consider financial support:
+
+</p>
+<p>
+  <a href="http://gum.co/RRZc" class="gumroad-button">Requests Pro</a><script type="text/javascript" src="https://gumroad.com/js/gumroad-button.js"></script>
 </p>
 
 <h3>Feedback</h3>

docs/_templates/sidebarlogo.html

@@ -4,7 +4,7 @@
   </a>
 </p>
 <p>
-<iframe src="http://markdotto.github.com/github-buttons/github-btn.html?user=kennethreitz&repo=requests&type=watch&count=true&size=large"
+<iframe src="http://ghbtns.com/github-btn.html?user=kennethreitz&repo=requests&type=watch&count=true&size=large"
   allowtransparency="true" frameborder="0" scrolling="0" width="200px" height="35px"></iframe>
 </p>
 
@@ -14,12 +14,21 @@
   development release.
 </p>
 
-<h3>Support Requests</h3>
+<h3>Donate</h3>
 <p>
-    If you love Requests, consider making a small donation <a href="https://flattr.com/thing/442264/Requests">on Flattr</a>:
+    If you love Requests, consider supporting the author <a href="https://www.gittip.com/kennethreitz/">on Gittip</a>:
 </p>
 <p>
-    <a class="FlattrButton" style="display:none;" rev="flattr;button:compact;" href="http://docs.python-requests.org/"></a>
-    <noscript><a href="http://flattr.com/thing/442264/Requests" target="_blank">
-    <img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a></noscript>
-</p>
+  <iframe style="border: 0; margin: 0; padding: 0;"
+      src="https://www.gittip.com/kennethreitz/widget.html"
+      width="48pt" height="20pt"></iframe>
+</p>
+
+<p>
+    If your organization uses Requests, consider financial support:
+
+</p>
+
+<p>
+  <a href="http://gum.co/RRZc" class="gumroad-button">Requests Pro</a><script type="text/javascript" src="https://gumroad.com/js/gumroad-button.js"></script>
+</p>

docs/_themes/LICENSE

@@ -1,9 +1,9 @@
-Modifications: 
+Modifications:
 
 Copyright (c) 2011 Kenneth Reitz.
 
 
-Original Project: 
+Original Project:
 
 Copyright (c) 2010 by Armin Ronacher.
 

docs/_themes/README.rst

@@ -1,7 +1,7 @@
 krTheme Sphinx Style
 ====================
 
-This repository contains sphinx styles Kenneth Reitz uses in most of 
+This repository contains sphinx styles Kenneth Reitz uses in most of
 his projects. It is a derivative of Mitsuhiko's themes for Flask and Flask related
 projects.  To use this style in your Sphinx documentation, follow
 this guide:
@@ -11,15 +11,15 @@ this guide:
 
 2. add this to your conf.py: ::
 
-	sys.path.append(os.path.abspath('_themes'))
-	html_theme_path = ['_themes']
-	html_theme = 'flask'
+    sys.path.append(os.path.abspath('_themes'))
+    html_theme_path = ['_themes']
+    html_theme = 'flask'
 
 The following themes exist:
 
 **kr**
-	the standard flask documentation theme for large projects
+    the standard flask documentation theme for large projects
 
 **kr_small**
-	small one-page theme.  Intended to be used by very small addon libraries.
+    small one-page theme.  Intended to be used by very small addon libraries.
 

docs/_themes/kr/theme.conf

@@ -4,4 +4,4 @@ stylesheet = flasky.css
 pygments_style = flask_theme_support.FlaskyStyle
 
 [options]
-touch_icon = 
+touch_icon =

docs/_themes/kr_small/static/flasky.css_t

@@ -8,11 +8,11 @@
  * :license: BSD, see LICENSE for details.
  *
  */
- 
+
 @import url("basic.css");
- 
+
 /* -- page layout ----------------------------------------------------------- */
- 
+
 body {
     font-family: 'Georgia', serif;
     font-size: 17px;
@@ -35,7 +35,7 @@ div.bodywrapper {
 hr {
     border: 1px solid #B1B4B6;
 }
- 
+
 div.body {
     background-color: #ffffff;
     color: #3E4349;
@@ -46,7 +46,7 @@ img.floatingflask {
     padding: 0 0 10px 10px;
     float: right;
 }
- 
+
 div.footer {
     text-align: right;
     color: #888;
@@ -55,12 +55,12 @@ div.footer {
     width: 650px;
     margin: 0 auto 40px auto;
 }
- 
+
 div.footer a {
     color: #888;
     text-decoration: underline;
 }
- 
+
 div.related {
     line-height: 32px;
     color: #888;
@@ -69,18 +69,18 @@ div.related {
 div.related ul {
     padding: 0 0 0 10px;
 }
- 
+
 div.related a {
     color: #444;
 }
- 
+
 /* -- body styles ----------------------------------------------------------- */
- 
+
 a {
     color: #004B6B;
     text-decoration: underline;
 }
- 
+
 a:hover {
     color: #6D4100;
     text-decoration: underline;
@@ -89,7 +89,7 @@ a:hover {
 div.body {
     padding-bottom: 40px; /* saved for footer */
 }
- 
+
 div.body h1,
 div.body h2,
 div.body h3,
@@ -109,24 +109,24 @@ div.indexwrapper h1 {
     height: {{ theme_index_logo_height }};
 }
 {% endif %}
- 
+
 div.body h2 { font-size: 180%; }
 div.body h3 { font-size: 150%; }
 div.body h4 { font-size: 130%; }
 div.body h5 { font-size: 100%; }
 div.body h6 { font-size: 100%; }
- 
+
 a.headerlink {
     color: white;
     padding: 0 4px;
     text-decoration: none;
 }
- 
+
 a.headerlink:hover {
     color: #444;
     background: #eaeaea;
 }
- 
+
 div.body p, div.body dd, div.body li {
     line-height: 1.4em;
 }
@@ -164,25 +164,25 @@ div.note {
     background-color: #eee;
     border: 1px solid #ccc;
 }
- 
+
 div.seealso {
     background-color: #ffc;
     border: 1px solid #ff6;
 }
- 
+
 div.topic {
     background-color: #eee;
 }
- 
+
 div.warning {
     background-color: #ffe4e4;
     border: 1px solid #f66;
 }
- 
+
 p.admonition-title {
     display: inline;
 }
- 
+
 p.admonition-title:after {
     content: ":";
 }
@@ -254,7 +254,7 @@ dl {
 dl dd {
     margin-left: 30px;
 }
- 
+
 pre {
     padding: 0;
     margin: 15px -30px;

docs/community/support.rst

@@ -34,4 +34,4 @@ IRC
 The official Freenode channel for Requests is
 `#python-requests <irc://irc.freenode.net/python-requests>`_
 
-I'm also available as **kennethreitz** on Freenode.
+I'm also available as **kennethreitz** on Freenode.

docs/conf.py

@@ -241,4 +241,4 @@ texinfo_appendices = []
 
 sys.path.append(os.path.abspath('_themes'))
 html_theme_path = ['_themes']
-html_theme = 'kr'
+html_theme = 'kr'

docs/dev/authors.rst

@@ -2,4 +2,4 @@ Authors
 =======
 
 
-.. include:: ../../AUTHORS.rst
+.. include:: ../../AUTHORS.rst

docs/index.rst

@@ -38,13 +38,7 @@ Requests takes all of the work out of Python HTTP/1.1 — making your integrati
 Testimonials
 ------------
 
-`Heroku <http://heroku.com>`_, `PayPal <https://www.paypal.com/>`_,
-`Transifex <https://www.transifex.net/>`_,
-`Native Instruments <http://www.native-instruments.com/>`_, `The Washington Post <http://www.washingtonpost.com/>`_,
-`Twitter, Inc <http://twitter.com>`_,
-`Readability <http://readability.com>`_, and
-Federal US Institutions
-use Requests internally. It has been installed over 100,000 times from PyPI.
+Amazon, Google, Twilio, Mozilla, Heroku, PayPal, Transifex, Native Instruments, The Washington Post, Twitter, SoundCloud, Kippt, Readability, and Federal US Institutions use Requests internally. It has been downloaded over 1,000,000 times from PyPI.
 
 **Armin Ronacher**
     Requests is the perfect example how beautiful an API can be with the

docs/user/advanced.rst

@@ -93,28 +93,43 @@ I don't have SSL setup on this domain, so it fails. Excellent. Github does thoug
 
 You can also pass ``verify`` the path to a CA_BUNDLE file for private certs. You can also set the ``REQUESTS_CA_BUNDLE`` environment variable.
 
+Requests can also ignore verifying the SSL certficate if you set ``verify`` to False.
+
+::
+
+    >>> requests.get('https://kennethreitz.com', verify=False)
+    <Response [200]>
+
+By default, ``verify`` is set to True. Option ``verify`` only applies to host certs.
+
+You can also specify the local cert file either as a path or key value pair::
+
+    >>> requests.get('https://kennethreitz.com', cert=('/path/server.crt', '/path/key'))
+    <Response [200]>
+
+If you specify a wrong path or an invalid cert::
+
+    >>> requests.get('https://kennethreitz.com', cert='/wrong_path/server.pem')
+    SSLError: [Errno 336265225] _ssl.c:347: error:140B0009:SSL routines:SSL_CTX_use_PrivateKey_file:PEM lib
+
 
 Body Content Workflow
 ---------------------
 
-By default, when you make a request, the body of the response isn't downloaded immediately. The response headers are downloaded when you make a request, but the content isn't downloaded until you access the :class:`Response.content` attribute.
-
-Let's walk through it::
+By default, when you make a request, the body of the response is downloaded immediately. You can override this behavior and defer downloading the response body until you access the :class:`Response.content` attribute with the ``prefetch`` parameter::
 
     tarball_url = 'https://github.com/kennethreitz/requests/tarball/master'
-    r = requests.get(tarball_url)
+    r = requests.get(tarball_url, prefetch=False)
 
-The request has been made, but the connection is still open. The response body has not been downloaded yet. ::
+At this point only the response headers have been downloaded and the connection remains open, hence allowing us to make content retrieval conditional::
 
-    r.content
+    if int(r.headers['content-length']) < TOO_LONG:
+      content = r.content
+      ...
 
-The content has been downloaded and cached.
-
-You can override this default behavior with the ``prefetch`` parameter::
-
-    r = requests.get(tarball_url, prefetch=True)
-    # Blocks until all of request body has been downloaded.
+You can further control the workflow by use of the :class:`Response.iter_content` and :class:`Response.iter_lines` methods, or reading from the underlying urllib3 :class:`urllib3.HTTPResponse` at :class:`Response.raw`.
 
+Note that in versions prior to 0.13.6 the ``prefetch`` default was set to ``False``.
 
 Configuring Requests
 --------------------
@@ -262,15 +277,13 @@ Streaming Requests
 With ``requests.Response.iter_lines()`` you can easily iterate over streaming
 APIs such as the `Twitter Streaming API <https://dev.twitter.com/docs/streaming-api>`_.
 
-To use the Twitter Streaming API to track the keyword "requests":
-
-::
+To use the Twitter Streaming API to track the keyword "requests"::
 
     import requests
     import json
 
     r = requests.post('https://stream.twitter.com/1/statuses/filter.json',
-        data={'track': 'requests'}, auth=('username', 'password'))
+        data={'track': 'requests'}, auth=('username', 'password'), prefetch=False)
 
     for line in r.iter_lines():
         if line: # filter out keep-alive new lines
@@ -295,9 +308,7 @@ Proxies
 -------
 
 If you need to use a proxy, you can configure individual requests with the
-``proxies`` argument to any request method:
-
-::
+``proxies`` argument to any request method::
 
     import requests
 
@@ -318,14 +329,37 @@ You can also configure proxies by environment variables ``HTTP_PROXY`` and ``HTT
     >>> import requests
     >>> requests.get("http://example.org")
 
-To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` syntax:
-
-::
+To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` syntax::
 
     proxies = {
         "http": "http://user:pass@10.10.1.10:3128/",
     }
 
+Compliance
+----------
+
+Requests is intended to be compliant with all relevant specifications and
+RFCs where that compliance will not cause difficulties for users. This
+attention to the specification can lead to some behaviour that may seem
+unusual to those not familiar with the relevant specification.
+
+Encodings
+^^^^^^^^^
+
+When you receive a response, Requests makes a guess at the encoding to use for
+decoding the response when you call the ``Response.text`` method. Requests
+will first check for an encoding in the HTTP header, and if none is present,
+will use `chardet <http://pypi.python.org/pypi/chardet>`_ to attempt to guess
+the encoding.
+
+The only time Requests will not do this is if no explicit charset is present
+in the HTTP headers **and** the ``Content-Type`` header contains ``text``. In
+this situation,
+`RFC 2616 <http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.7.1>`_
+specifies that the default charset must be ``ISO-8859-1``. Requests follows
+the specification in this case. If you require a different encoding, you can
+manually set the ``Response.encoding`` property, or use the raw
+``Response.content``.
 
 HTTP Verbs
 ----------
@@ -357,7 +391,9 @@ into Python objects. Because GitHub returned UTF-8, we should use the
 ``r.text`` method, not the ``r.content`` method. ``r.content`` returns a
 bytestring, while ``r.text`` returns a Unicode-encoded string. I have no plans
 to perform byte-manipulation on this response, so I want any Unicode code
-points encoded.::
+points encoded.
+
+::
 
     >>> import json
     >>> commit_data = json.loads(r.text)
@@ -371,7 +407,9 @@ points encoded.::
 So far, so simple. Well, let's investigate the GitHub API a little bit. Now,
 we could look at the documentation, but we might have a little more fun if we
 use Requests instead. We can take advantage of the Requests OPTIONS verb to
-see what kinds of HTTP methods are supported on the url we just used.::
+see what kinds of HTTP methods are supported on the url we just used.
+
+::
 
     >>> verbs = requests.options(r.url)
     >>> verbs.status_code
@@ -381,7 +419,9 @@ Uh, what? That's unhelpful! Turns out GitHub, like many API providers, don't
 actually implement the OPTIONS method. This is an annoying oversight, but it's
 OK, we can just use the boring documentation. If GitHub had correctly
 implemented OPTIONS, however, they should return the allowed methods in the
-headers, e.g.::
+headers, e.g.
+
+::
 
     >>> verbs = requests.options('http://a-good-website.com/api/cats')
     >>> print verbs.headers['allow']
@@ -393,7 +433,9 @@ we should probably avoid making ham-handed POSTS to it. Instead, let's play
 with the Issues feature of GitHub.
 
 This documentation was added in response to Issue #482. Given that this issue
-already exists, we will use it as an example. Let's start by getting it.::
+already exists, we will use it as an example. Let's start by getting it.
+
+::
 
     >>> r = requests.get('https://api.github.com/repos/kennethreitz/requests/issues/482')
     >>> r.status_code
@@ -404,7 +446,9 @@ already exists, we will use it as an example. Let's start by getting it.::
     >>> print issue[u'comments']
     3
 
-Cool, we have three comments. Let's take a look at the last of them.::
+Cool, we have three comments. Let's take a look at the last of them.
+
+::
 
     >>> r = requests.get(r.url + u'/comments')
     >>> r.status_code
@@ -416,14 +460,18 @@ Cool, we have three comments. Let's take a look at the last of them.::
     Probably in the "advanced" section
 
 Well, that seems like a silly place. Let's post a comment telling the poster
-that he's silly. Who is the poster, anyway?::
+that he's silly. Who is the poster, anyway?
+
+::
 
     >>> print comments[2][u'user'][u'login']
     kennethreitz
 
 OK, so let's tell this Kenneth guy that we think this example should go in the
 quickstart guide instead. According to the GitHub API doc, the way to do this
-is to POST to the thread. Let's do it.::
+is to POST to the thread. Let's do it.
+
+::
 
     >>> body = json.dumps({u"body": u"Sounds great! I'll get right on it!"})
     >>> url = u"https://api.github.com/repos/kennethreitz/requests/issues/482/comments"
@@ -433,7 +481,9 @@ is to POST to the thread. Let's do it.::
 
 Huh, that's weird. We probably need to authenticate. That'll be a pain, right?
 Wrong. Requests makes it easy to use many forms of authentication, including
-the very common Basic Auth.::
+the very common Basic Auth.
+
+::
 
     >>> from requests.auth import HTTPBasicAuth
     >>> auth = HTTPBasicAuth('fake@example.com', 'not_a_real_password')
@@ -447,7 +497,9 @@ the very common Basic Auth.::
 Brilliant. Oh, wait, no! I meant to add that it would take me a while, because
 I had to go feed my cat. If only I could edit this comment! Happily, GitHub
 allows us to use another HTTP verb, PATCH, to edit this comment. Let's do
-that.::
+that.
+
+::
 
     >>> print content[u"id"]
     5804413
@@ -460,7 +512,9 @@ that.::
 Excellent. Now, just to torture this Kenneth guy, I've decided to let him
 sweat and not tell him that I'm working on this. That means I want to delete
 this comment. GitHub lets us delete comments using the incredibly aptly named
-DELETE method. Let's get rid of it.::
+DELETE method. Let's get rid of it.
+
+::
 
     >>> r = requests.delete(url=url, auth=auth)
     >>> r.status_code
@@ -471,14 +525,37 @@ DELETE method. Let's get rid of it.::
 Excellent. All gone. The last thing I want to know is how much of my ratelimit
 I've used. Let's find out. GitHub sends that information in the headers, so
 rather than download the whole page I'll send a HEAD request to get the
-headers.::
+headers.
+
+::
 
     >>> r = requests.head(url=url, auth=auth)
     >>> print r.headers
-    // ...snip... //
+    ...
     'x-ratelimit-remaining': '4995'
     'x-ratelimit-limit': '5000'
-    // ...snip... //
+    ...
 
 Excellent. Time to write a Python program that abuses the GitHub API in all
 kinds of exciting ways, 4995 more times.
+
+Link Headers
+------------
+
+Many HTTP APIs feature Link headers. They make APIs more self describing and discoverable.
+
+GitHub uses these for `pagination <http://developer.github.com/v3/#pagination>`_ in their API, for example::
+
+    >>> url = 'https://api.github.com/users/kennethreitz/repos?page=1&per_page=10'
+    >>> r = requests.head(url=url)
+    >>> r.headers['link']
+    '<https://api.github.com/users/kennethreitz/repos?page=2&per_page=10>; rel="next", <https://api.github.com/users/kennethreitz/repos?page=6&per_page=10>; rel="last"'
+
+Requests will automatically parse these link headers and make them easily consumable::
+
+    >>> r.links['next']
+    'https://api.github.com/users/kennethreitz/repos?page=2&per_page=10'
+
+    >>> r.links['last']
+    'https://api.github.com/users/kennethreitz/repos?page=6&per_page=10'
+

docs/user/install.rst

@@ -25,10 +25,10 @@ But, you really `shouldn't do that <http://www.pip-installer.org/en/latest/other
 Cheeseshop Mirror
 -----------------
 
-If the Cheeseshop is down, you can also install Requests from Kenneth Reitz's
-personal `Cheeseshop mirror <http://pip.kennethreitz.com/>`_::
+If the Cheeseshop is down, you can also install Requests from one of the
+mirrors. `Crate.io <http://crate.io>`_ is one of them::
 
-    $ pip install -i http://pip.kennethreitz.com/simple requests
+    $ pip install -i http://simple.crate.io/ requests
 
 
 Get the Code
@@ -54,25 +54,3 @@ Once you have a copy of the source, you can embed it in your Python package,
 or install it into your site-packages easily::
 
     $ python setup.py install
-
-.. _gevent:
-
-Installing Gevent
------------------
-
-If you are using the ``requests.async`` module for making concurrent
-requests, you need to install gevent.
-
-To install gevent, you'll need ``libevent``.
-
-OSX::
-
-    $ brew install libevent
-
-Ubuntu::
-
-    $ apt-get install libevent-dev
-
-Once you have ``libevent``, you can install ``gevent`` with ``pip``::
-
-    $ pip install gevent

docs/user/intro.rst

@@ -17,6 +17,7 @@ Requests was developed with a few :pep:`20` idioms in mind.
 
 All contributions to Requests should keep these important rules in mind.
 
+.. _`isc`:
 
 ISC License
 -----------
@@ -41,9 +42,4 @@ Requests is released under terms of `The ISC License`_.
 Requests License
 ----------------
 
-    Copyright (c) 2011, Kenneth Reitz
-
-    Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
-
-    THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
+    .. include:: ../../LICENSE

docs/user/quickstart.rst

@@ -24,7 +24,7 @@ Make a Request
 Making a request with Requests is very simple.
 
 Begin by importing the Requests module::
-    
+
     >>> import requests
 
 Now, let's try to get a webpage. For this example, let's get GitHub's public
@@ -37,12 +37,12 @@ information we need from this object.
 
 Requests' simple API means that all forms of HTTP request are as obvious. For
 example, this is how you make an HTTP POST request::
-    
+
     >>> r = requests.post("http://httpbin.org/post")
 
 Nice, right? What about the other HTTP request types: PUT, DELETE, HEAD and
 OPTIONS? These are all just as simple::
-    
+
     >>> r = requests.put("http://httpbin.org/put")
     >>> r = requests.delete("http://httpbin.org/delete")
     >>> r = requests.head("http://httpbin.org/get")
@@ -70,7 +70,7 @@ You can see that the URL has been correctly encoded by printing the URL::
 
     >>> print r.url
     u'http://httpbin.org/get?key2=value2&key1=value1'
-    
+
 
 Response Content
 ----------------
@@ -86,12 +86,22 @@ again::
 Requests will automatically decode content from the server. Most unicode
 charsets are seamlessly decoded.
 
-When you make a request, ``r.encoding`` is set, based on the HTTP headers.
-Requests will use that encoding when you access ``r.text``.  If ``r.encoding``
-is ``None``, Requests will make an extremely educated guess of the encoding
-of the response body. You can manually set ``r.encoding`` to any encoding
-you'd like, and that charset will be used.
+When you make a request, Requests makes educated guesses about the encoding of
+the response based on the HTTP headers. The text encoding guessed by Requests
+is used when you access ``r.text``. You can find out what encoding Requests is
+using, and change it, using the ``r.encoding`` property::
 
+    >>> r.encoding
+    'utf-8'
+    >>> r.encoding = 'ISO-8859-1'
+
+If you change the encoding, Requests will use the new value of ``r.encoding``
+whenever you call ``r.text``.
+
+Requests will also use custom encodings in the event that you need them. If
+you have created your own encoding and registered it with the ``codecs``
+module, you can simply use the codec name as the value of ``r.encoding`` and
+Requests will handle the decoding for you.
 
 Binary Response Content
 -----------------------
@@ -189,14 +199,14 @@ POST a Multipart-Encoded File
 Requests makes it simple to upload Multipart-encoded files::
 
     >>> url = 'http://httpbin.org/post'
-    >>> files = {'report.xls': open('report.xls', 'rb')}
+    >>> files = {'file': open('report.xls', 'rb')}
 
     >>> r = requests.post(url, files=files)
     >>> r.text
     {
       // ...snip... //
       "files": {
-        "report.xls": "<censored...binary...data>"
+        "file": "<censored...binary...data>"
       },
       // ...snip... //
     }
@@ -219,7 +229,7 @@ You can set the filename explicitly::
 If you want, you can send strings to be received as files::
 
     >>> url = 'http://httpbin.org/post'
-    >>> files = {'file': ('report.csv', 'some,data,to,send\nanother,row,to,send\n')} 
+    >>> files = {'file': ('report.csv', 'some,data,to,send\nanother,row,to,send\n')}
 
     >>> r = requests.post(url, files=files)
     >>> r.text
@@ -250,11 +260,11 @@ reference::
 If we made a bad request (non-200 response), we can raise it with
 :class:`Response.raise_for_status()`::
 
-    >>> _r = requests.get('http://httpbin.org/status/404')
-    >>> _r.status_code
+    >>> bad_r = requests.get('http://httpbin.org/status/404')
+    >>> bad_r.status_code
     404
 
-    >>> _r.raise_for_status()
+    >>> bad_r.raise_for_status()
     Traceback (most recent call last):
       File "requests/models.py", line 832, in raise_for_status
         raise http_error
@@ -329,7 +339,7 @@ parameter::
 Basic Authentication
 --------------------
 
-Many web services require authentication. There many different types of
+Many web services require authentication. There are many different types of
 authentication, but the most common is HTTP Basic Auth.
 
 Making requests with Basic Auth is extremely simple::
@@ -362,10 +372,42 @@ Another popular form of web service protection is Digest Authentication::
 OAuth Authentication
 --------------------
 
-Miguel Araujo's `requests-oauth <http://pypi.python.org/pypi/requests-oauth>`_
-project provides a simple interface for establishing OAuth connections.
-Documentation and examples can be found on the requests-oauth
-`git repository <https://github.com/maraujop/requests-oauth>`_.
+Requests features robust, built-in OAuth support!
+
+OAuth takes many forms, so let's take a look at a few different forms::
+
+    import requests
+    from requests.auth import OAuth1
+
+    url = u'https://api.twitter.com/1/account/settings.json'
+
+    client_key = u'...'
+    client_secret = u'...'
+    resource_owner_key = u'...'
+    resource_owner_secret = u'...'
+
+
+Query signing::
+
+    queryoauth = OAuth1(client_key, client_secret,
+                        resource_owner_key, resource_owner_secret,
+                        signature_type='query')
+    r = requests.get(url, auth=queryoauth)
+
+Header signing::
+
+    headeroauth = OAuth1(client_key, client_secret,
+                         resource_owner_key, resource_owner_secret,
+                         signature_type='auth_header')
+    r = requests.get(url, auth=headeroauth)
+
+Body signing::
+
+    bodyoauth = OAuth1(client_key, client_secret,
+                       resource_owner_key, resource_owner_secret,
+                       signature_type='body')
+
+    r = requests.post(url, auth=bodyoauth)
 
 
 Redirection and History
@@ -386,7 +428,7 @@ of the Response object to track redirection. Let's see what Github does::
     [<Response [301]>]
 
 The :class:`Response.history` list contains a list of the
-:class:`Request` objects that were created in order to complete the request.
+:class:`Request` objects that were created in order to complete the request. The list is sorted from the oldest to the most recent request.
 
 If you're using GET or OPTIONS, you can disable redirection handling with the
 ``allow_redirects`` parameter::

requests/__init__.py

@@ -6,8 +6,35 @@
 #          /
 
 """
-requests
-~~~~~~~~
+requests HTTP library
+~~~~~~~~~~~~~~~~~~~~~
+
+Requests is an HTTP library, written in Python, for human beings. Basic GET
+usage:
+
+   >>> import requests
+   >>> r = requests.get('http://python.org')
+   >>> r.status_code
+   200
+   >>> 'Python is a programming language' in r.content
+   True
+
+... or POST:
+
+   >>> payload = dict(key1='value1', key2='value2')
+   >>> r = requests.post("http://httpbin.org/post", data=payload)
+   >>> print r.text
+   {
+     ...
+     "form": {
+       "key2": "value2",
+       "key1": "value1"
+     },
+     ...
+   }
+
+The other HTTP methods are supported - see `requests.api`. Full documentation
+is at <http://python-requests.org>.
 
 :copyright: (c) 2012 by Kenneth Reitz.
 :license: ISC, see LICENSE for more details.
@@ -15,8 +42,8 @@ requests
 """
 
 __title__ = 'requests'
-__version__ = '0.13.2'
-__build__ = 0x001302
+__version__ = '0.14.2'
+__build__ = 0x001402
 __author__ = 'Kenneth Reitz'
 __license__ = 'ISC'
 __copyright__ = 'Copyright 2012 Kenneth Reitz'

requests/_oauth.py

@@ -4,8 +4,8 @@
 requests._oauth
 ~~~~~~~~~~~~~~~
 
-This module comtains the path hack neccesary for oauthlib to be vendored into requests
-while allowing upstream changes.
+This module contains the path hack necessary for oauthlib to be vendored into
+requests while allowing upstream changes.
 """
 
 import os
@@ -16,8 +16,8 @@ try:
     from oauthlib.common import extract_params
     from oauthlib.oauth1.rfc5849 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)
 except ImportError:
-    path = os.path.abspath('/'.join(__file__.split('/')[:-1]+['packages']))
-    sys.path.insert(0, path)
+    from .packages import oauthlib
+    sys.modules['oauthlib'] = oauthlib
     from oauthlib.oauth1 import rfc5849
     from oauthlib.common import extract_params
-    from oauthlib.oauth1.rfc5849 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)
+    from oauthlib.oauth1.rfc5849 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)

requests/api.py

@@ -14,6 +14,7 @@ This module implements the Requests API.
 from . import sessions
 from .safe_mode import catch_exceptions_if_in_safe_mode
 
+
 @catch_exceptions_if_in_safe_mode
 def request(method, url, **kwargs):
     """Constructs and sends a :class:`Request <Request>`.
@@ -38,9 +39,19 @@ def request(method, url, **kwargs):
     :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
     """
 
-    s = kwargs.pop('session') if 'session' in kwargs else sessions.session()
-    return s.request(method=method, url=url, **kwargs)
+    # if this session was passed in, leave it open (and retain pooled connections);
+    # if we're making it just for this call, then close it when we're done.
+    adhoc_session = False
+    session = kwargs.pop('session', None)
+    if session is None:
+        session = sessions.session()
+        adhoc_session = True
 
+    try:
+        return session.request(method=method, url=url, **kwargs)
+    finally:
+        if adhoc_session:
+            session.close()
 
 
 def get(url, **kwargs):

requests/auth.py

@@ -8,8 +8,10 @@ This module contains the authentication handlers for Requests.
 """
 
 import os
+import re
 import time
 import hashlib
+import logging
 
 from base64 import b64encode
 
@@ -23,7 +25,16 @@ except (ImportError, SyntaxError):
     SIGNATURE_HMAC = None
     SIGNATURE_TYPE_AUTH_HEADER = None
 
+try:
+    import kerberos as k
+except ImportError as exc:
+    k = None
+
+log = logging.getLogger(__name__)
+
 CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
+CONTENT_TYPE_MULTI_PART = 'multipart/form-data'
+
 
 def _basic_auth_str(username, password):
     """Returns a Basic Auth string."""
@@ -64,29 +75,37 @@ class OAuth1(AuthBase):
         Parameters may be included from the body if the content-type is
         urlencoded, if no content type is set an educated guess is made.
         """
-        contenttype = r.headers.get('Content-Type', None)
+        # split(";") because Content-Type may be "multipart/form-data; boundary=xxxxx"
+        contenttype = r.headers.get('Content-Type', '').split(";")[0].lower()
         # extract_params will not give params unless the body is a properly
         # formatted string, a dictionary or a list of 2-tuples.
         decoded_body = extract_params(r.data)
-        if contenttype == None and decoded_body != None:
-            # extract_params can only check the present r.data and does not know
-            # of r.files, thus an extra check is performed. We know that
-            # if files are present the request will not have
-            # Content-type: x-www-form-urlencoded. We guess it will have
-            # a mimetype of multipart/form-encoded and if this is not the case
-            # we assume the correct header will be set later.
-            if r.files:
-                # Omit body data in the signing and since it will always
-                # be empty (cant add paras to body if multipart) and we wish
-                # to preserve body.
-                r.headers['Content-Type'] = 'multipart/form-encoded'
-                r.url, r.headers, _ = self.client.sign(
-                    unicode(r.full_url), unicode(r.method), None, r.headers)
-            else:
-                # Normal signing
-                r.headers['Content-Type'] = 'application/x-www-form-urlencoded'
-                r.url, r.headers, r.data = self.client.sign(
-                    unicode(r.full_url), unicode(r.method), r.data, r.headers)
+
+        # extract_params can only check the present r.data and does not know
+        # of r.files, thus an extra check is performed. We know that
+        # if files are present the request will not have
+        # Content-type: x-www-form-urlencoded. We guess it will have
+        # a mimetype of multipart/form-data and if this is not the case
+        # we assume the correct header will be set later.
+        _oauth_signed = True
+        if r.files and contenttype == CONTENT_TYPE_MULTI_PART:
+            # Omit body data in the signing and since it will always
+            # be empty (cant add paras to body if multipart) and we wish
+            # to preserve body.
+            r.url, r.headers, _ = self.client.sign(
+                unicode(r.full_url), unicode(r.method), None, r.headers)
+        elif decoded_body is not None and contenttype in (CONTENT_TYPE_FORM_URLENCODED, ''):
+            # Normal signing
+            if not contenttype:
+                r.headers['Content-Type'] = CONTENT_TYPE_FORM_URLENCODED
+            r.url, r.headers, r.data = self.client.sign(
+                unicode(r.full_url), unicode(r.method), r.data, r.headers)
+        else:
+            _oauth_signed = False
+        if _oauth_signed:
+            # Both flows add params to the URL by using r.full_url,
+            # so this prevents adding it again later
+            r.params = {}
 
             # Having the authorization header, key or value, in unicode will
             # result in UnicodeDecodeErrors when the request is concatenated
@@ -129,91 +148,101 @@ class HTTPDigestAuth(AuthBase):
     def __init__(self, username, password):
         self.username = username
         self.password = password
+        self.last_nonce = ''
+        self.nonce_count = 0
+        self.chal = {}
+
+    def build_digest_header(self, method, url):
+
+        realm = self.chal['realm']
+        nonce = self.chal['nonce']
+        qop = self.chal.get('qop')
+        algorithm = self.chal.get('algorithm', 'MD5')
+        opaque = self.chal.get('opaque', None)
+
+        algorithm = algorithm.upper()
+        # lambdas assume digest modules are imported at the top level
+        if algorithm == 'MD5':
+            def md5_utf8(x):
+                if isinstance(x, str):
+                    x = x.encode('utf-8')
+                return hashlib.md5(x).hexdigest()
+            hash_utf8 = md5_utf8
+        elif algorithm == 'SHA':
+            def sha_utf8(x):
+                if isinstance(x, str):
+                    x = x.encode('utf-8')
+                return hashlib.sha1(x).hexdigest()
+            hash_utf8 = sha_utf8
+        # XXX MD5-sess
+        KD = lambda s, d: hash_utf8("%s:%s" % (s, d))
+
+        if hash_utf8 is None:
+            return None
+
+        # XXX not implemented yet
+        entdig = None
+        p_parsed = urlparse(url)
+        path = p_parsed.path
+        if p_parsed.query:
+            path += '?' + p_parsed.query
+
+        A1 = '%s:%s:%s' % (self.username, realm, self.password)
+        A2 = '%s:%s' % (method, path)
+
+        if qop == 'auth':
+            if nonce == self.last_nonce:
+                self.nonce_count += 1
+            else:
+                self.nonce_count = 1
+
+            ncvalue = '%08x' % self.nonce_count
+            s = str(self.nonce_count).encode('utf-8')
+            s += nonce.encode('utf-8')
+            s += time.ctime().encode('utf-8')
+            s += os.urandom(8)
+
+            cnonce = (hashlib.sha1(s).hexdigest()[:16])
+            noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, hash_utf8(A2))
+            respdig = KD(hash_utf8(A1), noncebit)
+        elif qop is None:
+            respdig = KD(hash_utf8(A1), "%s:%s" % (nonce, hash_utf8(A2)))
+        else:
+            # XXX handle auth-int.
+            return None
+
+        self.last_nonce = nonce
+
+        # XXX should the partial digests be encoded too?
+        base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+           'response="%s"' % (self.username, realm, nonce, path, respdig)
+        if opaque:
+            base += ', opaque="%s"' % opaque
+        if entdig:
+            base += ', digest="%s"' % entdig
+            base += ', algorithm="%s"' % algorithm
+        if qop:
+            base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
+
+        return 'Digest %s' % (base)
 
     def handle_401(self, r):
         """Takes the given response and tries digest-auth, if needed."""
 
-        r.request.deregister_hook('response', self.handle_401)
+        num_401_calls = r.request.hooks['response'].count(self.handle_401)
 
         s_auth = r.headers.get('www-authenticate', '')
 
-        if 'digest' in s_auth.lower():
+        if 'digest' in s_auth.lower() and num_401_calls < 2:
 
-            last_nonce = ''
-            nonce_count = 0
+            self.chal = parse_dict_header(s_auth.replace('Digest ', ''))
 
-            chal = parse_dict_header(s_auth.replace('Digest ', ''))
+            # Consume content and release the original connection
+            # to allow our new request to reuse the same one.
+            r.content
+            r.raw.release_conn()
 
-            realm = chal['realm']
-            nonce = chal['nonce']
-            qop = chal.get('qop')
-            algorithm = chal.get('algorithm', 'MD5')
-            opaque = chal.get('opaque', None)
-
-            algorithm = algorithm.upper()
-            # lambdas assume digest modules are imported at the top level
-            if algorithm == 'MD5':
-                def md5_utf8(x):
-                    if isinstance(x, str):
-                        x = x.encode('utf-8')
-                    return hashlib.md5(x).hexdigest()
-                hash_utf8 = md5_utf8
-            elif algorithm == 'SHA':
-                def sha_utf8(x):
-                    if isinstance(x, str):
-                        x = x.encode('utf-8')
-                    return hashlib.sha1(x).hexdigest()
-                hash_utf8 = sha_utf8
-            # XXX MD5-sess
-            KD = lambda s, d: hash_utf8("%s:%s" % (s, d))
-
-            if hash_utf8 is None:
-                return None
-
-            # XXX not implemented yet
-            entdig = None
-            p_parsed = urlparse(r.request.url)
-            path = p_parsed.path
-            if p_parsed.query:
-                path += '?' + p_parsed.query
-
-            A1 = '%s:%s:%s' % (self.username, realm, self.password)
-            A2 = '%s:%s' % (r.request.method, path)
-
-            if qop == 'auth':
-                if nonce == last_nonce:
-                    nonce_count += 1
-                else:
-                    nonce_count = 1
-                    last_nonce = nonce
-
-                ncvalue = '%08x' % nonce_count
-                s = str(nonce_count).encode('utf-8')
-                s += nonce.encode('utf-8')
-                s += time.ctime().encode('utf-8')
-                s += os.urandom(8)
-
-                cnonce = (hashlib.sha1(s).hexdigest()[:16])
-                noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, hash_utf8(A2))
-                respdig = KD(hash_utf8(A1), noncebit)
-            elif qop is None:
-                respdig = KD(hash_utf8(A1), "%s:%s" % (nonce, hash_utf8(A2)))
-            else:
-                # XXX handle auth-int.
-                return None
-
-            # XXX should the partial digests be encoded too?
-            base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
-                   'response="%s"' % (self.username, realm, nonce, path, respdig)
-            if opaque:
-                base += ', opaque="%s"' % opaque
-            if entdig:
-                base += ', digest="%s"' % entdig
-            base += ', algorithm="%s"' % algorithm
-            if qop:
-                base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
-
-            r.request.headers['Authorization'] = 'Digest %s' % (base)
+            r.request.headers['Authorization'] = self.build_digest_header(r.request.method, r.request.url)
             r.request.send(anyway=True)
             _r = r.request.response
             _r.history.append(r)
@@ -223,5 +252,131 @@ class HTTPDigestAuth(AuthBase):
         return r
 
     def __call__(self, r):
+        # If we have a saved nonce, skip the 401
+        if self.last_nonce:
+            r.headers['Authorization'] = self.build_digest_header(r.method, r.url)
         r.register_hook('response', self.handle_401)
         return r
+
+
+def _negotiate_value(r):
+    """Extracts the gssapi authentication token from the appropriate header"""
+
+    authreq = r.headers.get('www-authenticate', None)
+
+    if authreq:
+        rx = re.compile('(?:.*,)*\s*Negotiate\s*([^,]*),?', re.I)
+        mo = rx.search(authreq)
+        if mo:
+            return mo.group(1)
+
+    return None
+
+
+class HTTPKerberosAuth(AuthBase):
+    """Attaches HTTP GSSAPI/Kerberos Authentication to the given Request object."""
+    def __init__(self, require_mutual_auth=True):
+        if k is None:
+            raise Exception("Kerberos libraries unavailable")
+        self.context = None
+        self.require_mutual_auth = require_mutual_auth
+
+    def generate_request_header(self, r):
+        """Generates the gssapi authentication token with kerberos"""
+
+        host = urlparse(r.url).netloc
+        tail, _, head = host.rpartition(':')
+        domain = tail if tail else head
+
+        result, self.context = k.authGSSClientInit("HTTP@%s" % domain)
+
+        if result < 1:
+            raise Exception("authGSSClientInit failed")
+
+        result = k.authGSSClientStep(self.context, _negotiate_value(r))
+
+        if result < 0:
+            raise Exception("authGSSClientStep failed")
+
+        response = k.authGSSClientResponse(self.context)
+
+        return "Negotiate %s" % response
+
+    def authenticate_user(self, r):
+        """Handles user authentication with gssapi/kerberos"""
+
+        auth_header = self.generate_request_header(r)
+        log.debug("authenticate_user(): Authorization header: %s" % auth_header)
+        r.request.headers['Authorization'] = auth_header
+        r.request.send(anyway=True)
+        _r = r.request.response
+        _r.history.append(r)
+        log.debug("authenticate_user(): returning %s" % _r)
+        return _r
+
+    def handle_401(self, r):
+        """Handles 401's, attempts to use gssapi/kerberos authentication"""
+
+        log.debug("handle_401(): Handling: 401")
+        if _negotiate_value(r) is not None:
+            _r = self.authenticate_user(r)
+            log.debug("handle_401(): returning %s" % _r)
+            return _r
+        else:
+            log.debug("handle_401(): Kerberos is not supported")
+            log.debug("handle_401(): returning %s" % r)
+            return r
+
+    def handle_other(self, r):
+        """Handles all responses with the exception of 401s.
+
+        This is necessary so that we can authenticate responses if requested"""
+
+        log.debug("handle_other(): Handling: %d" % r.status_code)
+        self.deregister(r)
+        if self.require_mutual_auth:
+            if _negotiate_value(r) is not None:
+                log.debug("handle_other(): Authenticating the server")
+                _r = self.authenticate_server(r)
+                log.debug("handle_other(): returning %s" % _r)
+                return _r
+            else:
+                log.error("handle_other(): Mutual authentication failed")
+                raise Exception("Mutual authentication failed")
+        else:
+            log.debug("handle_other(): returning %s" % r)
+            return r
+
+    def authenticate_server(self, r):
+        """Uses GSSAPI to authenticate the server"""
+
+        log.debug("authenticate_server(): Authenticate header: %s" % _negotiate_value(r))
+        result = k.authGSSClientStep(self.context, _negotiate_value(r))
+        if  result < 1:
+            raise Exception("authGSSClientStep failed")
+        _r = r.request.response
+        log.debug("authenticate_server(): returning %s" % _r)
+        return _r
+
+    def handle_response(self, r):
+        """Takes the given response and tries kerberos-auth, as needed."""
+
+        if r.status_code == 401:
+            _r = self.handle_401(r)
+            log.debug("handle_response returning %s" % _r)
+            return _r
+        else:
+            _r = self.handle_other(r)
+            log.debug("handle_response returning %s" % _r)
+            return _r
+
+        log.debug("handle_response returning %s" % r)
+        return r
+
+    def deregister(self, r):
+        """Deregisters the response handler"""
+        r.request.deregister_hook('response', self.handle_response)
+
+    def __call__(self, r):
+        r.register_hook('response', self.handle_response)
+        return r

requests/compat.py

@@ -72,6 +72,10 @@ is_osx = ('darwin' in str(sys.platform).lower())
 is_hpux = ('hpux' in str(sys.platform).lower())   # Complete guess.
 is_solaris = ('solar==' in str(sys.platform).lower())   # Complete guess.
 
+try:
+    import simplejson as json
+except ImportError:
+    import json
 
 # ---------
 # Specifics
@@ -79,29 +83,37 @@ is_solaris = ('solar==' in str(sys.platform).lower())   # Complete guess.
 
 
 if is_py2:
-    from urllib import quote, unquote, urlencode
-    from urlparse import urlparse, urlunparse, urljoin, urlsplit
+    from urllib import quote, unquote, quote_plus, unquote_plus, urlencode
+    from urlparse import urlparse, urlunparse, urljoin, urlsplit, urldefrag
     from urllib2 import parse_http_list
     import cookielib
     from Cookie import Morsel
     from StringIO import StringIO
-    from .packages import chardet
+    try:
+        import cchardet as chardet
+    except ImportError:
+        from .packages import chardet
+    from .packages.urllib3.packages.ordered_dict import OrderedDict
 
+    builtin_str = str
     bytes = str
     str = unicode
     basestring = basestring
+    numeric_types = (int, long, float)
 
 
 
 elif is_py3:
-    from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote
+    from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag
     from urllib.request import parse_http_list
     from http import cookiejar as cookielib
     from http.cookies import Morsel
     from io import StringIO
     from .packages import chardet2 as chardet
+    from collections import OrderedDict
 
+    builtin_str = str
     str = str
     bytes = bytes
-    basestring = (str,bytes)
-
+    basestring = (str, bytes)
+    numeric_types = (int, float)

requests/cookies.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 """
 Compatibility code to be able to use `cookielib.CookieJar` with requests.
 
@@ -14,6 +16,7 @@ try:
 except ImportError:
     import dummy_threading as threading
 
+
 class MockRequest(object):
     """Wraps a `requests.Request` to mimic a `urllib2.Request`.
 
@@ -66,6 +69,11 @@ class MockRequest(object):
     def get_new_headers(self):
         return self._new_headers
 
+    @property
+    def unverifiable(self):
+        return self.is_unverifiable()
+
+
 class MockResponse(object):
     """Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`.
 
@@ -86,6 +94,7 @@ class MockResponse(object):
     def getheaders(self, name):
         self._headers.getheaders(name)
 
+
 def extract_cookies_to_jar(jar, request, response):
     """Extract the cookies from the response into a CookieJar.
 
@@ -99,12 +108,14 @@ def extract_cookies_to_jar(jar, request, response):
     res = MockResponse(response._original_response.msg)
     jar.extract_cookies(res, req)
 
+
 def get_cookie_header(jar, request):
     """Produce an appropriate Cookie header string to be sent with `request`, or None."""
     r = MockRequest(request)
     jar.add_cookie_header(r)
     return r.get_new_headers().get('Cookie')
 
+
 def remove_cookie_by_name(cookiejar, name, domain=None, path=None):
     """Unsets a cookie by name, by default over all domains and paths.
 
@@ -120,10 +131,12 @@ def remove_cookie_by_name(cookiejar, name, domain=None, path=None):
     for domain, path, name in clearables:
         cookiejar.clear(domain, path, name)
 
+
 class CookieConflictError(RuntimeError):
-    """There are two cookies that meet the criteria specified in the cookie jar. 
+    """There are two cookies that meet the criteria specified in the cookie jar.
     Use .get and .set and include domain and path args in order to be more specific."""
 
+
 class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
     """Compatibility class; is a cookielib.CookieJar, but exposes a dict interface.
 
@@ -181,7 +194,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         for cookie in iter(self):
             values.append(cookie.value)
         return values
-    
+
     def items(self):
         """Dict-like items() that returns a list of name-value tuples from the jar.
         See keys() and values(). Allows client-code to call "dict(RequestsCookieJar)
@@ -215,14 +228,14 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
             if cookie.domain is not None and cookie.domain in domains:
                 return True
             domains.append(cookie.domain)
-        return False # there is only one domain in jar
+        return False  # there is only one domain in jar
 
     def get_dict(self, domain=None, path=None):
         """Takes as an argument an optional domain and path and returns a plain old
         Python dict of name-value pairs of cookies that meet the requirements."""
         dictionary = {}
         for cookie in iter(self):
-            if (domain == None or cookie.domain == domain) and (path == None 
+            if (domain is None or cookie.domain == domain) and (path is None
                                                 or cookie.path == path):
                 dictionary[cookie.name] = cookie.value
         return dictionary
@@ -244,7 +257,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         remove_cookie_by_name(self, name)
 
     def _find(self, name, domain=None, path=None):
-        """Requests uses this method internally to get cookie values. Takes as args name 
+        """Requests uses this method internally to get cookie values. Takes as args name
         and optional domain and path. Returns a cookie.value. If there are conflicting cookies,
         _find arbitrarily chooses one. See _find_no_duplicates if you want an exception thrown
         if there are conflicting cookies."""
@@ -257,18 +270,18 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path))
 
     def _find_no_duplicates(self, name, domain=None, path=None):
-        """__get_item__ and get call _find_no_duplicates -- never used in Requests internally. 
-        Takes as args name and optional domain and path. Returns a cookie.value. 
-        Throws KeyError if cookie is not found and CookieConflictError if there are 
+        """__get_item__ and get call _find_no_duplicates -- never used in Requests internally.
+        Takes as args name and optional domain and path. Returns a cookie.value.
+        Throws KeyError if cookie is not found and CookieConflictError if there are
         multiple cookies that match name and optionally domain and path."""
         toReturn = None
         for cookie in iter(self):
             if cookie.name == name:
                 if domain is None or cookie.domain == domain:
                     if path is None or cookie.path == path:
-                        if toReturn != None: # if there are multiple cookies that meet passed in criteria
+                        if toReturn is not None:  # if there are multiple cookies that meet passed in criteria
                             raise CookieConflictError('There are multiple cookies with name, %r' % (name))
-                        toReturn = cookie.value # we will eventually return this as long as no cookie conflict
+                        toReturn = cookie.value  # we will eventually return this as long as no cookie conflict
 
         if toReturn:
             return toReturn
@@ -291,6 +304,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
         """This is not implemented. Calling this will throw an exception."""
         raise NotImplementedError
 
+
 def create_cookie(name, value, **kwargs):
     """Make a cookie from underspecified parameters.
 
@@ -310,8 +324,7 @@ def create_cookie(name, value, **kwargs):
         comment=None,
         comment_url=None,
         rest={'HttpOnly': None},
-        rfc2109=False,
-        )
+        rfc2109=False,)
 
     badargs = set(kwargs) - set(result)
     if badargs:
@@ -326,6 +339,7 @@ def create_cookie(name, value, **kwargs):
 
     return cookielib.Cookie(**result)
 
+
 def morsel_to_cookie(morsel):
     """Convert a Morsel object into a Cookie containing the one k/v pair."""
     c = create_cookie(
@@ -345,10 +359,10 @@ def morsel_to_cookie(morsel):
         comment=morsel['comment'],
         comment_url=bool(morsel['comment']),
         rest={'HttpOnly': morsel['httponly']},
-        rfc2109=False,
-        )
+        rfc2109=False,)
     return c
 
+
 def cookiejar_from_dict(cookie_dict, cookiejar=None):
     """Returns a CookieJar from a key/value dictionary.
 

requests/defaults.py

@@ -20,20 +20,19 @@ Configurations:
 :pool_connections: The number of active HTTP connection pools to use.
 :encode_uri: If true, URIs will automatically be percent-encoded.
 :trust_env: If true, the surrouding environment will be trusted (environ, netrc).
-:param store_cookies: If false, the received cookies as part of the HTTP response would be ignored.
+:store_cookies: If false, the received cookies as part of the HTTP response would be ignored.
 
 """
 
 SCHEMAS = ['http', 'https']
 
-from . import __version__
+from .utils import default_user_agent
 
 defaults = dict()
 
-
 defaults['base_headers'] = {
-    'User-Agent': 'python-requests/%s' % __version__,
-    'Accept-Encoding': ', '.join(('identity', 'deflate', 'compress', 'gzip')),
+    'User-Agent': default_user_agent(),
+    'Accept-Encoding': ', '.join(('gzip', 'deflate', 'compress')),
     'Accept': '*/*'
 }
 
@@ -49,5 +48,3 @@ defaults['keep_alive'] = True
 defaults['encode_uri'] = True
 defaults['trust_env'] = True
 defaults['store_cookies'] = True
-
-

requests/exceptions.py

@@ -8,34 +8,44 @@ This module contains the set of Requests' exceptions.
 
 """
 
+
 class RequestException(RuntimeError):
     """There was an ambiguous exception that occurred while handling your
     request."""
 
+
 class HTTPError(RequestException):
     """An HTTP error occurred."""
     response = None
 
+
 class ConnectionError(RequestException):
     """A Connection error occurred."""
 
+
 class SSLError(ConnectionError):
     """An SSL error occurred."""
 
+
 class Timeout(RequestException):
     """The request timed out."""
 
+
 class URLRequired(RequestException):
     """A valid URL is required to make a request."""
 
+
 class TooManyRedirects(RequestException):
     """Too many redirects."""
 
+
 class MissingSchema(RequestException, ValueError):
     """The URL schema (e.g. http or https) is missing."""
 
+
 class InvalidSchema(RequestException, ValueError):
     """See defaults.py for valid schemas."""
 
+
 class InvalidURL(RequestException, ValueError):
     """ The URL provided was somehow invalid. """

requests/hooks.py

@@ -25,11 +25,10 @@ Available hooks:
 
 """
 
-import traceback
-
 
 HOOKS = ('args', 'pre_request', 'pre_send', 'post_request', 'response')
 
+
 def dispatch_hook(key, hooks, hook_data):
     """Dispatches a hook dictionary on a given piece of data."""
 
@@ -42,12 +41,8 @@ def dispatch_hook(key, hooks, hook_data):
             hooks = [hooks]
 
         for hook in hooks:
-            try:
-                _hook_data = hook(hook_data)
-                if _hook_data is not None:
-                    hook_data = _hook_data
-
-            except Exception:
-                traceback.print_exc()
+            _hook_data = hook(hook_data)
+            if _hook_data is not None:
+                hook_data = _hook_data
 
     return hook_data

requests/models.py

@@ -7,9 +7,10 @@ requests.models
 This module contains the primary objects that power Requests.
 """
 
-import json
 import os
+import socket
 from datetime import datetime
+from io import BytesIO
 
 from .hooks import dispatch_hook, HOOKS
 from .structures import CaseInsensitiveDict
@@ -18,6 +19,7 @@ from .status_codes import codes
 from .auth import HTTPBasicAuth, HTTPProxyAuth
 from .cookies import cookiejar_from_dict, extract_cookies_to_jar, get_cookie_header
 from .packages.urllib3.exceptions import MaxRetryError, LocationParseError
+from .packages.urllib3.exceptions import TimeoutError
 from .packages.urllib3.exceptions import SSLError as _SSLError
 from .packages.urllib3.exceptions import HTTPError as _HTTPError
 from .packages.urllib3 import connectionpool, poolmanager
@@ -29,17 +31,19 @@ from .exceptions import (
 from .utils import (
     get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri,
     stream_decode_response_unicode, get_netrc_auth, get_environ_proxies,
-    DEFAULT_CA_BUNDLE_PATH)
+    to_key_val_list, DEFAULT_CA_BUNDLE_PATH, parse_header_links, iter_slices,
+    guess_json_utf)
 from .compat import (
     cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,
-    StringIO, is_py2, chardet)
+    StringIO, is_py2, chardet, json, builtin_str, urldefrag)
 
 REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
 CONTENT_CHUNK_SIZE = 10 * 1024
 
+
 class Request(object):
-    """The :class:`Request <Request>` object. It carries out all functionality of
-    Requests. Recommended interface is with the Requests functions.
+    """The :class:`Request <Request>` object. It carries out all functionality
+    of Requests. Recommended interface is with the Requests functions.
     """
 
     def __init__(self,
@@ -57,7 +61,7 @@ class Request(object):
         proxies=None,
         hooks=None,
         config=None,
-        prefetch=False,
+        prefetch=True,
         _poolmanager=None,
         verify=None,
         session=None,
@@ -71,7 +75,14 @@ class Request(object):
         self.timeout = timeout
 
         #: Request URL.
-        self.url = url
+        #: Accept objects that have string representations.
+        try:
+            self.url = unicode(url)
+        except NameError:
+            # We're on Python 3.
+            self.url = str(url)
+        except UnicodeDecodeError:
+            self.url = url
 
         #: Dictionary of HTTP Headers to attach to the :class:`Request <Request>`.
         self.headers = dict(headers or [])
@@ -101,6 +112,10 @@ class Request(object):
         # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'})
         self.proxies = dict(proxies or [])
 
+        for proxy_type, uri_ref in list(self.proxies.items()):
+            if not uri_ref:
+                del self.proxies[proxy_type]
+
         # If no proxies are given, allow configuration by environment variables
         # HTTP_PROXY and HTTPS_PROXY.
         if not self.proxies and self.config.get('trust_env'):
@@ -183,7 +198,7 @@ class Request(object):
                 response.status_code = getattr(resp, 'status', None)
 
                 # Make headers case-insensitive.
-                response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None))
+                response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
 
                 # Set encoding.
                 response.encoding = get_encoding_from_headers(response.headers)
@@ -291,7 +306,8 @@ class Request(object):
                     proxies=self.proxies,
                     verify=self.verify,
                     session=self.session,
-                    cert=self.cert
+                    cert=self.cert,
+                    prefetch=self.prefetch,
                 )
 
                 request.send()
@@ -311,53 +327,60 @@ class Request(object):
         if parameters are supplied as a dict.
         """
 
-        if isinstance(data, bytes):
-            return data
-        if isinstance(data, str):
+        if isinstance(data, (str, bytes)):
             return data
         elif hasattr(data, 'read'):
             return data
         elif hasattr(data, '__iter__'):
-            try:
-                dict(data)
-            except ValueError:
-                raise ValueError('Unable to encode lists with elements that are not 2-tuples.')
-
-            params = list(data.items() if isinstance(data, dict) else data)
             result = []
-            for k, vs in params:
+            for k, vs in to_key_val_list(data):
                 for v in isinstance(vs, list) and vs or [vs]:
-                    result.append(
-                        (k.encode('utf-8') if isinstance(k, str) else k,
-                         v.encode('utf-8') if isinstance(v, str) else v))
+                    if v is not None:
+                        result.append(
+                            (k.encode('utf-8') if isinstance(k, str) else k,
+                             v.encode('utf-8') if isinstance(v, str) else v))
             return urlencode(result, doseq=True)
         else:
             return data
 
     def _encode_files(self, files):
+        """Build the body for a multipart/form-data request.
 
+        Will successfully encode files when passed as a dict or a list of
+        2-tuples. Order is retained if data is a list of 2-tuples but abritrary
+        if parameters are supplied as a dict.
+
+        """
         if (not files) or isinstance(self.data, str):
             return None
 
-        try:
-            fields = self.data.copy()
-        except AttributeError:
-            fields = dict(self.data)
+        new_fields = []
+        fields = to_key_val_list(self.data)
+        files = to_key_val_list(files)
 
-        for (k, v) in list(files.items()):
+        for field, val in fields:
+            if isinstance(val, list):
+                for v in val:
+                    new_fields.append((field, builtin_str(v)))
+            else:
+                new_fields.append((field, builtin_str(val)))
+
+        for (k, v) in files:
             # support for explicit filename
             if isinstance(v, (tuple, list)):
                 fn, fp = v
             else:
                 fn = guess_filename(v) or k
                 fp = v
-            if isinstance(fp, (bytes, str)):
+            if isinstance(fp, str):
                 fp = StringIO(fp)
-            fields.update({k: (fn, fp.read())})
+            if isinstance(fp, bytes):
+                fp = BytesIO(fp)
+            new_fields.append((k, (fn, fp.read())))
 
-        (body, content_type) = encode_multipart_formdata(fields)
+        body, content_type = encode_multipart_formdata(new_fields)
 
-        return (body, content_type)
+        return body, content_type
 
     @property
     def full_url(self):
@@ -377,7 +400,10 @@ class Request(object):
         if not scheme in SCHEMAS:
             raise InvalidSchema("Invalid scheme %r" % scheme)
 
-        netloc = netloc.encode('idna').decode('utf-8')
+        try:
+            netloc = netloc.encode('idna').decode('utf-8')
+        except UnicodeError:
+            raise InvalidURL('URL has an invalid label.')
 
         if not path:
             path = '/'
@@ -396,14 +422,14 @@ class Request(object):
             if isinstance(fragment, str):
                 fragment = fragment.encode('utf-8')
 
-        url = (urlunparse([scheme, netloc, path, params, query, fragment]))
-
         enc_params = self._encode_params(self.params)
         if enc_params:
-            if urlparse(url).query:
-                url = '%s&%s' % (url, enc_params)
+            if query:
+                query = '%s&%s' % (query, enc_params)
             else:
-                url = '%s?%s' % (url, enc_params)
+                query = enc_params
+
+        url = (urlunparse([scheme, netloc, path, params, query, fragment]))
 
         if self.config.get('encode_uri', True):
             url = requote_uri(url)
@@ -420,7 +446,9 @@ class Request(object):
 
         # Proxies use full URLs.
         if p.scheme in self.proxies:
-            return self.full_url
+            url_base, frag = urldefrag(self.full_url)
+            return url_base
+
 
         path = p.path
         if not path:
@@ -437,8 +465,10 @@ class Request(object):
 
     def register_hook(self, event, hook):
         """Properly register a hook."""
-
-        self.hooks[event].append(hook)
+        if callable(hook):
+            self.hooks[event].append(hook)
+        elif hasattr(hook, '__iter__'):
+            self.hooks[event].extend(h for h in hook if callable(h))
 
     def deregister_hook(self, event, hook):
         """Deregister a previously registered hook.
@@ -451,7 +481,7 @@ class Request(object):
         except ValueError:
             return False
 
-    def send(self, anyway=False, prefetch=False):
+    def send(self, anyway=False, prefetch=None):
         """Sends the request. Returns True if successful, False if not.
         If there was an HTTPError during transmission,
         self.response.status_code will contain the HTTPError code.
@@ -460,6 +490,9 @@ class Request(object):
 
         :param anyway: If True, request will be sent, even if it has
         already been sent.
+
+        :param prefetch: If not None, will override the request's own setting
+        for prefetch.
         """
 
         # Build the URL
@@ -475,10 +508,6 @@ class Request(object):
                 datetime.now().isoformat(), self.method, url
             ))
 
-        # Nottin' on you.
-        body = None
-        content_type = None
-
         # Use .netrc auth if none was provided.
         if not self.auth and self.config.get('trust_env'):
             self.auth = get_netrc_auth(url)
@@ -494,6 +523,10 @@ class Request(object):
             # Update self to reflect the auth changes.
             self.__dict__.update(r.__dict__)
 
+        # Nottin' on you.
+        body = None
+        content_type = None
+
         # Multi-part file uploads.
         if self.files:
             (body, content_type) = self._encode_files(self.files)
@@ -501,7 +534,7 @@ class Request(object):
             if self.data:
 
                 body = self._encode_params(self.data)
-                if isinstance(self.data, str) or hasattr(self.data, 'read'):
+                if isinstance(self.data, str) or isinstance(self.data, builtin_str) or hasattr(self.data, 'read'):
                     content_type = None
                 else:
                     content_type = 'application/x-www-form-urlencoded'
@@ -511,10 +544,10 @@ class Request(object):
             self.headers['Content-Type'] = content_type
 
         _p = urlparse(url)
-        no_proxy = filter(lambda x:x.strip(), self.proxies.get('no', '').split(','))
+        no_proxy = filter(lambda x: x.strip(), self.proxies.get('no', '').split(','))
         proxy = self.proxies.get(_p.scheme)
 
-        if proxy and not any(map(_p.netloc.endswith, no_proxy)):
+        if proxy and not any(map(_p.hostname.endswith, no_proxy)):
             conn = poolmanager.proxy_from_url(proxy)
             _proxy = urlparse(proxy)
             if '@' in _proxy.netloc:
@@ -597,14 +630,19 @@ class Request(object):
                 )
                 self.sent = True
 
+            except socket.error as sockerr:
+                raise ConnectionError(sockerr)
+
             except MaxRetryError as e:
                 raise ConnectionError(e)
 
             except (_SSLError, _HTTPError) as e:
-                if self.verify and isinstance(e, _SSLError):
+                if isinstance(e, _SSLError):
                     raise SSLError(e)
-
-                raise Timeout('Request timed out.')
+                elif isinstance(e, TimeoutError):
+                    raise Timeout(e)
+                else:
+                    raise Timeout('Request timed out.')
 
             # build_response can throw TooManyRedirects
             self._build_response(r)
@@ -617,7 +655,9 @@ class Request(object):
             self.__dict__.update(r.__dict__)
 
             # If prefetch is True, mark content as consumed.
-            if prefetch or self.prefetch:
+            if prefetch is None:
+                prefetch = self.prefetch
+            if prefetch:
                 # Save the response.
                 self.response.content
 
@@ -661,7 +701,7 @@ class Response(object):
 
         #: A list of :class:`Response <Response>` objects from
         #: the history of the Request. Any redirect responses will end
-        #: up here.
+        #: up here. The list is sorted from the oldest to the most recent request.
         self.history = []
 
         #: The :class:`Request <Request>` that created the Response.
@@ -699,9 +739,8 @@ class Response(object):
         length of each item returned as decoding can take place.
         """
         if self._content_consumed:
-            raise RuntimeError(
-                'The content for this response was already consumed'
-            )
+            # simulate reading small chunks of the content
+            return iter_slices(self._content, chunk_size)
 
         def generate():
             while 1:
@@ -765,6 +804,8 @@ class Response(object):
                 self._content = None
 
         self._content_consumed = True
+        # don't need to release the connection; that's been handled by urllib3
+        # since we exhausted the data.
         return self._content
 
     @property
@@ -790,10 +831,12 @@ class Response(object):
         # Decode unicode from given encoding.
         try:
             content = str(self.content, encoding, errors='replace')
-        except LookupError:
+        except (LookupError, TypeError):
             # A LookupError is raised if the encoding was not found which could
             # indicate a misspelling or similar mistake.
             #
+            # A TypeError can be raised if encoding is None
+            #
             # So we try blindly encoding.
             content = str(self.content, errors='replace')
 
@@ -801,12 +844,42 @@ class Response(object):
 
     @property
     def json(self):
-        """Returns the json-encoded content of a request, if any."""
+        """Returns the json-encoded content of a response, if any."""
+
+        if not self.encoding and len(self.content) > 3:
+            # No encoding set. JSON RFC 4627 section 3 states we should expect
+            # UTF-8, -16 or -32. Detect which one to use; If the detection or
+            # decoding fails, fall back to `self.text` (using chardet to make
+            # a best guess).
+            encoding = guess_json_utf(self.content)
+            if encoding is not None:
+                try:
+                    return json.loads(self.content.decode(encoding))
+                except (ValueError, UnicodeDecodeError):
+                    pass
         try:
             return json.loads(self.text or self.content)
         except ValueError:
             return None
 
+    @property
+    def links(self):
+        """Returns the parsed header links of the response, if any."""
+
+        header = self.headers['link']
+
+        # l = MultiDict()
+        l = {}
+
+        if header:
+            links = parse_header_links(header)
+
+            for link in links:
+                key = link.get('rel') or link.get('url')
+                l[key] = link
+
+        return l
+
     @property
     def reason(self):
         """The HTTP Reason for the response."""
@@ -818,17 +891,17 @@ class Response(object):
         if self.error:
             raise self.error
 
-        if (self.status_code >= 300) and (self.status_code < 400) and not allow_redirects:
-            http_error = HTTPError('%s Redirection' % self.status_code)
-            http_error.response = self
-            raise http_error
+        http_error_msg = ''
+        if 300 <= self.status_code < 400 and not allow_redirects:
+            http_error_msg = '%s Redirection: %s' % (self.status_code, self.reason)
 
-        elif (self.status_code >= 400) and (self.status_code < 500):
-            http_error = HTTPError('%s Client Error' % self.status_code)
-            http_error.response = self
-            raise http_error
+        elif 400 <= self.status_code < 500:
+            http_error_msg = '%s Client Error: %s' % (self.status_code, self.reason)
 
-        elif (self.status_code >= 500) and (self.status_code < 600):
-            http_error = HTTPError('%s Server Error' % self.status_code)
+        elif 500 <= self.status_code < 600:
+            http_error_msg = '%s Server Error: %s' % (self.status_code, self.reason)
+
+        if http_error_msg:
+            http_error = HTTPError(http_error_msg)
             http_error.response = self
             raise http_error

requests/packages/chardet2/test.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import sys, glob
 sys.path.insert(0, '..')
 from chardet.universaldetector import UniversalDetector

requests/packages/oauthlib/oauth1/rfc5849/__init__.py

@@ -17,6 +17,8 @@ from oauthlib.common import Request, urlencode, generate_nonce
 from oauthlib.common import generate_timestamp
 from . import parameters, signature, utils
 
+logger = logging.getLogger(__name__)
+
 SIGNATURE_HMAC = u"HMAC-SHA1"
 SIGNATURE_RSA = u"RSA-SHA1"
 SIGNATURE_PLAINTEXT = u"PLAINTEXT"
@@ -66,17 +68,17 @@ class Client(object):
             uri_query=urlparse.urlparse(uri).query,
             body=body,
             headers=headers)
-        logging.debug("Collected params: {0}".format(collected_params))
+        logger.debug("Collected params: {0}".format(collected_params))
 
         normalized_params = signature.normalize_parameters(collected_params)
         normalized_uri = signature.normalize_base_string_uri(request.uri)
-        logging.debug("Normalized params: {0}".format(normalized_params))
-        logging.debug("Normalized URI: {0}".format(normalized_uri))
+        logger.debug("Normalized params: {0}".format(normalized_params))
+        logger.debug("Normalized URI: {0}".format(normalized_uri))
 
         base_string = signature.construct_base_string(request.http_method,
             normalized_uri, normalized_params)
 
-        logging.debug("Base signing string: {0}".format(base_string))
+        logger.debug("Base signing string: {0}".format(base_string))
 
         if self.signature_method == SIGNATURE_HMAC:
             sig = signature.sign_hmac_sha1(base_string, self.client_secret,
@@ -87,7 +89,7 @@ class Client(object):
             sig = signature.sign_plaintext(self.client_secret,
                 self.resource_owner_secret)
 
-        logging.debug("Signature: {0}".format(sig))
+        logger.debug("Signature: {0}".format(sig))
         return sig
 
     def get_oauth_params(self):

requests/packages/urllib3/_collections.py

@@ -4,128 +4,91 @@
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
-from collections import deque
+from collections import MutableMapping
+from threading import Lock
+
+try: # Python 2.7+
+    from collections import OrderedDict
+except ImportError:
+    from .packages.ordered_dict import OrderedDict
 
-from threading import RLock
 
 __all__ = ['RecentlyUsedContainer']
 
 
-class AccessEntry(object):
-    __slots__ = ('key', 'is_valid')
-
-    def __init__(self, key, is_valid=True):
-        self.key = key
-        self.is_valid = is_valid
+_Null = object()
 
 
-class RecentlyUsedContainer(dict):
+class RecentlyUsedContainer(MutableMapping):
     """
-    Provides a dict-like that maintains up to ``maxsize`` keys while throwing
-    away the least-recently-used keys beyond ``maxsize``.
+    Provides a thread-safe dict-like container which maintains up to
+    ``maxsize`` keys while throwing away the least-recently-used keys beyond
+    ``maxsize``.
+
+    :param maxsize:
+        Maximum number of recent elements to retain.
+
+    :param dispose_func:
+        Every time an item is evicted from the container,
+        ``dispose_func(value)`` is called.  Callback which will get called
     """
 
-    # If len(self.access_log) exceeds self._maxsize * CLEANUP_FACTOR, then we
-    # will attempt to cleanup the invalidated entries in the access_log
-    # datastructure during the next 'get' operation.
-    CLEANUP_FACTOR = 10
+    ContainerCls = OrderedDict
 
-    def __init__(self, maxsize=10):
+    def __init__(self, maxsize=10, dispose_func=None):
         self._maxsize = maxsize
+        self.dispose_func = dispose_func
 
-        self._container = {}
-
-        # We use a deque to to store our keys ordered by the last access.
-        self.access_log = deque()
-        self.access_log_lock = RLock()
-
-        # We look up the access log entry by the key to invalidate it so we can
-        # insert a new authorative entry at the head without having to dig and
-        # find the old entry for removal immediately.
-        self.access_lookup = {}
-
-        # Trigger a heap cleanup when we get past this size
-        self.access_log_limit = maxsize * self.CLEANUP_FACTOR
-
-    def _invalidate_entry(self, key):
-        "If exists: Invalidate old entry and return it."
-        old_entry = self.access_lookup.get(key)
-        if old_entry:
-            old_entry.is_valid = False
-
-        return old_entry
-
-    def _push_entry(self, key):
-        "Push entry onto our access log, invalidate the old entry if exists."
-        self._invalidate_entry(key)
-
-        new_entry = AccessEntry(key)
-        self.access_lookup[key] = new_entry
-
-        self.access_log_lock.acquire()
-        self.access_log.appendleft(new_entry)
-        self.access_log_lock.release()
-
-    def _prune_entries(self, num):
-        "Pop entries from our access log until we popped ``num`` valid ones."
-        while num > 0:
-            self.access_log_lock.acquire()
-            p = self.access_log.pop()
-            self.access_log_lock.release()
-
-            if not p.is_valid:
-                continue # Invalidated entry, skip
-
-            dict.pop(self, p.key, None)
-            self.access_lookup.pop(p.key, None)
-            num -= 1
-
-    def _prune_invalidated_entries(self):
-        "Rebuild our access_log without the invalidated entries."
-        self.access_log_lock.acquire()
-        self.access_log = deque(e for e in self.access_log if e.is_valid)
-        self.access_log_lock.release()
-
-    def _get_ordered_access_keys(self):
-        "Return ordered access keys for inspection. Used for testing."
-        self.access_log_lock.acquire()
-        r = [e.key for e in self.access_log if e.is_valid]
-        self.access_log_lock.release()
-
-        return r
+        self._container = self.ContainerCls()
+        self._lock = Lock()
 
     def __getitem__(self, key):
-        item = dict.get(self, key)
+        # Re-insert the item, moving it to the end of the eviction line.
+        with self._lock:
+            item = self._container.pop(key)
+            self._container[key] = item
+            return item
 
-        if not item:
-            raise KeyError(key)
+    def __setitem__(self, key, value):
+        evicted_value = _Null
+        with self._lock:
+            # Possibly evict the existing value of 'key'
+            evicted_value = self._container.get(key, _Null)
+            self._container[key] = value
 
-        # Insert new entry with new high priority, also implicitly invalidates
-        # the old entry.
-        self._push_entry(key)
+            # If we didn't evict an existing value, we might have to evict the
+            # least recently used item from the beginning of the container.
+            if len(self._container) > self._maxsize:
+                _key, evicted_value = self._container.popitem(last=False)
 
-        if len(self.access_log) > self.access_log_limit:
-            # Heap is getting too big, try to clean up any tailing invalidated
-            # entries.
-            self._prune_invalidated_entries()
-
-        return item
-
-    def __setitem__(self, key, item):
-        # Add item to our container and access log
-        dict.__setitem__(self, key, item)
-        self._push_entry(key)
-
-        # Discard invalid and excess entries
-        self._prune_entries(len(self) - self._maxsize)
+        if self.dispose_func and evicted_value is not _Null:
+            self.dispose_func(evicted_value)
 
     def __delitem__(self, key):
-        self._invalidate_entry(key)
-        self.access_lookup.pop(key, None)
-        dict.__delitem__(self, key)
+        with self._lock:
+            value = self._container.pop(key)
 
-    def get(self, key, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            return default
+        if self.dispose_func:
+            self.dispose_func(value)
+
+    def __len__(self):
+        with self._lock:
+            return len(self._container)
+
+    def __iter__(self):
+        raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.')
+
+    def clear(self):
+        with self._lock:
+            # Copy pointers to all values, then wipe the mapping
+            # under Python 2, this copies the list of values twice :-|
+            values = list(self._container.values())
+            self._container.clear()
+
+        if self.dispose_func:
+            for value in values:
+                self.dispose_func(value)
+
+    def keys(self):
+        with self._lock:
+            return self._container.keys()

requests/packages/urllib3/connectionpool.py

@@ -7,27 +7,27 @@
 import logging
 import socket
 
-from socket import error as SocketError, timeout as SocketTimeout
+from socket import timeout as SocketTimeout
 
-try:   # Python 3
+try: # Python 3
     from http.client import HTTPConnection, HTTPException
     from http.client import HTTP_PORT, HTTPS_PORT
 except ImportError:
     from httplib import HTTPConnection, HTTPException
     from httplib import HTTP_PORT, HTTPS_PORT
 
-try:   # Python 3
+try: # Python 3
     from queue import LifoQueue, Empty, Full
 except ImportError:
     from Queue import LifoQueue, Empty, Full
 
 
-try:   # Compiled with SSL?
+try: # Compiled with SSL?
     HTTPSConnection = object
     BaseSSLError = None
     ssl = None
 
-    try:   # Python 3
+    try: # Python 3
         from http.client import HTTPSConnection
     except ImportError:
         from httplib import HTTPSConnection
@@ -35,7 +35,7 @@ try:   # Compiled with SSL?
     import ssl
     BaseSSLError = ssl.SSLError
 
-except (ImportError, AttributeError):
+except (ImportError, AttributeError): # Platform-specific: No SSL.
     pass
 
 
@@ -43,6 +43,7 @@ from .request import RequestMethods
 from .response import HTTPResponse
 from .util import get_host, is_connection_dropped
 from .exceptions import (
+    ClosedPoolError,
     EmptyPoolError,
     HostChangedError,
     MaxRetryError,
@@ -206,10 +207,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         try:
             conn = self.pool.get(block=self.block, timeout=timeout)
 
-            # If this is a persistent connection, check if it got disconnected
-            if conn and is_connection_dropped(conn):
-                log.info("Resetting dropped connection: %s" % self.host)
-                conn.close()
+        except AttributeError: # self.pool is None
+            raise ClosedPoolError(self, "Pool is closed.")
 
         except Empty:
             if self.block:
@@ -218,6 +217,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                                      "connections are allowed.")
             pass  # Oh well, we'll create a new connection then
 
+        # If this is a persistent connection, check if it got disconnected
+        if conn and is_connection_dropped(conn):
+            log.info("Resetting dropped connection: %s" % self.host)
+            conn.close()
+
         return conn or self._new_conn()
 
     def _put_conn(self, conn):
@@ -228,17 +232,26 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             Connection object for the current host and port as returned by
             :meth:`._new_conn` or :meth:`._get_conn`.
 
-        If the pool is already full, the connection is discarded because we
-        exceeded maxsize. If connections are discarded frequently, then maxsize
-        should be increased.
+        If the pool is already full, the connection is closed and discarded
+        because we exceeded maxsize. If connections are discarded frequently,
+        then maxsize should be increased.
+
+        If the pool is closed, then the connection will be closed and discarded.
         """
         try:
             self.pool.put(conn, block=False)
+            return # Everything is dandy, done.
+        except AttributeError:
+            # self.pool is None.
+            pass
         except Full:
             # This should never happen if self.block == True
             log.warning("HttpConnectionPool is full, discarding connection: %s"
                         % self.host)
 
+        # Connection never got put back into the pool, close it.
+        conn.close()
+
     def _make_request(self, conn, method, url, timeout=_Default,
                       **httplib_request_kw):
         """
@@ -268,15 +281,32 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         log.debug("\"%s %s %s\" %s %s" % (method, url, http_version,
                                           httplib_response.status,
                                           httplib_response.length))
-
         return httplib_response
 
+    def close(self):
+        """
+        Close all pooled connections and disable the pool.
+        """
+        # Disable access to the pool
+        old_pool, self.pool = self.pool, None
+
+        try:
+            while True:
+                conn = old_pool.get(block=False)
+                if conn:
+                    conn.close()
+
+        except Empty:
+            pass # Done.
 
     def is_same_host(self, url):
         """
         Check if the given ``url`` is a member of the same host as this
         connection pool.
         """
+        if url.startswith('/'):
+            return True
+
         # TODO: Add optional support for socket.gethostbyname checking.
         scheme, host, port = get_host(url)
 
@@ -284,8 +314,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # Use explicit default port for comparison when none is given.
             port = port_by_scheme.get(scheme)
 
-        return (url.startswith('/') or
-                (scheme, host, port) == (self.scheme, self.host, self.port))
+        return (scheme, host, port) == (self.scheme, self.host, self.port)
 
     def urlopen(self, method, url, body=None, headers=None, retries=3,
                 redirect=True, assert_same_host=True, timeout=_Default,
@@ -378,7 +407,6 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
 
         try:
             # Request a connection from the queue
-            # (Could raise SocketError: Bad file descriptor)
             conn = self._get_conn(timeout=pool_timeout)
 
             # Make the request on the httplib connection object
@@ -421,29 +449,38 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # Name mismatch
             raise SSLError(e)
 
-        except (HTTPException, SocketError) as e:
+        except HTTPException as e:
             # Connection broken, discard. It will be replaced next _get_conn().
             conn = None
             # This is necessary so we can access e below
             err = e
 
         finally:
-            if conn and release_conn:
-                # Put the connection back to be reused
+            if release_conn:
+                # Put the connection back to be reused. If the connection is
+                # expired then it will be None, which will get replaced with a
+                # fresh connection during _get_conn.
                 self._put_conn(conn)
 
         if not conn:
+            # Try again
             log.warn("Retrying (%d attempts remain) after connection "
                      "broken by '%r': %s" % (retries, err, url))
             return self.urlopen(method, url, body, headers, retries - 1,
-                                redirect, assert_same_host)  # Try again
+                                redirect, assert_same_host,
+                                timeout=timeout, pool_timeout=pool_timeout,
+                                release_conn=release_conn, **response_kw)
 
         # Handle redirect?
         redirect_location = redirect and response.get_redirect_location()
         if redirect_location:
+            if response.status == 303:
+                method = 'GET'
             log.info("Redirecting %s -> %s" % (url, redirect_location))
             return self.urlopen(method, redirect_location, body, headers,
-                                retries - 1, redirect, assert_same_host)
+                                retries - 1, redirect, assert_same_host,
+                                timeout=timeout, pool_timeout=pool_timeout,
+                                release_conn=release_conn, **response_kw)
 
         return response
 

requests/packages/urllib3/exceptions.py

@@ -24,6 +24,11 @@ class SSLError(HTTPError):
     pass
 
 
+class DecodeError(HTTPError):
+    "Raised when automatic decoding based on Content-Type fails."
+    pass
+
+
 ## Leaf Exceptions
 
 class MaxRetryError(PoolError):
@@ -57,6 +62,11 @@ class EmptyPoolError(PoolError):
     pass
 
 
+class ClosedPoolError(PoolError):
+    "Raised when a request enters a pool after the pool has been closed."
+    pass
+
+
 class LocationParseError(ValueError, HTTPError):
     "Raised when get_host or similar fails to parse the URL input."
 

requests/packages/urllib3/packages/ordered_dict.py

@@ -0,0 +1,260 @@
+# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
+# Passes Python2.7's test suite and incorporates all the latest updates.
+# Copyright 2009 Raymond Hettinger, released under the MIT License.
+# http://code.activestate.com/recipes/576693/
+
+try:
+    from thread import get_ident as _get_ident
+except ImportError:
+    from dummy_thread import get_ident as _get_ident
+
+try:
+    from _abcoll import KeysView, ValuesView, ItemsView
+except ImportError:
+    pass
+
+
+class OrderedDict(dict):
+    'Dictionary that remembers insertion order'
+    # An inherited dict maps keys to values.
+    # The inherited dict provides __getitem__, __len__, __contains__, and get.
+    # The remaining methods are order-aware.
+    # Big-O running times for all methods are the same as for regular dictionaries.
+
+    # The internal self.__map dictionary maps keys to links in a doubly linked list.
+    # The circular doubly linked list starts and ends with a sentinel element.
+    # The sentinel element never gets deleted (this simplifies the algorithm).
+    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+
+    def __init__(self, *args, **kwds):
+        '''Initialize an ordered dictionary.  Signature is the same as for
+        regular dictionaries, but keyword arguments are not recommended
+        because their insertion order is arbitrary.
+
+        '''
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        try:
+            self.__root
+        except AttributeError:
+            self.__root = root = []                     # sentinel node
+            root[:] = [root, root, None]
+            self.__map = {}
+        self.__update(*args, **kwds)
+
+    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
+        'od.__setitem__(i, y) <==> od[i]=y'
+        # Setting a new item creates a new link which goes at the end of the linked
+        # list, and the inherited dictionary is updated with the new key/value pair.
+        if key not in self:
+            root = self.__root
+            last = root[0]
+            last[1] = root[0] = self.__map[key] = [last, root, key]
+        dict_setitem(self, key, value)
+
+    def __delitem__(self, key, dict_delitem=dict.__delitem__):
+        'od.__delitem__(y) <==> del od[y]'
+        # Deleting an existing item uses self.__map to find the link which is
+        # then removed by updating the links in the predecessor and successor nodes.
+        dict_delitem(self, key)
+        link_prev, link_next, key = self.__map.pop(key)
+        link_prev[1] = link_next
+        link_next[0] = link_prev
+
+    def __iter__(self):
+        'od.__iter__() <==> iter(od)'
+        root = self.__root
+        curr = root[1]
+        while curr is not root:
+            yield curr[2]
+            curr = curr[1]
+
+    def __reversed__(self):
+        'od.__reversed__() <==> reversed(od)'
+        root = self.__root
+        curr = root[0]
+        while curr is not root:
+            yield curr[2]
+            curr = curr[0]
+
+    def clear(self):
+        'od.clear() -> None.  Remove all items from od.'
+        try:
+            for node in self.__map.itervalues():
+                del node[:]
+            root = self.__root
+            root[:] = [root, root, None]
+            self.__map.clear()
+        except AttributeError:
+            pass
+        dict.clear(self)
+
+    def popitem(self, last=True):
+        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
+        Pairs are returned in LIFO order if last is true or FIFO order if false.
+
+        '''
+        if not self:
+            raise KeyError('dictionary is empty')
+        root = self.__root
+        if last:
+            link = root[0]
+            link_prev = link[0]
+            link_prev[1] = root
+            root[0] = link_prev
+        else:
+            link = root[1]
+            link_next = link[1]
+            root[1] = link_next
+            link_next[0] = root
+        key = link[2]
+        del self.__map[key]
+        value = dict.pop(self, key)
+        return key, value
+
+    # -- the following methods do not depend on the internal structure --
+
+    def keys(self):
+        'od.keys() -> list of keys in od'
+        return list(self)
+
+    def values(self):
+        'od.values() -> list of values in od'
+        return [self[key] for key in self]
+
+    def items(self):
+        'od.items() -> list of (key, value) pairs in od'
+        return [(key, self[key]) for key in self]
+
+    def iterkeys(self):
+        'od.iterkeys() -> an iterator over the keys in od'
+        return iter(self)
+
+    def itervalues(self):
+        'od.itervalues -> an iterator over the values in od'
+        for k in self:
+            yield self[k]
+
+    def iteritems(self):
+        'od.iteritems -> an iterator over the (key, value) items in od'
+        for k in self:
+            yield (k, self[k])
+
+    def update(*args, **kwds):
+        '''od.update(E, **F) -> None.  Update od from dict/iterable E and F.
+
+        If E is a dict instance, does:           for k in E: od[k] = E[k]
+        If E has a .keys() method, does:         for k in E.keys(): od[k] = E[k]
+        Or if E is an iterable of items, does:   for k, v in E: od[k] = v
+        In either case, this is followed by:     for k, v in F.items(): od[k] = v
+
+        '''
+        if len(args) > 2:
+            raise TypeError('update() takes at most 2 positional '
+                            'arguments (%d given)' % (len(args),))
+        elif not args:
+            raise TypeError('update() takes at least 1 argument (0 given)')
+        self = args[0]
+        # Make progressively weaker assumptions about "other"
+        other = ()
+        if len(args) == 2:
+            other = args[1]
+        if isinstance(other, dict):
+            for key in other:
+                self[key] = other[key]
+        elif hasattr(other, 'keys'):
+            for key in other.keys():
+                self[key] = other[key]
+        else:
+            for key, value in other:
+                self[key] = value
+        for key, value in kwds.items():
+            self[key] = value
+
+    __update = update  # let subclasses override update without breaking __init__
+
+    __marker = object()
+
+    def pop(self, key, default=__marker):
+        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
+        If key is not found, d is returned if given, otherwise KeyError is raised.
+
+        '''
+        if key in self:
+            result = self[key]
+            del self[key]
+            return result
+        if default is self.__marker:
+            raise KeyError(key)
+        return default
+
+    def setdefault(self, key, default=None):
+        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
+        if key in self:
+            return self[key]
+        self[key] = default
+        return default
+
+    def __repr__(self, _repr_running={}):
+        'od.__repr__() <==> repr(od)'
+        call_key = id(self), _get_ident()
+        if call_key in _repr_running:
+            return '...'
+        _repr_running[call_key] = 1
+        try:
+            if not self:
+                return '%s()' % (self.__class__.__name__,)
+            return '%s(%r)' % (self.__class__.__name__, self.items())
+        finally:
+            del _repr_running[call_key]
+
+    def __reduce__(self):
+        'Return state information for pickling'
+        items = [[k, self[k]] for k in self]
+        inst_dict = vars(self).copy()
+        for k in vars(OrderedDict()):
+            inst_dict.pop(k, None)
+        if inst_dict:
+            return (self.__class__, (items,), inst_dict)
+        return self.__class__, (items,)
+
+    def copy(self):
+        'od.copy() -> a shallow copy of od'
+        return self.__class__(self)
+
+    @classmethod
+    def fromkeys(cls, iterable, value=None):
+        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
+        and values equal to v (which defaults to None).
+
+        '''
+        d = cls()
+        for key in iterable:
+            d[key] = value
+        return d
+
+    def __eq__(self, other):
+        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
+        while comparison to a regular mapping is order-insensitive.
+
+        '''
+        if isinstance(other, OrderedDict):
+            return len(self)==len(other) and self.items() == other.items()
+        return dict.__eq__(self, other)
+
+    def __ne__(self, other):
+        return not self == other
+
+    # -- the following methods are only used in Python 2.7 --
+
+    def viewkeys(self):
+        "od.viewkeys() -> a set-like object providing a view on od's keys"
+        return KeysView(self)
+
+    def viewvalues(self):
+        "od.viewvalues() -> an object providing a view on od's values"
+        return ValuesView(self)
+
+    def viewitems(self):
+        "od.viewitems() -> a set-like object providing a view on od's items"
+        return ItemsView(self)

requests/packages/urllib3/poolmanager.py

@@ -8,9 +8,9 @@ import logging
 
 from ._collections import RecentlyUsedContainer
 from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
-from .connectionpool import get_host, connection_from_url, port_by_scheme
-from .exceptions import HostChangedError
+from .connectionpool import connection_from_url, port_by_scheme
 from .request import RequestMethods
+from .util import parse_url
 
 
 __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
@@ -48,19 +48,29 @@ class PoolManager(RequestMethods):
 
     """
 
-    # TODO: Make sure there are no memory leaks here.
-
     def __init__(self, num_pools=10, **connection_pool_kw):
         self.connection_pool_kw = connection_pool_kw
-        self.pools = RecentlyUsedContainer(num_pools)
+        self.pools = RecentlyUsedContainer(num_pools,
+                                           dispose_func=lambda p: p.close())
 
-    def connection_from_host(self, host, port=80, scheme='http'):
+    def clear(self):
+        """
+        Empty our store of pools and direct them all to close.
+
+        This will not affect in-flight connections, but they will not be
+        re-used after completion.
+        """
+        self.pools.clear()
+
+    def connection_from_host(self, host, port=None, scheme='http'):
         """
         Get a :class:`ConnectionPool` based on the host, port, and scheme.
 
-        Note that an appropriate ``port`` value is required here to normalize
-        connection pools in our container most effectively.
+        If ``port`` isn't given, it will be derived from the ``scheme`` using
+        ``urllib3.connectionpool.port_by_scheme``.
         """
+        port = port or port_by_scheme.get(scheme, 80)
+
         pool_key = (scheme, host, port)
 
         # If the scheme, host, or port doesn't match existing open connections,
@@ -86,26 +96,36 @@ class PoolManager(RequestMethods):
         Additional parameters are taken from the :class:`.PoolManager`
         constructor.
         """
-        scheme, host, port = get_host(url)
+        u = parse_url(url)
+        return self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
 
-        port = port or port_by_scheme.get(scheme, 80)
-
-        return self.connection_from_host(host, port=port, scheme=scheme)
-
-    def urlopen(self, method, url, **kw):
+    def urlopen(self, method, url, redirect=True, **kw):
         """
-        Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`.
+        Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
+        with custom cross-host redirect logic and only sends the request-uri
+        portion of the ``url``.
 
-        ``url`` must be absolute, such that an appropriate
+        The given ``url`` parameter must be absolute, such that an appropriate
         :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
         """
-        conn = self.connection_from_url(url)
-        try:
-            return conn.urlopen(method, url, **kw)
+        u = parse_url(url)
+        conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
 
-        except HostChangedError as e:
-            kw['retries'] = e.retries # Persist retries countdown
-            return self.urlopen(method, e.url, **kw)
+        kw['assert_same_host'] = False
+        kw['redirect'] = False
+
+        response = conn.urlopen(method, u.request_uri, **kw)
+
+        redirect_location = redirect and response.get_redirect_location()
+        if not redirect_location:
+            return response
+
+        if response.status == 303:
+            method = 'GET'
+
+        log.info("Redirecting %s -> %s" % (url, redirect_location))
+        kw['retries'] = kw.get('retries', 3) - 1 # Persist retries countdown
+        return self.urlopen(method, redirect_location, **kw)
 
 
 class ProxyManager(RequestMethods):

requests/packages/urllib3/response.py

@@ -10,7 +10,7 @@ import zlib
 
 from io import BytesIO
 
-from .exceptions import HTTPError
+from .exceptions import DecodeError
 from .packages.six import string_types as basestring
 
 
@@ -148,9 +148,9 @@ class HTTPResponse(object):
             try:
                 if decode_content and decoder:
                     data = decoder(data)
-            except IOError:
-                raise HTTPError("Received response with content-encoding: %s, but "
-                                "failed to decode it." % content_encoding)
+            except (IOError, zlib.error):
+                raise DecodeError("Received response with content-encoding: %s, but "
+                                  "failed to decode it." % content_encoding)
 
             if cache_content:
                 self._body = data

requests/packages/urllib3/util.py

@@ -6,6 +6,8 @@
 
 
 from base64 import b64encode
+from collections import namedtuple
+from socket import error as SocketError
 
 try:
     from select import poll, POLLIN
@@ -20,6 +22,152 @@ from .packages import six
 from .exceptions import LocationParseError
 
 
+class Url(namedtuple('Url', ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'])):
+    """
+    Datastructure for representing an HTTP URL. Used as a return value for
+    :func:`parse_url`.
+    """
+    slots = ()
+
+    def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None):
+        return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment)
+
+    @property
+    def hostname(self):
+        """For backwards-compatibility with urlparse. We're nice like that."""
+        return self.host
+
+    @property
+    def request_uri(self):
+        """Absolute path including the query string."""
+        uri = self.path or '/'
+
+        if self.query is not None:
+            uri += '?' + self.query
+
+        return uri
+
+
+def split_first(s, delims):
+    """
+    Given a string and an iterable of delimiters, split on the first found
+    delimiter. Return two split parts and the matched delimiter.
+
+    If not found, then the first part is the full input string.
+
+    Example: ::
+
+        >>> split_first('foo/bar?baz', '?/=')
+        ('foo', 'bar?baz', '/')
+        >>> split_first('foo/bar?baz', '123')
+        ('foo/bar?baz', '', None)
+
+    Scales linearly with number of delims. Not ideal for large number of delims.
+    """
+    min_idx = None
+    min_delim = None
+    for d in delims:
+        idx = s.find(d)
+        if idx < 0:
+            continue
+
+        if min_idx is None or idx < min_idx:
+            min_idx = idx
+            min_delim = d
+
+    if min_idx is None or min_idx < 0:
+        return s, '', None
+
+    return s[:min_idx], s[min_idx+1:], min_delim
+
+
+def parse_url(url):
+    """
+    Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is
+    performed to parse incomplete urls. Fields not provided will be None.
+
+    Partly backwards-compatible with :mod:`urlparse`.
+
+    Example: ::
+
+        >>> parse_url('http://google.com/mail/')
+        Url(scheme='http', host='google.com', port=None, path='/', ...)
+        >>> prase_url('google.com:80')
+        Url(scheme=None, host='google.com', port=80, path=None, ...)
+        >>> prase_url('/foo?bar')
+        Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)
+    """
+
+    # While this code has overlap with stdlib's urlparse, it is much
+    # simplified for our needs and less annoying.
+    # Additionally, this imeplementations does silly things to be optimal
+    # on CPython.
+
+    scheme = None
+    auth = None
+    host = None
+    port = None
+    path = None
+    fragment = None
+    query = None
+
+    # Scheme
+    if '://' in url:
+        scheme, url = url.split('://', 1)
+
+    # Find the earliest Authority Terminator
+    # (http://tools.ietf.org/html/rfc3986#section-3.2)
+    url, path_, delim = split_first(url, ['/', '?', '#'])
+
+    if delim:
+        # Reassemble the path
+        path = delim + path_
+
+    # Auth
+    if '@' in url:
+        auth, url = url.split('@', 1)
+
+    # IPv6
+    if url and url[0] == '[':
+        host, url = url[1:].split(']', 1)
+
+    # Port
+    if ':' in url:
+        _host, port = url.split(':', 1)
+
+        if not host:
+            host = _host
+
+        if not port.isdigit():
+            raise LocationParseError("Failed to parse: %s" % url)
+
+        port = int(port)
+
+    elif not host and url:
+        host = url
+
+    if not path:
+        return Url(scheme, auth, host, port, path, query, fragment)
+
+    # Fragment
+    if '#' in path:
+        path, fragment = path.split('#', 1)
+
+    # Query
+    if '?' in path:
+        path, query = path.split('?', 1)
+
+    return Url(scheme, auth, host, port, path, query, fragment)
+
+
+def get_host(url):
+    """
+    Deprecated. Use :func:`.parse_url` instead.
+    """
+    p = parse_url(url)
+    return p.scheme or 'http', p.hostname, p.port
+
+
 def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
                  basic_auth=None):
     """
@@ -72,93 +220,12 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
     return headers
 
 
-def split_first(s, delims):
-    """
-    Given a string and an iterable of delimiters, split on the first found
-    delimiter. Return two split parts.
-
-    If not found, then the first part is the full input string.
-
-    Scales linearly with number of delims. Not ideal for large number of delims.
-    """
-    min_idx = None
-    for d in delims:
-        idx = s.find(d)
-        if idx < 0:
-            continue
-
-        if not min_idx:
-            min_idx = idx
-        else:
-            min_idx = min(idx, min_idx)
-
-    if min_idx < 0:
-        return s, ''
-
-    return s[:min_idx], s[min_idx+1:]
-
-
-def get_host(url):
-    """
-    Given a url, return its scheme, host and port (None if it's not there).
-
-    For example: ::
-
-        >>> get_host('http://google.com/mail/')
-        ('http', 'google.com', None)
-        >>> get_host('google.com:80')
-        ('http', 'google.com', 80)
-    """
-
-    # While this code has overlap with stdlib's urlparse, it is much
-    # simplified for our needs and less annoying.
-    # Additionally, this imeplementations does silly things to be optimal
-    # on CPython.
-
-    scheme = 'http'
-    host = None
-    port = None
-
-    # Scheme
-    if '://' in url:
-        scheme, url = url.split('://', 1)
-
-    # Find the earliest Authority Terminator
-    # (http://tools.ietf.org/html/rfc3986#section-3.2)
-    url, _path = split_first(url, ['/', '?', '#'])
-
-    # Auth
-    if '@' in url:
-        _auth, url = url.split('@', 1)
-
-    # IPv6
-    if url and url[0] == '[':
-        host, url = url[1:].split(']', 1)
-
-    # Port
-    if ':' in url:
-        _host, port = url.split(':', 1)
-
-        if not host:
-            host = _host
-
-        if not port.isdigit():
-            raise LocationParseError("Failed to parse: %s" % url)
-
-        port = int(port)
-
-    elif not host:
-        host = url
-
-    return scheme, host, port
-
-
 def is_connection_dropped(conn):
     """
     Returns True if the connection is dropped and should be closed.
 
     :param conn:
-        ``HTTPConnection`` object.
+        :class:`httplib.HTTPConnection` object.
 
     Note: For platforms like AppEngine, this will always return ``False`` to
     let the platform handle connection recycling transparently for us.
@@ -171,7 +238,10 @@ def is_connection_dropped(conn):
         if not select: # Platform-specific: AppEngine
             return False
 
-        return select([sock], [], [], 0.0)[0]
+        try:
+            return select([sock], [], [], 0.0)[0]
+        except SocketError:
+            return True
 
     # This version is better on platforms that support it.
     p = poll()

requests/safe_mode.py

@@ -16,23 +16,25 @@ from .packages.urllib3.response import HTTPResponse
 from .exceptions import RequestException, ConnectionError, HTTPError
 import socket
 
+
 def catch_exceptions_if_in_safe_mode(function):
     """New implementation of safe_mode. We catch all exceptions at the API level
     and then return a blank Response object with the error field filled. This decorator
     wraps request() in api.py.
     """
-    
+
     def wrapped(method, url, **kwargs):
         # if save_mode, we catch exceptions and fill error field
-        if (kwargs.get('config') and kwargs.get('config').get('safe_mode')) or (kwargs.get('session') 
+        if (kwargs.get('config') and kwargs.get('config').get('safe_mode')) or (kwargs.get('session')
                                             and kwargs.get('session').config.get('safe_mode')):
             try:
                 return function(method, url, **kwargs)
-            except (RequestException, ConnectionError, HTTPError, socket.timeout) as e:
+            except (RequestException, ConnectionError, HTTPError,
+                    socket.timeout, socket.gaierror) as e:
                 r = Response()
                 r.error = e
-                r.raw = HTTPResponse() # otherwise, tests fail
-                r.status_code = 0 # with this status_code, content returns None
+                r.raw = HTTPResponse()  # otherwise, tests fail
+                r.status_code = 0  # with this status_code, content returns None
                 return r
         return function(method, url, **kwargs)
     return wrapped

requests/sessions.py

@@ -15,9 +15,10 @@ from .cookies import cookiejar_from_dict, remove_cookie_by_name
 from .defaults import defaults
 from .models import Request
 from .hooks import dispatch_hook
-from .utils import header_expand
+from .utils import header_expand, from_key_val_list
 from .packages.urllib3.poolmanager import PoolManager
 
+
 def merge_kwargs(local_kwarg, default_kwarg):
     """Merges kwarg dictionaries.
 
@@ -37,12 +38,15 @@ def merge_kwargs(local_kwarg, default_kwarg):
     if not hasattr(default_kwarg, 'items'):
         return local_kwarg
 
+    default_kwarg = from_key_val_list(default_kwarg)
+    local_kwarg = from_key_val_list(local_kwarg)
+
     # Update new values.
     kwargs = default_kwarg.copy()
     kwargs.update(local_kwarg)
 
     # Remove keys that are set to None.
-    for (k, v) in list(local_kwarg.items()):
+    for (k, v) in local_kwarg.items():
         if v is None:
             del kwargs[k]
 
@@ -56,7 +60,6 @@ class Session(object):
         'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks',
         'params', 'config', 'verify', 'cert', 'prefetch']
 
-
     def __init__(self,
         headers=None,
         cookies=None,
@@ -66,17 +69,17 @@ class Session(object):
         hooks=None,
         params=None,
         config=None,
-        prefetch=False,
+        prefetch=True,
         verify=True,
         cert=None):
 
-        self.headers = headers or {}
+        self.headers = from_key_val_list(headers or [])
         self.auth = auth
         self.timeout = timeout
-        self.proxies = proxies or {}
-        self.hooks = hooks or {}
-        self.params = params or {}
-        self.config = config or {}
+        self.proxies = from_key_val_list(proxies or [])
+        self.hooks = from_key_val_list(hooks or {})
+        self.params = from_key_val_list(params or [])
+        self.config = from_key_val_list(config or {})
         self.prefetch = prefetch
         self.verify = verify
         self.cert = cert
@@ -105,7 +108,15 @@ class Session(object):
         return self
 
     def __exit__(self, *args):
-        pass
+        self.close()
+
+    def close(self):
+        """Dispose of any internal state.
+
+        Currently, this just closes the PoolManager, which closes pooled
+        connections.
+        """
+        self.poolmanager.clear()
 
     def request(self, method, url,
         params=None,
@@ -120,7 +131,7 @@ class Session(object):
         hooks=None,
         return_response=True,
         config=None,
-        prefetch=False,
+        prefetch=None,
         verify=None,
         cert=None):
 
@@ -140,7 +151,7 @@ class Session(object):
         :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
         :param return_response: (optional) If False, an un-sent Request object will returned.
         :param config: (optional) A configuration dictionary. See ``request.defaults`` for allowed keys and their default values.
-        :param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
+        :param prefetch: (optional) whether to immediately download the response content. Defaults to ``True``.
         :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
         :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
         """
@@ -148,12 +159,12 @@ class Session(object):
         method = str(method).upper()
 
         # Default empty dicts for dict params.
-        data = {} if data is None else data
-        files = {} if files is None else files
+        data = [] if data is None else data
+        files = [] if files is None else files
         headers = {} if headers is None else headers
         params = {} if params is None else params
         hooks = {} if hooks is None else hooks
-        prefetch = self.prefetch or prefetch
+        prefetch = prefetch if prefetch is not None else self.prefetch
 
         # use session's hooks as defaults
         for key, cb in list(self.hooks.items()):
@@ -161,23 +172,23 @@ class Session(object):
 
         # Expand header values.
         if headers:
-            for k, v in list(headers.items()) or {}:
+            for k, v in list(headers.items() or {}):
                 headers[k] = header_expand(v)
 
         args = dict(
             method=method,
             url=url,
             data=data,
-            params=params,
-            headers=headers,
+            params=from_key_val_list(params),
+            headers=from_key_val_list(headers),
             cookies=cookies,
             files=files,
             auth=auth,
-            hooks=hooks,
+            hooks=from_key_val_list(hooks),
             timeout=timeout,
             allow_redirects=allow_redirects,
-            proxies=proxies,
-            config=config,
+            proxies=from_key_val_list(proxies),
+            config=from_key_val_list(config),
             prefetch=prefetch,
             verify=verify,
             cert=cert,
@@ -232,7 +243,6 @@ class Session(object):
         # Return the response.
         return r.response
 
-
     def get(self, url, **kwargs):
         """Sends a GET request. Returns :class:`Response` object.
 
@@ -243,7 +253,6 @@ class Session(object):
         kwargs.setdefault('allow_redirects', True)
         return self.request('get', url, **kwargs)
 
-
     def options(self, url, **kwargs):
         """Sends a OPTIONS request. Returns :class:`Response` object.
 
@@ -254,7 +263,6 @@ class Session(object):
         kwargs.setdefault('allow_redirects', True)
         return self.request('options', url, **kwargs)
 
-
     def head(self, url, **kwargs):
         """Sends a HEAD request. Returns :class:`Response` object.
 
@@ -265,7 +273,6 @@ class Session(object):
         kwargs.setdefault('allow_redirects', False)
         return self.request('head', url, **kwargs)
 
-
     def post(self, url, data=None, **kwargs):
         """Sends a POST request. Returns :class:`Response` object.
 
@@ -276,7 +283,6 @@ class Session(object):
 
         return self.request('post', url, data=data, **kwargs)
 
-
     def put(self, url, data=None, **kwargs):
         """Sends a PUT request. Returns :class:`Response` object.
 
@@ -287,7 +293,6 @@ class Session(object):
 
         return self.request('put', url, data=data, **kwargs)
 
-
     def patch(self, url, data=None, **kwargs):
         """Sends a PATCH request. Returns :class:`Response` object.
 
@@ -298,7 +303,6 @@ class Session(object):
 
         return self.request('patch', url,  data=data, **kwargs)
 
-
     def delete(self, url, **kwargs):
         """Sends a DELETE request. Returns :class:`Response` object.
 

requests/status_codes.py

@@ -83,4 +83,4 @@ for (code, titles) in list(_codes.items()):
     for title in titles:
         setattr(codes, title, code)
         if not title.startswith('\\'):
-            setattr(codes, title.upper(), code)
+            setattr(codes, title.upper(), code)

requests/structures.py

@@ -47,6 +47,7 @@ class CaseInsensitiveDict(dict):
         else:
             return default
 
+
 class LookupDict(dict):
     """Dictionary lookup object."""
 

requests/utils.py

@@ -12,12 +12,15 @@ that are also useful for external consumption.
 import cgi
 import codecs
 import os
+import platform
 import re
+import sys
 import zlib
 from netrc import netrc, NetrcParseError
 
+from . import __version__
 from .compat import parse_http_list as _parse_list_header
-from .compat import quote, urlparse, basestring, bytes, str
+from .compat import quote, quote_plus, urlparse, basestring, bytes, str, OrderedDict
 from .cookies import RequestsCookieJar, cookiejar_from_dict
 
 _hush_pyflakes = (RequestsCookieJar,)
@@ -46,6 +49,7 @@ POSSIBLE_CA_BUNDLE_PATHS = [
         '/etc/ssl/ca-bundle.pem',
 ]
 
+
 def get_os_ca_bundle_path():
     """Try to pick an available CA certificate bundle provided by the OS."""
     for path in POSSIBLE_CA_BUNDLE_PATHS:
@@ -57,6 +61,7 @@ def get_os_ca_bundle_path():
 # otherwise, try and use the OS bundle
 DEFAULT_CA_BUNDLE_PATH = CERTIFI_BUNDLE_PATH or get_os_ca_bundle_path()
 
+
 def dict_to_sequence(d):
     """Returns an internal sequence dictionary update."""
 
@@ -98,7 +103,7 @@ def get_netrc_auth(url):
             pass
 
     # AppEngine hackiness.
-    except AttributeError:
+    except (ImportError, AttributeError):
         pass
 
 
@@ -109,6 +114,54 @@ def guess_filename(obj):
         return name
 
 
+def from_key_val_list(value):
+    """Take an object and test to see if it can be represented as a
+    dictionary. Unless it can not be represented as such, return an
+    OrderedDict, e.g.,
+
+    ::
+
+        >>> from_key_val_list([('key', 'val')])
+        OrderedDict([('key', 'val')])
+        >>> from_key_val_list('string')
+        ValueError: need more than 1 value to unpack
+        >>> from_key_val_list({'key': 'val'})
+        OrderedDict([('key', 'val')])
+    """
+    if value is None:
+        return None
+
+    if isinstance(value, (str, bytes, bool, int)):
+        raise ValueError('cannot encode objects that are not 2-tuples')
+
+    return OrderedDict(value)
+
+
+def to_key_val_list(value):
+    """Take an object and test to see if it can be represented as a
+    dictionary. If it can be, return a list of tuples, e.g.,
+
+    ::
+
+        >>> to_key_val_list([('key', 'val')])
+        [('key', 'val')]
+        >>> to_key_val_list({'key': 'val'})
+        [('key', 'val')]
+        >>> to_key_val_list('string')
+        ValueError: cannot encode objects that are not 2-tuples.
+    """
+    if value is None:
+        return None
+
+    if isinstance(value, (str, bytes, bool, int)):
+        raise ValueError('cannot encode objects that are not 2-tuples')
+
+    if isinstance(value, dict):
+        value = value.items()
+
+    return list(value)
+
+
 # From mitsuhiko/werkzeug (used with permission).
 def parse_list_header(value):
     """Parse lists as described by RFC 2068 Section 2.
@@ -258,11 +311,8 @@ def dict_from_cookiejar(cj):
 
     cookie_dict = {}
 
-    for _, cookies in list(cj._cookies.items()):
-        for _, cookies in list(cookies.items()):
-            for cookie in list(cookies.values()):
-                # print cookie
-                cookie_dict[cookie.name] = cookie.value
+    for cookie in cj:
+        cookie_dict[cookie.name] = cookie.value
 
     return cookie_dict
 
@@ -329,6 +379,14 @@ def stream_decode_response_unicode(iterator, r):
         yield rv
 
 
+def iter_slices(string, slice_length):
+    """Iterate over slices of a string."""
+    pos = 0
+    while pos < len(string):
+        yield string[pos:pos + slice_length]
+        pos += slice_length
+
+
 def get_unicode_from_response(r):
     """Returns the requested content back in unicode.
 
@@ -396,9 +454,10 @@ def stream_decompress(iterator, mode='gzip'):
 
 
 def stream_untransfer(gen, resp):
-    if 'gzip' in resp.headers.get('content-encoding', ''):
+    ce = resp.headers.get('content-encoding', '').lower()
+    if 'gzip' in ce:
         gen = stream_decompress(gen, mode='gzip')
-    elif 'deflate' in resp.headers.get('content-encoding', ''):
+    elif 'deflate' in ce:
         gen = stream_decompress(gen, mode='deflate')
 
     return gen
@@ -442,6 +501,7 @@ def requote_uri(uri):
     # or '%')
     return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~")
 
+
 def get_environ_proxies():
     """Return a dict of environment proxies."""
 
@@ -457,3 +517,100 @@ def get_environ_proxies():
     get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
     proxies = [(key, get_proxy(key + '_proxy')) for key in proxy_keys]
     return dict([(key, val) for (key, val) in proxies if val])
+
+
+def default_user_agent():
+    """Return a string representing the default user agent."""
+    _implementation = platform.python_implementation()
+
+    if _implementation == 'CPython':
+        _implementation_version = platform.python_version()
+    elif _implementation == 'PyPy':
+        _implementation_version = '%s.%s.%s' % (
+                                                sys.pypy_version_info.major,
+                                                sys.pypy_version_info.minor,
+                                                sys.pypy_version_info.micro
+                                            )
+        if sys.pypy_version_info.releaselevel != 'final':
+            _implementation_version = ''.join([_implementation_version, sys.pypy_version_info.releaselevel])
+    elif _implementation == 'Jython':
+        _implementation_version = platform.python_version()  # Complete Guess
+    elif _implementation == 'IronPython':
+        _implementation_version = platform.python_version()  # Complete Guess
+    else:
+        _implementation_version = 'Unknown'
+
+    return " ".join([
+            'python-requests/%s' % __version__,
+            '%s/%s' % (_implementation, _implementation_version),
+            '%s/%s' % (platform.system(), platform.release()),
+        ])
+
+
+def parse_header_links(value):
+    """Return a dict of parsed link headers proxies.
+
+    i.e. Link: <http:/.../front.jpeg>; rel=front; type="image/jpeg",<http://.../back.jpeg>; rel=back;type="image/jpeg"
+
+    """
+
+    links = []
+
+    replace_chars = " '\""
+
+    for val in value.split(","):
+        try:
+            url, params = val.split(";", 1)
+        except ValueError:
+            url, params = val, ''
+
+        link = {}
+
+        link["url"] = url.strip("<> '\"")
+
+        for param in params.split(";"):
+            try:
+                key,value = param.split("=")
+            except ValueError:
+                break
+
+            link[key.strip(replace_chars)] = value.strip(replace_chars)
+
+        links.append(link)
+
+    return links
+
+
+# Null bytes; no need to recreate these on each call to guess_json_utf
+_null = '\x00'.encode('ascii')  # encoding to ASCII for Python 3
+_null2 = _null * 2
+_null3 = _null * 3
+
+
+def guess_json_utf(data):
+    # JSON always starts with two ASCII characters, so detection is as
+    # easy as counting the nulls and from their location and count
+    # determine the encoding. Also detect a BOM, if present.
+    sample = data[:4]
+    if sample in (codecs.BOM_UTF32_LE, codecs.BOM32_BE):
+        return 'utf-32'     # BOM included
+    if sample[:3] == codecs.BOM_UTF8:
+        return 'utf-8-sig'  # BOM included, MS style (discouraged)
+    if sample[:2] in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE):
+        return 'utf-16'     # BOM included
+    nullcount = sample.count(_null)
+    if nullcount == 0:
+        return 'utf-8'
+    if nullcount == 2:
+        if sample[::2] == _null2:   # 1st and 3rd are null
+            return 'utf-16-be'
+        if sample[1::2] == _null2:  # 2nd and 4th are null
+            return 'utf-16-le'
+        # Did not detect 2 valid UTF-16 ascii-range characters
+    if nullcount == 3:
+        if sample[:3] == _null3:
+            return 'utf-32-be'
+        if sample[1:] == _null3:
+            return 'utf-32-le'
+        # Did not detect a valid UTF-32 ascii-range character
+    return None

setup.py

@@ -15,6 +15,8 @@ if sys.argv[-1] == 'publish':
     os.system('python setup.py sdist upload')
     sys.exit()
 
+os.environ['PYTHONDONTWRITEBYTECODE'] = '1'
+
 packages = [
     'requests',
     'requests.packages',
@@ -63,5 +65,9 @@ setup(
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.0',
         'Programming Language :: Python :: 3.1',
+        'Programming Language :: Python :: 3.2',
+        'Programming Language :: Python :: 3.3',
     ),
 )
+
+del os.environ['PYTHONDONTWRITEBYTECODE']

tests/informal/test_leaked_connections.py

@@ -0,0 +1,31 @@
+"""
+This is an informal test originally written by Bluehorn;
+it verifies that Requests does not leak connections when
+the body of the request is not read.
+"""
+
+import gc
+import os
+import requests
+import subprocess
+import sys
+
+
+def main():
+    gc.disable()
+
+    for x in range(20):
+        requests.head("http://www.google.com/")
+
+    print("Open sockets after 20 head requests:")
+    pid = os.getpid()
+    subprocess.call("lsof -p%d -a -iTCP" % (pid,), shell=True)
+
+    gcresult = gc.collect()
+    print("Garbage collection result: %s" % (gcresult,))
+
+    print("Open sockets after garbage collection:")
+    subprocess.call("lsof -p%d -a -iTCP" % (pid,), shell=True)
+
+if __name__ == '__main__':
+    sys.exit(main())

tests/test_cookies.py

@@ -16,6 +16,7 @@ from requests.compat import cookielib
 sys.path.append('.')
 from test_requests import httpbin, TestBaseMixin
 
+
 class CookieTests(TestBaseMixin, unittest.TestCase):
 
     def test_cookies_from_response(self):
@@ -106,22 +107,22 @@ class CookieTests(TestBaseMixin, unittest.TestCase):
     def test_disabled_cookie_persistence(self):
         """Test that cookies are not persisted when configured accordingly."""
 
-        config = {'store_cookies' : False}
+        config = {'store_cookies': False}
 
         # Check the case when no cookie is passed as part of the request and the one in response is ignored
-        cookies = requests.get(httpbin('cookies', 'set', 'key', 'value'), config = config).cookies
+        cookies = requests.get(httpbin('cookies', 'set', 'key', 'value'), config=config).cookies
         self.assertTrue(cookies.get("key") is None)
 
         # Test that the cookies passed while making the request still gets used and is available in response object.
         # only the ones received from server is not saved
-        cookies_2 = requests.get(httpbin('cookies', 'set', 'key', 'value'), config = config,\
-                                                cookies = {"key_2" : "value_2"}).cookies
+        cookies_2 = requests.get(httpbin('cookies', 'set', 'key', 'value'), config=config,\
+                                                cookies={"key_2": "value_2"}).cookies
         self.assertEqual(len(cookies_2), 1)
         self.assertEqual(cookies_2.get("key_2"), "value_2")
 
         # Use the session and make sure that the received cookie is not used in subsequent calls
         s = requests.session()
-        s.get(httpbin('cookies', 'set', 'key', 'value'), config = config)
+        s.get(httpbin('cookies', 'set', 'key', 'value'), config=config)
         r = s.get(httpbin('cookies'))
         self.assertEqual(json.loads(r.text)['cookies'], {})
 
@@ -134,7 +135,7 @@ class CookieTests(TestBaseMixin, unittest.TestCase):
         self.assertEqual(len(c), len(r.cookies.keys()))
         self.assertEqual(len(c), len(r.cookies.values()))
         self.assertEqual(len(c), len(r.cookies.items()))
-        
+
         # domain and path utility functions
         domain = r.cookies.list_domains()[0]
         path = r.cookies.list_paths()[0]
@@ -151,13 +152,14 @@ class CookieTests(TestBaseMixin, unittest.TestCase):
         # test keys, values, and items
         self.assertEqual(r.cookies.keys(), ['myname'])
         self.assertEqual(r.cookies.values(), ['myvalue'])
-        self.assertEqual(r.cookies.items(), [('myname','myvalue')])
-        
+        self.assertEqual(r.cookies.items(), [('myname', 'myvalue')])
+
         # test if we can convert jar to dict
         dictOfCookies = dict(r.cookies)
-        self.assertEqual(dictOfCookies, {'myname':'myvalue'})
+        self.assertEqual(dictOfCookies, {'myname': 'myvalue'})
         self.assertEqual(dictOfCookies, r.cookies.get_dict())
 
+
 class LWPCookieJarTest(TestBaseMixin, unittest.TestCase):
     """Check store/load of cookies to FileCookieJar's, specifically LWPCookieJar's."""
 
@@ -254,6 +256,7 @@ class LWPCookieJarTest(TestBaseMixin, unittest.TestCase):
         self.assertEqual(len(cookiejar_2), 1)
         self.assertCookieHas(list(cookiejar_2)[0], name='Persistent', value='CookiesAreScary')
 
+
 class MozCookieJarTest(LWPCookieJarTest):
     """Same test, but substitute MozillaCookieJar."""
 

tests/test_proxies.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import unittest
+
+# Path hack.
+sys.path.insert(0, os.path.abspath('..'))
+import requests
+
+
+class HTTPSProxyTest(unittest.TestCase):
+    """Smoke test for https functionality."""
+
+    smoke_url = "https://github.com"
+
+    def test_empty_https_proxy(self):
+        proxy = {"https": ""}
+        result = requests.get(self.smoke_url, verify=False, proxies=proxy)
+        self.assertEqual(result.status_code, 200)
+
+    def test_empty_http_proxy(self):
+        proxy = {"http": ""}
+        result = requests.get(self.smoke_url, proxies=proxy)
+        self.assertEqual(result.status_code, 200)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_requests.py

@@ -7,9 +7,7 @@
 import sys
 import os
 sys.path.insert(0, os.path.abspath('..'))
-
 import json
-import os
 import unittest
 import pickle
 import tempfile
@@ -20,6 +18,7 @@ from requests.compat import str, StringIO
 from requests import HTTPError
 from requests import get, post, head, put
 from requests.auth import HTTPBasicAuth, HTTPDigestAuth
+from requests.exceptions import InvalidURL
 
 if 'HTTPBIN_URL' not in os.environ:
     os.environ['HTTPBIN_URL'] = 'http://httpbin.org/'
@@ -52,6 +51,7 @@ class TestSetup(object):
             # time.sleep(1)
             _httpbin = True
 
+
 class TestBaseMixin(object):
 
     def assertCookieHas(self, cookie, **kwargs):
@@ -61,6 +61,7 @@ class TestBaseMixin(object):
             message = 'Failed comparison for %s: %s != %s' % (attr, cookie_attr, expected_value)
             self.assertEqual(cookie_attr, expected_value, message)
 
+
 class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
     """Requests test cases."""
 
@@ -83,6 +84,21 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
 
         self.assertEqual(request.path_url, "/get/test%20case")
 
+    def test_params_are_added_before_fragment(self):
+        request = requests.Request(
+            "http://example.com/path#fragment", params={"a": "b"})
+        self.assertEqual(request.full_url,
+            "http://example.com/path?a=b#fragment")
+        request = requests.Request(
+            "http://example.com/path?key=value#fragment", params={"a": "b"})
+        self.assertEqual(request.full_url,
+            "http://example.com/path?key=value&a=b#fragment")
+
+    def test_params_accepts_kv_list(self):
+        request = requests.Request('http://example.com/path',
+                params=[('a', 'b')])
+        self.assertEqual(request.full_url, 'http://example.com/path?a=b')
+
     def test_HTTP_200_OK_GET(self):
         r = get(httpbin('get'))
         self.assertEqual(r.status_code, 200)
@@ -105,7 +121,7 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
 
         r = get(httpbin('user-agent'), headers=heads)
 
-        assert heads['User-agent'] in r.text
+        self.assertTrue(heads['User-agent'] in r.text)
         self.assertEqual(r.status_code, 200)
 
     def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self):
@@ -265,6 +281,11 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
 
             r = get(url, auth=auth)
             self.assertEqual(r.status_code, 200)
+            self.assertEqual(len(r.history), 1)
+
+            r = get(url, auth=auth)
+            self.assertEqual(r.status_code, 200)
+            self.assertEqual(len(r.history), 0)
 
             r = get(url)
             self.assertEqual(r.status_code, 401)
@@ -299,22 +320,45 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
 
             with open(__file__) as f:
                 post2 = post(url, files={'some': f})
+                post3 = post(url, files=[('some', f)])
             self.assertEqual(post2.status_code, 200)
-
-            post3 = post(url, data='[{"some": "json"}]')
             self.assertEqual(post3.status_code, 200)
 
+            post4 = post(url, data='[{"some": "json"}]')
+            self.assertEqual(post4.status_code, 200)
+
+            try:
+                post(url, files=['bad file data'])
+            except ValueError:
+                pass
+
     def test_POSTBIN_GET_POST_FILES_WITH_PARAMS(self):
 
         for service in SERVICES:
 
             with open(__file__) as f:
                 url = service('post')
-                post1 = post(url,
-                             files={'some': f},
-                             data={'some': 'data'})
+                post1 = post(url, data={'some': 'data'}, files={'some': f})
+                post2 = post(url, data={'some': 'data'}, files=[('some', f)])
+                post3 = post(url, data=[('some', 'data')], files=[('some', f)])
 
             self.assertEqual(post1.status_code, 200)
+            self.assertEqual(post2.status_code, 200)
+            self.assertEqual(post3.status_code, 200)
+
+    def test_POSTBIN_GET_POST_FILES_WITH_CJK_PARAMS(self):
+
+        for service in SERVICES:
+
+            with open(__file__) as f:
+                url = service('post')
+                post1 = post(url, data={'some': '中文'}, files={'some': f})
+                post2 = post(url, data={'some': '日本語'}, files=[('some', f)])
+                post3 = post(url, data=[('some', '한국의')], files=[('some', f)])
+
+            self.assertEqual(post1.status_code, 200)
+            self.assertEqual(post2.status_code, 200)
+            self.assertEqual(post3.status_code, 200)
 
     def test_POSTBIN_GET_POST_FILES_WITH_HEADERS(self):
 
@@ -339,10 +383,12 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
             post1 = post(url, files={'fname.txt': 'fdata'})
             self.assertEqual(post1.status_code, 200)
 
-            post2 = post(url, files={'fname.txt': 'fdata', 'fname2.txt':'more fdata'})
+            post2 = post(url, files={'fname.txt': 'fdata',
+                    'fname2.txt': 'more fdata'})
             self.assertEqual(post2.status_code, 200)
 
-            post3 = post(url, files={'fname.txt': 'fdata', 'fname2.txt':open(__file__,'rb')})
+            post3 = post(url, files={'fname.txt': 'fdata',
+                    'fname2.txt': open(__file__, 'rb')})
             self.assertEqual(post3.status_code, 200)
 
             post4 = post(url, files={'fname.txt': 'fdata'})
@@ -364,9 +410,29 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
             post7 = post(url, files={'fname.txt': 'fdata to verify'})
             rbody = json.loads(post7.text)
             self.assertTrue(rbody.get('files', None))
-            self.assertTrue(rbody['files'].get('fname.txt'), None)
+            self.assertTrue(rbody['files'].get('fname.txt', None))
             self.assertEqual(rbody['files']['fname.txt'], 'fdata to verify')
 
+            post8 = post(url, files=[('fname.txt', 'fdata')])
+            self.assertEqual(post8.status_code, 200)
+            resp_body = post8.json
+            self.assertTrue(resp_body.get('files', None))
+            self.assertTrue(resp_body['files'].get('fname.txt', None))
+            self.assertEqual(resp_body['files']['fname.txt'], 'fdata')
+
+            post9 = post(url, files=[('fname.txt', fdata)])
+            self.assertEqual(post9.status_code, 200)
+
+            post10 = post(url, files=[('file',
+                        ('file.txt', 'more file data'))])
+            self.assertEqual(post10.status_code, 200)
+
+            post11 = post(url, files=[('fname.txt', 'fdata'),
+                    ('fname2.txt', 'more fdata')])
+            post12 = post(url, files=[('fname.txt', 'fdata'),
+                    ('fname2.txt', open(__file__, 'rb'))])
+            self.assertEqual(post11.status_code, 200)
+            self.assertEqual(post12.status_code, 200)
 
     def test_nonzero_evaluation(self):
 
@@ -417,7 +483,7 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
 
             response = get(url)
 
-            assert isinstance(response.url, str)
+            self.assertTrue(isinstance(response.url, str))
 
     def test_unicode_get(self):
 
@@ -468,7 +534,7 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
             # Body wasn't valid url encoded data, so the server returns None as
             # "form" and the raw body as "data".
 
-            assert rbody.get('form') in (None, {})
+            self.assertTrue(rbody.get('form') in (None, {}))
             self.assertEqual(rbody.get('data'), 'fooaowpeuf')
 
     def test_file_post_data(self):
@@ -491,7 +557,7 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
             self.assertEqual(r.url, service('post'))
 
             rbody = json.loads(r.text)
-            assert rbody.get('form') in (None, {})
+            self.assertTrue(rbody.get('form') in (None, {}))
             self.assertEqual(rbody.get('data'), filecontent.decode('ascii'))
         os.remove(testfile.name)
 
@@ -537,7 +603,7 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
 
             rbody = json.loads(r.text)
 
-            assert rbody.get('form') in (None, {})
+            self.assertTrue(rbody.get('form') in (None, {}))
             self.assertEqual(rbody.get('data'), 'foobar')
 
     def test_urlencoded_get_query_multivalued_param(self):
@@ -628,14 +694,14 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
 
         # Make 2 requests from Session object, should send header both times
         r1 = get(httpbin('user-agent'), session=s)
-        assert heads['User-agent'] in r1.text
+        self.assertTrue(heads['User-agent'] in r1.text)
 
         r2 = get(httpbin('user-agent'), session=s)
-        assert heads['User-agent'] in r2.text
+        self.assertTrue(heads['User-agent'] in r2.text)
 
         new_heads = {'User-agent': 'blah'}
         r3 = get(httpbin('user-agent'), headers=new_heads, session=s)
-        assert new_heads['User-agent'] in r3.text
+        self.assertTrue(new_heads['User-agent'] in r3.text)
 
         self.assertEqual(r2.status_code, 200)
 
@@ -655,7 +721,7 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
             url = service('headers')
             response = get(url=url, hooks={'args': add_foo_header})
 
-            assert 'foo' in response.text
+            self.assertTrue('foo' in response.text)
 
     def test_multiple_hooks(self):
 
@@ -688,8 +754,46 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
                 }
             )
 
-            assert 'foo' in response.text
-            assert 'bar' in response.text
+            for text in ('foo', 'bar'):
+                self.assertTrue(text in response.text)
+
+    def test_allow_list_of_hooks_to_register_hook(self):
+        """Issue 785: https://github.com/kennethreitz/requests/issues/785"""
+        def add_foo_header(args):
+            if not args.get('headers'):
+                args['headers'] = {}
+
+            args['headers'].update({
+                'X-Foo': 'foo'
+            })
+
+            return args
+
+        def add_bar_header(args):
+            if not args.get('headers'):
+                args['headers'] = {}
+
+            args['headers'].update({
+                'X-Bar': 'bar'
+            })
+
+            return args
+
+        def assert_hooks_are_callable(hooks):
+            for h in hooks['args']:
+                self.assertTrue(callable(h))
+
+        hooks = [add_foo_header, add_bar_header]
+        r = requests.models.Request()
+        r.register_hook('args', hooks)
+        assert_hooks_are_callable(r.hooks)
+
+        r = requests.models.Request(hooks={'args': hooks})
+        assert_hooks_are_callable(r.hooks)
+
+        hooks.append('string that should not be registered')
+        r = requests.models.Request(hooks={'args': hooks})
+        assert_hooks_are_callable(r.hooks)
 
     def test_session_persistent_cookies(self):
 
@@ -725,13 +829,13 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
         r = get(httpbin('cookies', 'set', 'k', 'v'), allow_redirects=True, session=s)
         c = json.loads(r.text).get('cookies')
 
-        assert 'k' in c
+        self.assertTrue('k' in c)
 
         # And server-set cookie persistience.
         r = get(httpbin('cookies'), session=s)
         c = json.loads(r.text).get('cookies')
 
-        assert 'k' in c
+        self.assertTrue('k' in c)
 
     def test_session_persistent_params(self):
 
@@ -742,21 +846,22 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
 
         # Make 2 requests from Session object, should send header both times
         r1 = get(httpbin('get'), session=s)
-        assert params['a'] in r1.text
+        self.assertTrue(params['a'] in r1.text)
 
         params2 = {'b': 'b_test'}
 
         r2 = get(httpbin('get'), params=params2, session=s)
-        assert params['a'] in r2.text
-        assert params2['b'] in r2.text
+        for param in (params['a'], params2['b']):
+            self.assertTrue(param in r2.text)
 
         params3 = {'b': 'b_test', 'a': None, 'c': 'c_test'}
 
         r3 = get(httpbin('get'), params=params3, session=s)
 
-        assert not params['a'] in r3.text
-        assert params3['b'] in r3.text
-        assert params3['c'] in r3.text
+        self.assertFalse(params['a'] in r3.text)
+
+        for param in (params3['b'], params3['c']):
+            self.assertTrue(param in r3.text)
 
     def test_session_cookies_with_return_response_false(self):
         s = requests.session()
@@ -765,8 +870,8 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
                  allow_redirects=True, session=s)
         rq.send(prefetch=True)
         c = rq.response.json.get('cookies')
-        assert 'k' in c
-        assert 'k' in s.cookies
+        self.assertTrue('k' in c)
+        self.assertTrue('k' in s.cookies)
 
     def test_session_pickling(self):
 
@@ -789,44 +894,57 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
         s = requests.session()
         r = get(httpbin('cookies', 'set', 'k', 'v'), allow_redirects=True, session=s)
         c = json.loads(r.text).get('cookies')
-        assert 'k' in c
+        self.assertTrue('k' in c)
 
         ds = pickle.loads(pickle.dumps(s))
         r = get(httpbin('cookies'), session=ds)
         c = json.loads(r.text).get('cookies')
-        assert 'k' in c
+        self.assertTrue('k' in c)
 
         ds1 = pickle.loads(pickle.dumps(requests.session()))
-        ds2 = pickle.loads(pickle.dumps(requests.session(prefetch=True)))
-        assert not ds1.prefetch
-        assert ds2.prefetch
+        ds2 = pickle.loads(pickle.dumps(requests.session(prefetch=False)))
+        self.assertTrue(ds1.prefetch)
+        self.assertFalse(ds2.prefetch)
 
-    def test_invalid_content(self):
-        # WARNING: if you're using a terrible DNS provider (comcast),
-        # this will fail.
+    def test_connection_error(self):
         try:
-            hah = 'http://somedomainthatclearlydoesntexistg.com'
-            r = get(hah, allow_redirects=False)
+            get('http://localhost:1/nope')
         except requests.ConnectionError:
-            pass   # \o/
+            pass
         else:
-            assert False
+            self.fail()
 
+    def test_connection_error_with_safe_mode(self):
         config = {'safe_mode': True}
-        r = get(hah, allow_redirects=False, config=config)
-        assert r.content == None
+        r = get('http://localhost:1/nope', allow_redirects=False, config=config)
+        self.assertTrue(r.content is None)
+
+    # def test_invalid_content(self):
+    #     # WARNING: if you're using a terrible DNS provider (comcast),
+    #     # this will fail.
+    #     try:
+    #         hah = 'http://somedomainthatclearlydoesntexistg.com'
+    #         r = get(hah, allow_redirects=False)
+    #     except requests.ConnectionError:
+    #         pass   # \o/
+    #     else:
+    #         assert False
+
+    #     config = {'safe_mode': True}
+    #     r = get(hah, allow_redirects=False, config=config)
+    #     assert r.content == None
 
     def test_cached_response(self):
 
         r1 = get(httpbin('get'), prefetch=False)
-        assert not r1._content
-        assert r1.content
-        assert r1.text
+        self.assertFalse(r1._content)
+        self.assertTrue(r1.content)
+        self.assertTrue(r1.text)
 
         r2 = get(httpbin('get'), prefetch=True)
-        assert r2._content
-        assert r2.content
-        assert r2.text
+        self.assertTrue(r2._content)
+        self.assertTrue(r2.content)
+        self.assertTrue(r2.text)
 
     def test_iter_lines(self):
 
@@ -848,7 +966,7 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
         )
 
         # Make a request and monkey-patch its contents
-        r = get(httpbin('get'))
+        r = get(httpbin('get'), prefetch=False)
         r.raw = StringIO(quote)
 
         lines = list(r.iter_lines())
@@ -858,31 +976,49 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
         joined = lines[0] + '\n' + lines[1] + '\r\n' + lines[2]
         self.assertEqual(joined, quote)
 
-    def test_safe_mode(self):
+    def test_permissive_iter_content(self):
+        """Test that iter_content and iter_lines work even after the body has been fetched."""
+        r = get(httpbin('stream', '10'), prefetch=True)
+        self.assertTrue(r._content_consumed)
+        # iter_lines should still work without crashing
+        self.assertEqual(len(list(r.iter_lines())), 10)
 
-        safe = requests.session(config=dict(safe_mode=True))
+        # iter_content should return a one-item iterator over the whole content
+        iter_content_list = list(r.iter_content(chunk_size=1))
+        self.assertTrue(all(len(item) == 1 for item in iter_content_list))
+        # when joined, it should be exactly the original content
+        self.assertEqual(bytes().join(iter_content_list), r.content)
 
-        # Safe mode creates empty responses for failed requests.
-        # Iterating on these responses should produce empty sequences
-        r = get('http://_/', session=safe)
-        self.assertEqual(list(r.iter_lines()), [])
-        assert isinstance(r.error, requests.exceptions.ConnectionError)
+        # test different chunk sizes:
+        for chunk_size in range(2, 20):
+            self.assertEqual(bytes().join(r.iter_content(chunk_size=chunk_size)), r.content)
 
-        r = get('http://_/', session=safe)
-        self.assertEqual(list(r.iter_content()), [])
-        assert isinstance(r.error, requests.exceptions.ConnectionError)
 
-        # When not in safe mode, should raise Timeout exception
-        self.assertRaises(
-            requests.exceptions.Timeout,
-            get,
-            httpbin('stream', '1000'), timeout=0.0001)
+    # def test_safe_mode(self):
 
-        # In safe mode, should return a blank response
-        r = get(httpbin('stream', '1000'), timeout=0.0001,
-                config=dict(safe_mode=True))
-        assert r.content is None
-        assert isinstance(r.error, requests.exceptions.Timeout)
+    #     safe = requests.session(config=dict(safe_mode=True))
+
+    #     # Safe mode creates empty responses for failed requests.
+    #     # Iterating on these responses should produce empty sequences
+    #     r = get('http://0.0.0.0:700/', session=safe)
+    #     self.assertEqual(list(r.iter_lines()), [])
+    #     assert isinstance(r.error, requests.exceptions.ConnectionError)
+
+    #     r = get('http://0.0.0.0:789/', session=safe)
+    #     self.assertEqual(list(r.iter_content()), [])
+    #     assert isinstance(r.error, requests.exceptions.ConnectionError)
+
+    #     # When not in safe mode, should raise Timeout exception
+    #     self.assertRaises(
+    #         requests.exceptions.Timeout,
+    #         get,
+    #         httpbin('stream', '1000'), timeout=0.0001)
+
+    #     # In safe mode, should return a blank response
+    #     r = get(httpbin('stream', '1000'), timeout=0.0001,
+    #             config=dict(safe_mode=True))
+    #     assert r.content is None
+    #     assert isinstance(r.error, requests.exceptions.Timeout)
 
     def test_upload_binary_data(self):
 
@@ -909,14 +1045,13 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
             # Don't choke on headers with none in the value.
             requests.get(httpbin('headers'), headers={'Foo': None})
         except TypeError:
-            self.fail()
+            self.fail('Not able to have none in header values')
 
     def test_danger_mode_redirects(self):
         s = requests.session()
         s.config['danger_mode'] = True
         s.get(httpbin('redirect', '4'))
 
-
     def test_empty_response(self):
         r = requests.get(httpbin('status', '404'))
         r.text
@@ -952,5 +1087,110 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
         r.content
         r.text
 
+    def test_post_fields_with_multiple_values_and_files(self):
+        """Test that it is possible to POST using the files argument and a
+        list for a value in the data argument."""
+
+        data = {'field': ['a', 'b']}
+        files = {'field': 'Garbled data'}
+        r = post(httpbin('post'), data=data, files=files)
+        t = json.loads(r.text)
+        self.assertEqual(t.get('form'), {'field': ['a', 'b']})
+        self.assertEqual(t.get('files'), files)
+        r = post(httpbin('post'), data=data, files=files.items())
+        self.assertEqual(t.get('files'), files)
+
+    def test_str_data_content_type(self):
+        data = 'test string data'
+        r = post(httpbin('post'), data=data)
+        t = json.loads(r.text)
+        self.assertEqual(t.get('headers').get('Content-Type'), '')
+
+    def test_prefetch_redirect_bug(self):
+        """Test that prefetch persists across redirections."""
+        res = get(httpbin('redirect/2'), prefetch=False)
+        # prefetch should persist across the redirect;
+        # the content should not have been consumed
+        self.assertFalse(res._content_consumed)
+        first_line = next(res.iter_lines())
+        self.assertTrue(first_line.strip().decode('utf-8').startswith('{'))
+
+    def test_prefetch_return_response_interaction(self):
+        """Test that prefetch can be overridden as a kwarg to `send`."""
+        req = requests.get(httpbin('get'), return_response=False)
+        req.send(prefetch=False)
+        # content should not have been prefetched
+        self.assertFalse(req.response._content_consumed)
+        first_line = next(req.response.iter_lines())
+        self.assertTrue(first_line.strip().decode('utf-8').startswith('{'))
+
+    def test_accept_objects_with_string_representations_as_urls(self):
+        """Test that URLs can be set to objects with string representations,
+        e.g. for use with furl."""
+        class URL():
+            def __unicode__(self):
+                # Can't have unicode literals in Python3, so avoid them.
+                # TODO: fixup when moving to Python 3.3
+                if (sys.version_info[0] == 2):
+                    return 'http://httpbin.org/get'.decode('utf-8')
+                else:
+                    return 'http://httpbin.org/get'
+
+            def __str__(self):
+                return 'http://httpbin.org/get'
+
+        r = get(URL())
+        self.assertEqual(r.status_code, 200)
+
+    def test_post_fields_with_multiple_values_and_files_as_tuples(self):
+        """Test that it is possible to POST multiple data and file fields
+        with the same name.
+        https://github.com/kennethreitz/requests/pull/746
+        """
+
+        fields = [
+            ('__field__', '__value__'),
+            ('__field__', '__value__'),
+        ]
+
+        r = post(httpbin('post'), data=fields, files=fields)
+        t = json.loads(r.text)
+
+        self.assertEqual(t.get('form'), {
+            '__field__': [
+                '__value__',
+                '__value__',
+            ]
+        })
+
+        # It's not currently possible to test for multiple file fields with
+        # the same name against httpbin so we need to inspect the encoded
+        # body manually.
+        request = r.request
+        body, content_type = request._encode_files(request.files)
+        file_field = (b'Content-Disposition: form-data;'
+                      b' name="__field__"; filename="__field__"')
+        self.assertEqual(body.count(b'__value__'), 4)
+        self.assertEqual(body.count(file_field), 2)
+
+    def test_bytes_files(self):
+        """Test that `bytes` can be used as the values of `files`."""
+        post(httpbin('post'), files={'test': b'test'})
+
+    def test_invalid_urls_throw_requests_exception(self):
+        """Test that URLs with invalid labels throw
+        Requests.exceptions.InvalidURL instead of UnicodeError."""
+        self.assertRaises(InvalidURL, get, 'http://.google.com/')
+
+    def test_none_vals_in_data_are_deleted(self):
+        """Test that keys with None as the value are removed instead of
+        being posted."""
+        data = {'key1': 'value1', 'key2': None}
+        r = post(httpbin('post'), data=data)
+        vals = r.json['form']
+        self.assertEqual(vals['key1'], 'value1')
+        # The 'key2' key should not have been sent.
+        self.assertTrue(vals.get('key2') is None)
+
 if __name__ == '__main__':
     unittest.main()

tests/test_requests_ext.py

@@ -2,7 +2,8 @@
 # -*- coding: utf-8 -*-
 
 # Path hack.
-import sys, os
+import os
+import sys
 sys.path.insert(0, os.path.abspath('..'))
 
 import unittest
@@ -23,19 +24,16 @@ class RequestsTestSuite(unittest.TestCase):
     _multiprocess_can_split_ = True
 
     def test_addition(self):
-        assert (1 + 1) == 2
-
+        self.assertEqual(2, 1 + 1)
 
     def test_ssl_hostname_ok(self):
         requests.get('https://github.com', verify=True)
 
-
     def test_ssl_hostname_not_ok(self):
         requests.get('https://kennethreitz.com', verify=False)
 
         self.assertRaises(requests.exceptions.SSLError, requests.get, 'https://kennethreitz.com')
 
-
     def test_ssl_hostname_session_not_ok(self):
 
         s = requests.session()
@@ -44,7 +42,6 @@ class RequestsTestSuite(unittest.TestCase):
 
         s.get('https://kennethreitz.com', verify=False)
 
-
     def test_binary_post(self):
         '''We need to be careful how we build the utf-8 string since
         unicode literals are a syntax error in python3
@@ -59,13 +56,10 @@ class RequestsTestSuite(unittest.TestCase):
             raise EnvironmentError('Flesh out this test for your environment.')
         requests.post('http://www.google.com/', data=utf8_string)
 
-
-
     def test_unicode_error(self):
         url = 'http://blip.fm/~1abvfu'
         requests.get(url)
 
-
     def test_chunked_head_redirect(self):
         url = "http://t.co/NFrx0zLG"
         r = requests.head(url, allow_redirects=True)
@@ -85,7 +79,7 @@ class RequestsTestSuite(unittest.TestCase):
         '''
         r = requests.get('http://www.marketwire.com/mw/release.' +
                          'do?id=1628202&sourceType=3')
-        assert r.ok
+        self.assertTrue(r.ok)
 
     def test_unicode_url_outright(self):
         '''This url visits in my browser'''
@@ -93,7 +87,7 @@ class RequestsTestSuite(unittest.TestCase):
                          'jp-morgan-behauptet-sich-der-spitze-euro' +
                          'p%C3%A4ischer-anleihe-analysten-laut-umf' +
                          'rageergebnissen-1628202.htm')
-        assert r.ok
+        self.assertTrue(r.ok)
 
     def test_redirect_encoding(self):
         '''This url redirects to
@@ -102,7 +96,7 @@ class RequestsTestSuite(unittest.TestCase):
         r = requests.get('http://feedproxy.google.com/~r/Dealipedia' +
                          'News/~3/BQtUJRJeZlo/deal_view_investment.' +
                          'php')
-        assert r.ok
+        self.assertTrue(r.ok)
 
     def test_cookies_on_redirects(self):
         """Test interaction between cookie handling and redirection."""
@@ -110,7 +104,7 @@ class RequestsTestSuite(unittest.TestCase):
         s = requests.session()
         s.get(url='http://tinyurl.com/preview.php?disable=1')
         # we should have set a cookie for tinyurl: preview=0
-        self.assertIn('preview', s.cookies)
+        self.assertTrue('preview' in s.cookies)
         self.assertEqual(s.cookies['preview'], '0')
         self.assertEqual(list(s.cookies)[0].name, 'preview')
         self.assertEqual(list(s.cookies)[0].domain, 'tinyurl.com')
@@ -118,14 +112,13 @@ class RequestsTestSuite(unittest.TestCase):
         # get cookies on another domain
         r2 = s.get(url='http://httpbin.org/cookies')
         # the cookie is not there
-        self.assertNotIn('preview', json.loads(r2.text)['cookies'])
+        self.assertTrue('preview' not in json.loads(r2.text)['cookies'])
 
         # this redirects to another domain, httpbin.org
         # cookies of the first domain should NOT be sent to the next one
         r3 = s.get(url='http://tinyurl.com/7zp3jnr')
-        assert r3.url == 'http://httpbin.org/cookies'
-        self.assertNotIn('preview', json.loads(r2.text)['cookies'])
+        self.assertEqual(r3.url, 'http://httpbin.org/cookies')
+        self.assertTrue('preview' not in json.loads(r3.text)['cookies'])
 
 if __name__ == '__main__':
     unittest.main()
-

tests/test_requests_https.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-import sys, os
-import json
+import os
+import sys
 import unittest
 
 # Path hack.
 sys.path.insert(0, os.path.abspath('..'))
 import requests
 
+
 class HTTPSTest(unittest.TestCase):
     """Smoke test for https functionality."""
 

tests/test_utils.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import codecs
+import os
+import sys
+import unittest
+import random
+
+# Path hack.
+sys.path.insert(0, os.path.abspath('..'))
+import requests.utils
+from requests.compat import is_py3, bytes
+
+
+if is_py3:
+    unichr = chr
+    byteschr = lambda c: bytes([c])
+else:
+    byteschr = chr
+
+
+class GuessJSONUTFTests(unittest.TestCase):
+    """Tests for the JSON UTF encoding guessing code."""
+
+    codecs = (
+        'utf-8', 'utf-8-sig',
+        'utf-16', 'utf-16-le', 'utf-16-be',
+        'utf-32', 'utf-32-le', 'utf-32-be'
+    )
+
+    def test_guess_encoding(self):
+        # Throw 4-character ASCII strings (encoded to a UTF encoding)
+        # at the guess routine; it should correctly guess all codecs.
+        guess = requests.utils.guess_json_utf
+        for c in range(33, 127):  # printable only
+            sample = unichr(c) * 4
+            for codec in self.codecs:
+                res = guess(sample.encode(codec))
+                self.assertEqual(res, codec)
+
+    def test_smoke_encoding(self):
+        # Throw random 4-byte strings at the guess function.
+        # Any guess for a UTF encoding is verified, a decode exception
+        # is a test failure.
+        guess = requests.utils.guess_json_utf
+        for i in range(1000):
+            sample = bytes().join(
+                [byteschr(random.randrange(256)) for _ in range(4)])
+            res = guess(sample)
+            if res is not None:
+                # This should decode without errors if this is *really*
+                # something in this encoding. However, UTF-8 is a lot
+                # more picky, so we expect errors there. UTF-16 surrogate
+                # pairs also fail
+                try:
+                    sample.decode(res)
+                except UnicodeDecodeError as e:
+                    self.assertEqual(e.args[0].replace('-', '').lower(),
+                                     res.replace('-', '').lower())
+                    if res == 'utf-8':
+                        self.assertTrue(e.args[-1], (
+                            'invalid continuation byte',
+                            'invalid start byte'))
+                        continue
+                    if res == 'utf-16':
+                        self.assertEqual(e.args[-1], 'unexpected end of data')
+                        self.assertTrue(sample[:2] in (
+                            codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE))
+                        # the second two bytes are in the range \ud800-\udfff
+                        # if someone wants to add tests for that as well. I don't
+                        # see the need; we are not testing UTF decoding here.
+                        continue
+                    raise
+
+if __name__ == '__main__':
+    unittest.main()