v0.13.4

Implement GSSAPI/Kerberos authentication for requests

AUTHORS.rst

@@ -105,3 +105,8 @@ Patches and Suggestions
 - Danilo Bargen (gwrtheyrn)
 - Torsten Landschoff
 - Michael Holler (apotheos)
+- Timnit Gebru
+- Sarah Gonzalez
+- Victoria Mo
+- Leila Muhtasib
+- Matthias Rahlf <matthias@webding.de>

HISTORY.rst

@@ -3,6 +3,24 @@
 History
 -------
 
+0.13.4 (2012-07-27)
++++++++++++++++++++
+
+- GSSAPI/Kerberos authentication!
+- App Engine 2.7 Fixes!
+- Fix leaking connections (from urllib3 update)
+- OAuthlib path hack fix
+- OAuthlib URL parameters fix.
+
+0.13.3 (2012-07-12)
++++++++++++++++++++
+
+- Use simplejson if available.
+- Do not hide SSLErrors behind Timeouts.
+- Fixed param handling with urls containing fragments.
+- Significantly improved information in User Agent.
+- client certificates are ignored when verify=False
+
 0.13.2 (2012-06-28)
 +++++++++++++++++++
 

README.rst

@@ -3,6 +3,7 @@ Requests: HTTP for Humans
 
 
 .. image:: https://secure.travis-ci.org/kennethreitz/requests.png?branch=develop
+        :target: https://secure.travis-ci.org/kennethreitz/requests
 
 Requests is an ISC Licensed HTTP library, written in Python, for human
 beings.

docs/index.rst

@@ -38,7 +38,7 @@ Requests takes all of the work out of Python HTTP/1.1 — making your integrati
 Testimonials
 ------------
 
-`Heroku <http://heroku.com>`_, `PayPal <https://www.paypal.com/>`_,
+`Kippt <http://kippt.com>`_, `Heroku <http://heroku.com>`_, `PayPal <https://www.paypal.com/>`_,
 `Transifex <https://www.transifex.net/>`_,
 `Native Instruments <http://www.native-instruments.com/>`_, `The Washington Post <http://www.washingtonpost.com/>`_,
 `Twitter, Inc <http://twitter.com>`_,

docs/user/advanced.rst

@@ -93,6 +93,23 @@ I don't have SSL setup on this domain, so it fails. Excellent. Github does thoug
 
 You can also pass ``verify`` the path to a CA_BUNDLE file for private certs. You can also set the ``REQUESTS_CA_BUNDLE`` environment variable.
 
+Requests can also ignore verifying the SSL certficate if you set ``verify`` to False. ::
+
+    >>> requests.get('https://kennethreitz.com', verify=False)
+    <Response [200]>
+
+By default, ``verify`` is set to True. Option ``verify`` only applies to host certs.
+
+You can also specify the local cert file either as a path or key value pair::
+
+    >>> requests.get('https://kennethreitz.com', cert=('/path/server.crt', '/path/key'))
+    <Response [200]>
+
+If you specify a wrong path or an invalid cert::
+
+    >>> requests.get('https://kennethreitz.com', cert='/wrong_path/server.pem')
+    SSLError: [Errno 336265225] _ssl.c:347: error:140B0009:SSL routines:SSL_CTX_use_PrivateKey_file:PEM lib
+
 
 Body Content Workflow
 ---------------------

docs/user/install.rst

@@ -54,25 +54,3 @@ Once you have a copy of the source, you can embed it in your Python package,
 or install it into your site-packages easily::
 
     $ python setup.py install
-
-.. _gevent:
-
-Installing Gevent
------------------
-
-If you are using the ``requests.async`` module for making concurrent
-requests, you need to install gevent.
-
-To install gevent, you'll need ``libevent``.
-
-OSX::
-
-    $ brew install libevent
-
-Ubuntu::
-
-    $ apt-get install libevent-dev
-
-Once you have ``libevent``, you can install ``gevent`` with ``pip``::
-
-    $ pip install gevent

docs/user/quickstart.rst

@@ -386,7 +386,7 @@ of the Response object to track redirection. Let's see what Github does::
     [<Response [301]>]
 
 The :class:`Response.history` list contains a list of the
-:class:`Request` objects that were created in order to complete the request.
+:class:`Request` objects that were created in order to complete the request. The list is sorted from the oldest to the most recent request.
 
 If you're using GET or OPTIONS, you can disable redirection handling with the
 ``allow_redirects`` parameter::

requests/__init__.py

@@ -15,8 +15,8 @@ requests
 """
 
 __title__ = 'requests'
-__version__ = '0.13.2'
-__build__ = 0x001302
+__version__ = '0.13.4'
+__build__ = 0x001304
 __author__ = 'Kenneth Reitz'
 __license__ = 'ISC'
 __copyright__ = 'Copyright 2012 Kenneth Reitz'

requests/_oauth.py

@@ -4,8 +4,8 @@
 requests._oauth
 ~~~~~~~~~~~~~~~
 
-This module comtains the path hack neccesary for oauthlib to be vendored into requests
-while allowing upstream changes.
+This module contains the path hack necessary for oauthlib to be vendored into
+requests while allowing upstream changes.
 """
 
 import os
@@ -16,8 +16,9 @@ try:
     from oauthlib.common import extract_params
     from oauthlib.oauth1.rfc5849 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)
 except ImportError:
-    path = os.path.abspath('/'.join(__file__.split('/')[:-1]+['packages']))
+    directory = os.path.dirname(__file__)
+    path = os.path.join(directory, 'packages')
     sys.path.insert(0, path)
     from oauthlib.oauth1 import rfc5849
     from oauthlib.common import extract_params
-    from oauthlib.oauth1.rfc5849 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)
+    from oauthlib.oauth1.rfc5849 import (Client, SIGNATURE_HMAC, SIGNATURE_TYPE_AUTH_HEADER)

requests/auth.py

@@ -8,8 +8,10 @@ This module contains the authentication handlers for Requests.
 """
 
 import os
+import re
 import time
 import hashlib
+import logging
 
 from base64 import b64encode
 
@@ -23,6 +25,13 @@ except (ImportError, SyntaxError):
     SIGNATURE_HMAC = None
     SIGNATURE_TYPE_AUTH_HEADER = None
 
+try:
+    import kerberos as k
+except ImportError as exc:
+    k = None
+
+log = logging.getLogger(__name__)
+
 CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
 
 def _basic_auth_str(username, password):
@@ -88,6 +97,10 @@ class OAuth1(AuthBase):
                 r.url, r.headers, r.data = self.client.sign(
                     unicode(r.full_url), unicode(r.method), r.data, r.headers)
 
+            # Both flows add params to the URL by using r.full_url,
+            # so this prevents adding it again later
+            r.params = {}
+
             # Having the authorization header, key or value, in unicode will
             # result in UnicodeDecodeErrors when the request is concatenated
             # by httplib. This can easily be seen when attaching files.
@@ -133,11 +146,11 @@ class HTTPDigestAuth(AuthBase):
     def handle_401(self, r):
         """Takes the given response and tries digest-auth, if needed."""
 
-        r.request.deregister_hook('response', self.handle_401)
+        num_401_calls = r.request.hooks['response'].count(self.handle_401)
 
         s_auth = r.headers.get('www-authenticate', '')
 
-        if 'digest' in s_auth.lower():
+        if 'digest' in s_auth.lower() and num_401_calls < 2:
 
             last_nonce = ''
             nonce_count = 0
@@ -225,3 +238,124 @@ class HTTPDigestAuth(AuthBase):
     def __call__(self, r):
         r.register_hook('response', self.handle_401)
         return r
+
+def _negotiate_value(r):
+    """Extracts the gssapi authentication token from the appropriate header"""
+
+    authreq = r.headers.get('www-authenticate', None)
+
+    if authreq:
+        rx = re.compile('(?:.*,)*\s*Negotiate\s*([^,]*),?', re.I)
+        mo = rx.search(authreq)
+        if mo:
+            return mo.group(1)
+
+    return None
+
+class HTTPKerberosAuth(AuthBase):
+    """Attaches HTTP GSSAPI/Kerberos Authentication to the given Request object."""
+    def __init__(self, require_mutual_auth=True):
+        if k is None:
+            raise Exception("Kerberos libraries unavailable")
+        self.context = None
+        self.require_mutual_auth = require_mutual_auth
+
+    def generate_request_header(self, r):
+        """Generates the gssapi authentication token with kerberos"""
+
+        host = urlparse(r.url).netloc
+        tail, _, head = host.rpartition(':')
+        domain = tail if tail else head
+
+        result, self.context = k.authGSSClientInit("HTTP@%s" % domain)
+
+        if result < 1:
+            raise Exception("authGSSClientInit failed")
+
+        result = k.authGSSClientStep(self.context, _negotiate_value(r))
+
+        if result < 0:
+            raise Exception("authGSSClientStep failed")
+
+        response = k.authGSSClientResponse(self.context)
+
+        return "Negotiate %s" % response
+
+    def authenticate_user(self, r):
+        """Handles user authentication with gssapi/kerberos"""
+
+        auth_header = self.generate_request_header(r)
+        log.debug("authenticate_user(): Authorization header: %s" % auth_header)
+        r.request.headers['Authorization'] = auth_header
+        r.request.send(anyway=True)
+        _r = r.request.response
+        _r.history.append(r)
+        log.debug("authenticate_user(): returning %s" % _r)
+        return _r
+
+    def handle_401(self, r):
+        """Handles 401's, attempts to use gssapi/kerberos authentication"""
+
+        log.debug("handle_401(): Handling: 401")
+        if _negotiate_value(r) is not None:
+            _r = self.authenticate_user(r)
+            log.debug("handle_401(): returning %s" % _r)
+            return _r
+        else:
+            log.debug("handle_401(): Kerberos is not supported")
+            log.debug("handle_401(): returning %s" % r)
+            return r
+
+    def handle_other(self, r):
+        """Handles all responses with the exception of 401s.
+
+        This is necessary so that we can authenticate responses if requested"""
+
+        log.debug("handle_other(): Handling: %d" % r.status_code)
+        self.deregister(r)
+        if self.require_mutual_auth:
+            if _negotiate_value(r) is not None:
+                log.debug("handle_other(): Authenticating the server")
+                _r = self.authenticate_server(r)
+                log.debug("handle_other(): returning %s" % _r)
+                return _r
+            else:
+                log.error("handle_other(): Mutual authentication failed")
+                raise Exception("Mutual authentication failed")
+        else:
+            log.debug("handle_other(): returning %s" % r)
+            return r
+
+    def authenticate_server(self, r):
+        """Uses GSSAPI to authenticate the server"""
+
+        log.debug("authenticate_server(): Authenticate header: %s" % _negotiate_value(r))
+        result = k.authGSSClientStep(self.context, _negotiate_value(r))
+        if  result < 1:
+            raise Exception("authGSSClientStep failed")
+        _r = r.request.response
+        log.debug("authenticate_server(): returning %s" % _r)
+        return _r
+
+    def handle_response(self, r):
+        """Takes the given response and tries kerberos-auth, as needed."""
+
+        if r.status_code == 401:
+            _r = self.handle_401(r)
+            log.debug("handle_response returning %s" % _r)
+            return _r
+        else:
+            _r = self.handle_other(r)
+            log.debug("handle_response returning %s" % _r)
+            return _r
+
+        log.debug("handle_response returning %s" % r)
+        return r
+
+    def deregister(self, r):
+        """Deregisters the response handler"""
+        r.request.deregister_hook('response', self.handle_response)
+
+    def __call__(self, r):
+        r.register_hook('response', self.handle_response)
+        return r

requests/compat.py

@@ -72,6 +72,10 @@ is_osx = ('darwin' in str(sys.platform).lower())
 is_hpux = ('hpux' in str(sys.platform).lower())   # Complete guess.
 is_solaris = ('solar==' in str(sys.platform).lower())   # Complete guess.
 
+try:
+    import simplejson as json
+except ImportError:
+    import json
 
 # ---------
 # Specifics

requests/defaults.py

@@ -26,13 +26,12 @@ Configurations:
 
 SCHEMAS = ['http', 'https']
 
-from . import __version__
+from .utils import default_user_agent
 
 defaults = dict()
 
-
 defaults['base_headers'] = {
-    'User-Agent': 'python-requests/%s' % __version__,
+    'User-Agent': default_user_agent(),
     'Accept-Encoding': ', '.join(('identity', 'deflate', 'compress', 'gzip')),
     'Accept': '*/*'
 }
@@ -49,5 +48,3 @@ defaults['keep_alive'] = True
 defaults['encode_uri'] = True
 defaults['trust_env'] = True
 defaults['store_cookies'] = True
-
-

requests/models.py

@@ -7,7 +7,6 @@ requests.models
 This module contains the primary objects that power Requests.
 """
 
-import json
 import os
 from datetime import datetime
 
@@ -18,6 +17,7 @@ from .status_codes import codes
 from .auth import HTTPBasicAuth, HTTPProxyAuth
 from .cookies import cookiejar_from_dict, extract_cookies_to_jar, get_cookie_header
 from .packages.urllib3.exceptions import MaxRetryError, LocationParseError
+from .packages.urllib3.exceptions import TimeoutError
 from .packages.urllib3.exceptions import SSLError as _SSLError
 from .packages.urllib3.exceptions import HTTPError as _HTTPError
 from .packages.urllib3 import connectionpool, poolmanager
@@ -32,7 +32,7 @@ from .utils import (
     DEFAULT_CA_BUNDLE_PATH)
 from .compat import (
     cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,
-    StringIO, is_py2, chardet)
+    StringIO, is_py2, chardet, json)
 
 REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
 CONTENT_CHUNK_SIZE = 10 * 1024
@@ -355,6 +355,13 @@ class Request(object):
                 fp = StringIO(fp)
             fields.update({k: (fn, fp.read())})
 
+        for field in fields:
+            if isinstance(fields[field], float):
+                fields[field] = str(fields[field])
+            if isinstance(fields[field], list):
+                newvalue = ', '.join(fields[field])
+                fields[field] = newvalue
+                
         (body, content_type) = encode_multipart_formdata(fields)
 
         return (body, content_type)
@@ -396,14 +403,14 @@ class Request(object):
             if isinstance(fragment, str):
                 fragment = fragment.encode('utf-8')
 
-        url = (urlunparse([scheme, netloc, path, params, query, fragment]))
-
         enc_params = self._encode_params(self.params)
         if enc_params:
-            if urlparse(url).query:
-                url = '%s&%s' % (url, enc_params)
+            if query:
+                query = '%s&%s' % (query, enc_params)
             else:
-                url = '%s?%s' % (url, enc_params)
+                query = enc_params
+
+        url = (urlunparse([scheme, netloc, path, params, query, fragment]))
 
         if self.config.get('encode_uri', True):
             url = requote_uri(url)
@@ -601,10 +608,12 @@ class Request(object):
                 raise ConnectionError(e)
 
             except (_SSLError, _HTTPError) as e:
-                if self.verify and isinstance(e, _SSLError):
+                if isinstance(e, _SSLError):
                     raise SSLError(e)
-
-                raise Timeout('Request timed out.')
+                elif isinstance(e, TimeoutError):
+                    raise Timeout(e)
+                else:
+                    raise Timeout('Request timed out.')
 
             # build_response can throw TooManyRedirects
             self._build_response(r)
@@ -661,7 +670,7 @@ class Response(object):
 
         #: A list of :class:`Response <Response>` objects from
         #: the history of the Request. Any redirect responses will end
-        #: up here.
+        #: up here. The list is sorted from the oldest to the most recent request.
         self.history = []
 
         #: The :class:`Request <Request>` that created the Response.
@@ -819,16 +828,16 @@ class Response(object):
             raise self.error
 
         if (self.status_code >= 300) and (self.status_code < 400) and not allow_redirects:
-            http_error = HTTPError('%s Redirection' % self.status_code)
+            http_error = HTTPError('%s Redirection: %s' % (self.status_code, self.reason))
             http_error.response = self
             raise http_error
 
         elif (self.status_code >= 400) and (self.status_code < 500):
-            http_error = HTTPError('%s Client Error' % self.status_code)
+            http_error = HTTPError('%s Client Error: %s' % (self.status_code, self.reason))
             http_error.response = self
             raise http_error
 
         elif (self.status_code >= 500) and (self.status_code < 600):
-            http_error = HTTPError('%s Server Error' % self.status_code)
+            http_error = HTTPError('%s Server Error: %s' % (self.status_code, self.reason))
             http_error.response = self
             raise http_error

requests/packages/chardet2/test.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import sys, glob
 sys.path.insert(0, '..')
 from chardet.universaldetector import UniversalDetector

requests/packages/urllib3/_collections.py

@@ -4,128 +4,91 @@
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
-from collections import deque
+from collections import MutableMapping
+from threading import Lock
+
+try: # Python 2.7+
+    from collections import OrderedDict
+except ImportError:
+    from .packages.ordered_dict import OrderedDict
 
-from threading import RLock
 
 __all__ = ['RecentlyUsedContainer']
 
 
-class AccessEntry(object):
-    __slots__ = ('key', 'is_valid')
-
-    def __init__(self, key, is_valid=True):
-        self.key = key
-        self.is_valid = is_valid
+_Null = object()
 
 
-class RecentlyUsedContainer(dict):
+class RecentlyUsedContainer(MutableMapping):
     """
-    Provides a dict-like that maintains up to ``maxsize`` keys while throwing
-    away the least-recently-used keys beyond ``maxsize``.
+    Provides a thread-safe dict-like container which maintains up to
+    ``maxsize`` keys while throwing away the least-recently-used keys beyond
+    ``maxsize``.
+
+    :param maxsize:
+        Maximum number of recent elements to retain.
+
+    :param dispose_func:
+        Every time an item is evicted from the container,
+        ``dispose_func(value)`` is called.  Callback which will get called
     """
 
-    # If len(self.access_log) exceeds self._maxsize * CLEANUP_FACTOR, then we
-    # will attempt to cleanup the invalidated entries in the access_log
-    # datastructure during the next 'get' operation.
-    CLEANUP_FACTOR = 10
+    ContainerCls = OrderedDict
 
-    def __init__(self, maxsize=10):
+    def __init__(self, maxsize=10, dispose_func=None):
         self._maxsize = maxsize
+        self.dispose_func = dispose_func
 
-        self._container = {}
-
-        # We use a deque to to store our keys ordered by the last access.
-        self.access_log = deque()
-        self.access_log_lock = RLock()
-
-        # We look up the access log entry by the key to invalidate it so we can
-        # insert a new authorative entry at the head without having to dig and
-        # find the old entry for removal immediately.
-        self.access_lookup = {}
-
-        # Trigger a heap cleanup when we get past this size
-        self.access_log_limit = maxsize * self.CLEANUP_FACTOR
-
-    def _invalidate_entry(self, key):
-        "If exists: Invalidate old entry and return it."
-        old_entry = self.access_lookup.get(key)
-        if old_entry:
-            old_entry.is_valid = False
-
-        return old_entry
-
-    def _push_entry(self, key):
-        "Push entry onto our access log, invalidate the old entry if exists."
-        self._invalidate_entry(key)
-
-        new_entry = AccessEntry(key)
-        self.access_lookup[key] = new_entry
-
-        self.access_log_lock.acquire()
-        self.access_log.appendleft(new_entry)
-        self.access_log_lock.release()
-
-    def _prune_entries(self, num):
-        "Pop entries from our access log until we popped ``num`` valid ones."
-        while num > 0:
-            self.access_log_lock.acquire()
-            p = self.access_log.pop()
-            self.access_log_lock.release()
-
-            if not p.is_valid:
-                continue # Invalidated entry, skip
-
-            dict.pop(self, p.key, None)
-            self.access_lookup.pop(p.key, None)
-            num -= 1
-
-    def _prune_invalidated_entries(self):
-        "Rebuild our access_log without the invalidated entries."
-        self.access_log_lock.acquire()
-        self.access_log = deque(e for e in self.access_log if e.is_valid)
-        self.access_log_lock.release()
-
-    def _get_ordered_access_keys(self):
-        "Return ordered access keys for inspection. Used for testing."
-        self.access_log_lock.acquire()
-        r = [e.key for e in self.access_log if e.is_valid]
-        self.access_log_lock.release()
-
-        return r
+        self._container = self.ContainerCls()
+        self._lock = Lock()
 
     def __getitem__(self, key):
-        item = dict.get(self, key)
+        # Re-insert the item, moving it to the end of the eviction line.
+        with self._lock:
+            item = self._container.pop(key)
+            self._container[key] = item
+            return item
 
-        if not item:
-            raise KeyError(key)
+    def __setitem__(self, key, value):
+        evicted_value = _Null
+        with self._lock:
+            # Possibly evict the existing value of 'key'
+            evicted_value = self._container.get(key, _Null)
+            self._container[key] = value
 
-        # Insert new entry with new high priority, also implicitly invalidates
-        # the old entry.
-        self._push_entry(key)
+            # If we didn't evict an existing value, we might have to evict the
+            # least recently used item from the beginning of the container.
+            if len(self._container) > self._maxsize:
+                _key, evicted_value = self._container.popitem(last=False)
 
-        if len(self.access_log) > self.access_log_limit:
-            # Heap is getting too big, try to clean up any tailing invalidated
-            # entries.
-            self._prune_invalidated_entries()
-
-        return item
-
-    def __setitem__(self, key, item):
-        # Add item to our container and access log
-        dict.__setitem__(self, key, item)
-        self._push_entry(key)
-
-        # Discard invalid and excess entries
-        self._prune_entries(len(self) - self._maxsize)
+        if self.dispose_func and evicted_value is not _Null:
+            self.dispose_func(evicted_value)
 
     def __delitem__(self, key):
-        self._invalidate_entry(key)
-        self.access_lookup.pop(key, None)
-        dict.__delitem__(self, key)
+        with self._lock:
+            value = self._container.pop(key)
 
-    def get(self, key, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            return default
+        if self.dispose_func:
+            self.dispose_func(value)
+
+    def __len__(self):
+        with self._lock:
+            return len(self._container)
+
+    def __iter__(self):
+        raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.')
+
+    def clear(self):
+        with self._lock:
+            # Copy pointers to all values, then wipe the mapping
+            # under Python 2, this copies the list of values twice :-|
+            values = list(self._container.values())
+            self._container.clear()
+
+        if self.dispose_func:
+            for value in values:
+                self.dispose_func(value)
+
+    def keys(self):
+        with self._lock:
+            return self._container.keys()

requests/packages/urllib3/connectionpool.py

@@ -7,27 +7,27 @@
 import logging
 import socket
 
-from socket import error as SocketError, timeout as SocketTimeout
+from socket import timeout as SocketTimeout
 
-try:   # Python 3
+try: # Python 3
     from http.client import HTTPConnection, HTTPException
     from http.client import HTTP_PORT, HTTPS_PORT
 except ImportError:
     from httplib import HTTPConnection, HTTPException
     from httplib import HTTP_PORT, HTTPS_PORT
 
-try:   # Python 3
+try: # Python 3
     from queue import LifoQueue, Empty, Full
 except ImportError:
     from Queue import LifoQueue, Empty, Full
 
 
-try:   # Compiled with SSL?
+try: # Compiled with SSL?
     HTTPSConnection = object
     BaseSSLError = None
     ssl = None
 
-    try:   # Python 3
+    try: # Python 3
         from http.client import HTTPSConnection
     except ImportError:
         from httplib import HTTPSConnection
@@ -35,7 +35,7 @@ try:   # Compiled with SSL?
     import ssl
     BaseSSLError = ssl.SSLError
 
-except (ImportError, AttributeError):
+except (ImportError, AttributeError): # Platform-specific: No SSL.
     pass
 
 
@@ -43,6 +43,7 @@ from .request import RequestMethods
 from .response import HTTPResponse
 from .util import get_host, is_connection_dropped
 from .exceptions import (
+    ClosedPoolError,
     EmptyPoolError,
     HostChangedError,
     MaxRetryError,
@@ -206,10 +207,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         try:
             conn = self.pool.get(block=self.block, timeout=timeout)
 
-            # If this is a persistent connection, check if it got disconnected
-            if conn and is_connection_dropped(conn):
-                log.info("Resetting dropped connection: %s" % self.host)
-                conn.close()
+        except AttributeError: # self.pool is None
+            raise ClosedPoolError(self, "Pool is closed.")
 
         except Empty:
             if self.block:
@@ -218,6 +217,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
                                      "connections are allowed.")
             pass  # Oh well, we'll create a new connection then
 
+        # If this is a persistent connection, check if it got disconnected
+        if conn and is_connection_dropped(conn):
+            log.info("Resetting dropped connection: %s" % self.host)
+            conn.close()
+
         return conn or self._new_conn()
 
     def _put_conn(self, conn):
@@ -228,17 +232,26 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             Connection object for the current host and port as returned by
             :meth:`._new_conn` or :meth:`._get_conn`.
 
-        If the pool is already full, the connection is discarded because we
-        exceeded maxsize. If connections are discarded frequently, then maxsize
-        should be increased.
+        If the pool is already full, the connection is closed and discarded
+        because we exceeded maxsize. If connections are discarded frequently,
+        then maxsize should be increased.
+
+        If the pool is closed, then the connection will be closed and discarded.
         """
         try:
             self.pool.put(conn, block=False)
+            return # Everything is dandy, done.
+        except AttributeError:
+            # self.pool is None.
+            pass
         except Full:
             # This should never happen if self.block == True
             log.warning("HttpConnectionPool is full, discarding connection: %s"
                         % self.host)
 
+        # Connection never got put back into the pool, close it.
+        conn.close()
+
     def _make_request(self, conn, method, url, timeout=_Default,
                       **httplib_request_kw):
         """
@@ -268,15 +281,32 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
         log.debug("\"%s %s %s\" %s %s" % (method, url, http_version,
                                           httplib_response.status,
                                           httplib_response.length))
-
         return httplib_response
 
+    def close(self):
+        """
+        Close all pooled connections and disable the pool.
+        """
+        # Disable access to the pool
+        old_pool, self.pool = self.pool, None
+
+        try:
+            while True:
+                conn = old_pool.get(block=False)
+                if conn:
+                    conn.close()
+
+        except Empty:
+            pass # Done.
 
     def is_same_host(self, url):
         """
         Check if the given ``url`` is a member of the same host as this
         connection pool.
         """
+        if url.startswith('/'):
+            return True
+
         # TODO: Add optional support for socket.gethostbyname checking.
         scheme, host, port = get_host(url)
 
@@ -284,8 +314,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # Use explicit default port for comparison when none is given.
             port = port_by_scheme.get(scheme)
 
-        return (url.startswith('/') or
-                (scheme, host, port) == (self.scheme, self.host, self.port))
+        return (scheme, host, port) == (self.scheme, self.host, self.port)
 
     def urlopen(self, method, url, body=None, headers=None, retries=3,
                 redirect=True, assert_same_host=True, timeout=_Default,
@@ -378,7 +407,6 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
 
         try:
             # Request a connection from the queue
-            # (Could raise SocketError: Bad file descriptor)
             conn = self._get_conn(timeout=pool_timeout)
 
             # Make the request on the httplib connection object
@@ -421,29 +449,38 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             # Name mismatch
             raise SSLError(e)
 
-        except (HTTPException, SocketError) as e:
+        except HTTPException as e:
             # Connection broken, discard. It will be replaced next _get_conn().
             conn = None
             # This is necessary so we can access e below
             err = e
 
         finally:
-            if conn and release_conn:
-                # Put the connection back to be reused
+            if release_conn:
+                # Put the connection back to be reused. If the connection is
+                # expired then it will be None, which will get replaced with a
+                # fresh connection during _get_conn.
                 self._put_conn(conn)
 
         if not conn:
+            # Try again
             log.warn("Retrying (%d attempts remain) after connection "
                      "broken by '%r': %s" % (retries, err, url))
             return self.urlopen(method, url, body, headers, retries - 1,
-                                redirect, assert_same_host)  # Try again
+                                redirect, assert_same_host,
+                                timeout=timeout, pool_timeout=pool_timeout,
+                                release_conn=release_conn, **response_kw)
 
         # Handle redirect?
         redirect_location = redirect and response.get_redirect_location()
         if redirect_location:
+            if response.status == 303:
+                method = 'GET'
             log.info("Redirecting %s -> %s" % (url, redirect_location))
             return self.urlopen(method, redirect_location, body, headers,
-                                retries - 1, redirect, assert_same_host)
+                                retries - 1, redirect, assert_same_host,
+                                timeout=timeout, pool_timeout=pool_timeout,
+                                release_conn=release_conn, **response_kw)
 
         return response
 

requests/packages/urllib3/exceptions.py

@@ -24,6 +24,11 @@ class SSLError(HTTPError):
     pass
 
 
+class DecodeError(HTTPError):
+    "Raised when automatic decoding based on Content-Type fails."
+    pass
+
+
 ## Leaf Exceptions
 
 class MaxRetryError(PoolError):
@@ -57,6 +62,11 @@ class EmptyPoolError(PoolError):
     pass
 
 
+class ClosedPoolError(PoolError):
+    "Raised when a request enters a pool after the pool has been closed."
+    pass
+
+
 class LocationParseError(ValueError, HTTPError):
     "Raised when get_host or similar fails to parse the URL input."
 

requests/packages/urllib3/packages/ordered_dict.py

@@ -0,0 +1,260 @@
+# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
+# Passes Python2.7's test suite and incorporates all the latest updates.
+# Copyright 2009 Raymond Hettinger, released under the MIT License.
+# http://code.activestate.com/recipes/576693/
+
+try:
+    from thread import get_ident as _get_ident
+except ImportError:
+    from dummy_thread import get_ident as _get_ident
+
+try:
+    from _abcoll import KeysView, ValuesView, ItemsView
+except ImportError:
+    pass
+
+
+class OrderedDict(dict):
+    'Dictionary that remembers insertion order'
+    # An inherited dict maps keys to values.
+    # The inherited dict provides __getitem__, __len__, __contains__, and get.
+    # The remaining methods are order-aware.
+    # Big-O running times for all methods are the same as for regular dictionaries.
+
+    # The internal self.__map dictionary maps keys to links in a doubly linked list.
+    # The circular doubly linked list starts and ends with a sentinel element.
+    # The sentinel element never gets deleted (this simplifies the algorithm).
+    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
+
+    def __init__(self, *args, **kwds):
+        '''Initialize an ordered dictionary.  Signature is the same as for
+        regular dictionaries, but keyword arguments are not recommended
+        because their insertion order is arbitrary.
+
+        '''
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        try:
+            self.__root
+        except AttributeError:
+            self.__root = root = []                     # sentinel node
+            root[:] = [root, root, None]
+            self.__map = {}
+        self.__update(*args, **kwds)
+
+    def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
+        'od.__setitem__(i, y) <==> od[i]=y'
+        # Setting a new item creates a new link which goes at the end of the linked
+        # list, and the inherited dictionary is updated with the new key/value pair.
+        if key not in self:
+            root = self.__root
+            last = root[0]
+            last[1] = root[0] = self.__map[key] = [last, root, key]
+        dict_setitem(self, key, value)
+
+    def __delitem__(self, key, dict_delitem=dict.__delitem__):
+        'od.__delitem__(y) <==> del od[y]'
+        # Deleting an existing item uses self.__map to find the link which is
+        # then removed by updating the links in the predecessor and successor nodes.
+        dict_delitem(self, key)
+        link_prev, link_next, key = self.__map.pop(key)
+        link_prev[1] = link_next
+        link_next[0] = link_prev
+
+    def __iter__(self):
+        'od.__iter__() <==> iter(od)'
+        root = self.__root
+        curr = root[1]
+        while curr is not root:
+            yield curr[2]
+            curr = curr[1]
+
+    def __reversed__(self):
+        'od.__reversed__() <==> reversed(od)'
+        root = self.__root
+        curr = root[0]
+        while curr is not root:
+            yield curr[2]
+            curr = curr[0]
+
+    def clear(self):
+        'od.clear() -> None.  Remove all items from od.'
+        try:
+            for node in self.__map.itervalues():
+                del node[:]
+            root = self.__root
+            root[:] = [root, root, None]
+            self.__map.clear()
+        except AttributeError:
+            pass
+        dict.clear(self)
+
+    def popitem(self, last=True):
+        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
+        Pairs are returned in LIFO order if last is true or FIFO order if false.
+
+        '''
+        if not self:
+            raise KeyError('dictionary is empty')
+        root = self.__root
+        if last:
+            link = root[0]
+            link_prev = link[0]
+            link_prev[1] = root
+            root[0] = link_prev
+        else:
+            link = root[1]
+            link_next = link[1]
+            root[1] = link_next
+            link_next[0] = root
+        key = link[2]
+        del self.__map[key]
+        value = dict.pop(self, key)
+        return key, value
+
+    # -- the following methods do not depend on the internal structure --
+
+    def keys(self):
+        'od.keys() -> list of keys in od'
+        return list(self)
+
+    def values(self):
+        'od.values() -> list of values in od'
+        return [self[key] for key in self]
+
+    def items(self):
+        'od.items() -> list of (key, value) pairs in od'
+        return [(key, self[key]) for key in self]
+
+    def iterkeys(self):
+        'od.iterkeys() -> an iterator over the keys in od'
+        return iter(self)
+
+    def itervalues(self):
+        'od.itervalues -> an iterator over the values in od'
+        for k in self:
+            yield self[k]
+
+    def iteritems(self):
+        'od.iteritems -> an iterator over the (key, value) items in od'
+        for k in self:
+            yield (k, self[k])
+
+    def update(*args, **kwds):
+        '''od.update(E, **F) -> None.  Update od from dict/iterable E and F.
+
+        If E is a dict instance, does:           for k in E: od[k] = E[k]
+        If E has a .keys() method, does:         for k in E.keys(): od[k] = E[k]
+        Or if E is an iterable of items, does:   for k, v in E: od[k] = v
+        In either case, this is followed by:     for k, v in F.items(): od[k] = v
+
+        '''
+        if len(args) > 2:
+            raise TypeError('update() takes at most 2 positional '
+                            'arguments (%d given)' % (len(args),))
+        elif not args:
+            raise TypeError('update() takes at least 1 argument (0 given)')
+        self = args[0]
+        # Make progressively weaker assumptions about "other"
+        other = ()
+        if len(args) == 2:
+            other = args[1]
+        if isinstance(other, dict):
+            for key in other:
+                self[key] = other[key]
+        elif hasattr(other, 'keys'):
+            for key in other.keys():
+                self[key] = other[key]
+        else:
+            for key, value in other:
+                self[key] = value
+        for key, value in kwds.items():
+            self[key] = value
+
+    __update = update  # let subclasses override update without breaking __init__
+
+    __marker = object()
+
+    def pop(self, key, default=__marker):
+        '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
+        If key is not found, d is returned if given, otherwise KeyError is raised.
+
+        '''
+        if key in self:
+            result = self[key]
+            del self[key]
+            return result
+        if default is self.__marker:
+            raise KeyError(key)
+        return default
+
+    def setdefault(self, key, default=None):
+        'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
+        if key in self:
+            return self[key]
+        self[key] = default
+        return default
+
+    def __repr__(self, _repr_running={}):
+        'od.__repr__() <==> repr(od)'
+        call_key = id(self), _get_ident()
+        if call_key in _repr_running:
+            return '...'
+        _repr_running[call_key] = 1
+        try:
+            if not self:
+                return '%s()' % (self.__class__.__name__,)
+            return '%s(%r)' % (self.__class__.__name__, self.items())
+        finally:
+            del _repr_running[call_key]
+
+    def __reduce__(self):
+        'Return state information for pickling'
+        items = [[k, self[k]] for k in self]
+        inst_dict = vars(self).copy()
+        for k in vars(OrderedDict()):
+            inst_dict.pop(k, None)
+        if inst_dict:
+            return (self.__class__, (items,), inst_dict)
+        return self.__class__, (items,)
+
+    def copy(self):
+        'od.copy() -> a shallow copy of od'
+        return self.__class__(self)
+
+    @classmethod
+    def fromkeys(cls, iterable, value=None):
+        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
+        and values equal to v (which defaults to None).
+
+        '''
+        d = cls()
+        for key in iterable:
+            d[key] = value
+        return d
+
+    def __eq__(self, other):
+        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
+        while comparison to a regular mapping is order-insensitive.
+
+        '''
+        if isinstance(other, OrderedDict):
+            return len(self)==len(other) and self.items() == other.items()
+        return dict.__eq__(self, other)
+
+    def __ne__(self, other):
+        return not self == other
+
+    # -- the following methods are only used in Python 2.7 --
+
+    def viewkeys(self):
+        "od.viewkeys() -> a set-like object providing a view on od's keys"
+        return KeysView(self)
+
+    def viewvalues(self):
+        "od.viewvalues() -> an object providing a view on od's values"
+        return ValuesView(self)
+
+    def viewitems(self):
+        "od.viewitems() -> a set-like object providing a view on od's items"
+        return ItemsView(self)

requests/packages/urllib3/poolmanager.py

@@ -8,9 +8,9 @@ import logging
 
 from ._collections import RecentlyUsedContainer
 from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
-from .connectionpool import get_host, connection_from_url, port_by_scheme
-from .exceptions import HostChangedError
+from .connectionpool import connection_from_url, port_by_scheme
 from .request import RequestMethods
+from .util import parse_url
 
 
 __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
@@ -48,19 +48,29 @@ class PoolManager(RequestMethods):
 
     """
 
-    # TODO: Make sure there are no memory leaks here.
-
     def __init__(self, num_pools=10, **connection_pool_kw):
         self.connection_pool_kw = connection_pool_kw
-        self.pools = RecentlyUsedContainer(num_pools)
+        self.pools = RecentlyUsedContainer(num_pools,
+                                           dispose_func=lambda p: p.close())
 
-    def connection_from_host(self, host, port=80, scheme='http'):
+    def clear(self):
+        """
+        Empty our store of pools and direct them all to close.
+
+        This will not affect in-flight connections, but they will not be
+        re-used after completion.
+        """
+        self.pools.clear()
+
+    def connection_from_host(self, host, port=None, scheme='http'):
         """
         Get a :class:`ConnectionPool` based on the host, port, and scheme.
 
-        Note that an appropriate ``port`` value is required here to normalize
-        connection pools in our container most effectively.
+        If ``port`` isn't given, it will be derived from the ``scheme`` using
+        ``urllib3.connectionpool.port_by_scheme``.
         """
+        port = port or port_by_scheme.get(scheme, 80)
+
         pool_key = (scheme, host, port)
 
         # If the scheme, host, or port doesn't match existing open connections,
@@ -86,26 +96,36 @@ class PoolManager(RequestMethods):
         Additional parameters are taken from the :class:`.PoolManager`
         constructor.
         """
-        scheme, host, port = get_host(url)
+        u = parse_url(url)
+        return self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
 
-        port = port or port_by_scheme.get(scheme, 80)
-
-        return self.connection_from_host(host, port=port, scheme=scheme)
-
-    def urlopen(self, method, url, **kw):
+    def urlopen(self, method, url, redirect=True, **kw):
         """
-        Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`.
+        Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
+        with custom cross-host redirect logic and only sends the request-uri
+        portion of the ``url``.
 
-        ``url`` must be absolute, such that an appropriate
+        The given ``url`` parameter must be absolute, such that an appropriate
         :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
         """
-        conn = self.connection_from_url(url)
-        try:
-            return conn.urlopen(method, url, **kw)
+        u = parse_url(url)
+        conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
 
-        except HostChangedError as e:
-            kw['retries'] = e.retries # Persist retries countdown
-            return self.urlopen(method, e.url, **kw)
+        kw['assert_same_host'] = False
+        kw['redirect'] = False
+
+        response = conn.urlopen(method, u.request_uri, **kw)
+
+        redirect_location = redirect and response.get_redirect_location()
+        if not redirect_location:
+            return response
+
+        if response.status == 303:
+            method = 'GET'
+
+        log.info("Redirecting %s -> %s" % (url, redirect_location))
+        kw['retries'] = kw.get('retries', 3) - 1 # Persist retries countdown
+        return self.urlopen(method, redirect_location, **kw)
 
 
 class ProxyManager(RequestMethods):

requests/packages/urllib3/response.py

@@ -10,7 +10,7 @@ import zlib
 
 from io import BytesIO
 
-from .exceptions import HTTPError
+from .exceptions import DecodeError
 from .packages.six import string_types as basestring
 
 
@@ -148,9 +148,9 @@ class HTTPResponse(object):
             try:
                 if decode_content and decoder:
                     data = decoder(data)
-            except IOError:
-                raise HTTPError("Received response with content-encoding: %s, but "
-                                "failed to decode it." % content_encoding)
+            except (IOError, zlib.error):
+                raise DecodeError("Received response with content-encoding: %s, but "
+                                  "failed to decode it." % content_encoding)
 
             if cache_content:
                 self._body = data

requests/packages/urllib3/util.py

@@ -6,6 +6,8 @@
 
 
 from base64 import b64encode
+from collections import namedtuple
+from socket import error as SocketError
 
 try:
     from select import poll, POLLIN
@@ -20,6 +22,152 @@ from .packages import six
 from .exceptions import LocationParseError
 
 
+class Url(namedtuple('Url', ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'])):
+    """
+    Datastructure for representing an HTTP URL. Used as a return value for
+    :func:`parse_url`.
+    """
+    slots = ()
+
+    def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None):
+        return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment)
+
+    @property
+    def hostname(self):
+        """For backwards-compatibility with urlparse. We're nice like that."""
+        return self.host
+
+    @property
+    def request_uri(self):
+        """Absolute path including the query string."""
+        uri = self.path or '/'
+
+        if self.query is not None:
+            uri += '?' + self.query
+
+        return uri
+
+
+def split_first(s, delims):
+    """
+    Given a string and an iterable of delimiters, split on the first found
+    delimiter. Return two split parts and the matched delimiter.
+
+    If not found, then the first part is the full input string.
+
+    Example: ::
+
+        >>> split_first('foo/bar?baz', '?/=')
+        ('foo', 'bar?baz', '/')
+        >>> split_first('foo/bar?baz', '123')
+        ('foo/bar?baz', '', None)
+
+    Scales linearly with number of delims. Not ideal for large number of delims.
+    """
+    min_idx = None
+    min_delim = None
+    for d in delims:
+        idx = s.find(d)
+        if idx < 0:
+            continue
+
+        if min_idx is None or idx < min_idx:
+            min_idx = idx
+            min_delim = d
+
+    if min_idx is None or min_idx < 0:
+        return s, '', None
+
+    return s[:min_idx], s[min_idx+1:], min_delim
+
+
+def parse_url(url):
+    """
+    Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is
+    performed to parse incomplete urls. Fields not provided will be None.
+
+    Partly backwards-compatible with :mod:`urlparse`.
+
+    Example: ::
+
+        >>> parse_url('http://google.com/mail/')
+        Url(scheme='http', host='google.com', port=None, path='/', ...)
+        >>> prase_url('google.com:80')
+        Url(scheme=None, host='google.com', port=80, path=None, ...)
+        >>> prase_url('/foo?bar')
+        Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...)
+    """
+
+    # While this code has overlap with stdlib's urlparse, it is much
+    # simplified for our needs and less annoying.
+    # Additionally, this imeplementations does silly things to be optimal
+    # on CPython.
+
+    scheme = None
+    auth = None
+    host = None
+    port = None
+    path = None
+    fragment = None
+    query = None
+
+    # Scheme
+    if '://' in url:
+        scheme, url = url.split('://', 1)
+
+    # Find the earliest Authority Terminator
+    # (http://tools.ietf.org/html/rfc3986#section-3.2)
+    url, path_, delim = split_first(url, ['/', '?', '#'])
+
+    if delim:
+        # Reassemble the path
+        path = delim + path_
+
+    # Auth
+    if '@' in url:
+        auth, url = url.split('@', 1)
+
+    # IPv6
+    if url and url[0] == '[':
+        host, url = url[1:].split(']', 1)
+
+    # Port
+    if ':' in url:
+        _host, port = url.split(':', 1)
+
+        if not host:
+            host = _host
+
+        if not port.isdigit():
+            raise LocationParseError("Failed to parse: %s" % url)
+
+        port = int(port)
+
+    elif not host and url:
+        host = url
+
+    if not path:
+        return Url(scheme, auth, host, port, path, query, fragment)
+
+    # Fragment
+    if '#' in path:
+        path, fragment = path.split('#', 1)
+
+    # Query
+    if '?' in path:
+        path, query = path.split('?', 1)
+
+    return Url(scheme, auth, host, port, path, query, fragment)
+
+
+def get_host(url):
+    """
+    Deprecated. Use :func:`.parse_url` instead.
+    """
+    p = parse_url(url)
+    return p.scheme or 'http', p.hostname, p.port
+
+
 def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
                  basic_auth=None):
     """
@@ -72,93 +220,12 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
     return headers
 
 
-def split_first(s, delims):
-    """
-    Given a string and an iterable of delimiters, split on the first found
-    delimiter. Return two split parts.
-
-    If not found, then the first part is the full input string.
-
-    Scales linearly with number of delims. Not ideal for large number of delims.
-    """
-    min_idx = None
-    for d in delims:
-        idx = s.find(d)
-        if idx < 0:
-            continue
-
-        if not min_idx:
-            min_idx = idx
-        else:
-            min_idx = min(idx, min_idx)
-
-    if min_idx < 0:
-        return s, ''
-
-    return s[:min_idx], s[min_idx+1:]
-
-
-def get_host(url):
-    """
-    Given a url, return its scheme, host and port (None if it's not there).
-
-    For example: ::
-
-        >>> get_host('http://google.com/mail/')
-        ('http', 'google.com', None)
-        >>> get_host('google.com:80')
-        ('http', 'google.com', 80)
-    """
-
-    # While this code has overlap with stdlib's urlparse, it is much
-    # simplified for our needs and less annoying.
-    # Additionally, this imeplementations does silly things to be optimal
-    # on CPython.
-
-    scheme = 'http'
-    host = None
-    port = None
-
-    # Scheme
-    if '://' in url:
-        scheme, url = url.split('://', 1)
-
-    # Find the earliest Authority Terminator
-    # (http://tools.ietf.org/html/rfc3986#section-3.2)
-    url, _path = split_first(url, ['/', '?', '#'])
-
-    # Auth
-    if '@' in url:
-        _auth, url = url.split('@', 1)
-
-    # IPv6
-    if url and url[0] == '[':
-        host, url = url[1:].split(']', 1)
-
-    # Port
-    if ':' in url:
-        _host, port = url.split(':', 1)
-
-        if not host:
-            host = _host
-
-        if not port.isdigit():
-            raise LocationParseError("Failed to parse: %s" % url)
-
-        port = int(port)
-
-    elif not host:
-        host = url
-
-    return scheme, host, port
-
-
 def is_connection_dropped(conn):
     """
     Returns True if the connection is dropped and should be closed.
 
     :param conn:
-        ``HTTPConnection`` object.
+        :class:`httplib.HTTPConnection` object.
 
     Note: For platforms like AppEngine, this will always return ``False`` to
     let the platform handle connection recycling transparently for us.
@@ -171,7 +238,10 @@ def is_connection_dropped(conn):
         if not select: # Platform-specific: AppEngine
             return False
 
-        return select([sock], [], [], 0.0)[0]
+        try:
+            return select([sock], [], [], 0.0)[0]
+        except SocketError:
+            return True
 
     # This version is better on platforms that support it.
     p = poll()

requests/utils.py

@@ -12,10 +12,13 @@ that are also useful for external consumption.
 import cgi
 import codecs
 import os
+import platform
 import re
+import sys
 import zlib
 from netrc import netrc, NetrcParseError
 
+from . import __version__
 from .compat import parse_http_list as _parse_list_header
 from .compat import quote, urlparse, basestring, bytes, str
 from .cookies import RequestsCookieJar, cookiejar_from_dict
@@ -98,7 +101,7 @@ def get_netrc_auth(url):
             pass
 
     # AppEngine hackiness.
-    except AttributeError:
+    except (ImportError, AttributeError):
         pass
 
 
@@ -457,3 +460,31 @@ def get_environ_proxies():
     get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
     proxies = [(key, get_proxy(key + '_proxy')) for key in proxy_keys]
     return dict([(key, val) for (key, val) in proxies if val])
+
+
+def default_user_agent():
+    """Return a string representing the default user agent."""
+    _implementation = platform.python_implementation()
+
+    if _implementation == 'CPython':
+        _implementation_version = platform.python_version()
+    elif _implementation == 'PyPy':
+        _implementation_version = '%s.%s.%s' % (
+                                                sys.pypy_version_info.major,
+                                                sys.pypy_version_info.minor,
+                                                sys.pypy_version_info.micro
+                                            )
+        if sys.pypy_version_info.releaselevel != 'final':
+            _implementation_version = ''.join([_implementation_version, sys.pypy_version_info.releaselevel])
+    elif _implementation == 'Jython':
+        _implementation_version = platform.python_version()  # Complete Guess
+    elif _implementation == 'IronPython':
+        _implementation_version = platform.python_version()  # Complete Guess
+    else:
+        _implementation_version = 'Unknown'
+
+    return " ".join([
+            'python-requests/%s' % __version__,
+            '%s/%s' % (_implementation, _implementation_version),
+            '%s/%s' % (platform.system(), platform.release()),
+        ])

tests/test_requests.py

@@ -83,6 +83,16 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
 
         self.assertEqual(request.path_url, "/get/test%20case")
 
+    def test_params_are_added_before_fragment(self):
+        request = requests.Request(
+            "http://example.com/path#fragment", params={"a": "b"})
+        self.assertEqual(request.full_url,
+            "http://example.com/path?a=b#fragment")
+        request = requests.Request(
+            "http://example.com/path?key=value#fragment", params={"a": "b"})
+        self.assertEqual(request.full_url,
+            "http://example.com/path?key=value&a=b#fragment")
+
     def test_HTTP_200_OK_GET(self):
         r = get(httpbin('get'))
         self.assertEqual(r.status_code, 200)
@@ -801,20 +811,20 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
         assert not ds1.prefetch
         assert ds2.prefetch
 
-    def test_invalid_content(self):
-        # WARNING: if you're using a terrible DNS provider (comcast),
-        # this will fail.
-        try:
-            hah = 'http://somedomainthatclearlydoesntexistg.com'
-            r = get(hah, allow_redirects=False)
-        except requests.ConnectionError:
-            pass   # \o/
-        else:
-            assert False
+    # def test_invalid_content(self):
+    #     # WARNING: if you're using a terrible DNS provider (comcast),
+    #     # this will fail.
+    #     try:
+    #         hah = 'http://somedomainthatclearlydoesntexistg.com'
+    #         r = get(hah, allow_redirects=False)
+    #     except requests.ConnectionError:
+    #         pass   # \o/
+    #     else:
+    #         assert False
 
-        config = {'safe_mode': True}
-        r = get(hah, allow_redirects=False, config=config)
-        assert r.content == None
+    #     config = {'safe_mode': True}
+    #     r = get(hah, allow_redirects=False, config=config)
+    #     assert r.content == None
 
     def test_cached_response(self):
 
@@ -858,31 +868,31 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
         joined = lines[0] + '\n' + lines[1] + '\r\n' + lines[2]
         self.assertEqual(joined, quote)
 
-    def test_safe_mode(self):
+    # def test_safe_mode(self):
 
-        safe = requests.session(config=dict(safe_mode=True))
+    #     safe = requests.session(config=dict(safe_mode=True))
 
-        # Safe mode creates empty responses for failed requests.
-        # Iterating on these responses should produce empty sequences
-        r = get('http://_/', session=safe)
-        self.assertEqual(list(r.iter_lines()), [])
-        assert isinstance(r.error, requests.exceptions.ConnectionError)
+    #     # Safe mode creates empty responses for failed requests.
+    #     # Iterating on these responses should produce empty sequences
+    #     r = get('http://0.0.0.0:700/', session=safe)
+    #     self.assertEqual(list(r.iter_lines()), [])
+    #     assert isinstance(r.error, requests.exceptions.ConnectionError)
 
-        r = get('http://_/', session=safe)
-        self.assertEqual(list(r.iter_content()), [])
-        assert isinstance(r.error, requests.exceptions.ConnectionError)
+    #     r = get('http://0.0.0.0:789/', session=safe)
+    #     self.assertEqual(list(r.iter_content()), [])
+    #     assert isinstance(r.error, requests.exceptions.ConnectionError)
 
-        # When not in safe mode, should raise Timeout exception
-        self.assertRaises(
-            requests.exceptions.Timeout,
-            get,
-            httpbin('stream', '1000'), timeout=0.0001)
+    #     # When not in safe mode, should raise Timeout exception
+    #     self.assertRaises(
+    #         requests.exceptions.Timeout,
+    #         get,
+    #         httpbin('stream', '1000'), timeout=0.0001)
 
-        # In safe mode, should return a blank response
-        r = get(httpbin('stream', '1000'), timeout=0.0001,
-                config=dict(safe_mode=True))
-        assert r.content is None
-        assert isinstance(r.error, requests.exceptions.Timeout)
+    #     # In safe mode, should return a blank response
+    #     r = get(httpbin('stream', '1000'), timeout=0.0001,
+    #             config=dict(safe_mode=True))
+    #     assert r.content is None
+    #     assert isinstance(r.error, requests.exceptions.Timeout)
 
     def test_upload_binary_data(self):