~bzr-pqm/bzr/bzr.dev

1540.3.3 by Martin Pool
Review updates of pycurl transport
1
# Copyright (C) 2005, 2006 Canonical Ltd
1540.3.18 by Martin Pool
Style review fixes (thanks robertc)
2
#
1540.3.3 by Martin Pool
Review updates of pycurl transport
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1540.3.18 by Martin Pool
Style review fixes (thanks robertc)
7
#
1540.3.3 by Martin Pool
Review updates of pycurl transport
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1540.3.18 by Martin Pool
Style review fixes (thanks robertc)
12
#
1540.3.3 by Martin Pool
Review updates of pycurl transport
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
2004.1.2 by vila
Implements a BasicAuthManager.
17
from cStringIO import StringIO
2363.4.9 by Vincent Ladeuil
Catch first succesful authentification to avoid further 401
18
import urllib
19
import urlparse
1540.3.3 by Martin Pool
Review updates of pycurl transport
20
2420.1.5 by Vincent Ladeuil
Refactor http and proxy authentication. Tests passing. proxy password can be prompted too.
21
from bzrlib import errors
1540.3.3 by Martin Pool
Review updates of pycurl transport
22
from bzrlib.trace import mutter
1636.1.2 by Robert Collins
More review fixen to the relpath at '/' fixes.
23
from bzrlib.transport import register_urlparse_netloc_protocol
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
24
from bzrlib.transport.http import HttpTransportBase
2004.1.9 by vila
Takes jam's remarks into account when possible, add TODOs for the rest.
25
# TODO: handle_response should be integrated into the _urllib2_wrappers
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
26
from bzrlib.transport.http.response import handle_response
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
27
from bzrlib.transport.http._urllib2_wrappers import (
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
28
    Opener,
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
29
    Request,
2420.1.5 by Vincent Ladeuil
Refactor http and proxy authentication. Tests passing. proxy password can be prompted too.
30
    extract_authentication_uri,
2420.1.3 by Vincent Ladeuil
Implement http proxy basic authentication.
31
    extract_credentials,
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
32
    )
33
1540.3.3 by Martin Pool
Review updates of pycurl transport
34
1636.1.2 by Robert Collins
More review fixen to the relpath at '/' fixes.
35
register_urlparse_netloc_protocol('http+urllib')
1636.1.1 by Robert Collins
Fix calling relpath() and abspath() on transports at their root.
36
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
37
1540.3.26 by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet
38
class HttpTransport_urllib(HttpTransportBase):
1786.1.33 by John Arbash Meinel
Cleanup pass #2
39
    """Python urllib transport for http and https."""
1540.3.3 by Martin Pool
Review updates of pycurl transport
40
2004.1.9 by vila
Takes jam's remarks into account when possible, add TODOs for the rest.
41
    # In order to debug we have to issue our traces in sync with
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
42
    # httplib, which use print :(
43
    _debuglevel = 0
2004.3.1 by vila
Test ConnectionError exceptions.
44
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
45
    _opener_class = Opener
46
47
    def __init__(self, base, from_transport=None):
1540.3.3 by Martin Pool
Review updates of pycurl transport
48
        """Set the base path where files will be stored."""
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
49
        if from_transport is not None:
2363.4.9 by Vincent Ladeuil
Catch first succesful authentification to avoid further 401
50
            super(HttpTransport_urllib, self).__init__(base, from_transport)
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
51
            self._connection = from_transport._connection
2420.1.5 by Vincent Ladeuil
Refactor http and proxy authentication. Tests passing. proxy password can be prompted too.
52
            self._auth = from_transport._auth
53
            self._proxy_auth = from_transport._proxy_auth
54
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
55
            self._opener = from_transport._opener
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
56
        else:
2363.4.12 by Vincent Ladeuil
Take jam's review comments into account. Fix typos, give better
57
            # urllib2 will be confused if it find authentication
58
            # info in the urls. So we handle them separatly.
59
            # Note: we don't need to when cloning because it was
60
            # already done.
2420.1.3 by Vincent Ladeuil
Implement http proxy basic authentication.
61
            clean_base, user, password = extract_credentials(base)
2363.4.9 by Vincent Ladeuil
Catch first succesful authentification to avoid further 401
62
            super(HttpTransport_urllib, self).__init__(clean_base,
63
                                                       from_transport)
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
64
            self._connection = None
2420.1.5 by Vincent Ladeuil
Refactor http and proxy authentication. Tests passing. proxy password can be prompted too.
65
            self._opener = self._opener_class()
66
2420.1.11 by Vincent Ladeuil
Implement digest authentication. Test suite passes. Tested against apache-2.x.
67
            authuri = extract_authentication_uri(self._real_abspath(self._path))
2420.1.6 by Vincent Ladeuil
Update NEWS to explain the intent of the modification. Also, use dicts
68
            self._auth = {'user': user, 'password': password,
69
                          'authuri': authuri}
2363.4.10 by Vincent Ladeuil
Complete tests.
70
            if user and password is not None: # '' is a valid password
2363.4.9 by Vincent Ladeuil
Catch first succesful authentification to avoid further 401
71
                # Make the (user, password) available to urllib2
2420.1.6 by Vincent Ladeuil
Update NEWS to explain the intent of the modification. Also, use dicts
72
                # We default to a realm of None to catch them all.
73
                self._opener.password_manager.add_password(None, authuri,
74
                                                           user, password)
75
            self._proxy_auth = {}
2363.4.9 by Vincent Ladeuil
Catch first succesful authentification to avoid further 401
76
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
77
    def _perform(self, request):
2004.1.28 by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code
78
        """Send the request to the server and handles common errors.
79
80
        :returns: urllib2 Response object
81
        """
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
82
        if self._connection is not None:
2004.1.2 by vila
Implements a BasicAuthManager.
83
            # Give back shared info
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
84
            request.connection = self._connection
2363.4.12 by Vincent Ladeuil
Take jam's review comments into account. Fix typos, give better
85
        # Ensure authentication info is provided
2420.1.5 by Vincent Ladeuil
Refactor http and proxy authentication. Tests passing. proxy password can be prompted too.
86
        request.auth = self._auth
87
        request.proxy_auth = self._proxy_auth
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
88
89
        mutter('%s: [%s]' % (request.method, request.get_full_url()))
90
        if self._debuglevel > 0:
91
            print 'perform: %s base: %s, url: %s' % (request.method, self.base,
92
                                                     request.get_full_url())
93
        response = self._opener.open(request)
94
        if self._connection is None:
95
            # Acquire connection when the first request is able
96
            # to connect to the server
97
            self._connection = request.connection
2420.1.16 by Vincent Ladeuil
Handle nonce changes. Fix a nasty bug breaking the auth parameters sharing.
98
        # Always get auth parameters, they may change
99
        self._auth = request.auth
100
        self._proxy_auth = request.proxy_auth
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
101
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
102
        code = response.code
2164.2.11 by Vincent Ladeuil
Explicit tests.
103
        if request.follow_redirections is False \
104
                and code in (301, 302, 303, 307):
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
105
            raise errors.RedirectRequested(request.get_full_url(),
106
                                           request.redirected_to,
2164.2.13 by v.ladeuil+lp at free
Add tests for redirection. Preserve transport decorations.
107
                                           is_permament=(code == 301),
108
                                           qual_proto=self._qualified_proto)
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
109
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
110
        if request.redirected_to is not None:
111
            mutter('redirected from: %s to: %s' % (request.get_full_url(),
112
                                                   request.redirected_to))
113
114
        return response
1540.3.26 by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet
115
2164.2.15 by Vincent Ladeuil
Http redirections are not followed by default. Do not use hints
116
    def _get(self, relpath, ranges, tail_amount=0):
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
117
        """See HttpTransport._get"""
118
119
        abspath = self._real_abspath(relpath)
120
        headers = {}
1786.1.8 by John Arbash Meinel
[merge] Johan Rydberg test updates
121
        if ranges or tail_amount:
2004.1.30 by v.ladeuil+lp at free
Fix #62276 and #62029 by providing a more robust http range handling.
122
            range_header = self.attempted_range_header(ranges, tail_amount)
123
            if range_header is not None:
124
                bytes = 'bytes=' + range_header
125
                headers = {'Range': bytes}
2004.3.1 by vila
Test ConnectionError exceptions.
126
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
127
        request = Request('GET', abspath, None, headers)
128
        response = self._perform(request)
129
130
        code = response.code
131
        if code == 404: # not found
132
            self._connection.fake_close()
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
133
            raise errors.NoSuchFile(abspath)
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
134
135
        data = handle_response(abspath, code, response.headers, response)
136
        # Close response to free the httplib.HTTPConnection pipeline
137
        self._connection.fake_close()
138
        return code, data
1540.3.3 by Martin Pool
Review updates of pycurl transport
139
2018.2.7 by Andrew Bennetts
Implement _post on HttpTransport_urllib.
140
    def _post(self, body_bytes):
2004.1.28 by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code
141
        abspath = self._real_abspath('.bzr/smart')
142
        response = self._perform(Request('POST', abspath, body_bytes))
143
        code = response.code
144
        data = handle_response(abspath, code, response.headers, response)
145
        # Close response to free the httplib.HTTPConnection pipeline
146
        self._connection.fake_close()
147
        return code, data
2018.2.7 by Andrew Bennetts
Implement _post on HttpTransport_urllib.
148
1540.3.3 by Martin Pool
Review updates of pycurl transport
149
    def should_cache(self):
150
        """Return True if the data pulled across should be cached locally.
151
        """
152
        return True
153
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
154
    def _head(self, relpath):
155
        """Request the HEAD of a file.
156
157
        Performs the request and leaves callers handle the results.
158
        """
159
        abspath = self._real_abspath(relpath)
160
        request = Request('HEAD', abspath)
161
        response = self._perform(request)
162
163
        self._connection.fake_close()
164
        return response
165
1540.3.3 by Martin Pool
Review updates of pycurl transport
166
    def has(self, relpath):
167
        """Does the target location exist?
168
        """
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
169
        response = self._head(relpath)
170
171
        code = response.code
2164.2.16 by Vincent Ladeuil
Add tests.
172
        if code == 200: # "ok",
1540.3.3 by Martin Pool
Review updates of pycurl transport
173
            return True
174
        else:
2164.2.16 by Vincent Ladeuil
Add tests.
175
            assert(code == 404, 'Only 200 or 404 are correct')
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
176
            return False
1540.3.25 by Martin Pool
New 'http+urllib' scheme
177
178
1540.3.6 by Martin Pool
[merge] update from bzr.dev
179
def get_test_permutations():
180
    """Return the permutations to be used in testing."""
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
181
    from bzrlib.tests.HttpServer import HttpServer_urllib
1540.3.26 by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet
182
    return [(HttpTransport_urllib, HttpServer_urllib),
1540.3.6 by Martin Pool
[merge] update from bzr.dev
183
            ]