~bzr-pqm/bzr/bzr.dev

3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
1540.3.18 by Martin Pool
Style review fixes (thanks robertc)
2
#
1540.3.3 by Martin Pool
Review updates of pycurl transport
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1540.3.18 by Martin Pool
Style review fixes (thanks robertc)
7
#
1540.3.3 by Martin Pool
Review updates of pycurl transport
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1540.3.18 by Martin Pool
Style review fixes (thanks robertc)
12
#
1540.3.3 by Martin Pool
Review updates of pycurl transport
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
2004.1.2 by vila
Implements a BasicAuthManager.
17
from cStringIO import StringIO
2363.4.9 by Vincent Ladeuil
Catch first succesful authentification to avoid further 401
18
import urllib
19
import urlparse
1540.3.3 by Martin Pool
Review updates of pycurl transport
20
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
21
from bzrlib import (
22
    errors,
3052.3.3 by Vincent Ladeuil
Add -Dhttp support.
23
    trace,
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
24
    urlutils,
25
    )
3052.3.3 by Vincent Ladeuil
Add -Dhttp support.
26
from bzrlib.transport import http
3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
27
# TODO: handle_response should be integrated into the http/__init__.py
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
28
from bzrlib.transport.http.response import handle_response
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
29
from bzrlib.transport.http._urllib2_wrappers import (
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
30
    Opener,
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
31
    Request,
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
32
    )
33
1540.3.3 by Martin Pool
Review updates of pycurl transport
34
3052.3.3 by Vincent Ladeuil
Add -Dhttp support.
35
class HttpTransport_urllib(http.HttpTransportBase):
1786.1.33 by John Arbash Meinel
Cleanup pass #2
36
    """Python urllib transport for http and https."""
1540.3.3 by Martin Pool
Review updates of pycurl transport
37
2004.1.9 by vila
Takes jam's remarks into account when possible, add TODOs for the rest.
38
    # In order to debug we have to issue our traces in sync with
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
39
    # httplib, which use print :(
40
    _debuglevel = 0
2004.3.1 by vila
Test ConnectionError exceptions.
41
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
42
    _opener_class = Opener
43
2485.8.59 by Vincent Ladeuil
Update from review comments.
44
    def __init__(self, base, _from_transport=None):
45
        super(HttpTransport_urllib, self).__init__(
46
            base, _from_transport=_from_transport)
47
        if _from_transport is not None:
48
            self._opener = _from_transport._opener
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
49
        else:
2420.1.5 by Vincent Ladeuil
Refactor http and proxy authentication. Tests passing. proxy password can be prompted too.
50
            self._opener = self._opener_class()
51
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
52
    def _perform(self, request):
53
        """Send the request to the server and handles common errors.
54
55
        :returns: urllib2 Response object
56
        """
57
        connection = self._get_connection()
58
        if connection is not None:
59
            # Give back shared info
60
            request.connection = connection
61
            (auth, proxy_auth) = self._get_credentials()
3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
62
            # Clean the httplib.HTTPConnection pipeline in case the previous
63
            # request couldn't do it
64
            connection.cleanup_pipe()
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
65
        else:
3133.1.2 by Vincent Ladeuil
Fix #177643 by making pycurl handle url-embedded credentials again.
66
            # First request, initialize credentials.
2900.2.16 by Vincent Ladeuil
Make hhtp proxy aware of AuthenticationConfig (for password).
67
            # scheme and realm will be set by the _urllib2_wrappers.AuthHandler
3133.1.2 by Vincent Ladeuil
Fix #177643 by making pycurl handle url-embedded credentials again.
68
            auth = self._create_auth()
69
            # Proxy initialization will be done by the first proxied request
2900.2.24 by Vincent Ladeuil
Review feedback.
70
            proxy_auth = dict()
2363.4.12 by Vincent Ladeuil
Take jam's review comments into account. Fix typos, give better
71
        # Ensure authentication info is provided
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
72
        request.auth = auth
73
        request.proxy_auth = proxy_auth
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
74
75
        if self._debuglevel > 0:
76
            print 'perform: %s base: %s, url: %s' % (request.method, self.base,
77
                                                     request.get_full_url())
78
        response = self._opener.open(request)
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
79
        if self._get_connection() is not request.connection:
80
            # First connection or reconnection
81
            self._set_connection(request.connection,
82
                                 (request.auth, request.proxy_auth))
83
        else:
84
            # http may change the credentials while keeping the
85
            # connection opened
86
            self._update_credentials((request.auth, request.proxy_auth))
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
87
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
88
        code = response.code
2164.2.11 by Vincent Ladeuil
Explicit tests.
89
        if request.follow_redirections is False \
90
                and code in (301, 302, 303, 307):
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
91
            raise errors.RedirectRequested(request.get_full_url(),
92
                                           request.redirected_to,
2949.4.1 by Vincent Ladeuil
Fix typo (is_permament => is_permanent) reported on IRC
93
                                           is_permanent=(code == 301),
2485.8.24 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
94
                                           qual_proto=self._scheme)
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
95
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
96
        if request.redirected_to is not None:
3052.3.3 by Vincent Ladeuil
Add -Dhttp support.
97
            trace.mutter('redirected from: %s to: %s' % (request.get_full_url(),
98
                                                         request.redirected_to))
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
99
100
        return response
1540.3.26 by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet
101
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
102
    def _get(self, relpath, offsets, tail_amount=0):
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
103
        """See HttpTransport._get"""
104
2485.8.25 by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different.
105
        abspath = self._remote_path(relpath)
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
106
        headers = {}
2520.2.2 by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request
107
        accepted_errors = [200, 404]
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
108
        if offsets or tail_amount:
109
            range_header = self._attempted_range_header(offsets, tail_amount)
2004.1.30 by v.ladeuil+lp at free
Fix #62276 and #62029 by providing a more robust http range handling.
110
            if range_header is not None:
2520.2.2 by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request
111
                accepted_errors.append(206)
112
                accepted_errors.append(400)
3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
113
                accepted_errors.append(416)
2004.1.30 by v.ladeuil+lp at free
Fix #62276 and #62029 by providing a more robust http range handling.
114
                bytes = 'bytes=' + range_header
115
                headers = {'Range': bytes}
2004.3.1 by vila
Test ConnectionError exceptions.
116
2520.2.2 by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request
117
        request = Request('GET', abspath, None, headers,
118
                          accepted_errors=accepted_errors)
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
119
        response = self._perform(request)
120
121
        code = response.code
122
        if code == 404: # not found
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
123
            raise errors.NoSuchFile(abspath)
3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
124
        elif code in (400, 416):
125
            # We don't know which, but one of the ranges we specified was
126
            # wrong.
127
            raise errors.InvalidHttpRange(abspath, range_header,
128
                                          'Server return code %d' % code)
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
129
3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
130
        data = handle_response(abspath, code, response.info(), response)
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
131
        return code, data
1540.3.3 by Martin Pool
Review updates of pycurl transport
132
2018.2.7 by Andrew Bennetts
Implement _post on HttpTransport_urllib.
133
    def _post(self, body_bytes):
2485.8.25 by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different.
134
        abspath = self._remote_path('.bzr/smart')
3430.3.5 by Vincent Ladeuil
Fixed as per Andrew's review.
135
        # We include 403 in accepted_errors so that send_http_smart_request can
136
        # handle a 403.  Otherwise a 403 causes an unhandled TransportError.
3430.3.1 by Vincent Ladeuil
Fix #230223 by making both http implementations raise appropriate exceptions.
137
        response = self._perform(Request('POST', abspath, body_bytes,
138
                                         accepted_errors=[200, 403]))
2004.1.28 by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code
139
        code = response.code
3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
140
        data = handle_response(abspath, code, response.info(), response)
2004.1.28 by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code
141
        return code, data
2018.2.7 by Andrew Bennetts
Implement _post on HttpTransport_urllib.
142
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
143
    def _head(self, relpath):
144
        """Request the HEAD of a file.
145
146
        Performs the request and leaves callers handle the results.
147
        """
2485.8.25 by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different.
148
        abspath = self._remote_path(relpath)
2520.2.2 by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request
149
        request = Request('HEAD', abspath,
150
                          accepted_errors=[200, 404])
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
151
        response = self._perform(request)
152
153
        return response
154
1540.3.3 by Martin Pool
Review updates of pycurl transport
155
    def has(self, relpath):
156
        """Does the target location exist?
157
        """
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
158
        response = self._head(relpath)
159
160
        code = response.code
2164.2.16 by Vincent Ladeuil
Add tests.
161
        if code == 200: # "ok",
1540.3.3 by Martin Pool
Review updates of pycurl transport
162
            return True
163
        else:
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
164
            return False
1540.3.25 by Martin Pool
New 'http+urllib' scheme
165
166
1540.3.6 by Martin Pool
[merge] update from bzr.dev
167
def get_test_permutations():
168
    """Return the permutations to be used in testing."""
3102.1.1 by Vincent Ladeuil
Rename bzrlib/test/HTTPTestUtils.py to bzrlib/tests/http_utils.py and fix
169
    from bzrlib.tests.http_server import HttpServer_urllib
1540.3.26 by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet
170
    return [(HttpTransport_urllib, HttpServer_urllib),
1540.3.6 by Martin Pool
[merge] update from bzr.dev
171
            ]