~bzr-pqm/bzr/bzr.dev

4763.2.4 by John Arbash Meinel
merge bzr.2.1 in preparation for NEWS entry.
1
# Copyright (C) 2006-2010 Canonical Ltd
1540.3.18 by Martin Pool
Style review fixes (thanks robertc)
2
#
1540.3.3 by Martin Pool
Review updates of pycurl transport
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1540.3.18 by Martin Pool
Style review fixes (thanks robertc)
7
#
1540.3.3 by Martin Pool
Review updates of pycurl transport
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1540.3.18 by Martin Pool
Style review fixes (thanks robertc)
12
#
1540.3.3 by Martin Pool
Review updates of pycurl transport
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
1540.3.3 by Martin Pool
Review updates of pycurl transport
16
6379.6.3 by Jelmer Vernooij
Use absolute_import.
17
from __future__ import absolute_import
18
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
19
from bzrlib import (
20
    errors,
3052.3.3 by Vincent Ladeuil
Add -Dhttp support.
21
    trace,
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
22
    )
3052.3.3 by Vincent Ladeuil
Add -Dhttp support.
23
from bzrlib.transport import http
3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
24
# TODO: handle_response should be integrated into the http/__init__.py
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
25
from bzrlib.transport.http.response import handle_response
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
26
from bzrlib.transport.http._urllib2_wrappers import (
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
27
    Opener,
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
28
    Request,
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
29
    )
30
1540.3.3 by Martin Pool
Review updates of pycurl transport
31
3052.3.3 by Vincent Ladeuil
Add -Dhttp support.
32
class HttpTransport_urllib(http.HttpTransportBase):
1786.1.33 by John Arbash Meinel
Cleanup pass #2
33
    """Python urllib transport for http and https."""
1540.3.3 by Martin Pool
Review updates of pycurl transport
34
2004.1.9 by vila
Takes jam's remarks into account when possible, add TODOs for the rest.
35
    # In order to debug we have to issue our traces in sync with
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
36
    # httplib, which use print :(
37
    _debuglevel = 0
2004.3.1 by vila
Test ConnectionError exceptions.
38
2004.2.1 by John Arbash Meinel
Cleanup of urllib functions
39
    _opener_class = Opener
40
6238.2.22 by Vincent Ladeuil
Create a specific test permutation for urllib https so we can inject our test ca certs. The wiring in _urllib2_wrappers is a bit hackish and it will need to use auth instead so different certs can be used for proxies and real servers but this could wait until authentication.conf is migrated to the config stacks. With this change in place, all https tests pass without the need to create a dedicated GlobalStore.
41
    def __init__(self, base, _from_transport=None, ca_certs=None):
2485.8.59 by Vincent Ladeuil
Update from review comments.
42
        super(HttpTransport_urllib, self).__init__(
3878.4.6 by Vincent Ladeuil
Fix bug #270863 by preserving 'bzr+http[s]' decorator.
43
            base, 'urllib', _from_transport=_from_transport)
2485.8.59 by Vincent Ladeuil
Update from review comments.
44
        if _from_transport is not None:
45
            self._opener = _from_transport._opener
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
46
        else:
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
47
            self._opener = self._opener_class(
6238.2.22 by Vincent Ladeuil
Create a specific test permutation for urllib https so we can inject our test ca certs. The wiring in _urllib2_wrappers is a bit hackish and it will need to use auth instead so different certs can be used for proxies and real servers but this could wait until authentication.conf is migrated to the config stacks. With this change in place, all https tests pass without the need to create a dedicated GlobalStore.
48
                report_activity=self._report_activity, ca_certs=ca_certs)
2420.1.5 by Vincent Ladeuil
Refactor http and proxy authentication. Tests passing. proxy password can be prompted too.
49
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
50
    def _perform(self, request):
51
        """Send the request to the server and handles common errors.
52
53
        :returns: urllib2 Response object
54
        """
55
        connection = self._get_connection()
56
        if connection is not None:
57
            # Give back shared info
58
            request.connection = connection
59
            (auth, proxy_auth) = self._get_credentials()
3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
60
            # Clean the httplib.HTTPConnection pipeline in case the previous
61
            # request couldn't do it
62
            connection.cleanup_pipe()
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
63
        else:
3133.1.2 by Vincent Ladeuil
Fix #177643 by making pycurl handle url-embedded credentials again.
64
            # First request, initialize credentials.
2900.2.16 by Vincent Ladeuil
Make hhtp proxy aware of AuthenticationConfig (for password).
65
            # scheme and realm will be set by the _urllib2_wrappers.AuthHandler
3133.1.2 by Vincent Ladeuil
Fix #177643 by making pycurl handle url-embedded credentials again.
66
            auth = self._create_auth()
67
            # Proxy initialization will be done by the first proxied request
2900.2.24 by Vincent Ladeuil
Review feedback.
68
            proxy_auth = dict()
2363.4.12 by Vincent Ladeuil
Take jam's review comments into account. Fix typos, give better
69
        # Ensure authentication info is provided
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
70
        request.auth = auth
71
        request.proxy_auth = proxy_auth
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
72
73
        if self._debuglevel > 0:
74
            print 'perform: %s base: %s, url: %s' % (request.method, self.base,
75
                                                     request.get_full_url())
76
        response = self._opener.open(request)
2485.8.41 by Vincent Ladeuil
Finish http refactoring. Test suite passing.
77
        if self._get_connection() is not request.connection:
78
            # First connection or reconnection
79
            self._set_connection(request.connection,
80
                                 (request.auth, request.proxy_auth))
81
        else:
82
            # http may change the credentials while keeping the
83
            # connection opened
84
            self._update_credentials((request.auth, request.proxy_auth))
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
85
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
86
        code = response.code
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
87
        if (request.follow_redirections is False
88
            and code in (301, 302, 303, 307)):
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
89
            raise errors.RedirectRequested(request.get_full_url(),
90
                                           request.redirected_to,
3878.4.4 by Vincent Ladeuil
Cleanup.
91
                                           is_permanent=(code == 301))
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
92
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
93
        if request.redirected_to is not None:
3052.3.3 by Vincent Ladeuil
Add -Dhttp support.
94
            trace.mutter('redirected from: %s to: %s' % (request.get_full_url(),
95
                                                         request.redirected_to))
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
96
97
        return response
1540.3.26 by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet
98
5247.2.12 by Vincent Ladeuil
Ensure that all transports close their underlying connection.
99
    def disconnect(self):
100
        connection = self._get_connection()
101
        if connection is not None:
102
            connection.close()
103
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
104
    def _get(self, relpath, offsets, tail_amount=0):
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
105
        """See HttpTransport._get"""
2485.8.25 by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different.
106
        abspath = self._remote_path(relpath)
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
107
        headers = {}
2520.2.2 by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request
108
        accepted_errors = [200, 404]
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
109
        if offsets or tail_amount:
110
            range_header = self._attempted_range_header(offsets, tail_amount)
2004.1.30 by v.ladeuil+lp at free
Fix #62276 and #62029 by providing a more robust http range handling.
111
            if range_header is not None:
2520.2.2 by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request
112
                accepted_errors.append(206)
113
                accepted_errors.append(400)
3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
114
                accepted_errors.append(416)
2004.1.30 by v.ladeuil+lp at free
Fix #62276 and #62029 by providing a more robust http range handling.
115
                bytes = 'bytes=' + range_header
116
                headers = {'Range': bytes}
2004.3.1 by vila
Test ConnectionError exceptions.
117
2520.2.2 by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request
118
        request = Request('GET', abspath, None, headers,
119
                          accepted_errors=accepted_errors)
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
120
        response = self._perform(request)
121
122
        code = response.code
123
        if code == 404: # not found
2164.2.1 by v.ladeuil+lp at free
First rough http branch redirection implementation.
124
            raise errors.NoSuchFile(abspath)
3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
125
        elif code in (400, 416):
126
            # We don't know which, but one of the ranges we specified was
127
            # wrong.
128
            raise errors.InvalidHttpRange(abspath, range_header,
129
                                          'Server return code %d' % code)
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
130
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
131
        data = handle_response(abspath, code, response.info(), response)
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
132
        return code, data
1540.3.3 by Martin Pool
Review updates of pycurl transport
133
2018.2.7 by Andrew Bennetts
Implement _post on HttpTransport_urllib.
134
    def _post(self, body_bytes):
2485.8.25 by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different.
135
        abspath = self._remote_path('.bzr/smart')
3430.3.5 by Vincent Ladeuil
Fixed as per Andrew's review.
136
        # We include 403 in accepted_errors so that send_http_smart_request can
137
        # handle a 403.  Otherwise a 403 causes an unhandled TransportError.
5514.1.1 by Vincent Ladeuil
Correctly set the Content-Type header when POSTing.
138
        response = self._perform(
139
            Request('POST', abspath, body_bytes,
140
                    {'Content-Type': 'application/octet-stream'},
141
                    accepted_errors=[200, 403]))
2004.1.28 by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code
142
        code = response.code
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
143
        data = handle_response(abspath, code, response.info(), response)
2004.1.28 by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code
144
        return code, data
2018.2.7 by Andrew Bennetts
Implement _post on HttpTransport_urllib.
145
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
146
    def _head(self, relpath):
147
        """Request the HEAD of a file.
148
149
        Performs the request and leaves callers handle the results.
150
        """
2485.8.25 by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different.
151
        abspath = self._remote_path(relpath)
2520.2.2 by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request
152
        request = Request('HEAD', abspath,
153
                          accepted_errors=[200, 404])
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
154
        response = self._perform(request)
155
156
        return response
157
1540.3.3 by Martin Pool
Review updates of pycurl transport
158
    def has(self, relpath):
159
        """Does the target location exist?
160
        """
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
161
        response = self._head(relpath)
162
163
        code = response.code
2164.2.16 by Vincent Ladeuil
Add tests.
164
        if code == 200: # "ok",
1540.3.3 by Martin Pool
Review updates of pycurl transport
165
            return True
166
        else:
2004.1.1 by vila
Connection sharing, with redirection. without authentification.
167
            return False
1540.3.25 by Martin Pool
New 'http+urllib' scheme
168
169
1540.3.6 by Martin Pool
[merge] update from bzr.dev
170
def get_test_permutations():
171
    """Return the permutations to be used in testing."""
5967.12.1 by Martin Pool
Move all test features into bzrlib.tests.features
172
    from bzrlib.tests import (
173
        features,
174
        http_server,
175
        )
2929.3.10 by Vincent Ladeuil
Add a fake https server and test facilities.
176
    permutations = [(HttpTransport_urllib, http_server.HttpServer_urllib),]
5967.12.1 by Martin Pool
Move all test features into bzrlib.tests.features
177
    if features.HTTPSServerFeature.available():
6238.2.22 by Vincent Ladeuil
Create a specific test permutation for urllib https so we can inject our test ca certs. The wiring in _urllib2_wrappers is a bit hackish and it will need to use auth instead so different certs can be used for proxies and real servers but this could wait until authentication.conf is migrated to the config stacks. With this change in place, all https tests pass without the need to create a dedicated GlobalStore.
178
        from bzrlib.tests import (
179
            https_server,
180
            ssl_certs,
181
            )
182
183
        class HTTPS_urllib_transport(HttpTransport_urllib):
184
185
            def __init__(self, base, _from_transport=None):
186
                super(HTTPS_urllib_transport, self).__init__(
187
                    base, _from_transport=_from_transport,
188
                    ca_certs=ssl_certs.build_path('ca.crt'))
189
190
        permutations.append((HTTPS_urllib_transport,
2929.3.10 by Vincent Ladeuil
Add a fake https server and test facilities.
191
                             https_server.HTTPSServer_urllib))
192
    return permutations