4763.2.4
by John Arbash Meinel
merge bzr.2.1 in preparation for NEWS entry. |
1 |
# Copyright (C) 2006-2010 Canonical Ltd
|
1540.3.18
by Martin Pool
Style review fixes (thanks robertc) |
2 |
#
|
1540.3.3
by Martin Pool
Review updates of pycurl transport |
3 |
# This program is free software; you can redistribute it and/or modify
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
1540.3.18
by Martin Pool
Style review fixes (thanks robertc) |
7 |
#
|
1540.3.3
by Martin Pool
Review updates of pycurl transport |
8 |
# This program is distributed in the hope that it will be useful,
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
1540.3.18
by Martin Pool
Style review fixes (thanks robertc) |
12 |
#
|
1540.3.3
by Martin Pool
Review updates of pycurl transport |
13 |
# You should have received a copy of the GNU General Public License
|
14 |
# along with this program; if not, write to the Free Software
|
|
4183.7.1
by Sabin Iacob
update FSF mailing address |
15 |
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
1540.3.3
by Martin Pool
Review updates of pycurl transport |
16 |
|
6379.6.3
by Jelmer Vernooij
Use absolute_import. |
17 |
from __future__ import absolute_import |
18 |
||
2485.8.41
by Vincent Ladeuil
Finish http refactoring. Test suite passing. |
19 |
from bzrlib import ( |
20 |
errors, |
|
3052.3.3
by Vincent Ladeuil
Add -Dhttp support. |
21 |
trace, |
2485.8.41
by Vincent Ladeuil
Finish http refactoring. Test suite passing. |
22 |
)
|
3052.3.3
by Vincent Ladeuil
Add -Dhttp support. |
23 |
from bzrlib.transport import http |
3059.2.2
by Vincent Ladeuil
Read http responses on demand without buffering the whole body |
24 |
# TODO: handle_response should be integrated into the http/__init__.py
|
2004.2.1
by John Arbash Meinel
Cleanup of urllib functions |
25 |
from bzrlib.transport.http.response import handle_response |
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
26 |
from bzrlib.transport.http._urllib2_wrappers import ( |
2004.2.1
by John Arbash Meinel
Cleanup of urllib functions |
27 |
Opener, |
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
28 |
Request, |
2004.2.1
by John Arbash Meinel
Cleanup of urllib functions |
29 |
)
|
30 |
||
1540.3.3
by Martin Pool
Review updates of pycurl transport |
31 |
|
3052.3.3
by Vincent Ladeuil
Add -Dhttp support. |
32 |
class HttpTransport_urllib(http.HttpTransportBase): |
1786.1.33
by John Arbash Meinel
Cleanup pass #2 |
33 |
"""Python urllib transport for http and https."""
|
1540.3.3
by Martin Pool
Review updates of pycurl transport |
34 |
|
2004.1.9
by vila
Takes jam's remarks into account when possible, add TODOs for the rest. |
35 |
# In order to debug we have to issue our traces in sync with
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
36 |
# httplib, which use print :(
|
37 |
_debuglevel = 0 |
|
2004.3.1
by vila
Test ConnectionError exceptions. |
38 |
|
2004.2.1
by John Arbash Meinel
Cleanup of urllib functions |
39 |
_opener_class = Opener |
40 |
||
6238.2.22
by Vincent Ladeuil
Create a specific test permutation for urllib https so we can inject our test ca certs. The wiring in _urllib2_wrappers is a bit hackish and it will need to use auth instead so different certs can be used for proxies and real servers but this could wait until authentication.conf is migrated to the config stacks. With this change in place, all https tests pass without the need to create a dedicated GlobalStore. |
41 |
def __init__(self, base, _from_transport=None, ca_certs=None): |
2485.8.59
by Vincent Ladeuil
Update from review comments. |
42 |
super(HttpTransport_urllib, self).__init__( |
3878.4.6
by Vincent Ladeuil
Fix bug #270863 by preserving 'bzr+http[s]' decorator. |
43 |
base, 'urllib', _from_transport=_from_transport) |
2485.8.59
by Vincent Ladeuil
Update from review comments. |
44 |
if _from_transport is not None: |
45 |
self._opener = _from_transport._opener |
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
46 |
else: |
3945.1.5
by Vincent Ladeuil
Start implementing http activity reporting at socket level. |
47 |
self._opener = self._opener_class( |
6238.2.22
by Vincent Ladeuil
Create a specific test permutation for urllib https so we can inject our test ca certs. The wiring in _urllib2_wrappers is a bit hackish and it will need to use auth instead so different certs can be used for proxies and real servers but this could wait until authentication.conf is migrated to the config stacks. With this change in place, all https tests pass without the need to create a dedicated GlobalStore. |
48 |
report_activity=self._report_activity, ca_certs=ca_certs) |
2420.1.5
by Vincent Ladeuil
Refactor http and proxy authentication. Tests passing. proxy password can be prompted too. |
49 |
|
2485.8.41
by Vincent Ladeuil
Finish http refactoring. Test suite passing. |
50 |
def _perform(self, request): |
51 |
"""Send the request to the server and handles common errors.
|
|
52 |
||
53 |
:returns: urllib2 Response object
|
|
54 |
"""
|
|
55 |
connection = self._get_connection() |
|
56 |
if connection is not None: |
|
57 |
# Give back shared info
|
|
58 |
request.connection = connection |
|
59 |
(auth, proxy_auth) = self._get_credentials() |
|
3059.2.2
by Vincent Ladeuil
Read http responses on demand without buffering the whole body |
60 |
# Clean the httplib.HTTPConnection pipeline in case the previous
|
61 |
# request couldn't do it
|
|
62 |
connection.cleanup_pipe() |
|
2485.8.41
by Vincent Ladeuil
Finish http refactoring. Test suite passing. |
63 |
else: |
3133.1.2
by Vincent Ladeuil
Fix #177643 by making pycurl handle url-embedded credentials again. |
64 |
# First request, initialize credentials.
|
2900.2.16
by Vincent Ladeuil
Make hhtp proxy aware of AuthenticationConfig (for password). |
65 |
# scheme and realm will be set by the _urllib2_wrappers.AuthHandler
|
3133.1.2
by Vincent Ladeuil
Fix #177643 by making pycurl handle url-embedded credentials again. |
66 |
auth = self._create_auth() |
67 |
# Proxy initialization will be done by the first proxied request
|
|
2900.2.24
by Vincent Ladeuil
Review feedback. |
68 |
proxy_auth = dict() |
2363.4.12
by Vincent Ladeuil
Take jam's review comments into account. Fix typos, give better |
69 |
# Ensure authentication info is provided
|
2485.8.41
by Vincent Ladeuil
Finish http refactoring. Test suite passing. |
70 |
request.auth = auth |
71 |
request.proxy_auth = proxy_auth |
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
72 |
|
73 |
if self._debuglevel > 0: |
|
74 |
print 'perform: %s base: %s, url: %s' % (request.method, self.base, |
|
75 |
request.get_full_url()) |
|
76 |
response = self._opener.open(request) |
|
2485.8.41
by Vincent Ladeuil
Finish http refactoring. Test suite passing. |
77 |
if self._get_connection() is not request.connection: |
78 |
# First connection or reconnection
|
|
79 |
self._set_connection(request.connection, |
|
80 |
(request.auth, request.proxy_auth)) |
|
81 |
else: |
|
82 |
# http may change the credentials while keeping the
|
|
83 |
# connection opened
|
|
84 |
self._update_credentials((request.auth, request.proxy_auth)) |
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
85 |
|
2164.2.1
by v.ladeuil+lp at free
First rough http branch redirection implementation. |
86 |
code = response.code |
4795.4.5
by Vincent Ladeuil
Make sure all redirection code paths can handle authentication. |
87 |
if (request.follow_redirections is False |
88 |
and code in (301, 302, 303, 307)): |
|
2164.2.1
by v.ladeuil+lp at free
First rough http branch redirection implementation. |
89 |
raise errors.RedirectRequested(request.get_full_url(), |
90 |
request.redirected_to, |
|
3878.4.4
by Vincent Ladeuil
Cleanup. |
91 |
is_permanent=(code == 301)) |
2164.2.1
by v.ladeuil+lp at free
First rough http branch redirection implementation. |
92 |
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
93 |
if request.redirected_to is not None: |
3052.3.3
by Vincent Ladeuil
Add -Dhttp support. |
94 |
trace.mutter('redirected from: %s to: %s' % (request.get_full_url(), |
95 |
request.redirected_to)) |
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
96 |
|
97 |
return response |
|
1540.3.26
by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet |
98 |
|
5247.2.12
by Vincent Ladeuil
Ensure that all transports close their underlying connection. |
99 |
def disconnect(self): |
100 |
connection = self._get_connection() |
|
101 |
if connection is not None: |
|
102 |
connection.close() |
|
103 |
||
2520.2.1
by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports. |
104 |
def _get(self, relpath, offsets, tail_amount=0): |
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
105 |
"""See HttpTransport._get"""
|
2485.8.25
by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different. |
106 |
abspath = self._remote_path(relpath) |
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
107 |
headers = {} |
2520.2.2
by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request |
108 |
accepted_errors = [200, 404] |
2520.2.1
by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports. |
109 |
if offsets or tail_amount: |
110 |
range_header = self._attempted_range_header(offsets, tail_amount) |
|
2004.1.30
by v.ladeuil+lp at free
Fix #62276 and #62029 by providing a more robust http range handling. |
111 |
if range_header is not None: |
2520.2.2
by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request |
112 |
accepted_errors.append(206) |
113 |
accepted_errors.append(400) |
|
3059.2.2
by Vincent Ladeuil
Read http responses on demand without buffering the whole body |
114 |
accepted_errors.append(416) |
2004.1.30
by v.ladeuil+lp at free
Fix #62276 and #62029 by providing a more robust http range handling. |
115 |
bytes = 'bytes=' + range_header |
116 |
headers = {'Range': bytes} |
|
2004.3.1
by vila
Test ConnectionError exceptions. |
117 |
|
2520.2.2
by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request |
118 |
request = Request('GET', abspath, None, headers, |
119 |
accepted_errors=accepted_errors) |
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
120 |
response = self._perform(request) |
121 |
||
122 |
code = response.code |
|
123 |
if code == 404: # not found |
|
2164.2.1
by v.ladeuil+lp at free
First rough http branch redirection implementation. |
124 |
raise errors.NoSuchFile(abspath) |
3059.2.2
by Vincent Ladeuil
Read http responses on demand without buffering the whole body |
125 |
elif code in (400, 416): |
126 |
# We don't know which, but one of the ranges we specified was
|
|
127 |
# wrong.
|
|
128 |
raise errors.InvalidHttpRange(abspath, range_header, |
|
129 |
'Server return code %d' % code) |
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
130 |
|
3945.1.5
by Vincent Ladeuil
Start implementing http activity reporting at socket level. |
131 |
data = handle_response(abspath, code, response.info(), response) |
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
132 |
return code, data |
1540.3.3
by Martin Pool
Review updates of pycurl transport |
133 |
|
2018.2.7
by Andrew Bennetts
Implement _post on HttpTransport_urllib. |
134 |
def _post(self, body_bytes): |
2485.8.25
by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different. |
135 |
abspath = self._remote_path('.bzr/smart') |
3430.3.5
by Vincent Ladeuil
Fixed as per Andrew's review. |
136 |
# We include 403 in accepted_errors so that send_http_smart_request can
|
137 |
# handle a 403. Otherwise a 403 causes an unhandled TransportError.
|
|
5514.1.1
by Vincent Ladeuil
Correctly set the Content-Type header when POSTing. |
138 |
response = self._perform( |
139 |
Request('POST', abspath, body_bytes, |
|
140 |
{'Content-Type': 'application/octet-stream'}, |
|
141 |
accepted_errors=[200, 403])) |
|
2004.1.28
by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code |
142 |
code = response.code |
3945.1.5
by Vincent Ladeuil
Start implementing http activity reporting at socket level. |
143 |
data = handle_response(abspath, code, response.info(), response) |
2004.1.28
by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code |
144 |
return code, data |
2018.2.7
by Andrew Bennetts
Implement _post on HttpTransport_urllib. |
145 |
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
146 |
def _head(self, relpath): |
147 |
"""Request the HEAD of a file.
|
|
148 |
||
149 |
Performs the request and leaves callers handle the results.
|
|
150 |
"""
|
|
2485.8.25
by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different. |
151 |
abspath = self._remote_path(relpath) |
2520.2.2
by Vincent Ladeuil
Fix #115209 by issuing a single range request on 400: Bad Request |
152 |
request = Request('HEAD', abspath, |
153 |
accepted_errors=[200, 404]) |
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
154 |
response = self._perform(request) |
155 |
||
156 |
return response |
|
157 |
||
1540.3.3
by Martin Pool
Review updates of pycurl transport |
158 |
def has(self, relpath): |
159 |
"""Does the target location exist?
|
|
160 |
"""
|
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
161 |
response = self._head(relpath) |
162 |
||
163 |
code = response.code |
|
2164.2.16
by Vincent Ladeuil
Add tests. |
164 |
if code == 200: # "ok", |
1540.3.3
by Martin Pool
Review updates of pycurl transport |
165 |
return True |
166 |
else: |
|
2004.1.1
by vila
Connection sharing, with redirection. without authentification. |
167 |
return False |
1540.3.25
by Martin Pool
New 'http+urllib' scheme |
168 |
|
169 |
||
1540.3.6
by Martin Pool
[merge] update from bzr.dev |
170 |
def get_test_permutations(): |
171 |
"""Return the permutations to be used in testing."""
|
|
5967.12.1
by Martin Pool
Move all test features into bzrlib.tests.features |
172 |
from bzrlib.tests import ( |
173 |
features, |
|
174 |
http_server, |
|
175 |
)
|
|
2929.3.10
by Vincent Ladeuil
Add a fake https server and test facilities. |
176 |
permutations = [(HttpTransport_urllib, http_server.HttpServer_urllib),] |
5967.12.1
by Martin Pool
Move all test features into bzrlib.tests.features |
177 |
if features.HTTPSServerFeature.available(): |
6238.2.22
by Vincent Ladeuil
Create a specific test permutation for urllib https so we can inject our test ca certs. The wiring in _urllib2_wrappers is a bit hackish and it will need to use auth instead so different certs can be used for proxies and real servers but this could wait until authentication.conf is migrated to the config stacks. With this change in place, all https tests pass without the need to create a dedicated GlobalStore. |
178 |
from bzrlib.tests import ( |
179 |
https_server, |
|
180 |
ssl_certs, |
|
181 |
)
|
|
182 |
||
183 |
class HTTPS_urllib_transport(HttpTransport_urllib): |
|
184 |
||
185 |
def __init__(self, base, _from_transport=None): |
|
186 |
super(HTTPS_urllib_transport, self).__init__( |
|
187 |
base, _from_transport=_from_transport, |
|
188 |
ca_certs=ssl_certs.build_path('ca.crt')) |
|
189 |
||
190 |
permutations.append((HTTPS_urllib_transport, |
|
2929.3.10
by Vincent Ladeuil
Add a fake https server and test facilities. |
191 |
https_server.HTTPSServer_urllib)) |
192 |
return permutations |