1
# Copyright (C) 2005, 2006 Canonical Ltd
1
# Copyright (C) 2005 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
6
6
# (at your option) any later version.
8
8
# This program is distributed in the hope that it will be useful,
9
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
11
# GNU General Public License for more details.
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Base implementation of Transport over http.
19
There are separate implementation modules for each http client implementation.
16
"""Implementation of Transport over http.
19
from bzrlib.transport import Transport, register_transport
20
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
21
NonRelativePath, TransportError)
22
23
from cStringIO import StringIO
29
from bzrlib import errors, ui
30
from bzrlib.smart import medium
31
from bzrlib.symbol_versioning import (
27
from bzrlib.errors import BzrError, BzrCheckError
28
from bzrlib.branch import Branch
35
29
from bzrlib.trace import mutter
36
from bzrlib.transport import (
42
# TODO: This is not used anymore by HttpTransport_urllib
43
# (extracting the auth info and prompting the user for a password
44
# have been split), only the tests still use it. It should be
45
# deleted and the tests rewritten ASAP to stay in sync.
46
def extract_auth(url, password_manager):
47
"""Extract auth parameters from am HTTP/HTTPS url and add them to the given
48
password manager. Return the url, minus those auth parameters (which
51
assert re.match(r'^(https?)(\+\w+)?://', url), \
52
'invalid absolute url %r' % url
53
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
56
auth, netloc = netloc.split('@', 1)
58
username, password = auth.split(':', 1)
60
username, password = auth, None
62
host = netloc.split(':', 1)[0]
65
username = urllib.unquote(username)
66
if password is not None:
67
password = urllib.unquote(password)
69
password = ui.ui_factory.get_password(
70
prompt='HTTP %(user)s@%(host)s password',
71
user=username, host=host)
72
password_manager.add_password(None, host, username, password)
73
url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
77
def _extract_headers(header_text, url):
78
"""Extract the mapping for an rfc2822 header
80
This is a helper function for the test suite and for _pycurl.
81
(urllib already parses the headers for us)
83
In the case that there are multiple headers inside the file,
84
the last one is returned.
86
:param header_text: A string of header information.
87
This expects that the first line of a header will always be HTTP ...
88
:param url: The url we are parsing, so we can raise nice errors
89
:return: mimetools.Message object, which basically acts like a case
90
insensitive dictionary.
93
remaining = header_text
96
raise errors.InvalidHttpResponse(url, 'Empty headers')
99
header_file = StringIO(remaining)
100
first_line = header_file.readline()
101
if not first_line.startswith('HTTP'):
102
if first_header: # The first header *must* start with HTTP
103
raise errors.InvalidHttpResponse(url,
104
'Opening header line did not start with HTTP: %s'
107
break # We are done parsing
109
m = mimetools.Message(header_file)
111
# mimetools.Message parses the first header up to a blank line
112
# So while there is remaining data, it probably means there is
113
# another header to be parsed.
114
# Get rid of any preceeding whitespace, which if it is all whitespace
115
# will get rid of everything.
116
remaining = header_file.read().lstrip()
120
class HttpTransportBase(Transport, medium.SmartClientMedium):
121
"""Base class for http implementations.
123
Does URL parsing, etc, but not any network IO.
125
The protocol can be given as e.g. http+urllib://host/ to use a particular
129
# _proto: "http" or "https"
130
# _qualified_proto: may have "+pycurl", etc
132
def __init__(self, base, from_transport=None):
31
# velocitynet.com.au transparently proxies connections and thereby
32
# breaks keep-alive -- sucks!
37
mutter("get_url %s" % url)
38
url_f = urllib2.urlopen(url)
41
class HttpTransportError(TransportError):
44
class HttpTransport(Transport):
45
"""This is the transport agent for http:// access.
47
TODO: Implement pipelined versions of all of the *_multi() functions.
50
def __init__(self, base):
133
51
"""Set the base path where files will be stored."""
134
proto_match = re.match(r'^(https?)(\+\w+)?://', base)
136
raise AssertionError("not a http url: %r" % base)
137
self._proto = proto_match.group(1)
138
impl_name = proto_match.group(2)
140
impl_name = impl_name[1:]
141
self._impl_name = impl_name
144
super(HttpTransportBase, self).__init__(base)
145
(apparent_proto, self._host,
52
assert base.startswith('http://') or base.startswith('https://')
53
super(HttpTransport, self).__init__(base)
54
# In the future we might actually connect to the remote host
55
# rather than using get_url
56
# self._connection = None
57
(self._proto, self._host,
146
58
self._path, self._parameters,
147
59
self._query, self._fragment) = urlparse.urlparse(self.base)
148
self._qualified_proto = apparent_proto
149
# range hint is handled dynamically throughout the life
150
# of the transport object. We start by trying multi-range
151
# requests and if the server returns bogus results, we
152
# retry with single range requests and, finally, we
153
# forget about range if the server really can't
154
# understand. Once acquired, this piece of info is
155
# propagated to clones.
156
if from_transport is not None:
157
self._range_hint = from_transport._range_hint
61
def should_cache(self):
62
"""Return True if the data pulled across should be cached locally.
66
def clone(self, offset=None):
67
"""Return a new HttpTransport with root at self.base + offset
68
For now HttpTransport does not actually connect, so just return
69
a new HttpTransport object.
72
return HttpTransport(self.base)
159
self._range_hint = 'multi'
74
return HttpTransport(self.abspath(offset))
161
76
def abspath(self, relpath):
162
77
"""Return the full url to the given relative path.
164
This can be supplied with a string or a list.
166
The URL returned always has the protocol scheme originally used to
167
construct the transport, even if that includes an explicit
168
implementation qualifier.
78
This can be supplied with a string or a list
170
assert isinstance(relpath, basestring)
171
if isinstance(relpath, unicode):
172
raise errors.InvalidURL(relpath, 'paths must not be unicode.')
173
80
if isinstance(relpath, basestring):
174
relpath_parts = relpath.split('/')
176
# TODO: Don't call this with an array - no magic interfaces
177
relpath_parts = relpath[:]
178
if relpath.startswith('/'):
181
# Except for the root, no trailing slashes are allowed
182
if len(relpath_parts) > 1 and relpath_parts[-1] == '':
184
"path %r within branch %r seems to be a directory"
185
% (relpath, self._path))
186
basepath = self._path.split('/')
187
if len(basepath) > 0 and basepath[-1] == '':
188
basepath = basepath[:-1]
82
basepath = self._path.split('/')
83
if len(basepath) > 0 and basepath[-1] == '':
84
basepath = basepath[:-1]
190
for p in relpath_parts:
192
if len(basepath) == 0:
193
89
# In most filesystems, a request for the parent
194
90
# of root, just returns root.
197
elif p == '.' or p == '':
200
97
basepath.append(p)
201
99
# Possibly, we could use urlparse.urljoin() here, but
202
100
# I'm concerned about when it chooses to strip the last
203
101
# portion of the path, and when it doesn't.
204
102
path = '/'.join(basepath)
207
result = urlparse.urlunparse((self._qualified_proto,
208
self._host, path, '', '', ''))
103
return urlparse.urlunparse((self._proto,
104
self._host, path, '', '', ''))
211
def _real_abspath(self, relpath):
212
"""Produce absolute path, adjusting protocol if needed"""
213
abspath = self.abspath(relpath)
214
qp = self._qualified_proto
216
if self._qualified_proto != self._proto:
217
abspath = rp + abspath[len(qp):]
218
if not isinstance(abspath, str):
219
# escaping must be done at a higher level
220
abspath = abspath.encode('ascii')
106
def relpath(self, abspath):
107
if not abspath.startswith(self.base):
108
raise NonRelativePath('path %r is not under base URL %r'
109
% (abspath, self.base))
111
return abspath[pl:].lstrip('/')
223
113
def has(self, relpath):
224
raise NotImplementedError("has() is abstract on %r" % self)
226
def get(self, relpath):
114
"""Does the target location exist?
116
TODO: HttpTransport.has() should use a HEAD request,
117
not a full GET request.
119
TODO: This should be changed so that we don't use
120
urllib2 and get an exception, the code path would be
121
cleaner if we just do an http HEAD request, and parse
125
f = get_url(self.abspath(relpath))
126
# Without the read and then close()
127
# we tend to have busy sockets.
133
except urllib2.URLError:
136
if e.errno == errno.ENOENT:
138
raise HttpTransportError(orig_error=e)
140
def get(self, relpath, decode=False):
227
141
"""Get the file at the given relative path.
229
143
:param relpath: The relative path to the file
231
code, response_file = self._get(relpath, None)
234
def _get(self, relpath, ranges, tail_amount=0):
235
"""Get a file, or part of a file.
237
:param relpath: Path relative to transport base URL
238
:param ranges: None to get the whole file;
239
or a list of _CoalescedOffset to fetch parts of a file.
240
:param tail_amount: The amount to get from the end of the file.
242
:returns: (http_code, result_file)
244
raise NotImplementedError(self._get)
246
def get_request(self):
247
return SmartClientHTTPMediumRequest(self)
249
def get_smart_medium(self):
250
"""See Transport.get_smart_medium.
252
HttpTransportBase directly implements the minimal interface of
253
SmartMediumClient, so this returns self.
257
def _degrade_range_hint(self, relpath, ranges, exc_info):
258
if self._range_hint == 'multi':
259
self._range_hint = 'single'
260
mutter('Retry "%s" with single range request' % relpath)
261
elif self._range_hint == 'single':
262
self._range_hint = None
263
mutter('Retry "%s" without ranges' % relpath)
265
# We tried all the tricks, but nothing worked. We re-raise original
266
# exception; the 'mutter' calls above will indicate that further
267
# tries were unsuccessful
268
raise exc_info[0], exc_info[1], exc_info[2]
270
def _get_ranges_hinted(self, relpath, ranges):
271
"""Issue a ranged GET request taking server capabilities into account.
273
Depending of the errors returned by the server, we try several GET
274
requests, trying to minimize the data transferred.
276
:param relpath: Path relative to transport base URL
277
:param ranges: None to get the whole file;
278
or a list of _CoalescedOffset to fetch parts of a file.
279
:returns: A file handle containing at least the requested ranges.
286
code, f = self._get(relpath, ranges)
287
except errors.InvalidRange, e:
289
exc_info = sys.exc_info()
290
self._degrade_range_hint(relpath, ranges, exc_info)
146
return get_url(self.abspath(relpath))
147
except (BzrError, urllib2.URLError, IOError), e:
148
raise NoSuchFile(msg = "Error retrieving %s"
149
% self.abspath(relpath),
152
def get_partial(self, relpath, start, length=None):
153
"""Get just part of a file.
155
:param relpath: Path to the file, relative to base
156
:param start: The starting position to read from
157
:param length: The length to read. A length of None indicates
158
read to the end of the file.
159
:return: A file-like object containing at least the specified bytes.
160
Some implementations may return objects which can be read
161
past this length, but this is not guaranteed.
163
# TODO: You can make specialized http requests for just
164
# a portion of the file. Figure out how to do that.
165
# For now, urllib2 returns files that cannot seek() so
166
# we just read bytes off the beginning, until we
167
# get to the point that we care about.
168
f = self.get(relpath)
169
# TODO: read in smaller chunks, in case things are
170
# buffered internally.
294
# _coalesce_offsets is a helper for readv, it try to combine ranges without
295
# degrading readv performances. _bytes_to_read_before_seek is the value
296
# used for the limit parameter and has been tuned for other transports. For
297
# HTTP, the name is inappropriate but the parameter is still useful and
298
# helps reduce the number of chunks in the response. The overhead for a
299
# chunk (headers, length, footer around the data itself is variable but
300
# around 50 bytes. We use 128 to reduce the range specifiers that appear in
301
# the header, some servers (notably Apache) enforce a maximum length for a
302
# header and issue a '400: Bad request' error when too much ranges are
304
_bytes_to_read_before_seek = 128
305
# No limit on the offset number that get combined into one, we are trying
306
# to avoid downloading the whole file.
307
_max_readv_combined = 0
309
def readv(self, relpath, offsets):
310
"""Get parts of the file at the given relative path.
312
:param offsets: A list of (offset, size) tuples.
313
:param return: A list or generator of (offset, data) tuples
315
sorted_offsets = sorted(list(offsets))
316
fudge = self._bytes_to_read_before_seek
317
coalesced = self._coalesce_offsets(sorted_offsets,
318
limit=self._max_readv_combine,
320
coalesced = list(coalesced)
321
mutter('http readv of %s offsets => %s collapsed %s',
322
relpath, len(offsets), len(coalesced))
324
f = self._get_ranges_hinted(relpath, coalesced)
325
for start, size in offsets:
329
f.seek(start, ((start < 0) and 2) or 0)
333
if len(data) != size:
334
raise errors.ShortReadvError(relpath, start, size,
336
except errors.ShortReadvError, e:
337
self._degrade_range_hint(relpath, coalesced, sys.exc_info())
339
# Since the offsets and the ranges may not be in the same
340
# order, we don't try to calculate a restricted single
341
# range encompassing unprocessed offsets.
343
# Note: we replace 'f' here, it may need cleaning one day
344
# before being thrown that way.
345
f = self._get_ranges_hinted(relpath, coalesced)
348
# After one or more tries, we get the data.
352
@deprecated_method(zero_seventeen)
353
def offsets_to_ranges(offsets):
354
"""Turn a list of offsets and sizes into a list of byte ranges.
356
:param offsets: A list of tuples of (start, size). An empty list
358
:return: a list of inclusive byte ranges (start, end)
359
Adjacent ranges will be combined.
361
# Make sure we process sorted offsets
362
offsets = sorted(offsets)
367
for start, size in offsets:
368
end = start + size - 1
370
combined.append([start, end])
371
elif start <= prev_end + 1:
372
combined[-1][1] = end
374
combined.append([start, end])
379
def _post(self, body_bytes):
380
"""POST body_bytes to .bzr/smart on this transport.
382
:returns: (response code, response body file-like object).
384
# TODO: Requiring all the body_bytes to be available at the beginning of
385
# the POST may require large client buffers. It would be nice to have
386
# an interface that allows streaming via POST when possible (and
387
# degrades to a local buffer when not).
388
raise NotImplementedError(self._post)
390
def put_file(self, relpath, f, mode=None):
391
"""Copy the file-like object into the location.
174
def put(self, relpath, f):
175
"""Copy the file-like or string object into the location.
393
177
:param relpath: Location to put the contents, relative to base.
394
:param f: File-like object.
178
:param f: File-like or string object.
396
raise errors.TransportNotPossible('http PUT not supported')
180
raise TransportNotPossible('http PUT not supported')
398
def mkdir(self, relpath, mode=None):
182
def mkdir(self, relpath):
399
183
"""Create a directory at the given path."""
400
raise errors.TransportNotPossible('http does not support mkdir()')
402
def rmdir(self, relpath):
403
"""See Transport.rmdir."""
404
raise errors.TransportNotPossible('http does not support rmdir()')
406
def append_file(self, relpath, f, mode=None):
184
raise TransportNotPossible('http does not support mkdir()')
186
def append(self, relpath, f):
407
187
"""Append the text in the file-like object into the final
410
raise errors.TransportNotPossible('http does not support append()')
190
raise TransportNotPossible('http does not support append()')
412
192
def copy(self, rel_from, rel_to):
413
193
"""Copy the item at rel_from to the location at rel_to"""
414
raise errors.TransportNotPossible('http does not support copy()')
194
raise TransportNotPossible('http does not support copy()')
416
def copy_to(self, relpaths, other, mode=None, pb=None):
196
def copy_to(self, relpaths, other, pb=None):
417
197
"""Copy a set of entries from self into another Transport.
419
199
:param relpaths: A list/generator of entries to be copied.
added
removed
bzrlib/transport/http.py
