72
55
if password is not None:
73
56
password = urllib.unquote(password)
75
password = ui.ui_factory.get_password(
76
prompt='HTTP %(user)s@%(host)s password',
77
user=username, host=host)
58
password = ui_factory.get_password(prompt='HTTP %(user)@%(host) password',
59
user=username, host=host)
78
60
password_manager.add_password(None, host, username, password)
79
61
url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
83
class HttpTransportBase(ConnectedTransport):
84
"""Base class for http implementations.
86
Does URL parsing, etc, but not any network IO.
88
The protocol can be given as e.g. http+urllib://host/ to use a particular
65
class Request(urllib2.Request):
66
"""Request object for urllib2 that allows the method to be overridden."""
71
if self.method is not None:
74
return urllib2.Request.get_method(self)
77
def get_url(url, method=None):
79
mutter("get_url %s", url)
80
manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
81
url = extract_auth(url, manager)
82
auth_handler = urllib2.HTTPBasicAuthHandler(manager)
83
opener = urllib2.build_opener(auth_handler)
85
request = Request(url)
86
request.method = method
87
request.add_header('User-Agent', 'bzr/%s' % bzrlib.__version__)
88
response = opener.open(request)
92
class HttpTransport(Transport):
93
"""This is the transport agent for http:// access.
95
TODO: Implement pipelined versions of all of the *_multi() functions.
92
# _unqualified_scheme: "http" or "https"
93
# _scheme: may have "+pycurl", etc
95
def __init__(self, base, _from_transport=None):
98
def __init__(self, base):
96
99
"""Set the base path where files will be stored."""
97
proto_match = re.match(r'^(https?)(\+\w+)?://', base)
99
raise AssertionError("not a http url: %r" % base)
100
self._unqualified_scheme = proto_match.group(1)
101
impl_name = proto_match.group(2)
103
impl_name = impl_name[1:]
104
self._impl_name = impl_name
105
super(HttpTransportBase, self).__init__(base,
106
_from_transport=_from_transport)
108
# range hint is handled dynamically throughout the life
109
# of the transport object. We start by trying multi-range
110
# requests and if the server returns bogus results, we
111
# retry with single range requests and, finally, we
112
# forget about range if the server really can't
113
# understand. Once acquired, this piece of info is
114
# propagated to clones.
115
if _from_transport is not None:
116
self._range_hint = _from_transport._range_hint
118
self._range_hint = 'multi'
100
assert base.startswith('http://') or base.startswith('https://')
103
super(HttpTransport, self).__init__(base)
104
# In the future we might actually connect to the remote host
105
# rather than using get_url
106
# self._connection = None
107
(self._proto, self._host,
108
self._path, self._parameters,
109
self._query, self._fragment) = urlparse.urlparse(self.base)
111
def should_cache(self):
112
"""Return True if the data pulled across should be cached locally.
116
def clone(self, offset=None):
117
"""Return a new HttpTransport with root at self.base + offset
118
For now HttpTransport does not actually connect, so just return
119
a new HttpTransport object.
122
return HttpTransport(self.base)
124
return HttpTransport(self.abspath(offset))
126
def abspath(self, relpath):
127
"""Return the full url to the given relative path.
128
This can be supplied with a string or a list
130
assert isinstance(relpath, basestring)
131
if isinstance(relpath, basestring):
132
relpath_parts = relpath.split('/')
134
# TODO: Don't call this with an array - no magic interfaces
135
relpath_parts = relpath[:]
136
if len(relpath_parts) > 1:
137
if relpath_parts[0] == '':
138
raise ValueError("path %r within branch %r seems to be absolute"
139
% (relpath, self._path))
140
if relpath_parts[-1] == '':
141
raise ValueError("path %r within branch %r seems to be a directory"
142
% (relpath, self._path))
143
basepath = self._path.split('/')
144
if len(basepath) > 0 and basepath[-1] == '':
145
basepath = basepath[:-1]
146
for p in relpath_parts:
148
if len(basepath) == 0:
149
# In most filesystems, a request for the parent
150
# of root, just returns root.
153
elif p == '.' or p == '':
157
# Possibly, we could use urlparse.urljoin() here, but
158
# I'm concerned about when it chooses to strip the last
159
# portion of the path, and when it doesn't.
160
path = '/'.join(basepath)
161
return urlparse.urlunparse((self._proto,
162
self._host, path, '', '', ''))
120
164
def has(self, relpath):
121
raise NotImplementedError("has() is abstract on %r" % self)
123
def get(self, relpath):
165
"""Does the target location exist?
167
TODO: This should be changed so that we don't use
168
urllib2 and get an exception, the code path would be
169
cleaner if we just do an http HEAD request, and parse
174
path = self.abspath(relpath)
175
f = get_url(path, method='HEAD')
176
# Without the read and then close()
177
# we tend to have busy sockets.
181
except urllib2.URLError, e:
182
mutter('url error code: %s for has url: %r', e.code, path)
187
mutter('io error: %s %s for has url: %r',
188
e.errno, errno.errorcode.get(e.errno), path)
189
if e.errno == errno.ENOENT:
191
raise TransportError(orig_error=e)
193
def get(self, relpath, decode=False):
124
194
"""Get the file at the given relative path.
126
196
:param relpath: The relative path to the file
128
code, response_file = self._get(relpath, None)
129
# FIXME: some callers want an iterable... One step forward, three steps
130
# backwards :-/ And not only an iterable, but an iterable that can be
131
# seeked backwards, so we will never be able to do that. One such
132
# known client is bzrlib.bundle.serializer.v4.get_bundle_reader. At the
133
# time of this writing it's even the only known client -- vila20071203
134
return StringIO(response_file.read())
136
def _get(self, relpath, ranges, tail_amount=0):
137
"""Get a file, or part of a file.
139
:param relpath: Path relative to transport base URL
140
:param ranges: None to get the whole file;
141
or a list of _CoalescedOffset to fetch parts of a file.
142
:param tail_amount: The amount to get from the end of the file.
144
:returns: (http_code, result_file)
146
raise NotImplementedError(self._get)
148
def _remote_path(self, relpath):
149
"""See ConnectedTransport._remote_path.
151
user and passwords are not embedded in the path provided to the server.
153
relative = urlutils.unescape(relpath).encode('utf-8')
154
path = self._combine_paths(self._path, relative)
155
return self._unsplit_url(self._unqualified_scheme,
156
None, None, self._host, self._port, path)
158
def _create_auth(self):
159
"""Returns a dict returning the credentials provided at build time."""
160
auth = dict(host=self._host, port=self._port,
161
user=self._user, password=self._password,
162
protocol=self._unqualified_scheme,
166
def get_smart_medium(self):
167
"""See Transport.get_smart_medium."""
168
if self._medium is None:
169
# Since medium holds some state (smart server probing at least), we
170
# need to keep it around. Note that this is needed because medium
171
# has the same 'base' attribute as the transport so it can't be
172
# shared between transports having different bases.
173
self._medium = SmartClientHTTPMedium(self)
177
def _degrade_range_hint(self, relpath, ranges, exc_info):
178
if self._range_hint == 'multi':
179
self._range_hint = 'single'
180
mutter('Retry "%s" with single range request' % relpath)
181
elif self._range_hint == 'single':
182
self._range_hint = None
183
mutter('Retry "%s" without ranges' % relpath)
185
# We tried all the tricks, but nothing worked. We re-raise the
186
# original exception; the 'mutter' calls above will indicate that
187
# further tries were unsuccessful
188
raise exc_info[0], exc_info[1], exc_info[2]
190
# _coalesce_offsets is a helper for readv, it try to combine ranges without
191
# degrading readv performances. _bytes_to_read_before_seek is the value
192
# used for the limit parameter and has been tuned for other transports. For
193
# HTTP, the name is inappropriate but the parameter is still useful and
194
# helps reduce the number of chunks in the response. The overhead for a
195
# chunk (headers, length, footer around the data itself is variable but
196
# around 50 bytes. We use 128 to reduce the range specifiers that appear in
197
# the header, some servers (notably Apache) enforce a maximum length for a
198
# header and issue a '400: Bad request' error when too much ranges are
200
_bytes_to_read_before_seek = 128
201
# No limit on the offset number that get combined into one, we are trying
202
# to avoid downloading the whole file.
203
_max_readv_combine = 0
204
# By default Apache has a limit of ~400 ranges before replying with a 400
205
# Bad Request. So we go underneath that amount to be safe.
206
_max_get_ranges = 200
207
# We impose no limit on the range size. But see _pycurl.py for a different
211
def _readv(self, relpath, offsets):
212
"""Get parts of the file at the given relative path.
214
:param offsets: A list of (offset, size) tuples.
215
:param return: A list or generator of (offset, data) tuples
218
# offsets may be a generator, we will iterate it several times, so
220
offsets = list(offsets)
223
retried_offset = None
227
# Coalesce the offsets to minimize the GET requests issued
228
sorted_offsets = sorted(offsets)
229
coalesced = self._coalesce_offsets(
230
sorted_offsets, limit=self._max_readv_combine,
231
fudge_factor=self._bytes_to_read_before_seek,
232
max_size=self._get_max_size)
234
# Turn it into a list, we will iterate it several times
235
coalesced = list(coalesced)
236
if 'http' in debug.debug_flags:
237
mutter('http readv of %s offsets => %s collapsed %s',
238
relpath, len(offsets), len(coalesced))
240
# Cache the data read, but only until it's been used
242
# We will iterate on the data received from the GET requests and
243
# serve the corresponding offsets respecting the initial order. We
244
# need an offset iterator for that.
245
iter_offsets = iter(offsets)
246
cur_offset_and_size = iter_offsets.next()
249
for cur_coal, rfile in self._coalesce_readv(relpath, coalesced):
250
# Split the received chunk
251
for offset, size in cur_coal.ranges:
252
start = cur_coal.start + offset
254
data = rfile.read(size)
257
raise errors.ShortReadvError(relpath, start, size,
259
if (start, size) == cur_offset_and_size:
260
# The offset requested are sorted as the coalesced
261
# ones, no need to cache. Win !
262
yield cur_offset_and_size[0], data
263
cur_offset_and_size = iter_offsets.next()
265
# Different sorting. We need to cache.
266
data_map[(start, size)] = data
268
# Yield everything we can
269
while cur_offset_and_size in data_map:
270
# Clean the cached data since we use it
271
# XXX: will break if offsets contains duplicates --
273
this_data = data_map.pop(cur_offset_and_size)
274
yield cur_offset_and_size[0], this_data
275
cur_offset_and_size = iter_offsets.next()
277
except (errors.ShortReadvError, errors.InvalidRange,
278
errors.InvalidHttpRange), e:
279
mutter('Exception %r: %s during http._readv',e, e)
280
if (not isinstance(e, errors.ShortReadvError)
281
or retried_offset == cur_offset_and_size):
282
# We don't degrade the range hint for ShortReadvError since
283
# they do not indicate a problem with the server ability to
284
# handle ranges. Except when we fail to get back a required
285
# offset twice in a row. In that case, falling back to
286
# single range or whole file should help or end up in a
288
self._degrade_range_hint(relpath, coalesced, sys.exc_info())
289
# Some offsets may have been already processed, so we retry
290
# only the unsuccessful ones.
291
offsets = [cur_offset_and_size] + [o for o in iter_offsets]
292
retried_offset = cur_offset_and_size
295
def _coalesce_readv(self, relpath, coalesced):
296
"""Issue several GET requests to satisfy the coalesced offsets"""
298
def get_and_yield(relpath, coalesced):
300
# Note that the _get below may raise
301
# errors.InvalidHttpRange. It's the caller's responsibility to
302
# decide how to retry since it may provide different coalesced
304
code, rfile = self._get(relpath, coalesced)
305
for coal in coalesced:
308
if self._range_hint is None:
309
# Download whole file
310
for c, rfile in get_and_yield(relpath, coalesced):
313
total = len(coalesced)
314
if self._range_hint == 'multi':
315
max_ranges = self._max_get_ranges
316
elif self._range_hint == 'single':
319
raise AssertionError("Unknown _range_hint %r"
320
% (self._range_hint,))
321
# TODO: Some web servers may ignore the range requests and return
322
# the whole file, we may want to detect that and avoid further
324
# Hint: test_readv_multiple_get_requests will fail once we do that
327
for coal in coalesced:
328
if ((self._get_max_size > 0
329
and cumul + coal.length > self._get_max_size)
330
or len(ranges) >= max_ranges):
331
# Get that much and yield
332
for c, rfile in get_and_yield(relpath, ranges):
334
# Restart with the current offset
340
# Get the rest and yield
341
for c, rfile in get_and_yield(relpath, ranges):
344
def recommended_page_size(self):
345
"""See Transport.recommended_page_size().
347
For HTTP we suggest a large page size to reduce the overhead
348
introduced by latency.
352
def _post(self, body_bytes):
353
"""POST body_bytes to .bzr/smart on this transport.
355
:returns: (response code, response body file-like object).
357
# TODO: Requiring all the body_bytes to be available at the beginning of
358
# the POST may require large client buffers. It would be nice to have
359
# an interface that allows streaming via POST when possible (and
360
# degrades to a local buffer when not).
361
raise NotImplementedError(self._post)
363
def put_file(self, relpath, f, mode=None):
364
"""Copy the file-like object into the location.
200
path = self.abspath(relpath)
202
except urllib2.HTTPError, e:
203
mutter('url error code: %s for has url: %r', e.code, path)
205
raise NoSuchFile(path, extra=e)
207
except (BzrError, IOError), e:
208
if hasattr(e, 'errno'):
209
mutter('io error: %s %s for has url: %r',
210
e.errno, errno.errorcode.get(e.errno), path)
211
if e.errno == errno.ENOENT:
212
raise NoSuchFile(path, extra=e)
213
raise ConnectionError(msg = "Error retrieving %s: %s"
214
% (self.abspath(relpath), str(e)),
217
def put(self, relpath, f, mode=None):
218
"""Copy the file-like or string object into the location.
366
220
:param relpath: Location to put the contents, relative to base.
367
:param f: File-like object.
221
:param f: File-like or string object.
369
raise errors.TransportNotPossible('http PUT not supported')
223
raise TransportNotPossible('http PUT not supported')
371
225
def mkdir(self, relpath, mode=None):
372
226
"""Create a directory at the given path."""
373
raise errors.TransportNotPossible('http does not support mkdir()')
227
raise TransportNotPossible('http does not support mkdir()')
375
229
def rmdir(self, relpath):
376
230
"""See Transport.rmdir."""
377
raise errors.TransportNotPossible('http does not support rmdir()')
231
raise TransportNotPossible('http does not support rmdir()')
379
def append_file(self, relpath, f, mode=None):
233
def append(self, relpath, f):
380
234
"""Append the text in the file-like object into the final
383
raise errors.TransportNotPossible('http does not support append()')
237
raise TransportNotPossible('http does not support append()')
385
239
def copy(self, rel_from, rel_to):
386
240
"""Copy the item at rel_from to the location at rel_to"""
387
raise errors.TransportNotPossible('http does not support copy()')
241
raise TransportNotPossible('http does not support copy()')
389
243
def copy_to(self, relpaths, other, mode=None, pb=None):
390
244
"""Copy a set of entries from self into another Transport.
450
297
:return: A lock object, which should be passed to Transport.unlock()
452
raise errors.TransportNotPossible('http does not support lock_write()')
454
def clone(self, offset=None):
455
"""Return a new HttpTransportBase with root at self.base + offset
457
We leave the daughter classes take advantage of the hint
458
that it's a cloning not a raw creation.
461
return self.__class__(self.base, self)
463
return self.__class__(self.abspath(offset), self)
465
def _attempted_range_header(self, offsets, tail_amount):
466
"""Prepare a HTTP Range header at a level the server should accept.
468
:return: the range header representing offsets/tail_amount or None if
469
no header can be built.
472
if self._range_hint == 'multi':
473
# Generate the header describing all offsets
474
return self._range_header(offsets, tail_amount)
475
elif self._range_hint == 'single':
476
# Combine all the requested ranges into a single
479
if tail_amount not in (0, None):
480
# Nothing we can do here to combine ranges with tail_amount
481
# in a single range, just returns None. The whole file
482
# should be downloaded.
485
start = offsets[0].start
487
end = last.start + last.length - 1
488
whole = self._coalesce_offsets([(start, end - start + 1)],
489
limit=0, fudge_factor=0)
490
return self._range_header(list(whole), 0)
299
raise TransportNotPossible('http does not support lock_write()')
302
#---------------- test server facilities ----------------
303
import BaseHTTPServer, SimpleHTTPServer, socket, time
307
class WebserverNotAvailable(Exception):
311
class BadWebserverPath(ValueError):
313
return 'path %s is not in %s' % self.args
316
class TestingHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
318
def log_message(self, format, *args):
319
self.server.test_case.log('webserver - %s - - [%s] %s "%s" "%s"',
320
self.address_string(),
321
self.log_date_time_string(),
323
self.headers.get('referer', '-'),
324
self.headers.get('user-agent', '-'))
326
def handle_one_request(self):
327
"""Handle a single HTTP request.
329
You normally don't need to override this method; see the class
330
__doc__ string for information on how to handle specific HTTP
331
commands such as GET and POST.
334
for i in xrange(1,11): # Don't try more than 10 times
336
self.raw_requestline = self.rfile.readline()
337
except socket.error, e:
338
if e.args[0] in (errno.EAGAIN, errno.EWOULDBLOCK):
339
# omitted for now because some tests look at the log of
340
# the server and expect to see no errors. see recent
341
# email thread. -- mbp 20051021.
342
## self.log_message('EAGAIN (%d) while reading from raw_requestline' % i)
492
# Only tail_amount, requested, leave range_header
494
return self._range_header(offsets, tail_amount)
348
if not self.raw_requestline:
349
self.close_connection = 1
351
if not self.parse_request(): # An error code has been sent, just exit
353
mname = 'do_' + self.command
354
if not hasattr(self, mname):
355
self.send_error(501, "Unsupported method (%r)" % self.command)
357
method = getattr(self, mname)
361
class TestingHTTPServer(BaseHTTPServer.HTTPServer):
362
def __init__(self, server_address, RequestHandlerClass, test_case):
363
BaseHTTPServer.HTTPServer.__init__(self, server_address,
365
self.test_case = test_case
368
class HttpServer(Server):
369
"""A test server for http transports."""
371
def _http_start(self):
373
httpd = TestingHTTPServer(('localhost', 0),
374
TestingHTTPRequestHandler,
376
host, port = httpd.socket.getsockname()
377
self._http_base_url = 'http://localhost:%s/' % port
378
self._http_starting.release()
379
httpd.socket.settimeout(0.1)
381
while self._http_running:
383
httpd.handle_request()
384
except socket.timeout:
387
def _get_remote_url(self, path):
388
path_parts = path.split(os.path.sep)
389
if os.path.isabs(path):
390
if path_parts[:len(self._local_path_parts)] != \
391
self._local_path_parts:
392
raise BadWebserverPath(path, self.test_dir)
393
remote_path = '/'.join(path_parts[len(self._local_path_parts):])
499
def _range_header(ranges, tail_amount):
500
"""Turn a list of bytes ranges into a HTTP Range header value.
502
:param ranges: A list of _CoalescedOffset
503
:param tail_amount: The amount to get from the end of the file.
505
:return: HTTP range header string.
507
At least a non-empty ranges *or* a tail_amount must be
511
for offset in ranges:
512
strings.append('%d-%d' % (offset.start,
513
offset.start + offset.length - 1))
516
strings.append('-%d' % tail_amount)
518
return ','.join(strings)
521
# TODO: May be better located in smart/medium.py with the other
522
# SmartMedium classes
523
class SmartClientHTTPMedium(medium.SmartClientMedium):
525
def __init__(self, http_transport):
526
super(SmartClientHTTPMedium, self).__init__(http_transport.base)
527
# We don't want to create a circular reference between the http
528
# transport and its associated medium. Since the transport will live
529
# longer than the medium, the medium keep only a weak reference to its
531
self._http_transport_ref = weakref.ref(http_transport)
533
def get_request(self):
534
return SmartClientHTTPMediumRequest(self)
536
def should_probe(self):
539
def remote_path_from_transport(self, transport):
540
# Strip the optional 'bzr+' prefix from transport so it will have the
541
# same scheme as self.
542
transport_base = transport.base
543
if transport_base.startswith('bzr+'):
544
transport_base = transport_base[4:]
545
rel_url = urlutils.relative_url(self.base, transport_base)
546
return urllib.unquote(rel_url)
548
def send_http_smart_request(self, bytes):
550
# Get back the http_transport hold by the weak reference
551
t = self._http_transport_ref()
552
code, body_filelike = t._post(bytes)
554
raise InvalidHttpResponse(
555
t._remote_path('.bzr/smart'),
556
'Expected 200 response code, got %r' % (code,))
557
except errors.InvalidHttpResponse, e:
558
raise errors.SmartProtocolError(str(e))
562
# TODO: May be better located in smart/medium.py with the other
563
# SmartMediumRequest classes
564
class SmartClientHTTPMediumRequest(medium.SmartClientMediumRequest):
565
"""A SmartClientMediumRequest that works with an HTTP medium."""
567
def __init__(self, client_medium):
568
medium.SmartClientMediumRequest.__init__(self, client_medium)
571
def _accept_bytes(self, bytes):
572
self._buffer += bytes
574
def _finished_writing(self):
575
data = self._medium.send_http_smart_request(self._buffer)
576
self._response_body = data
578
def _read_bytes(self, count):
579
"""See SmartClientMediumRequest._read_bytes."""
580
return self._response_body.read(count)
582
def _read_line(self):
583
line, excess = medium._get_line(self._response_body.read)
585
raise AssertionError(
586
'_get_line returned excess bytes, but this mediumrequest '
587
'cannot handle excess. (%r)' % (excess,))
590
def _finished_reading(self):
591
"""See SmartClientMediumRequest._finished_reading."""
395
remote_path = '/'.join(path_parts)
397
self._http_starting.acquire()
398
self._http_starting.release()
399
return self._http_base_url + remote_path
401
def log(self, format, *args):
402
"""Capture Server log output."""
403
self.logs.append(format % args)
406
"""See bzrlib.transport.Server.setUp."""
407
self._home_dir = os.getcwdu()
408
self._local_path_parts = self._home_dir.split(os.path.sep)
409
self._http_starting = threading.Lock()
410
self._http_starting.acquire()
411
self._http_running = True
412
self._http_base_url = None
413
self._http_thread = threading.Thread(target=self._http_start)
414
self._http_thread.setDaemon(True)
415
self._http_thread.start()
416
self._http_proxy = os.environ.get("http_proxy")
417
if self._http_proxy is not None:
418
del os.environ["http_proxy"]
422
"""See bzrlib.transport.Server.tearDown."""
423
self._http_running = False
424
self._http_thread.join()
425
if self._http_proxy is not None:
427
os.environ["http_proxy"] = self._http_proxy
430
"""See bzrlib.transport.Server.get_url."""
431
return self._get_remote_url(self._home_dir)
433
def get_bogus_url(self):
434
"""See bzrlib.transport.Server.get_bogus_url."""
435
return 'http://jasldkjsalkdjalksjdkljasd'
438
def get_test_permutations():
439
"""Return the permutations to be used in testing."""
440
warn("There are no HTTPS transport provider tests yet.")
441
return [(HttpTransport, HttpServer),
added
removed
bzrlib/transport/http.py
