63
61
if password is not None:
64
62
password = urllib.unquote(password)
66
password = ui_factory.get_password(prompt='HTTP %(user)@%(host) password',
67
user=username, host=host)
64
password = ui.ui_factory.get_password(
65
prompt='HTTP %(user)s@%(host)s password',
66
user=username, host=host)
68
67
password_manager.add_password(None, host, username, password)
69
68
url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
73
class HttpTransportBase(Transport):
72
def _extract_headers(header_text, url):
73
"""Extract the mapping for an rfc2822 header
75
This is a helper function for the test suite and for _pycurl.
76
(urllib already parses the headers for us)
78
In the case that there are multiple headers inside the file,
79
the last one is returned.
81
:param header_text: A string of header information.
82
This expects that the first line of a header will always be HTTP ...
83
:param url: The url we are parsing, so we can raise nice errors
84
:return: mimetools.Message object, which basically acts like a case
85
insensitive dictionary.
88
remaining = header_text
91
raise errors.InvalidHttpResponse(url, 'Empty headers')
94
header_file = StringIO(remaining)
95
first_line = header_file.readline()
96
if not first_line.startswith('HTTP'):
97
if first_header: # The first header *must* start with HTTP
98
raise errors.InvalidHttpResponse(url,
99
'Opening header line did not start with HTTP: %s'
101
assert False, 'Opening header line was not HTTP'
103
break # We are done parsing
105
m = mimetools.Message(header_file)
107
# mimetools.Message parses the first header up to a blank line
108
# So while there is remaining data, it probably means there is
109
# another header to be parsed.
110
# Get rid of any preceeding whitespace, which if it is all whitespace
111
# will get rid of everything.
112
remaining = header_file.read().lstrip()
116
class HttpTransportBase(Transport, smart.SmartClientMedium):
74
117
"""Base class for http implementations.
76
119
Does URL parsing, etc, but not any network IO.
187
241
raise NotImplementedError(self._get)
243
def get_request(self):
244
return SmartClientHTTPMediumRequest(self)
246
def get_smart_medium(self):
247
"""See Transport.get_smart_medium.
249
HttpTransportBase directly implements the minimal interface of
250
SmartMediumClient, so this returns self.
254
def _retry_get(self, relpath, ranges, exc_info):
255
"""A GET request have failed, let's retry with a simpler request."""
258
# The server does not gives us enough data or
259
# bogus-looking result, let's try again with
260
# a simpler request if possible.
261
if self._range_hint == 'multi':
262
self._range_hint = 'single'
263
mutter('Retry %s with single range request' % relpath)
265
elif self._range_hint == 'single':
266
self._range_hint = None
267
mutter('Retry %s without ranges' % relpath)
270
# Note that since the offsets and the ranges may not
271
# be in the same order, we don't try to calculate a
272
# restricted single range encompassing unprocessed
274
code, f = self._get(relpath, ranges)
275
return try_again, code, f
277
# We tried all the tricks, but nothing worked. We
278
# re-raise original exception; the 'mutter' calls
279
# above will indicate that further tries were
281
raise exc_info[0], exc_info[1], exc_info[2]
189
283
def readv(self, relpath, offsets):
190
284
"""Get parts of the file at the given relative path.
192
286
:param offsets: A list of (offset, size) tuples.
193
287
:param return: A list or generator of (offset, data) tuples
195
# Ideally we would pass one big request asking for all the ranges in
196
# one go; however then the server will give a multipart mime response
197
# back, and we can't parse them yet. So instead we just get one range
198
# per region, and try to coallesce the regions as much as possible.
200
# The read-coallescing code is not quite regular enough to have a
201
# single driver routine and
202
# helper method in Transport.
203
def do_combined_read(combined_offsets):
204
# read one coalesced block
206
for offset, size in combined_offsets:
208
mutter('readv coalesced %d reads.', len(combined_offsets))
209
offset = combined_offsets[0][0]
210
byte_range = (offset, offset + total_size - 1)
211
code, result_file = self._get(relpath, [byte_range])
213
for off, size in combined_offsets:
214
result_bytes = result_file.read(size)
215
assert len(result_bytes) == size
216
yield off, result_bytes
218
data = result_file.read(offset + total_size)[offset:offset + total_size]
220
for offset, size in combined_offsets:
221
yield offset, data[pos:pos + size]
226
pending_offsets = deque(offsets)
227
combined_offsets = []
228
while len(pending_offsets):
229
offset, size = pending_offsets.popleft()
230
if not combined_offsets:
231
combined_offsets = [[offset, size]]
289
ranges = self.offsets_to_ranges(offsets)
290
mutter('http readv of %s collapsed %s offsets => %s',
291
relpath, len(offsets), ranges)
297
code, f = self._get(relpath, ranges)
298
except (errors.InvalidRange, errors.ShortReadvError), e:
299
try_again, code, f = self._retry_get(relpath, ranges,
302
for start, size in offsets:
306
f.seek(start, (start < 0) and 2 or 0)
310
if len(data) != size:
311
raise errors.ShortReadvError(relpath, start, size,
313
except (errors.InvalidRange, errors.ShortReadvError), e:
314
# Note that we replace 'f' here and that it
315
# may need cleaning one day before being
317
try_again, code, f = self._retry_get(relpath, ranges,
319
# After one or more tries, we get the data.
323
def offsets_to_ranges(offsets):
324
"""Turn a list of offsets and sizes into a list of byte ranges.
326
:param offsets: A list of tuples of (start, size). An empty list
328
:return: a list of inclusive byte ranges (start, end)
329
Adjacent ranges will be combined.
331
# Make sure we process sorted offsets
332
offsets = sorted(offsets)
337
for start, size in offsets:
338
end = start + size - 1
340
combined.append([start, end])
341
elif start <= prev_end + 1:
342
combined[-1][1] = end
233
if (len (combined_offsets) < 500 and
234
combined_offsets[-1][0] + combined_offsets[-1][1] == offset):
236
combined_offsets.append([offset, size])
238
# incompatible, or over the threshold issue a read and yield
239
pending_offsets.appendleft((offset, size))
240
for result in do_combined_read(combined_offsets):
242
combined_offsets = []
243
# whatever is left is a single coalesced request
244
if len(combined_offsets):
245
for result in do_combined_read(combined_offsets):
248
def put(self, relpath, f, mode=None):
249
"""Copy the file-like or string object into the location.
344
combined.append([start, end])
349
def _post(self, body_bytes):
350
"""POST body_bytes to .bzr/smart on this transport.
352
:returns: (response code, response body file-like object).
354
# TODO: Requiring all the body_bytes to be available at the beginning of
355
# the POST may require large client buffers. It would be nice to have
356
# an interface that allows streaming via POST when possible (and
357
# degrades to a local buffer when not).
358
raise NotImplementedError(self._post)
360
def put_file(self, relpath, f, mode=None):
361
"""Copy the file-like object into the location.
251
363
:param relpath: Location to put the contents, relative to base.
252
:param f: File-like or string object.
364
:param f: File-like object.
254
raise TransportNotPossible('http PUT not supported')
366
raise errors.TransportNotPossible('http PUT not supported')
256
368
def mkdir(self, relpath, mode=None):
257
369
"""Create a directory at the given path."""
258
raise TransportNotPossible('http does not support mkdir()')
370
raise errors.TransportNotPossible('http does not support mkdir()')
260
372
def rmdir(self, relpath):
261
373
"""See Transport.rmdir."""
262
raise TransportNotPossible('http does not support rmdir()')
374
raise errors.TransportNotPossible('http does not support rmdir()')
264
def append(self, relpath, f):
376
def append_file(self, relpath, f, mode=None):
265
377
"""Append the text in the file-like object into the final
268
raise TransportNotPossible('http does not support append()')
380
raise errors.TransportNotPossible('http does not support append()')
270
382
def copy(self, rel_from, rel_to):
271
383
"""Copy the item at rel_from to the location at rel_to"""
272
raise TransportNotPossible('http does not support copy()')
384
raise errors.TransportNotPossible('http does not support copy()')
274
386
def copy_to(self, relpaths, other, mode=None, pb=None):
275
387
"""Copy a set of entries from self into another Transport.
329
442
:return: A lock object, which should be passed to Transport.unlock()
331
raise TransportNotPossible('http does not support lock_write()')
444
raise errors.TransportNotPossible('http does not support lock_write()')
333
446
def clone(self, offset=None):
334
447
"""Return a new HttpTransportBase with root at self.base + offset
335
For now HttpTransportBase does not actually connect, so just return
336
a new HttpTransportBase object.
449
We leave the daughter classes take advantage of the hint
450
that it's a cloning not a raw creation.
338
452
if offset is None:
339
return self.__class__(self.base)
341
return self.__class__(self.abspath(offset))
343
#---------------- test server facilities ----------------
344
# TODO: load these only when running tests
347
class WebserverNotAvailable(Exception):
351
class BadWebserverPath(ValueError):
353
return 'path %s is not in %s' % self.args
356
class TestingHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
358
def log_message(self, format, *args):
359
self.server.test_case.log('webserver - %s - - [%s] %s "%s" "%s"',
360
self.address_string(),
361
self.log_date_time_string(),
363
self.headers.get('referer', '-'),
364
self.headers.get('user-agent', '-'))
366
def handle_one_request(self):
367
"""Handle a single HTTP request.
369
You normally don't need to override this method; see the class
370
__doc__ string for information on how to handle specific HTTP
371
commands such as GET and POST.
453
return self.__class__(self.base, self)
455
return self.__class__(self.abspath(offset), self)
457
def attempted_range_header(self, ranges, tail_amount):
458
"""Prepare a HTTP Range header at a level the server should accept"""
460
if self._range_hint == 'multi':
462
return self.range_header(ranges, tail_amount)
463
elif self._range_hint == 'single':
464
# Combine all the requested ranges into a single
467
start, ignored = ranges[0]
468
ignored, end = ranges[-1]
469
if tail_amount not in (0, None):
470
# Nothing we can do here to combine ranges
471
# with tail_amount, just returns None. The
472
# whole file should be downloaded.
475
return self.range_header([(start, end)], 0)
477
# Only tail_amount, requested, leave range_header
479
return self.range_header(ranges, tail_amount)
484
def range_header(ranges, tail_amount):
485
"""Turn a list of bytes ranges into a HTTP Range header value.
487
:param ranges: A list of byte ranges, (start, end).
488
:param tail_amount: The amount to get from the end of the file.
490
:return: HTTP range header string.
492
At least a non-empty ranges *or* a tail_amount must be
374
for i in xrange(1,11): # Don't try more than 10 times
376
self.raw_requestline = self.rfile.readline()
377
except socket.error, e:
378
if e.args[0] in (errno.EAGAIN, errno.EWOULDBLOCK):
379
# omitted for now because some tests look at the log of
380
# the server and expect to see no errors. see recent
381
# email thread. -- mbp 20051021.
382
## self.log_message('EAGAIN (%d) while reading from raw_requestline' % i)
388
if not self.raw_requestline:
389
self.close_connection = 1
391
if not self.parse_request(): # An error code has been sent, just exit
393
mname = 'do_' + self.command
394
if not hasattr(self, mname):
395
self.send_error(501, "Unsupported method (%r)" % self.command)
397
method = getattr(self, mname)
401
class TestingHTTPServer(BaseHTTPServer.HTTPServer):
402
def __init__(self, server_address, RequestHandlerClass, test_case):
403
BaseHTTPServer.HTTPServer.__init__(self, server_address,
405
self.test_case = test_case
407
class HttpServer(Server):
408
"""A test server for http transports."""
410
# used to form the url that connects to this server
411
_url_protocol = 'http'
413
def _http_start(self):
415
httpd = TestingHTTPServer(('localhost', 0),
416
TestingHTTPRequestHandler,
418
host, port = httpd.socket.getsockname()
419
self._http_base_url = '%s://localhost:%s/' % (self._url_protocol, port)
420
self._http_starting.release()
421
httpd.socket.settimeout(0.1)
423
while self._http_running:
425
httpd.handle_request()
426
except socket.timeout:
429
def _get_remote_url(self, path):
430
path_parts = path.split(os.path.sep)
431
if os.path.isabs(path):
432
if path_parts[:len(self._local_path_parts)] != \
433
self._local_path_parts:
434
raise BadWebserverPath(path, self.test_dir)
435
remote_path = '/'.join(path_parts[len(self._local_path_parts):])
437
remote_path = '/'.join(path_parts)
439
self._http_starting.acquire()
440
self._http_starting.release()
441
return self._http_base_url + remote_path
443
def log(self, format, *args):
444
"""Capture Server log output."""
445
self.logs.append(format % args)
448
"""See bzrlib.transport.Server.setUp."""
449
self._home_dir = os.getcwdu()
450
self._local_path_parts = self._home_dir.split(os.path.sep)
451
self._http_starting = threading.Lock()
452
self._http_starting.acquire()
453
self._http_running = True
454
self._http_base_url = None
455
self._http_thread = threading.Thread(target=self._http_start)
456
self._http_thread.setDaemon(True)
457
self._http_thread.start()
458
self._http_proxy = os.environ.get("http_proxy")
459
if self._http_proxy is not None:
460
del os.environ["http_proxy"]
464
"""See bzrlib.transport.Server.tearDown."""
465
self._http_running = False
466
self._http_thread.join()
467
if self._http_proxy is not None:
469
os.environ["http_proxy"] = self._http_proxy
472
"""See bzrlib.transport.Server.get_url."""
473
return self._get_remote_url(self._home_dir)
475
def get_bogus_url(self):
476
"""See bzrlib.transport.Server.get_bogus_url."""
477
# this is chosen to try to prevent trouble with proxies, wierd dns,
479
return 'http://127.0.0.1:1/'
496
for start, end in ranges:
497
strings.append('%d-%d' % (start, end))
500
strings.append('-%d' % tail_amount)
502
return ','.join(strings)
504
def send_http_smart_request(self, bytes):
505
code, body_filelike = self._post(bytes)
506
assert code == 200, 'unexpected HTTP response code %r' % (code,)
510
class SmartClientHTTPMediumRequest(smart.SmartClientMediumRequest):
511
"""A SmartClientMediumRequest that works with an HTTP medium."""
513
def __init__(self, medium):
514
smart.SmartClientMediumRequest.__init__(self, medium)
517
def _accept_bytes(self, bytes):
518
self._buffer += bytes
520
def _finished_writing(self):
521
data = self._medium.send_http_smart_request(self._buffer)
522
self._response_body = data
524
def _read_bytes(self, count):
525
return self._response_body.read(count)
527
def _finished_reading(self):
528
"""See SmartClientMediumRequest._finished_reading."""
added
removed
bzrlib/transport/http/__init__.py
