31
31
from warnings import warn
33
from bzrlib.transport import Transport, register_transport, Server
33
# TODO: load these only when running http tests
34
import BaseHTTPServer, SimpleHTTPServer, socket, time
37
from bzrlib import errors
34
38
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
35
39
TransportError, ConnectionError, InvalidURL)
36
40
from bzrlib.branch import Branch
37
41
from bzrlib.trace import mutter
38
# TODO: load these only when running http tests
39
import BaseHTTPServer, SimpleHTTPServer, socket, time
42
from bzrlib.transport import Transport, register_transport, Server
43
from bzrlib.transport.http.response import (HttpMultipartRangeResponse,
41
45
from bzrlib.ui import ui_factory
78
def _extract_headers(header_text, url):
79
"""Extract the mapping for an rfc2822 header
81
This is a helper function for the test suite and for _pycurl.
82
(urllib already parses the headers for us)
84
In the case that there are multiple headers inside the file,
85
the last one is returned.
87
:param header_text: A string of header information.
88
This expects that the first line of a header will always be HTTP ...
89
:param url: The url we are parsing, so we can raise nice errors
90
:return: mimetools.Message object, which basically acts like a case
91
insensitive dictionary.
94
remaining = header_text
97
raise errors.InvalidHttpResponse(url, 'Empty headers')
100
header_file = StringIO(remaining)
101
first_line = header_file.readline()
102
if not first_line.startswith('HTTP'):
103
if first_header: # The first header *must* start with HTTP
104
raise errors.InvalidHttpResponse(url,
105
'Opening header line did not start with HTTP: %s'
107
assert False, 'Opening header line was not HTTP'
109
break # We are done parsing
111
m = mimetools.Message(header_file)
113
# mimetools.Message parses the first header up to a blank line
114
# So while there is remaining data, it probably means there is
115
# another header to be parsed.
116
# Get rid of any preceeding whitespace, which if it is all whitespace
117
# will get rid of everything.
118
remaining = header_file.read().lstrip()
74
122
class HttpTransportBase(Transport):
75
123
"""Base class for http implementations.
195
243
:param offsets: A list of (offset, size) tuples.
196
244
:param return: A list or generator of (offset, data) tuples
198
# Ideally we would pass one big request asking for all the ranges in
199
# one go; however then the server will give a multipart mime response
200
# back, and we can't parse them yet. So instead we just get one range
201
# per region, and try to coallesce the regions as much as possible.
203
# The read-coallescing code is not quite regular enough to have a
204
# single driver routine and
205
# helper method in Transport.
206
def do_combined_read(combined_offsets):
207
# read one coalesced block
209
for offset, size in combined_offsets:
211
mutter('readv coalesced %d reads.', len(combined_offsets))
212
offset = combined_offsets[0][0]
213
byte_range = (offset, offset + total_size - 1)
214
code, result_file = self._get(relpath, [byte_range])
216
for off, size in combined_offsets:
217
result_bytes = result_file.read(size)
218
assert len(result_bytes) == size
219
yield off, result_bytes
221
data = result_file.read(offset + total_size)[offset:offset + total_size]
223
for offset, size in combined_offsets:
224
yield offset, data[pos:pos + size]
229
pending_offsets = deque(offsets)
230
combined_offsets = []
231
while len(pending_offsets):
232
offset, size = pending_offsets.popleft()
233
if not combined_offsets:
234
combined_offsets = [[offset, size]]
246
ranges = self.offsets_to_ranges(offsets)
247
mutter('http readv of %s collapsed %s offsets => %s',
248
relpath, len(offsets), ranges)
249
code, f = self._get(relpath, ranges)
250
for start, size in offsets:
251
f.seek(start, (start < 0) and 2 or 0)
254
assert len(data) == size
258
def offsets_to_ranges(offsets):
259
"""Turn a list of offsets and sizes into a list of byte ranges.
261
:param offsets: A list of tuples of (start, size). An empty list
263
:return: a list of inclusive byte ranges (start, end)
264
Adjacent ranges will be combined.
266
# Make sure we process sorted offsets
267
offsets = sorted(offsets)
272
for start, size in offsets:
273
end = start + size - 1
275
combined.append([start, end])
276
elif start <= prev_end + 1:
277
combined[-1][1] = end
236
if (len (combined_offsets) < 500 and
237
combined_offsets[-1][0] + combined_offsets[-1][1] == offset):
239
combined_offsets.append([offset, size])
241
# incompatible, or over the threshold issue a read and yield
242
pending_offsets.appendleft((offset, size))
243
for result in do_combined_read(combined_offsets):
245
combined_offsets = []
246
# whatever is left is a single coalesced request
247
if len(combined_offsets):
248
for result in do_combined_read(combined_offsets):
279
combined.append([start, end])
251
284
def put(self, relpath, f, mode=None):
252
285
"""Copy the file-like or string object into the location.
344
377
return self.__class__(self.abspath(offset))
380
def range_header(ranges, tail_amount):
381
"""Turn a list of bytes ranges into a HTTP Range header value.
383
:param offsets: A list of byte ranges, (start, end). An empty list
386
:return: HTTP range header string.
389
for start, end in ranges:
390
strings.append('%d-%d' % (start, end))
393
strings.append('-%d' % tail_amount)
395
return ','.join(strings)
346
398
#---------------- test server facilities ----------------
347
399
# TODO: load these only when running tests
added
removed
bzrlib/transport/http/__init__.py
