« back to all changes in this revision
Viewing changes to bzrlib/transport/http/__init__.py
- browse files at revision 1869
- compare with another revision
- compare with revision 2485.8.58
- download diff
- download tarball
- view history from revision 1869
- Committer: Canonical.com Patch Queue Manager
- Date: 2006-07-19 20:32:38 UTC
- mfrom: (1786.1.43 http)
- Revision ID: pqm@pqm.ubuntu.com-20060719203238-892f369155dca5dc
(jam, mpee, jrydberg) update http with keep-alive and multi-range requests. http operations 2-4x faster
- files added:
- bzrlib/tests/test_http_response.py
- files modified:
- NEWS
added
removed
bzrlib/transport/http/__init__.py
19
19
There are separate implementation modules for each http client implementation.
20
20
"""
21
21
22
from collections import deque
23
22
from cStringIO import StringIO
24
23
import errno
24
import mimetools
25
25
import os
26
26
import posixpath
27
27
import re
30
30
import urllib
31
31
from warnings import warn
32
32
33
from bzrlib.transport import Transport, register_transport, Server
33
# TODO: load these only when running http tests
34
import BaseHTTPServer, SimpleHTTPServer, socket, time
35
import threading
36
37
from bzrlib import errors
34
38
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
35
39
TransportError, ConnectionError, InvalidURL)
36
40
from bzrlib.branch import Branch
37
41
from bzrlib.trace import mutter
38
# TODO: load these only when running http tests
39
import BaseHTTPServer, SimpleHTTPServer, socket, time
40
import threading
42
from bzrlib.transport import Transport, register_transport, Server
43
from bzrlib.transport.http.response import (HttpMultipartRangeResponse,
44
HttpRangeResponse)
41
45
from bzrlib.ui import ui_factory
42
46
43
47
71
75
return url
72
76
73
77
78
def _extract_headers(header_text, url):
79
"""Extract the mapping for an rfc2822 header
80
81
This is a helper function for the test suite and for _pycurl.
82
(urllib already parses the headers for us)
83
84
In the case that there are multiple headers inside the file,
85
the last one is returned.
86
87
:param header_text: A string of header information.
88
This expects that the first line of a header will always be HTTP ...
89
:param url: The url we are parsing, so we can raise nice errors
90
:return: mimetools.Message object, which basically acts like a case
91
insensitive dictionary.
92
"""
93
first_header = True
94
remaining = header_text
95
96
if not remaining:
97
raise errors.InvalidHttpResponse(url, 'Empty headers')
98
99
while remaining:
100
header_file = StringIO(remaining)
101
first_line = header_file.readline()
102
if not first_line.startswith('HTTP'):
103
if first_header: # The first header *must* start with HTTP
104
raise errors.InvalidHttpResponse(url,
105
'Opening header line did not start with HTTP: %s'
106
% (first_line,))
107
assert False, 'Opening header line was not HTTP'
108
else:
109
break # We are done parsing
110
first_header = False
111
m = mimetools.Message(header_file)
112
113
# mimetools.Message parses the first header up to a blank line
114
# So while there is remaining data, it probably means there is
115
# another header to be parsed.
116
# Get rid of any preceeding whitespace, which if it is all whitespace
117
# will get rid of everything.
118
remaining = header_file.read().lstrip()
119
return m
120
121
74
122
class HttpTransportBase(Transport):
75
123
"""Base class for http implementations.
76
124
195
243
:param offsets: A list of (offset, size) tuples.
196
244
:param return: A list or generator of (offset, data) tuples
197
245
"""
198
# Ideally we would pass one big request asking for all the ranges in
199
# one go; however then the server will give a multipart mime response
200
# back, and we can't parse them yet. So instead we just get one range
201
# per region, and try to coallesce the regions as much as possible.
202
#
203
# The read-coallescing code is not quite regular enough to have a
204
# single driver routine and
205
# helper method in Transport.
206
def do_combined_read(combined_offsets):
207
# read one coalesced block
208
total_size = 0
209
for offset, size in combined_offsets:
210
total_size += size
211
mutter('readv coalesced %d reads.', len(combined_offsets))
212
offset = combined_offsets[0][0]
213
byte_range = (offset, offset + total_size - 1)
214
code, result_file = self._get(relpath, [byte_range])
215
if code == 206:
216
for off, size in combined_offsets:
217
result_bytes = result_file.read(size)
218
assert len(result_bytes) == size
219
yield off, result_bytes
220
elif code == 200:
221
data = result_file.read(offset + total_size)[offset:offset + total_size]
222
pos = 0
223
for offset, size in combined_offsets:
224
yield offset, data[pos:pos + size]
225
pos += size
226
del data
227
if not len(offsets):
228
return
229
pending_offsets = deque(offsets)
230
combined_offsets = []
231
while len(pending_offsets):
232
offset, size = pending_offsets.popleft()
233
if not combined_offsets:
234
combined_offsets = [[offset, size]]
246
ranges = self.offsets_to_ranges(offsets)
247
mutter('http readv of %s collapsed %s offsets => %s',
248
relpath, len(offsets), ranges)
249
code, f = self._get(relpath, ranges)
250
for start, size in offsets:
251
f.seek(start, (start < 0) and 2 or 0)
252
start = f.tell()
253
data = f.read(size)
254
assert len(data) == size
255
yield start, data
256
257
@staticmethod
258
def offsets_to_ranges(offsets):
259
"""Turn a list of offsets and sizes into a list of byte ranges.
260
261
:param offsets: A list of tuples of (start, size). An empty list
262
is not accepted.
263
:return: a list of inclusive byte ranges (start, end)
264
Adjacent ranges will be combined.
265
"""
266
# Make sure we process sorted offsets
267
offsets = sorted(offsets)
268
269
prev_end = None
270
combined = []
271
272
for start, size in offsets:
273
end = start + size - 1
274
if prev_end is None:
275
combined.append([start, end])
276
elif start <= prev_end + 1:
277
combined[-1][1] = end
235
278
else:
236
if (len (combined_offsets) < 500 and
237
combined_offsets[-1][0] + combined_offsets[-1][1] == offset):
238
# combatible offset:
239
combined_offsets.append([offset, size])
240
else:
241
# incompatible, or over the threshold issue a read and yield
242
pending_offsets.appendleft((offset, size))
243
for result in do_combined_read(combined_offsets):
244
yield result
245
combined_offsets = []
246
# whatever is left is a single coalesced request
247
if len(combined_offsets):
248
for result in do_combined_read(combined_offsets):
249
yield result
279
combined.append([start, end])
280
prev_end = end
281
282
return combined
250
283
251
284
def put(self, relpath, f, mode=None):
252
285
"""Copy the file-like or string object into the location.
343
376
else:
344
377
return self.__class__(self.abspath(offset))
345
378
379
@staticmethod
380
def range_header(ranges, tail_amount):
381
"""Turn a list of bytes ranges into a HTTP Range header value.
382
383
:param offsets: A list of byte ranges, (start, end). An empty list
384
is not accepted.
385
386
:return: HTTP range header string.
387
"""
388
strings = []
389
for start, end in ranges:
390
strings.append('%d-%d' % (start, end))
391
392
if tail_amount:
393
strings.append('-%d' % tail_amount)
394
395
return ','.join(strings)
396
397
346
398
#---------------- test server facilities ----------------
347
399
# TODO: load these only when running tests
348
400
435
487
RequestHandlerClass)
436
488
self.test_case = test_case
437
489
490
438
491
class HttpServer(Server):
439
492
"""A test server for http transports."""
440
493
Loggerhead is a web-based interface for Breezy
Version: 2.0.1