25
from bisect import bisect
26
from cStringIO import StringIO
29
from bzrlib import errors
30
from bzrlib.trace import mutter
33
class ResponseRange(object):
34
"""A range in a RangeFile-object."""
36
__slots__ = ['_ent_start', '_ent_end', '_data_start']
38
def __init__(self, ent_start, ent_end, data_start):
39
self._ent_start = ent_start
40
self._ent_end = ent_end
41
self._data_start = data_start
43
def __cmp__(self, other):
44
"""Compare this to other.
46
We need this both for sorting, and so that we can
47
bisect the list of ranges.
49
if isinstance(other, int):
50
# Later on we bisect for a starting point
51
# so we allow comparing against a single integer
52
return cmp(self._ent_start, other)
54
return cmp((self._ent_start, self._ent_end, self._data_start),
55
(other._ent_start, other._ent_end, other._data_start))
58
return "%s(%s-%s,%s)" % (self.__class__.__name__,
59
self._ent_start, self._ent_end,
33
# A RangeFile expects the following grammar (simplified to outline the
34
# assumptions we rely upon).
40
# whole_file: [content_length_header] data
42
# single_range: content_range_header data
44
# multiple_range: boundary_header boundary (content_range_header data boundary)+
65
46
class RangeFile(object):
66
47
"""File-like object that allow access to partial available data.
68
Specified by a set of ranges.
49
All accesses should happen sequentially since the acquisition occurs during
50
an http response reception (as sockets can't be seeked, we simulate the
51
seek by just reading and discarding the data).
53
The access pattern is defined by a set of ranges discovered as reading
54
progress. Only one range is available at a given time, so all accesses
55
should happen with monotonically increasing offsets.
71
def __init__(self, path, input_file):
58
# in _checked_read() below, we may have to discard several MB in the worst
59
# case. To avoid buffering that much, we read and discard by chunks
60
# instead. The underlying file is either a socket or a StringIO, so reading
61
# 8k chunks should be fine.
62
_discarded_buf_size = 8192
64
def __init__(self, path, infile):
67
:param path: File url, for error reports.
68
:param infile: File-like socket set at body start.
76
self._data = input_file.read()
78
def _add_range(self, ent_start, ent_end, data_start):
79
"""Add an entity range.
81
:param ent_start: Start offset of entity
82
:param ent_end: End offset of entity (inclusive)
83
:param data_start: Start offset of data in data stream.
85
self._ranges.append(ResponseRange(ent_start, ent_end, data_start))
86
self._len = max(self._len, ent_end)
88
def _finish_ranges(self):
73
# When using multi parts response, this will be set with the headers
74
# associated with the range currently read.
76
# Default to the whole file of unspecified size
79
def set_range(self, start, size):
80
"""Change the range mapping"""
83
# Set the new _pos since that's what we want to expose
84
self._pos = self._start
86
def set_boundary(self, boundary):
87
"""Define the boundary used in a multi parts message.
89
The file should be at the beginning of the body, the first range
90
definition is read and taken into account.
92
self._boundary = boundary
93
# Decode the headers and setup the first range
95
self.read_range_definition()
97
def read_boundary(self):
98
"""Read the boundary headers defining a new range"""
99
boundary_line = '\r\n'
100
while boundary_line == '\r\n':
101
# RFC2616 19.2 Additional CRLFs may precede the first boundary
103
# To be on the safe side we allow it before any boundary line
104
boundary_line = self._file.readline()
105
if boundary_line != '--' + self._boundary + '\r\n':
106
raise errors.InvalidHttpResponse(
108
"Expected a boundary (%s) line, got '%s'" % (self._boundary,
111
def read_range_definition(self):
112
"""Read a new range definition in a multi parts message.
114
Parse the headers including the empty line following them so that we
115
are ready to read the data itself.
117
self._headers = httplib.HTTPMessage(self._file, seekable=0)
118
# Extract the range definition
119
content_range = self._headers.getheader('content-range', None)
120
if content_range is None:
121
raise errors.InvalidHttpResponse(
123
'Content-Range header missing in a multi-part response')
124
self.set_range_from_header(content_range)
126
def set_range_from_header(self, content_range):
127
"""Helper to set the new range from its description in the headers"""
129
rtype, values = content_range.split()
131
raise errors.InvalidHttpRange(self._path, content_range,
134
raise errors.InvalidHttpRange(self._path, content_range,
135
"Unsupported range type '%s'" % rtype)
137
# We don't need total, but note that it may be either the file size
138
# or '*' if the server can't or doesn't want to return the file
140
start_end, total = values.split('/')
141
start, end = start_end.split('-')
145
raise errors.InvalidHttpRange(self._path, content_range,
146
'Invalid range values')
147
size = end - start + 1
149
raise errors.InvalidHttpRange(self._path, content_range,
150
'Invalid range, size <= 0')
151
self.set_range(start, size)
153
def _checked_read(self, size):
154
"""Read the file checking for short reads.
156
The data read is discarded along the way.
161
data = self._file.read(min(remaining, self._discarded_buf_size))
162
remaining -= len(data)
164
raise errors.ShortReadvError(self._path, pos, size,
168
def _seek_to_next_range(self):
169
# We will cross range boundaries
170
if self._boundary is None:
171
# If we don't have a boundary, we can't find another range
172
raise errors.InvalidRange(self._path, self._pos,
173
"Range (%s, %s) exhausted"
174
% (self._start, self._size))
176
self.read_range_definition()
178
def read(self, size=-1):
92
179
"""Read size bytes from the current position in the file.
94
Reading across ranges is not supported.
181
Reading across ranges is not supported. We rely on the underlying http
182
client to clean the socket if we leave bytes unread. This may occur for
183
the final boundary line of a multipart response or for any range
184
request not entirely consumed by the client (due to offset coalescing)
96
# find the last range which has a start <= pos
97
i = bisect(self._ranges, self._pos) - 1
99
if i < 0 or self._pos > self._ranges[i]._ent_end:
100
mutter('Bisect for pos: %s failed. Found offset: %d, ranges:%s',
101
self._pos, i, self._ranges)
102
raise errors.InvalidRange(self._path, self._pos)
106
# mutter('found range %s %s for pos %s', i, self._ranges[i], self._pos)
108
if (self._pos + size - 1) > r._ent_end:
109
raise errors.InvalidRange(self._path, self._pos)
111
start = r._data_start + (self._pos - r._ent_start)
113
# mutter("range read %d bytes at %d == %d-%d", size, self._pos,
115
self._pos += (end-start)
116
return self._data[start:end]
187
and self._pos == self._start + self._size):
191
self._seek_to_next_range()
192
elif self._pos < self._start:
193
raise errors.InvalidRange(
194
self._path, self._pos,
195
"Can't read %s bytes before range (%s, %s)"
196
% (size, self._start, self._size))
198
if size > 0 and self._pos + size > self._start + self._size:
199
raise errors.InvalidRange(
200
self._path, self._pos,
201
"Can't read %s bytes across range (%s, %s)"
202
% (size, self._start, self._size))
205
# Don't read past the range definition
206
limited = self._start + self._size - self._pos
208
limited = min(limited, size)
209
data = self._file.read(limited)
211
# Size of file unknown, the user may have specified a size or not,
212
# we delegate that to the filesocket object (-1 means read until
214
data = self._file.read(size)
215
# Update _pos respecting the data effectively read
216
self._pos += len(data)
118
219
def seek(self, offset, whence=0):
220
start_pos = self._pos
121
223
elif whence == 1:
224
final_pos = start_pos + offset
123
225
elif whence == 2:
124
self._pos = self._len + offset
227
final_pos = self._start + self._size + offset # offset < 0
229
raise errors.InvalidRange(
230
self._path, self._pos,
231
"RangeFile: can't seek from end while size is unknown")
126
233
raise ValueError("Invalid value %s for whence." % whence)
235
if final_pos < self._pos:
236
# Can't seek backwards
237
raise errors.InvalidRange(
238
self._path, self._pos,
239
'RangeFile: trying to seek backwards to %s' % final_pos)
242
cur_limit = self._start + self._size
243
while final_pos > cur_limit:
244
# We will cross range boundaries
245
remain = cur_limit - self._pos
247
# Finish reading the current range
248
self._checked_read(remain)
249
self._seek_to_next_range()
250
cur_limit = self._start + self._size
252
size = final_pos - self._pos
253
if size > 0: # size can be < 0 if we crossed a range boundary
254
# We don't need the data, just read it and throw it away
255
self._checked_read(size)
135
class HttpRangeResponse(RangeFile):
136
"""A single-range HTTP response."""
138
# TODO: jam 20060706 Consider compiling these regexes on demand
139
_CONTENT_RANGE_RE = re.compile(
140
r'\s*([^\s]+)\s+([0-9]+)-([0-9]+)/([0-9]+)\s*$')
142
def __init__(self, path, content_range, input_file):
143
# mutter("parsing 206 non-multipart response for %s", path)
144
RangeFile.__init__(self, path, input_file)
145
start, end = self._parse_range(content_range, path)
146
self._add_range(start, end, 0)
147
self._finish_ranges()
150
def _parse_range(range, path='<unknown>'):
151
"""Parse an http Content-range header and return start + end
153
:param range: The value for Content-range
154
:param path: Provide to give better error messages.
155
:return: (start, end) A tuple of integers
157
match = HttpRangeResponse._CONTENT_RANGE_RE.match(range)
159
raise errors.InvalidHttpRange(path, range,
160
"Invalid Content-range")
162
rtype, start, end, total = match.groups()
165
raise errors.InvalidHttpRange(path, range,
166
"Unsupported range type '%s'" % (rtype,))
171
except ValueError, e:
172
raise errors.InvalidHttpRange(path, range, str(e))
177
class HttpMultipartRangeResponse(RangeFile):
178
"""A multi-range HTTP response."""
180
_CONTENT_TYPE_RE = re.compile(
181
r'^\s*multipart/byteranges\s*;\s*boundary\s*=\s*("?)([^"]*?)\1\s*$')
183
# Start with --<boundary>\r\n
184
# and ignore all headers ending in \r\n
185
# except for content-range:
186
# and find the two trailing \r\n separators
187
# indicating the start of the text
188
# TODO: jam 20060706 This requires exact conformance
189
# to the spec, we probably could relax the requirement
190
# of \r\n, and use something more like (\r?\n)
192
"^--%s(?:\r\n(?:(?:content-range:([^\r]+))|[^\r]+))+\r\n\r\n")
194
def __init__(self, path, content_type, input_file):
195
# mutter("parsing 206 multipart response for %s", path)
196
# TODO: jam 20060706 Is it valid to initialize a
197
# grandparent without initializing parent?
198
RangeFile.__init__(self, path, input_file)
200
self.boundary_regex = self._parse_boundary(content_type, path)
201
# mutter('response:\n%r', self._data)
203
for match in self.boundary_regex.finditer(self._data):
204
ent_start, ent_end = HttpRangeResponse._parse_range(match.group(1),
206
self._add_range(ent_start, ent_end, match.end())
208
self._finish_ranges()
211
def _parse_boundary(ctype, path='<unknown>'):
212
"""Parse the Content-type field.
214
This expects a multipart Content-type, and returns a
215
regex which is capable of finding the boundaries
216
in the multipart data.
218
match = HttpMultipartRangeResponse._CONTENT_TYPE_RE.match(ctype)
220
raise errors.InvalidHttpContentType(path, ctype,
221
"Expected multipart/byteranges with boundary")
223
boundary = match.group(2)
224
# mutter('multipart boundary is %s', boundary)
225
pattern = HttpMultipartRangeResponse._BOUNDARY_PATT
226
return re.compile(pattern % re.escape(boundary),
227
re.IGNORECASE | re.MULTILINE)
230
def _is_multipart(content_type):
231
return content_type.startswith('multipart/byteranges;')
234
def handle_response(url, code, headers, data):
235
"""Interpret the code & headers and return a HTTP response.
237
This is a factory method which returns an appropriate HTTP response
238
based on the code & headers it's given.
261
def handle_response(url, code, msg, data):
262
"""Interpret the code & headers and wrap the provided data in a RangeFile.
264
This is a factory method which returns an appropriate RangeFile based on
265
the code & headers it's given.
240
267
:param url: The url being processed. Mostly for error reporting
241
268
:param code: The integer HTTP response code
242
:param headers: A dict-like object that contains the HTTP response headers
269
:param msg: An HTTPMessage containing the headers for the response
243
270
:param data: A file-like object that can be read() to get the
245
272
:return: A file-like object that can seek()+read() the
246
273
ranges indicated by the headers.
251
content_type = headers['Content-Type']
253
# When there is no content-type header we treat
254
# the response as being of type 'application/octet-stream' as per
255
# RFC2616 section 7.2.1.
275
rfile = RangeFile(url, data)
278
size = msg.getheader('content-length', None)
283
rfile.set_range(0, size)
285
content_type = msg.getheader('content-type', None)
286
if content_type is None:
287
# When there is no content-type header we treat the response as
288
# being of type 'application/octet-stream' as per RFC2616 section
256
290
# Therefore it is obviously not multipart
257
291
content_type = 'application/octet-stream'
258
292
is_multipart = False
260
is_multipart = _is_multipart(content_type)
294
is_multipart = (msg.getmaintype() == 'multipart'
295
and msg.getsubtype() == 'byteranges')
263
298
# Full fledged multipart response
264
return HttpMultipartRangeResponse(url, content_type, data)
299
rfile.set_boundary(msg.getparam('boundary'))
266
301
# A response to a range request, but not multipart
268
content_range = headers['Content-Range']
302
content_range = msg.getheader('content-range', None)
303
if content_range is None:
270
304
raise errors.InvalidHttpResponse(url,
271
305
'Missing the Content-Range header in a 206 range response')
272
return HttpRangeResponse(url, content_range, data)
274
# A regular non-range response, unfortunately the result from
275
# urllib doesn't support seek, so we wrap it in a StringIO
276
tell = getattr(data, 'tell', None)
278
return StringIO(data.read())
281
raise errors.NoSuchFile(url)
282
# Some servers will retun "400: Bad Request" when too much ranges are
284
elif code in (400, 416):
285
# We don't know which, but one of the ranges we specified
286
# was wrong. So we raise with 0 for a lack of a better
288
raise errors.InvalidRange(url,0)
290
# TODO: jam 20060713 Properly handle redirects (302 Found, etc)
291
# The '_get' code says to follow redirects, we probably
292
# should actually handle the return values
306
rfile.set_range_from_header(content_range)
294
raise errors.InvalidHttpResponse(url, "Unknown response code %s"
308
raise errors.InvalidHttpResponse(url,
309
'Unknown response code %s' % code)
added
removed
bzrlib/transport/http/response.py
