24
from __future__ import absolute_import
25
from bisect import bisect
26
28
from cStringIO import StringIO
29
from bzrlib import errors
30
from bzrlib.trace import mutter
33
class ResponseRange(object):
34
"""A range in a RangeFile-object."""
36
__slots__ = ['_ent_start', '_ent_end', '_data_start']
38
def __init__(self, ent_start, ent_end, data_start):
39
self._ent_start = ent_start
40
self._ent_end = ent_end
41
self._data_start = data_start
43
def __cmp__(self, other):
44
"""Compare this to other.
46
We need this both for sorting, and so that we can
47
bisect the list of ranges.
37
class ResponseFile(object):
38
"""A wrapper around the http socket containing the result of a GET request.
40
Only read() and seek() (forward) are supported.
42
def __init__(self, path, infile):
45
:param path: File url, for error reports.
47
:param infile: File-like socket set at body start.
49
if isinstance(other, int):
50
# Later on we bisect for a starting point
51
# so we allow comparing against a single integer
52
return cmp(self._ent_start, other)
54
return cmp((self._ent_start, self._ent_end, self._data_start),
55
(other._ent_start, other._ent_end, other._data_start))
58
return "%s(%s-%s,%s)" % (self.__class__.__name__,
59
self._ent_start, self._ent_end,
63
class RangeFile(object):
64
"""File-like object that allow access to partial available data.
66
Specified by a set of ranges.
69
def __init__(self, path, input_file):
74
self._data = input_file.read()
76
def _add_range(self, ent_start, ent_end, data_start):
77
"""Add an entity range.
79
:param ent_start: Start offset of entity
80
:param ent_end: End offset of entity (inclusive)
81
:param data_start: Start offset of data in data stream.
83
self._ranges.append(ResponseRange(ent_start, ent_end, data_start))
84
self._len = max(self._len, ent_end)
86
def _finish_ranges(self):
90
"""Read size bytes from the current position in the file.
92
Reading across ranges is not supported.
94
# find the last range which has a start <= pos
95
i = bisect(self._ranges, self._pos) - 1
97
if i < 0 or self._pos > self._ranges[i]._ent_end:
98
raise errors.InvalidRange(self._path, self._pos)
102
# mutter('found range %s %s for pos %s', i, self._ranges[i], self._pos)
104
if (self._pos + size - 1) > r._ent_end:
105
raise errors.InvalidRange(self._path, self._pos)
107
start = r._data_start + (self._pos - r._ent_start)
109
# mutter("range read %d bytes at %d == %d-%d", size, self._pos,
111
self._pos += (end-start)
112
return self._data[start:end]
56
Dummy implementation for consistency with the 'file' API.
59
def read(self, size=-1):
60
"""Read size bytes from the current position in the file.
62
:param size: The number of bytes to read. Leave unspecified or pass
65
data = self._file.read(size)
66
self._pos += len(data)
70
data = self._file.readline()
71
self._pos += len(data)
77
def seek(self, offset, whence=os.SEEK_SET):
78
if whence == os.SEEK_SET:
79
if offset < self._pos:
81
"Can't seek backwards, pos: %s, offset: %s"
82
% (self._pos, offset))
83
to_discard = offset - self._pos
84
elif whence == os.SEEK_CUR:
87
raise AssertionError("Can't seek backwards")
89
# Just discard the unwanted bytes
92
# A RangeFile expects the following grammar (simplified to outline the
93
# assumptions we rely upon).
98
# single_range: content_range_header data
100
# multiple_range: boundary_header boundary (content_range_header data boundary)+
102
class RangeFile(ResponseFile):
103
"""File-like object that allow access to partial available data.
105
All accesses should happen sequentially since the acquisition occurs during
106
an http response reception (as sockets can't be seeked, we simulate the
107
seek by just reading and discarding the data).
109
The access pattern is defined by a set of ranges discovered as reading
110
progress. Only one range is available at a given time, so all accesses
111
should happen with monotonically increasing offsets.
114
# in _checked_read() below, we may have to discard several MB in the worst
115
# case. To avoid buffering that much, we read and discard by chunks
116
# instead. The underlying file is either a socket or a StringIO, so reading
117
# 8k chunks should be fine.
118
_discarded_buf_size = 8192
120
# maximum size of read requests -- used to avoid MemoryError issues in recv
121
_max_read_size = 512 * 1024
123
def __init__(self, path, infile):
126
:param path: File url, for error reports.
128
:param infile: File-like socket set at body start.
130
super(RangeFile, self).__init__(path, infile)
131
self._boundary = None
132
# When using multi parts response, this will be set with the headers
133
# associated with the range currently read.
135
# Default to the whole file of unspecified size
136
self.set_range(0, -1)
138
def set_range(self, start, size):
139
"""Change the range mapping"""
142
# Set the new _pos since that's what we want to expose
143
self._pos = self._start
145
def set_boundary(self, boundary):
146
"""Define the boundary used in a multi parts message.
148
The file should be at the beginning of the body, the first range
149
definition is read and taken into account.
151
self._boundary = boundary
152
# Decode the headers and setup the first range
154
self.read_range_definition()
156
def read_boundary(self):
157
"""Read the boundary headers defining a new range"""
158
boundary_line = '\r\n'
159
while boundary_line == '\r\n':
160
# RFC2616 19.2 Additional CRLFs may precede the first boundary
162
# To be on the safe side we allow it before any boundary line
163
boundary_line = self._file.readline()
165
if boundary_line == '':
166
# A timeout in the proxy server caused the response to end early.
167
# See launchpad bug 198646.
168
raise errors.HttpBoundaryMissing(
172
if boundary_line != '--' + self._boundary + '\r\n':
173
# rfc822.unquote() incorrectly unquotes strings enclosed in <>
174
# IIS 6 and 7 incorrectly wrap boundary strings in <>
175
# together they make a beautiful bug, which we will be gracious
177
if (self._unquote_boundary(boundary_line) !=
178
'--' + self._boundary + '\r\n'):
179
raise errors.InvalidHttpResponse(
181
"Expected a boundary (%s) line, got '%s'"
182
% (self._boundary, boundary_line))
184
def _unquote_boundary(self, b):
185
return b[:2] + rfc822.unquote(b[2:-2]) + b[-2:]
187
def read_range_definition(self):
188
"""Read a new range definition in a multi parts message.
190
Parse the headers including the empty line following them so that we
191
are ready to read the data itself.
193
self._headers = httplib.HTTPMessage(self._file, seekable=0)
194
# Extract the range definition
195
content_range = self._headers.getheader('content-range', None)
196
if content_range is None:
197
raise errors.InvalidHttpResponse(
199
'Content-Range header missing in a multi-part response')
200
self.set_range_from_header(content_range)
202
def set_range_from_header(self, content_range):
203
"""Helper to set the new range from its description in the headers"""
205
rtype, values = content_range.split()
207
raise errors.InvalidHttpRange(self._path, content_range,
210
raise errors.InvalidHttpRange(self._path, content_range,
211
"Unsupported range type '%s'" % rtype)
213
# We don't need total, but note that it may be either the file size
214
# or '*' if the server can't or doesn't want to return the file
216
start_end, total = values.split('/')
217
start, end = start_end.split('-')
221
raise errors.InvalidHttpRange(self._path, content_range,
222
'Invalid range values')
223
size = end - start + 1
225
raise errors.InvalidHttpRange(self._path, content_range,
226
'Invalid range, size <= 0')
227
self.set_range(start, size)
229
def _checked_read(self, size):
230
"""Read the file checking for short reads.
232
The data read is discarded along the way.
237
data = self._file.read(min(remaining, self._discarded_buf_size))
238
remaining -= len(data)
240
raise errors.ShortReadvError(self._path, pos, size,
244
def _seek_to_next_range(self):
245
# We will cross range boundaries
246
if self._boundary is None:
247
# If we don't have a boundary, we can't find another range
248
raise errors.InvalidRange(self._path, self._pos,
249
"Range (%s, %s) exhausted"
250
% (self._start, self._size))
252
self.read_range_definition()
254
def read(self, size=-1):
255
"""Read size bytes from the current position in the file.
257
Reading across ranges is not supported. We rely on the underlying http
258
client to clean the socket if we leave bytes unread. This may occur for
259
the final boundary line of a multipart response or for any range
260
request not entirely consumed by the client (due to offset coalescing)
262
:param size: The number of bytes to read. Leave unspecified or pass
266
and self._pos == self._start + self._size):
270
self._seek_to_next_range()
271
elif self._pos < self._start:
272
raise errors.InvalidRange(
273
self._path, self._pos,
274
"Can't read %s bytes before range (%s, %s)"
275
% (size, self._start, self._size))
277
if size > 0 and self._pos + size > self._start + self._size:
278
raise errors.InvalidRange(
279
self._path, self._pos,
280
"Can't read %s bytes across range (%s, %s)"
281
% (size, self._start, self._size))
283
# read data from file
287
# Don't read past the range definition
288
limited = self._start + self._size - self._pos
290
limited = min(limited, size)
291
osutils.pumpfile(self._file, buffer, limited, self._max_read_size)
292
data = buffer.getvalue()
294
# Update _pos respecting the data effectively read
295
self._pos += len(data)
114
298
def seek(self, offset, whence=0):
299
start_pos = self._pos
117
302
elif whence == 1:
303
final_pos = start_pos + offset
119
304
elif whence == 2:
120
self._pos = self._len + offset
306
final_pos = self._start + self._size + offset # offset < 0
308
raise errors.InvalidRange(
309
self._path, self._pos,
310
"RangeFile: can't seek from end while size is unknown")
122
312
raise ValueError("Invalid value %s for whence." % whence)
314
if final_pos < self._pos:
315
# Can't seek backwards
316
raise errors.InvalidRange(
317
self._path, self._pos,
318
'RangeFile: trying to seek backwards to %s' % final_pos)
321
cur_limit = self._start + self._size
322
while final_pos > cur_limit:
323
# We will cross range boundaries
324
remain = cur_limit - self._pos
326
# Finish reading the current range
327
self._checked_read(remain)
328
self._seek_to_next_range()
329
cur_limit = self._start + self._size
331
size = final_pos - self._pos
332
if size > 0: # size can be < 0 if we crossed a range boundary
333
# We don't need the data, just read it and throw it away
334
self._checked_read(size)
131
class HttpRangeResponse(RangeFile):
132
"""A single-range HTTP response."""
134
# TODO: jam 20060706 Consider compiling these regexes on demand
135
_CONTENT_RANGE_RE = re.compile(
136
'\s*([^\s]+)\s+([0-9]+)-([0-9]+)/([0-9]+)\s*$')
138
def __init__(self, path, content_range, input_file):
139
# mutter("parsing 206 non-multipart response for %s", path)
140
RangeFile.__init__(self, path, input_file)
141
start, end = self._parse_range(content_range, path)
142
self._add_range(start, end, 0)
143
self._finish_ranges()
146
def _parse_range(range, path='<unknown>'):
147
"""Parse an http Content-range header and return start + end
149
:param range: The value for Content-range
150
:param path: Provide to give better error messages.
151
:return: (start, end) A tuple of integers
153
match = HttpRangeResponse._CONTENT_RANGE_RE.match(range)
155
raise errors.InvalidHttpRange(path, range,
156
"Invalid Content-range")
158
rtype, start, end, total = match.groups()
161
raise errors.InvalidHttpRange(path, range,
162
"Unsupported range type '%s'" % (rtype,))
167
except ValueError, e:
168
raise errors.InvalidHttpRange(path, range, str(e))
173
class HttpMultipartRangeResponse(RangeFile):
174
"""A multi-range HTTP response."""
176
_CONTENT_TYPE_RE = re.compile(
177
'^\s*multipart/byteranges\s*;\s*boundary\s*=\s*(.*?)\s*$')
179
# Start with --<boundary>\r\n
180
# and ignore all headers ending in \r\n
181
# except for content-range:
182
# and find the two trailing \r\n separators
183
# indicating the start of the text
184
# TODO: jam 20060706 This requires exact conformance
185
# to the spec, we probably could relax the requirement
186
# of \r\n, and use something more like (\r?\n)
188
"^--%s(?:\r\n(?:(?:content-range:([^\r]+))|[^\r]+))+\r\n\r\n")
190
def __init__(self, path, content_type, input_file):
191
# mutter("parsing 206 multipart response for %s", path)
192
# TODO: jam 20060706 Is it valid to initialize a
193
# grandparent without initializing parent?
194
RangeFile.__init__(self, path, input_file)
196
self.boundary_regex = self._parse_boundary(content_type, path)
198
for match in self.boundary_regex.finditer(self._data):
199
ent_start, ent_end = HttpRangeResponse._parse_range(match.group(1), path)
200
self._add_range(ent_start, ent_end, match.end())
202
self._finish_ranges()
205
def _parse_boundary(ctype, path='<unknown>'):
206
"""Parse the Content-type field.
208
This expects a multipart Content-type, and returns a
209
regex which is capable of finding the boundaries
210
in the multipart data.
212
match = HttpMultipartRangeResponse._CONTENT_TYPE_RE.match(ctype)
214
raise errors.InvalidHttpContentType(path, ctype,
215
"Expected multipart/byteranges with boundary")
217
boundary = match.group(1)
218
# mutter('multipart boundary is %s', boundary)
219
pattern = HttpMultipartRangeResponse._BOUNDARY_PATT
220
return re.compile(pattern % re.escape(boundary),
221
re.IGNORECASE | re.MULTILINE)
224
def _is_multipart(content_type):
225
return content_type.startswith('multipart/byteranges;')
228
def handle_response(url, code, headers, data):
229
"""Interpret the code & headers and return a HTTP response.
231
This is a factory method which returns an appropriate HTTP response
232
based on the code & headers it's given.
340
def handle_response(url, code, msg, data):
341
"""Interpret the code & headers and wrap the provided data in a RangeFile.
343
This is a factory method which returns an appropriate RangeFile based on
344
the code & headers it's given.
234
346
:param url: The url being processed. Mostly for error reporting
235
347
:param code: The integer HTTP response code
236
:param headers: A dict-like object that contains the HTTP response headers
348
:param msg: An HTTPMessage containing the headers for the response
237
349
:param data: A file-like object that can be read() to get the
239
:return: A file-like object that can seek()+read() the
351
:return: A file-like object that can seek()+read() the
240
352
ranges indicated by the headers.
245
content_type = headers['Content-Type']
247
raise errors.InvalidHttpContentType(url, '',
248
msg = 'Missing Content-Type')
250
if _is_multipart(content_type):
356
rfile = ResponseFile(url, data)
358
rfile = RangeFile(url, data)
359
content_type = msg.getheader('content-type', None)
360
if content_type is None:
361
# When there is no content-type header we treat the response as
362
# being of type 'application/octet-stream' as per RFC2616 section
364
# Therefore it is obviously not multipart
365
content_type = 'application/octet-stream'
368
is_multipart = (msg.getmaintype() == 'multipart'
369
and msg.getsubtype() == 'byteranges')
251
372
# Full fledged multipart response
252
return HttpMultipartRangeResponse(url, content_type, data)
373
rfile.set_boundary(msg.getparam('boundary'))
254
375
# A response to a range request, but not multipart
256
content_range = headers['Content-Range']
376
content_range = msg.getheader('content-range', None)
377
if content_range is None:
258
378
raise errors.InvalidHttpResponse(url,
259
379
'Missing the Content-Range header in a 206 range response')
260
return HttpRangeResponse(url, content_range, data)
262
# A regular non-range response, unfortunately the result from
263
# urllib doesn't support seek, so we wrap it in a StringIO
264
tell = getattr(data, 'tell', None)
266
return StringIO(data.read())
269
raise errors.NoSuchFile(url)
380
rfile.set_range_from_header(content_range)
382
raise errors.InvalidHttpResponse(url,
383
'Unknown response code %s' % code)
271
raise errors.InvalidHttpResponse(url, "Unknown response code %s" % (code,))
added
removed
bzrlib/transport/http/response.py
