24
from __future__ import absolute_import
26
from bisect import bisect
27
27
from cStringIO import StringIO
36
# A RangeFile expects the following grammar (simplified to outline the
37
# assumptions we rely upon).
43
# whole_file: [content_length_header] data
45
# single_range: content_range_header data
47
# multiple_range: boundary_header boundary (content_range_header data boundary)+
30
from bzrlib import errors
31
from bzrlib.trace import mutter
34
class ResponseRange(object):
35
"""A range in a RangeFile-object."""
37
__slots__ = ['_ent_start', '_ent_end', '_data_start']
39
def __init__(self, ent_start, ent_end, data_start):
40
self._ent_start = ent_start
41
self._ent_end = ent_end
42
self._data_start = data_start
44
def __cmp__(self, other):
45
"""Compare this to other.
47
We need this both for sorting, and so that we can
48
bisect the list of ranges.
50
if isinstance(other, int):
51
# Later on we bisect for a starting point
52
# so we allow comparing against a single integer
53
return cmp(self._ent_start, other)
55
return cmp((self._ent_start, self._ent_end, self._data_start),
56
(other._ent_start, other._ent_end, other._data_start))
59
return "%s(%s-%s,%s)" % (self.__class__.__name__,
60
self._ent_start, self._ent_end,
49
64
class RangeFile(object):
50
65
"""File-like object that allow access to partial available data.
52
All accesses should happen sequentially since the acquisition occurs during
53
an http response reception (as sockets can't be seeked, we simulate the
54
seek by just reading and discarding the data).
56
The access pattern is defined by a set of ranges discovered as reading
57
progress. Only one range is available at a given time, so all accesses
58
should happen with monotonically increasing offsets.
67
Specified by a set of ranges.
61
# in _checked_read() below, we may have to discard several MB in the worst
62
# case. To avoid buffering that much, we read and discard by chunks
63
# instead. The underlying file is either a socket or a StringIO, so reading
64
# 8k chunks should be fine.
65
_discarded_buf_size = 8192
67
# maximum size of read requests -- used to avoid MemoryError issues in recv
68
_max_read_size = 512 * 1024
70
def __init__(self, path, infile):
73
:param path: File url, for error reports.
74
:param infile: File-like socket set at body start.
70
def __init__(self, path, input_file):
79
# When using multi parts response, this will be set with the headers
80
# associated with the range currently read.
82
# Default to the whole file of unspecified size
88
Dummy implementation for consistency with the 'file' API.
91
def set_range(self, start, size):
92
"""Change the range mapping"""
95
# Set the new _pos since that's what we want to expose
96
self._pos = self._start
98
def set_boundary(self, boundary):
99
"""Define the boundary used in a multi parts message.
101
The file should be at the beginning of the body, the first range
102
definition is read and taken into account.
104
self._boundary = boundary
105
# Decode the headers and setup the first range
107
self.read_range_definition()
109
def read_boundary(self):
110
"""Read the boundary headers defining a new range"""
111
boundary_line = '\r\n'
112
while boundary_line == '\r\n':
113
# RFC2616 19.2 Additional CRLFs may precede the first boundary
115
# To be on the safe side we allow it before any boundary line
116
boundary_line = self._file.readline()
118
if boundary_line == '':
119
# A timeout in the proxy server caused the response to end early.
120
# See launchpad bug 198646.
121
raise errors.HttpBoundaryMissing(
125
if boundary_line != '--' + self._boundary + '\r\n':
126
# rfc822.unquote() incorrectly unquotes strings enclosed in <>
127
# IIS 6 and 7 incorrectly wrap boundary strings in <>
128
# together they make a beautiful bug, which we will be gracious
130
if (self._unquote_boundary(boundary_line) !=
131
'--' + self._boundary + '\r\n'):
132
raise errors.InvalidHttpResponse(
134
"Expected a boundary (%s) line, got '%s'"
135
% (self._boundary, boundary_line))
137
def _unquote_boundary(self, b):
138
return b[:2] + rfc822.unquote(b[2:-2]) + b[-2:]
140
def read_range_definition(self):
141
"""Read a new range definition in a multi parts message.
143
Parse the headers including the empty line following them so that we
144
are ready to read the data itself.
146
self._headers = httplib.HTTPMessage(self._file, seekable=0)
147
# Extract the range definition
148
content_range = self._headers.getheader('content-range', None)
149
if content_range is None:
150
raise errors.InvalidHttpResponse(
152
'Content-Range header missing in a multi-part response')
153
self.set_range_from_header(content_range)
155
def set_range_from_header(self, content_range):
156
"""Helper to set the new range from its description in the headers"""
158
rtype, values = content_range.split()
160
raise errors.InvalidHttpRange(self._path, content_range,
163
raise errors.InvalidHttpRange(self._path, content_range,
164
"Unsupported range type '%s'" % rtype)
166
# We don't need total, but note that it may be either the file size
167
# or '*' if the server can't or doesn't want to return the file
169
start_end, total = values.split('/')
170
start, end = start_end.split('-')
174
raise errors.InvalidHttpRange(self._path, content_range,
175
'Invalid range values')
176
size = end - start + 1
178
raise errors.InvalidHttpRange(self._path, content_range,
179
'Invalid range, size <= 0')
180
self.set_range(start, size)
182
def _checked_read(self, size):
183
"""Read the file checking for short reads.
185
The data read is discarded along the way.
190
data = self._file.read(min(remaining, self._discarded_buf_size))
191
remaining -= len(data)
193
raise errors.ShortReadvError(self._path, pos, size,
197
def _seek_to_next_range(self):
198
# We will cross range boundaries
199
if self._boundary is None:
200
# If we don't have a boundary, we can't find another range
201
raise errors.InvalidRange(self._path, self._pos,
202
"Range (%s, %s) exhausted"
203
% (self._start, self._size))
205
self.read_range_definition()
207
def read(self, size=-1):
75
self._data = input_file.read()
77
def _add_range(self, ent_start, ent_end, data_start):
78
"""Add an entity range.
80
:param ent_start: Start offset of entity
81
:param ent_end: End offset of entity (inclusive)
82
:param data_start: Start offset of data in data stream.
84
self._ranges.append(ResponseRange(ent_start, ent_end, data_start))
85
self._len = max(self._len, ent_end)
87
def _finish_ranges(self):
208
91
"""Read size bytes from the current position in the file.
210
Reading across ranges is not supported. We rely on the underlying http
211
client to clean the socket if we leave bytes unread. This may occur for
212
the final boundary line of a multipart response or for any range
213
request not entirely consumed by the client (due to offset coalescing)
215
:param size: The number of bytes to read. Leave unspecified or pass
93
Reading across ranges is not supported.
219
and self._pos == self._start + self._size):
223
self._seek_to_next_range()
224
elif self._pos < self._start:
225
raise errors.InvalidRange(
226
self._path, self._pos,
227
"Can't read %s bytes before range (%s, %s)"
228
% (size, self._start, self._size))
230
if size > 0 and self._pos + size > self._start + self._size:
231
raise errors.InvalidRange(
232
self._path, self._pos,
233
"Can't read %s bytes across range (%s, %s)"
234
% (size, self._start, self._size))
236
# read data from file
240
# Don't read past the range definition
241
limited = self._start + self._size - self._pos
243
limited = min(limited, size)
244
osutils.pumpfile(self._file, buffer, limited, self._max_read_size)
245
data = buffer.getvalue()
247
# Update _pos respecting the data effectively read
248
self._pos += len(data)
95
# find the last range which has a start <= pos
96
i = bisect(self._ranges, self._pos) - 1
98
if i < 0 or self._pos > self._ranges[i]._ent_end:
99
raise errors.InvalidRange(self._path, self._pos)
103
# mutter('found range %s %s for pos %s', i, self._ranges[i], self._pos)
105
if (self._pos + size - 1) > r._ent_end:
106
raise errors.InvalidRange(self._path, self._pos)
108
start = r._data_start + (self._pos - r._ent_start)
110
# mutter("range read %d bytes at %d == %d-%d", size, self._pos,
112
self._pos += (end-start)
113
return self._data[start:end]
251
115
def seek(self, offset, whence=0):
252
start_pos = self._pos
255
118
elif whence == 1:
256
final_pos = start_pos + offset
257
120
elif whence == 2:
259
final_pos = self._start + self._size + offset # offset < 0
261
raise errors.InvalidRange(
262
self._path, self._pos,
263
"RangeFile: can't seek from end while size is unknown")
121
self._pos = self._len + offset
265
123
raise ValueError("Invalid value %s for whence." % whence)
267
if final_pos < self._pos:
268
# Can't seek backwards
269
raise errors.InvalidRange(
270
self._path, self._pos,
271
'RangeFile: trying to seek backwards to %s' % final_pos)
274
cur_limit = self._start + self._size
275
while final_pos > cur_limit:
276
# We will cross range boundaries
277
remain = cur_limit - self._pos
279
# Finish reading the current range
280
self._checked_read(remain)
281
self._seek_to_next_range()
282
cur_limit = self._start + self._size
284
size = final_pos - self._pos
285
if size > 0: # size can be < 0 if we crossed a range boundary
286
# We don't need the data, just read it and throw it away
287
self._checked_read(size)
293
def handle_response(url, code, msg, data):
294
"""Interpret the code & headers and wrap the provided data in a RangeFile.
296
This is a factory method which returns an appropriate RangeFile based on
297
the code & headers it's given.
132
class HttpRangeResponse(RangeFile):
133
"""A single-range HTTP response."""
135
# TODO: jam 20060706 Consider compiling these regexes on demand
136
_CONTENT_RANGE_RE = re.compile(
137
'\s*([^\s]+)\s+([0-9]+)-([0-9]+)/([0-9]+)\s*$')
139
def __init__(self, path, content_range, input_file):
140
# mutter("parsing 206 non-multipart response for %s", path)
141
RangeFile.__init__(self, path, input_file)
142
start, end = self._parse_range(content_range, path)
143
self._add_range(start, end, 0)
144
self._finish_ranges()
147
def _parse_range(range, path='<unknown>'):
148
"""Parse an http Content-range header and return start + end
150
:param range: The value for Content-range
151
:param path: Provide to give better error messages.
152
:return: (start, end) A tuple of integers
154
match = HttpRangeResponse._CONTENT_RANGE_RE.match(range)
156
raise errors.InvalidHttpRange(path, range,
157
"Invalid Content-range")
159
rtype, start, end, total = match.groups()
162
raise errors.InvalidHttpRange(path, range,
163
"Unsupported range type '%s'" % (rtype,))
168
except ValueError, e:
169
raise errors.InvalidHttpRange(path, range, str(e))
174
class HttpMultipartRangeResponse(RangeFile):
175
"""A multi-range HTTP response."""
177
_CONTENT_TYPE_RE = re.compile(
178
'^\s*multipart/byteranges\s*;\s*boundary\s*=\s*(.*?)\s*$')
180
# Start with --<boundary>\r\n
181
# and ignore all headers ending in \r\n
182
# except for content-range:
183
# and find the two trailing \r\n separators
184
# indicating the start of the text
185
# TODO: jam 20060706 This requires exact conformance
186
# to the spec, we probably could relax the requirement
187
# of \r\n, and use something more like (\r?\n)
189
"^--%s(?:\r\n(?:(?:content-range:([^\r]+))|[^\r]+))+\r\n\r\n")
191
def __init__(self, path, content_type, input_file):
192
# mutter("parsing 206 multipart response for %s", path)
193
# TODO: jam 20060706 Is it valid to initialize a
194
# grandparent without initializing parent?
195
RangeFile.__init__(self, path, input_file)
197
self.boundary_regex = self._parse_boundary(content_type, path)
199
for match in self.boundary_regex.finditer(self._data):
200
ent_start, ent_end = HttpRangeResponse._parse_range(match.group(1),
202
self._add_range(ent_start, ent_end, match.end())
204
self._finish_ranges()
207
def _parse_boundary(ctype, path='<unknown>'):
208
"""Parse the Content-type field.
210
This expects a multipart Content-type, and returns a
211
regex which is capable of finding the boundaries
212
in the multipart data.
214
match = HttpMultipartRangeResponse._CONTENT_TYPE_RE.match(ctype)
216
raise errors.InvalidHttpContentType(path, ctype,
217
"Expected multipart/byteranges with boundary")
219
boundary = match.group(1)
220
# mutter('multipart boundary is %s', boundary)
221
pattern = HttpMultipartRangeResponse._BOUNDARY_PATT
222
return re.compile(pattern % re.escape(boundary),
223
re.IGNORECASE | re.MULTILINE)
226
def _is_multipart(content_type):
227
return content_type.startswith('multipart/byteranges;')
230
def handle_response(url, code, headers, data):
231
"""Interpret the code & headers and return a HTTP response.
233
This is a factory method which returns an appropriate HTTP response
234
based on the code & headers it's given.
299
236
:param url: The url being processed. Mostly for error reporting
300
237
:param code: The integer HTTP response code
301
:param msg: An HTTPMessage containing the headers for the response
238
:param headers: A dict-like object that contains the HTTP response headers
302
239
:param data: A file-like object that can be read() to get the
304
:return: A file-like object that can seek()+read() the
241
:return: A file-like object that can seek()+read() the
305
242
ranges indicated by the headers.
307
rfile = RangeFile(url, data)
310
size = msg.getheader('content-length', None)
315
rfile.set_range(0, size)
317
content_type = msg.getheader('content-type', None)
318
if content_type is None:
319
# When there is no content-type header we treat the response as
320
# being of type 'application/octet-stream' as per RFC2616 section
322
# Therefore it is obviously not multipart
323
content_type = 'application/octet-stream'
326
is_multipart = (msg.getmaintype() == 'multipart'
327
and msg.getsubtype() == 'byteranges')
247
content_type = headers['Content-Type']
249
raise errors.InvalidHttpContentType(url, '',
250
msg='Missing Content-Type')
252
if _is_multipart(content_type):
330
253
# Full fledged multipart response
331
rfile.set_boundary(msg.getparam('boundary'))
254
return HttpMultipartRangeResponse(url, content_type, data)
333
256
# A response to a range request, but not multipart
334
content_range = msg.getheader('content-range', None)
335
if content_range is None:
258
content_range = headers['Content-Range']
336
260
raise errors.InvalidHttpResponse(url,
337
261
'Missing the Content-Range header in a 206 range response')
338
rfile.set_range_from_header(content_range)
262
return HttpRangeResponse(url, content_range, data)
264
# A regular non-range response, unfortunately the result from
265
# urllib doesn't support seek, so we wrap it in a StringIO
266
tell = getattr(data, 'tell', None)
268
return StringIO(data.read())
271
raise errors.NoSuchFile(url)
273
# TODO: jam 20060713 Properly handle redirects (302 Found, etc)
274
# The '_get' code says to follow redirects, we probably
275
# should actually handle the return values
340
raise errors.InvalidHttpResponse(url,
341
'Unknown response code %s' % code)
277
raise errors.InvalidHttpResponse(url, "Unknown response code %s"
added
removed
bzrlib/transport/http/response.py
