1
# Copyright (C) 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Handlers for HTTP Responses.
19
The purpose of these classes is to provide a uniform interface for clients
20
to standard HTTP responses, single range responses and multipart range
26
from cStringIO import StringIO
35
# A RangeFile expects the following grammar (simplified to outline the
36
# assumptions we rely upon).
42
# whole_file: [content_length_header] data
44
# single_range: content_range_header data
46
# multiple_range: boundary_header boundary (content_range_header data boundary)+
48
class RangeFile(object):
49
"""File-like object that allow access to partial available data.
51
All accesses should happen sequentially since the acquisition occurs during
52
an http response reception (as sockets can't be seeked, we simulate the
53
seek by just reading and discarding the data).
55
The access pattern is defined by a set of ranges discovered as reading
56
progress. Only one range is available at a given time, so all accesses
57
should happen with monotonically increasing offsets.
60
# in _checked_read() below, we may have to discard several MB in the worst
61
# case. To avoid buffering that much, we read and discard by chunks
62
# instead. The underlying file is either a socket or a StringIO, so reading
63
# 8k chunks should be fine.
64
_discarded_buf_size = 8192
66
# maximum size of read requests -- used to avoid MemoryError issues in recv
67
_max_read_size = 512 * 1024
69
def __init__(self, path, infile):
72
:param path: File url, for error reports.
73
:param infile: File-like socket set at body start.
78
# When using multi parts response, this will be set with the headers
79
# associated with the range currently read.
81
# Default to the whole file of unspecified size
84
def set_range(self, start, size):
85
"""Change the range mapping"""
88
# Set the new _pos since that's what we want to expose
89
self._pos = self._start
91
def set_boundary(self, boundary):
92
"""Define the boundary used in a multi parts message.
94
The file should be at the beginning of the body, the first range
95
definition is read and taken into account.
97
self._boundary = boundary
98
# Decode the headers and setup the first range
100
self.read_range_definition()
102
def read_boundary(self):
103
"""Read the boundary headers defining a new range"""
104
boundary_line = '\r\n'
105
while boundary_line == '\r\n':
106
# RFC2616 19.2 Additional CRLFs may precede the first boundary
108
# To be on the safe side we allow it before any boundary line
109
boundary_line = self._file.readline()
110
if boundary_line != '--' + self._boundary + '\r\n':
111
raise errors.InvalidHttpResponse(
113
"Expected a boundary (%s) line, got '%s'" % (self._boundary,
116
def read_range_definition(self):
117
"""Read a new range definition in a multi parts message.
119
Parse the headers including the empty line following them so that we
120
are ready to read the data itself.
122
self._headers = httplib.HTTPMessage(self._file, seekable=0)
123
# Extract the range definition
124
content_range = self._headers.getheader('content-range', None)
125
if content_range is None:
126
raise errors.InvalidHttpResponse(
128
'Content-Range header missing in a multi-part response')
129
self.set_range_from_header(content_range)
131
def set_range_from_header(self, content_range):
132
"""Helper to set the new range from its description in the headers"""
134
rtype, values = content_range.split()
136
raise errors.InvalidHttpRange(self._path, content_range,
139
raise errors.InvalidHttpRange(self._path, content_range,
140
"Unsupported range type '%s'" % rtype)
142
# We don't need total, but note that it may be either the file size
143
# or '*' if the server can't or doesn't want to return the file
145
start_end, total = values.split('/')
146
start, end = start_end.split('-')
150
raise errors.InvalidHttpRange(self._path, content_range,
151
'Invalid range values')
152
size = end - start + 1
154
raise errors.InvalidHttpRange(self._path, content_range,
155
'Invalid range, size <= 0')
156
self.set_range(start, size)
158
def _checked_read(self, size):
159
"""Read the file checking for short reads.
161
The data read is discarded along the way.
166
data = self._file.read(min(remaining, self._discarded_buf_size))
167
remaining -= len(data)
169
raise errors.ShortReadvError(self._path, pos, size,
173
def _seek_to_next_range(self):
174
# We will cross range boundaries
175
if self._boundary is None:
176
# If we don't have a boundary, we can't find another range
177
raise errors.InvalidRange(self._path, self._pos,
178
"Range (%s, %s) exhausted"
179
% (self._start, self._size))
181
self.read_range_definition()
183
def read(self, size=-1):
184
"""Read size bytes from the current position in the file.
186
Reading across ranges is not supported. We rely on the underlying http
187
client to clean the socket if we leave bytes unread. This may occur for
188
the final boundary line of a multipart response or for any range
189
request not entirely consumed by the client (due to offset coalescing)
191
:param size: The number of bytes to read. Leave unspecified or pass
195
and self._pos == self._start + self._size):
199
self._seek_to_next_range()
200
elif self._pos < self._start:
201
raise errors.InvalidRange(
202
self._path, self._pos,
203
"Can't read %s bytes before range (%s, %s)"
204
% (size, self._start, self._size))
206
if size > 0 and self._pos + size > self._start + self._size:
207
raise errors.InvalidRange(
208
self._path, self._pos,
209
"Can't read %s bytes across range (%s, %s)"
210
% (size, self._start, self._size))
212
# read data from file
216
# Don't read past the range definition
217
limited = self._start + self._size - self._pos
219
limited = min(limited, size)
220
osutils.pumpfile(self._file, buffer, limited, self._max_read_size)
221
data = buffer.getvalue()
223
# Update _pos respecting the data effectively read
224
self._pos += len(data)
227
def seek(self, offset, whence=0):
228
start_pos = self._pos
232
final_pos = start_pos + offset
235
final_pos = self._start + self._size + offset # offset < 0
237
raise errors.InvalidRange(
238
self._path, self._pos,
239
"RangeFile: can't seek from end while size is unknown")
241
raise ValueError("Invalid value %s for whence." % whence)
243
if final_pos < self._pos:
244
# Can't seek backwards
245
raise errors.InvalidRange(
246
self._path, self._pos,
247
'RangeFile: trying to seek backwards to %s' % final_pos)
250
cur_limit = self._start + self._size
251
while final_pos > cur_limit:
252
# We will cross range boundaries
253
remain = cur_limit - self._pos
255
# Finish reading the current range
256
self._checked_read(remain)
257
self._seek_to_next_range()
258
cur_limit = self._start + self._size
260
size = final_pos - self._pos
261
if size > 0: # size can be < 0 if we crossed a range boundary
262
# We don't need the data, just read it and throw it away
263
self._checked_read(size)
269
def handle_response(url, code, msg, data):
270
"""Interpret the code & headers and wrap the provided data in a RangeFile.
272
This is a factory method which returns an appropriate RangeFile based on
273
the code & headers it's given.
275
:param url: The url being processed. Mostly for error reporting
276
:param code: The integer HTTP response code
277
:param msg: An HTTPMessage containing the headers for the response
278
:param data: A file-like object that can be read() to get the
280
:return: A file-like object that can seek()+read() the
281
ranges indicated by the headers.
283
rfile = RangeFile(url, data)
286
size = msg.getheader('content-length', None)
291
rfile.set_range(0, size)
293
content_type = msg.getheader('content-type', None)
294
if content_type is None:
295
# When there is no content-type header we treat the response as
296
# being of type 'application/octet-stream' as per RFC2616 section
298
# Therefore it is obviously not multipart
299
content_type = 'application/octet-stream'
302
is_multipart = (msg.getmaintype() == 'multipart'
303
and msg.getsubtype() == 'byteranges')
306
# Full fledged multipart response
307
rfile.set_boundary(msg.getparam('boundary'))
309
# A response to a range request, but not multipart
310
content_range = msg.getheader('content-range', None)
311
if content_range is None:
312
raise errors.InvalidHttpResponse(url,
313
'Missing the Content-Range header in a 206 range response')
314
rfile.set_range_from_header(content_range)
316
raise errors.InvalidHttpResponse(url,
317
'Unknown response code %s' % code)
added
removed
bzrlib/transport/http/response.py
