« back to all changes in this revision
Viewing changes to bzrlib/transport/http/response.py
- browse files at revision 2929.3.16
- compare with another revision
- compare with revision 4969.2.5
- download diff
- download tarball
- view history from revision 2929.3.16
- Committer: Vincent Ladeuil
- Date: 2007-12-09 18:00:14 UTC
- mfrom: (3059.2.17 173010)
- mto: (3928.1.1 bzr.integration)
- mto: This revision was merged to the branch mainline in revision 3929.
- Revision ID: v.ladeuil+lp@free.fr-20071209180014-hig14lbi1rap06qu
merge fix for bug 173010 since it changes the http response handling in a good way
- files modified:
- NEWS
added
removed
bzrlib/transport/http/response.py
1
# Copyright (C) 2006 Canonical Ltd
1
# Copyright (C) 2006, 2007 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
22
22
"""
23
23
24
24
25
from bisect import bisect
26
from cStringIO import StringIO
27
import re
28
29
from bzrlib import errors
30
from bzrlib.trace import mutter
31
32
33
class ResponseRange(object):
34
"""A range in a RangeFile-object."""
35
36
__slots__ = ['_ent_start', '_ent_end', '_data_start']
37
38
def __init__(self, ent_start, ent_end, data_start):
39
self._ent_start = ent_start
40
self._ent_end = ent_end
41
self._data_start = data_start
42
43
def __cmp__(self, other):
44
"""Compare this to other.
45
46
We need this both for sorting, and so that we can
47
bisect the list of ranges.
48
"""
49
if isinstance(other, int):
50
# Later on we bisect for a starting point
51
# so we allow comparing against a single integer
52
return cmp(self._ent_start, other)
53
else:
54
return cmp((self._ent_start, self._ent_end, self._data_start),
55
(other._ent_start, other._ent_end, other._data_start))
56
57
def __str__(self):
58
return "%s(%s-%s,%s)" % (self.__class__.__name__,
59
self._ent_start, self._ent_end,
60
self._data_start)
61
62
__repr__ = __str__
63
25
import httplib
26
27
from bzrlib import (
28
errors,
29
trace,
30
)
31
32
33
# A RangeFile epxects the following grammar (simplified to outline the
34
# assumptions we rely upon).
35
36
# file: whole_file
37
# | single_range
38
# | multiple_range
39
40
# whole_file: [content_length_header] data
41
42
# single_range: content_range_header data
43
44
# multiple_range: boundary_header boundary (content_range_header data boundary)+
64
45
65
46
class RangeFile(object):
66
47
"""File-like object that allow access to partial available data.
67
48
68
Specified by a set of ranges.
49
All accesses should happen sequentially since the acquisition occurs during
50
an http response reception (as sockets can't be seeked, we simulate the
51
seek by just reading and discarding the data).
52
53
The access pattern is defined by a set of ranges discovered as reading
54
progress. Only one range is available at a given time, so all accesses
55
should happen with monotonically increasing offsets.
69
56
"""
70
57
71
def __init__(self, path, input_file):
58
def __init__(self, path, infile):
59
"""Constructor.
60
61
:param path: File url, for error reports.
62
:param infile: File-like socket set at body start.
63
"""
72
64
self._path = path
73
self._pos = 0
74
self._len = 0
75
self._ranges = []
76
self._data = input_file.read()
77
78
def _add_range(self, ent_start, ent_end, data_start):
79
"""Add an entity range.
80
81
:param ent_start: Start offset of entity
82
:param ent_end: End offset of entity (inclusive)
83
:param data_start: Start offset of data in data stream.
84
"""
85
self._ranges.append(ResponseRange(ent_start, ent_end, data_start))
86
self._len = max(self._len, ent_end)
87
88
def _finish_ranges(self):
89
self._ranges.sort()
90
91
def read(self, size):
65
self._file = infile
66
self._boundary = None
67
# When using multi parts response, this will be set with the headers
68
# associated with the range currently read.
69
self._headers = None
70
# Default to the whole file of unspecified size
71
self.set_range(0, -1)
72
73
def set_range(self, start, size):
74
"""Change the range mapping"""
75
self._start = start
76
self._size = size
77
# Set the new _pos since that's what we want to expose
78
self._pos = self._start
79
80
def set_boundary(self, boundary):
81
"""Define the boundary used in a multi parts message.
82
83
The file should be at the beggining of the body, the first range
84
definition is read and taken into account.
85
"""
86
self._boundary = boundary
87
# Decode the headers and setup the first range
88
self.read_boundary()
89
self.read_range_definition()
90
91
def read_boundary(self):
92
"""Read the boundary headers defining a new range"""
93
boundary_line = '\r\n'
94
while boundary_line == '\r\n':
95
# RFC2616 19.2 Additional CRLFs may preceed the first boundary
96
# string entity.
97
# To be on the safe side we allow it before any boundary line
98
boundary_line = self._file.readline()
99
if boundary_line != '--' + self._boundary + '\r\n':
100
raise errors.InvalidHttpResponse(
101
self._path,
102
"Expected a boundary (%s) line, got '%s'" % (self._boundary,
103
boundary_line))
104
105
def read_range_definition(self):
106
"""Read a new range definition in a multi parts message.
107
108
Parse the headers including the empty line following them so that we
109
are ready to read the data itself.
110
"""
111
self._headers = httplib.HTTPMessage(self._file, seekable=0)
112
# Extract the range definition
113
content_range = self._headers.getheader('content-range', None)
114
if content_range is None:
115
raise errors.InvalidHttpResponse(
116
self._path,
117
'Content-Range header missing in a multi-part response')
118
self.set_range_from_header(content_range)
119
120
def set_range_from_header(self, content_range):
121
"""Helper to set the new range from its description in the headers"""
122
try:
123
rtype, values = content_range.split()
124
except ValueError:
125
raise errors.InvalidHttpRange(self._path, content_range,
126
'Malformed header')
127
if rtype != 'bytes':
128
raise errors.InvalidHttpRange(self._path, content_range,
129
"Unsupported range type '%s'" % rtype)
130
try:
131
# We don't need total, but note that it may be either the file size
132
# or '*' if the server can't or doesn't want to return the file
133
# size.
134
start_end, total = values.split('/')
135
start, end = start_end.split('-')
136
start = int(start)
137
end = int(end)
138
except ValueError:
139
raise errors.InvalidHttpRange(self._path, content_range,
140
'Invalid range values')
141
size = end - start + 1
142
if size <= 0:
143
raise errors.InvalidHttpRange(self._path, content_range,
144
'Invalid range, size <= 0')
145
self.set_range(start, size)
146
147
def _checked_read(self, size):
148
"""Read the file checking for short reads"""
149
pos = self._pos
150
data = self._file.read(size)
151
data_len = len(data)
152
if size > 0 and data_len != size:
153
raise errors.ShortReadvError(self._path, pos, size, data_len)
154
self._pos += data_len
155
return data
156
157
def read(self, size=-1):
92
158
"""Read size bytes from the current position in the file.
93
159
94
Reading across ranges is not supported.
160
Reading across ranges is not supported. We rely on the underlying http
161
client to clean the socket if we leave bytes unread. This may occur for
162
the final boundary line of a multipart response or for any range
163
request not entirely consumed by the client (due to offset coalescing)
95
164
"""
96
# find the last range which has a start <= pos
97
i = bisect(self._ranges, self._pos) - 1
98
99
if i < 0 or self._pos > self._ranges[i]._ent_end:
100
mutter('Bisect for pos: %s failed. Found offset: %d, ranges:%s',
101
self._pos, i, self._ranges)
102
raise errors.InvalidRange(self._path, self._pos)
103
104
r = self._ranges[i]
105
106
# mutter('found range %s %s for pos %s', i, self._ranges[i], self._pos)
107
108
if (self._pos + size - 1) > r._ent_end:
109
raise errors.InvalidRange(self._path, self._pos)
110
111
start = r._data_start + (self._pos - r._ent_start)
112
end = start + size
113
# mutter("range read %d bytes at %d == %d-%d", size, self._pos,
114
# start, end)
115
self._pos += (end-start)
116
return self._data[start:end]
165
if self._pos < self._start:
166
raise errors.InvalidRange(self._path, self._pos,
167
"Can't read before range (%s, %s)"
168
% (self._start, self._size))
169
if self._size > 0:
170
if size > 0 and self._pos + size > self._start + self._size:
171
raise errors.InvalidRange(
172
self._path, self._pos,
173
"Can't read %s bytes across range (%s, %s)"
174
% (size, self._start, self._size))
175
176
if self._size > 0:
177
# Don't read past the range definition
178
limited = self._start + self._size - self._pos
179
if size > 0:
180
limited = min(limited, size)
181
data = self._file.read(limited)
182
else:
183
# Size of file unknown
184
data = self._file.read(size)
185
# Update _pos respecting the data effectively read
186
self._pos += len(data)
187
return data
117
188
118
189
def seek(self, offset, whence=0):
190
start_pos = self._pos
119
191
if whence == 0:
120
self._pos = offset
192
final_pos = offset
121
193
elif whence == 1:
122
self._pos += offset
194
final_pos = start_pos + offset
123
195
elif whence == 2:
124
self._pos = self._len + offset
196
if self._size > 0:
197
final_pos = self._start + self._size + offset # offset < 0
198
else:
199
raise errors.InvalidRange(
200
self._path, self._pos,
201
"RangeFile: can't seek from end while size is unknown")
125
202
else:
126
203
raise ValueError("Invalid value %s for whence." % whence)
127
204
128
if self._pos < 0:
129
self._pos = 0
205
if final_pos < self._pos:
206
# Can't seek backwards
207
raise errors.InvalidRange(
208
self._path, self._pos,
209
'RangeFile: trying to seek backwards to %s' % final_pos)
210
211
if self._size > 0:
212
cur_limit = self._start + self._size
213
while final_pos >= cur_limit:
214
# We will cross range boundaries
215
remain = cur_limit - self._pos
216
if remain > 0:
217
# Finish reading the current range
218
self._checked_read(remain)
219
if self._boundary is None:
220
# If we don't have a boundary, we can't find another range
221
raise errors.InvalidRange(self._path, self._pos,
222
'Range (%s, %s) exhausted'
223
% (self._start, self._size))
224
self.read_boundary()
225
self.read_range_definition()
226
cur_limit = self._start + self._size
227
228
size = final_pos - self._pos
229
if size > 0: # size can be < 0 if we crossed a range boundary
230
# We don't need the data, just read it and throw it away
231
self._checked_read(size)
130
232
131
233
def tell(self):
132
234
return self._pos
133
235
134
236
135
class HttpRangeResponse(RangeFile):
136
"""A single-range HTTP response."""
137
138
# TODO: jam 20060706 Consider compiling these regexes on demand
139
_CONTENT_RANGE_RE = re.compile(
140
r'\s*([^\s]+)\s+([0-9]+)-([0-9]+)/([0-9]+)\s*$')
141
142
def __init__(self, path, content_range, input_file):
143
# mutter("parsing 206 non-multipart response for %s", path)
144
RangeFile.__init__(self, path, input_file)
145
start, end = self._parse_range(content_range, path)
146
self._add_range(start, end, 0)
147
self._finish_ranges()
148
149
@staticmethod
150
def _parse_range(range, path='<unknown>'):
151
"""Parse an http Content-range header and return start + end
152
153
:param range: The value for Content-range
154
:param path: Provide to give better error messages.
155
:return: (start, end) A tuple of integers
156
"""
157
match = HttpRangeResponse._CONTENT_RANGE_RE.match(range)
158
if not match:
159
raise errors.InvalidHttpRange(path, range,
160
"Invalid Content-range")
161
162
rtype, start, end, total = match.groups()
163
164
if rtype != 'bytes':
165
raise errors.InvalidHttpRange(path, range,
166
"Unsupported range type '%s'" % (rtype,))
167
168
try:
169
start = int(start)
170
end = int(end)
171
except ValueError, e:
172
raise errors.InvalidHttpRange(path, range, str(e))
173
174
return start, end
175
176
177
class HttpMultipartRangeResponse(RangeFile):
178
"""A multi-range HTTP response."""
179
180
_CONTENT_TYPE_RE = re.compile(
181
r'^\s*multipart/byteranges\s*;\s*boundary\s*=\s*("?)([^"]*?)\1\s*$')
182
183
# Start with --<boundary>\r\n
184
# and ignore all headers ending in \r\n
185
# except for content-range:
186
# and find the two trailing \r\n separators
187
# indicating the start of the text
188
# TODO: jam 20060706 This requires exact conformance
189
# to the spec, we probably could relax the requirement
190
# of \r\n, and use something more like (\r?\n)
191
_BOUNDARY_PATT = (
192
"^--%s(?:\r\n(?:(?:content-range:([^\r]+))|[^\r]+))+\r\n\r\n")
193
194
def __init__(self, path, content_type, input_file):
195
# mutter("parsing 206 multipart response for %s", path)
196
# TODO: jam 20060706 Is it valid to initialize a
197
# grandparent without initializing parent?
198
RangeFile.__init__(self, path, input_file)
199
200
self.boundary_regex = self._parse_boundary(content_type, path)
201
# mutter('response:\n%r', self._data)
202
203
for match in self.boundary_regex.finditer(self._data):
204
ent_start, ent_end = HttpRangeResponse._parse_range(match.group(1),
205
path)
206
self._add_range(ent_start, ent_end, match.end())
207
208
self._finish_ranges()
209
210
@staticmethod
211
def _parse_boundary(ctype, path='<unknown>'):
212
"""Parse the Content-type field.
213
214
This expects a multipart Content-type, and returns a
215
regex which is capable of finding the boundaries
216
in the multipart data.
217
"""
218
match = HttpMultipartRangeResponse._CONTENT_TYPE_RE.match(ctype)
219
if not match:
220
raise errors.InvalidHttpContentType(path, ctype,
221
"Expected multipart/byteranges with boundary")
222
223
boundary = match.group(2)
224
# mutter('multipart boundary is %s', boundary)
225
pattern = HttpMultipartRangeResponse._BOUNDARY_PATT
226
return re.compile(pattern % re.escape(boundary),
227
re.IGNORECASE | re.MULTILINE)
228
229
230
def _is_multipart(content_type):
231
return content_type.startswith('multipart/byteranges;')
232
233
234
def handle_response(url, code, headers, data):
235
"""Interpret the code & headers and return a HTTP response.
236
237
This is a factory method which returns an appropriate HTTP response
238
based on the code & headers it's given.
237
def handle_response(url, code, msg, data):
238
"""Interpret the code & headers and wrap the provided data in a RangeFile.
239
240
This is a factory method which returns an appropriate RangeFile based on
241
the code & headers it's given.
239
242
240
243
:param url: The url being processed. Mostly for error reporting
241
244
:param code: The integer HTTP response code
242
:param headers: A dict-like object that contains the HTTP response headers
245
:param msg: An HTTPMessage containing the headers for the response
243
246
:param data: A file-like object that can be read() to get the
244
247
requested data
245
248
:return: A file-like object that can seek()+read() the
246
249
ranges indicated by the headers.
247
250
"""
248
249
if code == 206:
250
try:
251
content_type = headers['Content-Type']
252
except KeyError:
253
# When there is no content-type header we treat
254
# the response as being of type 'application/octet-stream' as per
255
# RFC2616 section 7.2.1.
251
rfile = RangeFile(url, data)
252
if code == 200:
253
# A whole file
254
size = msg.getheader('content-length', None)
255
if size is None:
256
size = -1
257
else:
258
size = int(size)
259
rfile.set_range(0, size)
260
elif code == 206:
261
content_type = msg.getheader('content-type', None)
262
if content_type is None:
263
# When there is no content-type header we treat the response as
264
# being of type 'application/octet-stream' as per RFC2616 section
265
# 7.2.1.
256
266
# Therefore it is obviously not multipart
257
267
content_type = 'application/octet-stream'
258
268
is_multipart = False
259
269
else:
260
is_multipart = _is_multipart(content_type)
270
is_multipart = (msg.getmaintype() == 'multipart'
271
and msg.getsubtype() == 'byteranges')
261
272
262
273
if is_multipart:
263
274
# Full fledged multipart response
264
return HttpMultipartRangeResponse(url, content_type, data)
275
rfile.set_boundary(msg.getparam('boundary'))
265
276
else:
266
277
# A response to a range request, but not multipart
267
try:
268
content_range = headers['Content-Range']
269
except KeyError:
278
content_range = msg.getheader('content-range', None)
279
if content_range is None:
270
280
raise errors.InvalidHttpResponse(url,
271
281
'Missing the Content-Range header in a 206 range response')
272
return HttpRangeResponse(url, content_range, data)
273
elif code == 200:
274
# A regular non-range response, unfortunately the result from
275
# urllib doesn't support seek, so we wrap it in a StringIO
276
tell = getattr(data, 'tell', None)
277
if tell is None:
278
return StringIO(data.read())
279
return data
280
elif code == 404:
281
raise errors.NoSuchFile(url)
282
# Some servers will retun "400: Bad Request" when too much ranges are
283
# specified
284
elif code in (400, 416):
285
# We don't know which, but one of the ranges we specified
286
# was wrong. So we raise with 0 for a lack of a better
287
# magic value.
288
raise errors.InvalidRange(url,0)
289
282
rfile.set_range_from_header(content_range)
290
283
else:
291
raise errors.InvalidHttpResponse(url, "Unknown response code %s"
292
% (code,))
284
raise errors.InvalidHttpResponse(url,
285
'Unknown response code %s' % code)
286
287
return rfile
293
288
Loggerhead is a web-based interface for Breezy
Version: 2.0.1