13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Tests from HTTP response parsing."""
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Tests from HTTP response parsing.
19
The handle_response method read the response body of a GET request an returns
20
the corresponding RangeFile.
22
There are four different kinds of RangeFile:
23
- a whole file whose size is unknown, seen as a simple byte stream,
24
- a whole file whose size is known, we can't read past its end,
25
- a single range file, a part of a file with a start and a size,
26
- a multiple range file, several consecutive parts with known start offset
29
Some properties are common to all kinds:
30
- seek can only be forward (its really a socket underneath),
31
- read can't cross ranges,
32
- successive ranges are taken into account transparently,
34
- the expected pattern of use is either seek(offset)+read(size) or a single
35
read with no size specified. For multiple range files, multiple read() will
36
return the corresponding ranges, trying to read further will raise
19
40
from cStringIO import StringIO
22
from bzrlib import errors
23
from bzrlib.transport import http
24
from bzrlib.transport.http import response
25
from bzrlib.tests import TestCase
28
class TestResponseRange(TestCase):
29
"""Test the ResponseRange class."""
32
RR = response.ResponseRange
35
self.assertTrue(r1 < r2)
36
self.assertFalse(r1 > r2)
37
self.assertTrue(r1 < 5)
38
self.assertFalse(r2 < 5)
40
self.assertEqual(RR(0, 10, 5), RR(0, 10, 5))
41
self.assertNotEqual(RR(0, 10, 5), RR(0, 8, 5))
42
self.assertNotEqual(RR(0, 10, 5), RR(0, 10, 6))
44
def test_sort_list(self):
45
"""Ensure longer ranges are sorted after shorter ones"""
46
RR = response.ResponseRange
47
lst = [RR(3, 8, 0), 5, RR(3, 7, 0), 6]
49
self.assertEqual([RR(3,7,0), RR(3,8,0), 5, 6], lst)
52
class TestRangeFile(TestCase):
56
content = "abcdefghijklmnopqrstuvwxyz"
57
self.fp = response.RangeFile('foo', StringIO(content))
58
self.fp._add_range(0, 9, 0)
59
self.fp._add_range(20, 29, 10)
60
self.fp._add_range(30, 39, 15)
62
def test_valid_accesses(self):
63
"""Test so that valid accesses work to the file."""
65
self.assertEquals(self.fp.read(3), 'abc')
66
self.assertEquals(self.fp.read(3), 'def')
67
self.assertEquals(self.fp.tell(), 6)
69
self.assertEquals(self.fp.read(3), 'klm')
70
self.assertEquals(self.fp.read(2), 'no')
71
self.assertEquals(self.fp.tell(), 25)
72
# should wrap over to 30-39 entity
73
self.assertEquals(self.fp.read(3), 'pqr')
75
self.assertEquals(self.fp.read(3), 'def')
76
self.assertEquals(self.fp.tell(), 6)
78
def test_invalid_accesses(self):
79
"""Test so that invalid accesses trigger errors."""
81
self.assertRaises(errors.InvalidRange, self.fp.read, 2)
83
self.assertRaises(errors.InvalidRange, self.fp.read, 2)
85
self.assertRaises(errors.InvalidRange, self.fp.read, 2)
87
def test__finish_ranges(self):
88
"""Test that after RangeFile._finish_ranges the list is sorted."""
89
self.fp._add_range(1, 2, 3)
90
self.fp._add_range(8, 9, 10)
91
self.fp._add_range(3, 4, 5)
93
# TODO: jam 20060706 If we switch to inserting
94
# in sorted order, remove this test
95
self.assertNotEqual(self.fp._ranges, sorted(self.fp._ranges))
97
self.fp._finish_ranges()
98
self.assertEqual(self.fp._ranges, sorted(self.fp._ranges))
100
def test_seek_and_tell(self):
101
# Check for seeking before start
103
self.assertEqual(0, self.fp.tell())
106
self.assertEqual(5, self.fp.tell())
109
self.assertEqual(3, self.fp.tell())
111
# TODO: jam 20060706 following tests will fail if this
112
# is not true, and would be difficult to debug
113
# but it is a layering violation
114
self.assertEqual(39, self.fp._len)
117
self.assertEqual(39, self.fp.tell())
120
self.assertEqual(29, self.fp.tell())
122
self.assertRaises(ValueError, self.fp.seek, 0, 4)
123
self.assertRaises(ValueError, self.fp.seek, 0, -1)
126
class TestRegexes(TestCase):
128
def assertRegexMatches(self, groups, text):
129
"""Check that the regex matches and returns the right values"""
130
m = self.regex.match(text)
131
self.assertNotEqual(None, m, "text %s did not match regex" % (text,))
133
self.assertEqual(groups, m.groups())
135
def test_range_re(self):
136
"""Test that we match valid ranges."""
137
self.regex = response.HttpRangeResponse._CONTENT_RANGE_RE
138
self.assertRegexMatches(('bytes', '1', '10', '11'),
140
self.assertRegexMatches(('bytes', '1', '10', '11'),
142
self.assertRegexMatches(('bytes', '2123', '4242', '1231'),
143
'\tbytes 2123-4242/1231 ')
144
self.assertRegexMatches(('chars', '1', '2', '3'),
147
def test_content_type_re(self):
148
self.regex = response.HttpMultipartRangeResponse._CONTENT_TYPE_RE
149
self.assertRegexMatches(('xxyyzz',),
150
'multipart/byteranges; boundary = xxyyzz')
151
self.assertRegexMatches(('xxyyzz',),
152
'multipart/byteranges;boundary=xxyyzz')
153
self.assertRegexMatches(('xx yy zz',),
154
' multipart/byteranges ; boundary= xx yy zz ')
155
self.assertEqual(None,
156
self.regex.match('multipart byteranges;boundary=xx'))
162
Content-range: bytes 1-10/20\r
166
Content-Range: bytes 21-30/20\r
171
content-range: bytes 41-50/20\r
175
content-range: bytes 51-60/20\r
181
class TestHelpers(TestCase):
182
"""Test the helper functions"""
184
def test__parse_range(self):
185
"""Test that _parse_range acts reasonably."""
186
content = StringIO('')
187
parse_range = response.HttpRangeResponse._parse_range
188
self.assertEqual((1,2), parse_range('bytes 1-2/3'))
189
self.assertEqual((10,20), parse_range('bytes 10-20/2'))
191
self.assertRaises(errors.InvalidHttpRange, parse_range, 'char 1-3/2')
192
self.assertRaises(errors.InvalidHttpRange, parse_range, 'bytes a-3/2')
195
parse_range('bytes x-10/3', path='http://foo/bar')
196
except errors.InvalidHttpRange, e:
197
self.assertContainsRe(str(e), 'http://foo/bar')
198
self.assertContainsRe(str(e), 'bytes x-10/3')
200
self.fail('Did not raise InvalidHttpRange')
202
def test__parse_boundary_simple(self):
203
"""Test that _parse_boundary handles Content-type properly"""
204
parse_boundary = response.HttpMultipartRangeResponse._parse_boundary
205
m = parse_boundary(' multipart/byteranges; boundary=xxyyzz')
206
self.assertNotEqual(None, m)
207
# Check that the returned regex is capable of splitting simple_data
208
matches = list(m.finditer(simple_data))
209
self.assertEqual(4, len(matches))
211
# match.group() should be the content-range entry
212
# and match.end() should be the start of the content
213
self.assertEqual(' bytes 1-10/20', matches[0].group(1))
214
self.assertEqual(simple_data.find('1234567890'), matches[0].end())
215
self.assertEqual(' bytes 21-30/20', matches[1].group(1))
216
self.assertEqual(simple_data.find('abcdefghij'), matches[1].end())
217
self.assertEqual(' bytes 41-50/20', matches[2].group(1))
218
self.assertEqual(simple_data.find('zyxwvutsrq'), matches[2].end())
219
self.assertEqual(' bytes 51-60/20', matches[3].group(1))
220
self.assertEqual(simple_data.find('xxyyzz fbd'), matches[3].end())
222
def test__parse_boundary_invalid(self):
223
parse_boundary = response.HttpMultipartRangeResponse._parse_boundary
225
parse_boundary(' multipart/bytes;boundary=xxyyzz',
226
path='http://foo/bar')
227
except errors.InvalidHttpContentType, e:
228
self.assertContainsRe(str(e), 'http://foo/bar')
229
self.assertContainsRe(str(e), 'multipart/bytes;boundary=xxyyzz')
231
self.fail('Did not raise InvalidHttpContentType')
234
class TestHttpRangeResponse(TestCase):
236
def test_smoketest(self):
237
"""A basic test that HttpRangeResponse is reasonable."""
238
content = StringIO('0123456789')
239
f = response.HttpRangeResponse('http://foo', 'bytes 1-10/9', content)
240
self.assertEqual([response.ResponseRange(1,10,0)], f._ranges)
47
from bzrlib.transport.http import (
51
from bzrlib.tests.file_utils import (
56
class ReadSocket(object):
57
"""A socket-like object that can be given a predefined content."""
59
def __init__(self, data):
60
self.readfile = StringIO(data)
62
def makefile(self, mode='r', bufsize=None):
66
class FakeHTTPConnection(_urllib2_wrappers.HTTPConnection):
68
def __init__(self, sock):
69
_urllib2_wrappers.HTTPConnection.__init__(self, 'localhost')
70
# Set the socket to bypass the connection
74
"""Ignores the writes on the socket."""
78
class TestHTTPConnection(tests.TestCase):
80
def test_cleanup_pipe(self):
81
sock = ReadSocket("""HTTP/1.1 200 OK\r
82
Content-Type: text/plain; charset=UTF-8\r
87
conn = FakeHTTPConnection(sock)
88
# Simulate the request sending so that the connection will be able to
90
conn.putrequest('GET', 'http://localhost/fictious')
92
# Now, get the response
93
resp = conn.getresponse()
94
# Read part of the response
95
self.assertEquals('0123456789\n', resp.read(11))
96
# Override the thresold to force the warning emission
97
conn._range_warning_thresold = 6 # There are 7 bytes pending
99
self.assertContainsRe(self._get_log(keep_log_file=True),
100
'Got a 200 response when asking')
103
class TestRangeFileMixin(object):
104
"""Tests for accessing the first range in a RangeFile."""
106
# A simple string used to represent a file part (also called a range), in
107
# which offsets are easy to calculate for test writers. It's used as a
108
# building block with slight variations but basically 'a' is the first char
109
# of the range and 'z' is the last.
110
alpha = 'abcdefghijklmnopqrstuvwxyz'
112
def test_can_read_at_first_access(self):
113
"""Test that the just created file can be read."""
114
self.assertEquals(self.alpha, self._file.read())
116
def test_seek_read(self):
117
"""Test seek/read inside the range."""
119
start = self.first_range_start
120
# Before any use, tell() should be at the range start
121
self.assertEquals(start, f.tell())
122
cur = start # For an overall offset assertion
125
self.assertEquals('def', f.read(3))
129
self.assertEquals('klmn', f.read(4))
131
# read(0) in the middle of a range
132
self.assertEquals('', f.read(0))
136
self.assertEquals(here, f.tell())
137
self.assertEquals(cur, f.tell())
139
def test_read_zero(self):
141
start = self.first_range_start
142
self.assertEquals('', f.read(0))
144
self.assertEquals('', f.read(0))
146
def test_seek_at_range_end(self):
150
def test_read_at_range_end(self):
151
"""Test read behaviour at range end."""
153
self.assertEquals(self.alpha, f.read())
154
self.assertEquals('', f.read(0))
155
self.assertRaises(errors.InvalidRange, f.read, 1)
157
def test_unbounded_read_after_seek(self):
160
# Should not cross ranges
161
self.assertEquals('yz', f.read())
163
def test_seek_backwards(self):
165
start = self.first_range_start
168
self.assertRaises(errors.InvalidRange, f.seek, start + 5)
170
def test_seek_outside_single_range(self):
172
if f._size == -1 or f._boundary is not None:
173
raise tests.TestNotApplicable('Needs a fully defined range')
174
# Will seek past the range and then errors out
175
self.assertRaises(errors.InvalidRange,
176
f.seek, self.first_range_start + 27)
178
def test_read_past_end_of_range(self):
181
raise tests.TestNotApplicable("Can't check an unknown size")
182
start = self.first_range_start
184
self.assertRaises(errors.InvalidRange, f.read, 10)
186
def test_seek_from_end(self):
187
"""Test seeking from the end of the file.
189
The semantic is unclear in case of multiple ranges. Seeking from end
190
exists only for the http transports, cannot be used if the file size is
191
unknown and is not used in bzrlib itself. This test must be (and is)
192
overridden by daughter classes.
194
Reading from end makes sense only when a range has been requested from
195
the end of the file (see HttpTransportBase._get() when using the
196
'tail_amount' parameter). The HTTP response can only be a whole file or
201
self.assertEquals('yz', f.read())
204
class TestRangeFileSizeUnknown(tests.TestCase, TestRangeFileMixin):
205
"""Test a RangeFile for a whole file whose size is not known."""
208
super(TestRangeFileSizeUnknown, self).setUp()
209
self._file = response.RangeFile('Whole_file_size_known',
210
StringIO(self.alpha))
211
# We define no range, relying on RangeFile to provide default values
212
self.first_range_start = 0 # It's the whole file
214
def test_seek_from_end(self):
215
"""See TestRangeFileMixin.test_seek_from_end.
217
The end of the file can't be determined since the size is unknown.
219
self.assertRaises(errors.InvalidRange, self._file.seek, -1, 2)
221
def test_read_at_range_end(self):
222
"""Test read behaviour at range end."""
224
self.assertEquals(self.alpha, f.read())
225
self.assertEquals('', f.read(0))
226
self.assertEquals('', f.read(1))
229
class TestRangeFileSizeKnown(tests.TestCase, TestRangeFileMixin):
230
"""Test a RangeFile for a whole file whose size is known."""
233
super(TestRangeFileSizeKnown, self).setUp()
234
self._file = response.RangeFile('Whole_file_size_known',
235
StringIO(self.alpha))
236
self._file.set_range(0, len(self.alpha))
237
self.first_range_start = 0 # It's the whole file
240
class TestRangeFileSingleRange(tests.TestCase, TestRangeFileMixin):
241
"""Test a RangeFile for a single range."""
244
super(TestRangeFileSingleRange, self).setUp()
245
self._file = response.RangeFile('Single_range_file',
246
StringIO(self.alpha))
247
self.first_range_start = 15
248
self._file.set_range(self.first_range_start, len(self.alpha))
251
def test_read_before_range(self):
252
# This can't occur under normal circumstances, we have to force it
254
f._pos = 0 # Force an invalid pos
255
self.assertRaises(errors.InvalidRange, f.read, 2)
258
class TestRangeFileMultipleRanges(tests.TestCase, TestRangeFileMixin):
259
"""Test a RangeFile for multiple ranges.
261
The RangeFile used for the tests contains three ranges:
263
- at offset 25: alpha
264
- at offset 100: alpha
265
- at offset 126: alpha.upper()
267
The two last ranges are contiguous. This only rarely occurs (should not in
268
fact) in real uses but may lead to hard to track bugs.
271
# The following is used to represent the boundary paramter defined
272
# in HTTP response headers and the boundary lines that separate
275
boundary = "separation"
278
super(TestRangeFileMultipleRanges, self).setUp()
280
boundary = self.boundary
283
self.first_range_start = 25
284
file_size = 200 # big enough to encompass all ranges
285
for (start, part) in [(self.first_range_start, self.alpha),
286
# Two contiguous ranges
288
(126, self.alpha.upper())]:
289
content += self._multipart_byterange(part, start, boundary,
292
content += self._boundary_line()
294
self._file = response.RangeFile('Multiple_ranges_file',
296
self.set_file_boundary()
298
def _boundary_line(self):
299
"""Helper to build the formatted boundary line."""
300
return '--' + self.boundary + '\r\n'
302
def set_file_boundary(self):
303
# Ranges are set by decoding the range headers, the RangeFile user is
304
# supposed to call the following before using seek or read since it
305
# requires knowing the *response* headers (in that case the boundary
306
# which is part of the Content-Type header).
307
self._file.set_boundary(self.boundary)
309
def _multipart_byterange(self, data, offset, boundary, file_size='*'):
310
"""Encode a part of a file as a multipart/byterange MIME type.
312
When a range request is issued, the HTTP response body can be
313
decomposed in parts, each one representing a range (start, size) in a
316
:param data: The payload.
317
:param offset: where data starts in the file
318
:param boundary: used to separate the parts
319
:param file_size: the size of the file containing the range (default to
322
:return: a string containing the data encoded as it will appear in the
325
bline = self._boundary_line()
326
# Each range begins with a boundary line
328
# A range is described by a set of headers, but only 'Content-Range' is
329
# required for our implementation (TestHandleResponse below will
330
# exercise ranges with multiple or missing headers')
331
range += 'Content-Range: bytes %d-%d/%d\r\n' % (offset,
335
# Finally the raw bytes
339
def test_read_all_ranges(self):
341
self.assertEquals(self.alpha, f.read()) # Read first range
342
f.seek(100) # Trigger the second range recognition
343
self.assertEquals(self.alpha, f.read()) # Read second range
344
self.assertEquals(126, f.tell())
345
f.seek(126) # Start of third range which is also the current pos !
346
self.assertEquals('A', f.read(1))
348
self.assertEquals('LMN', f.read(3))
350
def test_seek_from_end(self):
351
"""See TestRangeFileMixin.test_seek_from_end."""
352
# The actual implementation will seek from end for the first range only
353
# and then fail. Since seeking from end is intended to be used for a
354
# single range only anyway, this test just document the actual
358
self.assertEquals('yz', f.read())
359
self.assertRaises(errors.InvalidRange, f.seek, -2, 2)
361
def test_seek_into_void(self):
363
start = self.first_range_start
365
# Seeking to a point between two ranges is possible (only once) but
366
# reading there is forbidden
368
# We crossed a range boundary, so now the file is positioned at the
369
# start of the new range (i.e. trying to seek below 100 will error out)
373
def test_seek_across_ranges(self):
375
start = self.first_range_start
376
f.seek(126) # skip the two first ranges
377
self.assertEquals('AB', f.read(2))
379
def test_checked_read_dont_overflow_buffers(self):
381
start = self.first_range_start
382
# We force a very low value to exercise all code paths in _checked_read
383
f._discarded_buf_size = 8
384
f.seek(126) # skip the two first ranges
385
self.assertEquals('AB', f.read(2))
387
def test_seek_twice_between_ranges(self):
389
start = self.first_range_start
390
f.seek(start + 40) # Past the first range but before the second
391
# Now the file is positioned at the second range start (100)
392
self.assertRaises(errors.InvalidRange, f.seek, start + 41)
394
def test_seek_at_range_end(self):
395
"""Test seek behavior at range end."""
401
def test_read_at_range_end(self):
403
self.assertEquals(self.alpha, f.read())
404
self.assertEquals(self.alpha, f.read())
405
self.assertEquals(self.alpha.upper(), f.read())
406
self.assertRaises(errors.InvalidHttpResponse, f.read, 1)
409
class TestRangeFileMultipleRangesQuotedBoundaries(TestRangeFileMultipleRanges):
410
"""Perform the same tests as TestRangeFileMultipleRanges, but uses
411
an angle-bracket quoted boundary string like IIS 6.0 and 7.0
412
(but not IIS 5, which breaks the RFC in a different way
413
by using square brackets, not angle brackets)
415
This reveals a bug caused by
417
- The bad implementation of RFC 822 unquoting in Python (angles are not
418
quotes), coupled with
420
- The bad implementation of RFC 2046 in IIS (angles are not permitted chars
424
# The boundary as it appears in boundary lines
425
# IIS 6 and 7 use this value
426
_boundary_trimmed = "q1w2e3r4t5y6u7i8o9p0zaxscdvfbgnhmjklkl"
427
boundary = '<' + _boundary_trimmed + '>'
429
def set_file_boundary(self):
430
# Emulate broken rfc822.unquote() here by removing angles
431
self._file.set_boundary(self._boundary_trimmed)
434
class TestRangeFileVarious(tests.TestCase):
435
"""Tests RangeFile aspects not covered elsewhere."""
437
def test_seek_whence(self):
438
"""Test the seek whence parameter values."""
439
f = response.RangeFile('foo', StringIO('abc'))
243
self.assertRaises(errors.InvalidRange, f.read, 2)
245
self.assertEqual('012345', f.read(6))
247
def test_invalid(self):
249
f = response.HttpRangeResponse('http://foo', 'bytes x-10/9',
250
StringIO('0123456789'))
251
except errors.InvalidHttpRange, e:
252
self.assertContainsRe(str(e), 'http://foo')
253
self.assertContainsRe(str(e), 'bytes x-10/9')
255
self.fail('Failed to raise InvalidHttpRange')
258
class TestHttpMultipartRangeResponse(TestCase):
259
"""Test the handling of multipart range responses"""
261
def test_simple(self):
262
content = StringIO(simple_data)
263
multi = response.HttpMultipartRangeResponse('http://foo',
264
'multipart/byteranges; boundary = xxyyzz', content)
266
self.assertEqual(4, len(multi._ranges))
269
self.assertEqual('1234567890', multi.read(10))
271
self.assertEqual('abcdefghij', multi.read(10))
273
self.assertEqual('zyxwvutsrq', multi.read(10))
275
self.assertEqual('xxyyzz fbd', multi.read(10))
276
# TODO: jam 20060706 Currently RangeFile does not support
277
# reading across ranges. Consider adding it.
279
# self.assertEqual('zyxwvutsrqxxyyzz fbd', multi.read(20))
280
self.assertRaises(errors.InvalidRange, multi.read, 20)
283
self.assertRaises(errors.InvalidRange, multi.read, 11)
285
self.assertRaises(errors.InvalidRange, multi.read, 10)
287
def test_invalid(self):
288
content = StringIO('')
290
response.HttpMultipartRangeResponse('http://foo',
291
'multipart/byte;boundary=invalid', content)
292
except errors.InvalidHttpContentType, e:
293
self.assertContainsRe(str(e), 'http://foo')
294
self.assertContainsRe(str(e), 'multipart/byte;')
444
self.assertRaises(ValueError, f.seek, 0, 14)
446
def test_range_syntax(self):
447
"""Test the Content-Range scanning."""
449
f = response.RangeFile('foo', StringIO())
451
def ok(expected, header_value):
452
f.set_range_from_header(header_value)
453
# Slightly peek under the covers to get the size
454
self.assertEquals(expected, (f.tell(), f._size))
456
ok((1, 10), 'bytes 1-10/11')
457
ok((1, 10), 'bytes 1-10/*')
458
ok((12, 2), '\tbytes 12-13/*')
459
ok((28, 1), ' bytes 28-28/*')
460
ok((2123, 2120), 'bytes 2123-4242/12310')
461
ok((1, 10), 'bytes 1-10/ttt') # We don't check total (ttt)
463
def nok(header_value):
464
self.assertRaises(errors.InvalidHttpRange,
465
f.set_range_from_header, header_value)
469
nok('bytes xx-yyy/zzz')
470
nok('bytes xx-12/zzz')
471
nok('bytes 11-yy/zzz')
297
475
# Taken from real request responses
440
# This should be in test_http.py, but the headers we
441
# want to parse are here
442
class TestExtractHeader(TestCase):
444
def use_response(self, response):
445
self.headers = http._extract_headers(response[1], 'http://foo')
447
def check_header(self, header, value):
448
self.assertEqual(value, self.headers[header])
450
def test_full_text(self):
451
self.use_response(_full_text_response)
453
self.check_header('Date', 'Tue, 11 Jul 2006 04:32:56 GMT')
454
self.check_header('date', 'Tue, 11 Jul 2006 04:32:56 GMT')
455
self.check_header('Content-Length', '35')
456
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
457
self.check_header('content-type', 'text/plain; charset=UTF-8')
459
def test_missing_response(self):
460
self.use_response(_missing_response)
462
self.check_header('Content-Length', '336')
463
self.check_header('Content-Type', 'text/html; charset=iso-8859-1')
465
def test_single_range(self):
466
self.use_response(_single_range_response)
468
self.check_header('Content-Length', '100')
469
self.check_header('Content-Range', 'bytes 100-199/93890')
470
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
472
def test_multi_range(self):
473
self.use_response(_multipart_range_response)
475
self.check_header('Content-Length', '1534')
476
self.check_header('Content-Type',
477
'multipart/byteranges; boundary=418470f848b63279b')
479
def test_redirect(self):
480
"""We default to returning the last group of headers in the file."""
481
self.use_response(_redirect_response)
482
self.check_header('Content-Range', 'bytes 8623075-8623499/8623500')
483
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
485
def test_empty(self):
486
self.assertRaises(errors.InvalidHttpResponse,
487
http._extract_headers, '', 'bad url')
489
def test_no_opening_http(self):
490
# Remove the HTTP line from the header
491
first, txt = _full_text_response[1].split('\r\n', 1)
492
self.assertRaises(errors.InvalidHttpResponse,
493
http._extract_headers, txt, 'missing HTTTP')
495
def test_trailing_whitespace(self):
496
# Test that we ignore bogus whitespace on the end
497
code, txt, body = _full_text_response
498
txt += '\r\n\n\n\n\n'
499
self.use_response((code, txt, body))
501
self.check_header('Date', 'Tue, 11 Jul 2006 04:32:56 GMT')
502
self.check_header('Content-Length', '35')
503
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
505
def test_trailing_non_http(self):
506
# Test that we ignore bogus stuff on the end
507
code, txt, body = _full_text_response
508
txt = txt + 'Foo: Bar\r\nBaz: Bling\r\n\r\n'
509
self.use_response((code, txt, body))
511
self.check_header('Date', 'Tue, 11 Jul 2006 04:32:56 GMT')
512
self.check_header('Content-Length', '35')
513
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
514
self.assertRaises(KeyError, self.headers.__getitem__, 'Foo')
516
def test_extra_whitespace(self):
517
# Test that we read an HTTP response, even with extra whitespace
518
code, txt, body = _redirect_response
519
# Find the second HTTP location
520
loc = txt.find('HTTP', 5)
521
txt = txt[:loc] + '\r\n\n' + txt[loc:]
522
self.use_response((code, txt, body))
523
self.check_header('Content-Range', 'bytes 8623075-8623499/8623500')
524
self.check_header('Content-Type', 'text/plain; charset=UTF-8')
527
class TestHandleResponse(TestCase):
671
_multipart_no_content_range = (206, """HTTP/1.0 206 Partial Content\r
672
Content-Type: multipart/byteranges; boundary=THIS_SEPARATES\r
673
Content-Length: 598\r
678
Content-Type: text/plain\r
685
_multipart_no_boundary = (206, """HTTP/1.0 206 Partial Content\r
686
Content-Type: multipart/byteranges; boundary=THIS_SEPARATES\r
687
Content-Length: 598\r
692
Content-Type: text/plain\r
693
Content-Range: bytes 0-18/18672\r
697
The range ended at the line above, this text is garbage instead of a boundary
702
class TestHandleResponse(tests.TestCase):
704
def _build_HTTPMessage(self, raw_headers):
705
status_and_headers = StringIO(raw_headers)
706
# Get rid of the status line
707
status_and_headers.readline()
708
msg = httplib.HTTPMessage(status_and_headers)
529
711
def get_response(self, a_response):
530
712
"""Process a supplied response, and return the result."""
531
headers = http._extract_headers(a_response[1], 'http://foo')
532
return response.handle_response('http://foo', a_response[0], headers,
713
code, raw_headers, body = a_response
714
msg = self._build_HTTPMessage(raw_headers)
715
return response.handle_response('http://foo', code, msg,
533
716
StringIO(a_response[2]))
535
718
def test_full_text(self):
750
def test_multi_squid_range(self):
751
out = self.get_response(_multipart_squid_range_response)
753
# Just make sure we can read the right contents
564
760
def test_invalid_response(self):
565
761
self.assertRaises(errors.InvalidHttpResponse,
566
self.get_response, _invalid_response)
762
self.get_response, _invalid_response)
568
764
def test_full_text_no_content_type(self):
569
765
# We should not require Content-Type for a full response
570
a_response = _full_text_response
571
headers = http._extract_headers(a_response[1], 'http://foo')
572
del headers['Content-Type']
573
out = response.handle_response('http://foo', a_response[0], headers,
574
StringIO(a_response[2]))
575
self.assertEqual(_full_text_response[2], out.read())
577
def test_missing_no_content_type(self):
578
# Without Content-Type we should still raise NoSuchFile on a 404
579
a_response = _missing_response
580
headers = http._extract_headers(a_response[1], 'http://missing')
581
del headers['Content-Type']
582
self.assertRaises(errors.NoSuchFile,
583
response.handle_response, 'http://missing', a_response[0], headers,
584
StringIO(a_response[2]))
586
def test_missing_content_type(self):
587
a_response = _single_range_response
588
headers = http._extract_headers(a_response[1], 'http://nocontent')
589
del headers['Content-Type']
590
self.assertRaises(errors.InvalidHttpContentType,
591
response.handle_response, 'http://nocontent', a_response[0],
592
headers, StringIO(a_response[2]))
766
code, raw_headers, body = _full_text_response_no_content_type
767
msg = self._build_HTTPMessage(raw_headers)
768
out = response.handle_response('http://foo', code, msg, StringIO(body))
769
self.assertEqual(body, out.read())
771
def test_full_text_no_content_length(self):
772
code, raw_headers, body = _full_text_response_no_content_length
773
msg = self._build_HTTPMessage(raw_headers)
774
out = response.handle_response('http://foo', code, msg, StringIO(body))
775
self.assertEqual(body, out.read())
594
777
def test_missing_content_range(self):
595
a_response = _single_range_response
596
headers = http._extract_headers(a_response[1], 'http://nocontent')
597
del headers['Content-Range']
598
self.assertRaises(errors.InvalidHttpResponse,
599
response.handle_response, 'http://nocontent', a_response[0],
600
headers, StringIO(a_response[2]))
778
code, raw_headers, body = _single_range_no_content_range
779
msg = self._build_HTTPMessage(raw_headers)
780
self.assertRaises(errors.InvalidHttpResponse,
781
response.handle_response,
782
'http://bogus', code, msg, StringIO(body))
784
def test_multipart_no_content_range(self):
785
code, raw_headers, body = _multipart_no_content_range
786
msg = self._build_HTTPMessage(raw_headers)
787
self.assertRaises(errors.InvalidHttpResponse,
788
response.handle_response,
789
'http://bogus', code, msg, StringIO(body))
791
def test_multipart_no_boundary(self):
792
out = self.get_response(_multipart_no_boundary)
793
out.read() # Read the whole range
794
# Fail to find the boundary line
795
self.assertRaises(errors.InvalidHttpResponse, out.seek, 1, 1)
798
class TestRangeFileSizeReadLimited(tests.TestCase):
799
"""Test RangeFile _max_read_size functionality which limits the size of
800
read blocks to prevent MemoryError messages in socket.recv.
804
tests.TestCase.setUp(self)
805
# create a test datablock larger than _max_read_size.
806
chunk_size = response.RangeFile._max_read_size
807
test_pattern = '0123456789ABCDEF'
808
self.test_data = test_pattern * (3 * chunk_size / len(test_pattern))
809
self.test_data_len = len(self.test_data)
811
def test_max_read_size(self):
812
"""Read data in blocks and verify that the reads are not larger than
813
the maximum read size.
815
# retrieve data in large blocks from response.RangeFile object
816
mock_read_file = FakeReadFile(self.test_data)
817
range_file = response.RangeFile('test_max_read_size', mock_read_file)
818
response_data = range_file.read(self.test_data_len)
820
# verify read size was equal to the maximum read size
821
self.assertTrue(mock_read_file.get_max_read_size() > 0)
822
self.assertEqual(mock_read_file.get_max_read_size(),
823
response.RangeFile._max_read_size)
824
self.assertEqual(mock_read_file.get_read_count(), 3)
826
# report error if the data wasn't equal (we only report the size due
827
# to the length of the data)
828
if response_data != self.test_data:
829
message = "Data not equal. Expected %d bytes, received %d."
830
self.fail(message % (len(response_data), self.test_data_len))