191
95
self.received_bytes = ''
195
return '%s://%s:%s/' % (self.scheme, self.host, self.port)
197
def start_server(self):
198
98
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
199
99
self._sock.bind(('127.0.0.1', 0))
200
100
self.host, self.port = self._sock.getsockname()
201
101
self._ready = threading.Event()
202
self._thread = test_server.TestThread(
203
sync_event=self._ready, target=self._accept_read_and_reply)
102
self._thread = threading.Thread(target=self._accept_read_and_reply)
103
self._thread.setDaemon(True)
204
104
self._thread.start()
205
if 'threads' in tests.selftest_debug_flags:
206
sys.stderr.write('Thread started: %s\n' % (self._thread.ident,))
209
107
def _accept_read_and_reply(self):
210
108
self._sock.listen(1)
211
109
self._ready.set()
212
conn, address = self._sock.accept()
213
if self._expect_body_tail is not None:
110
self._sock.settimeout(5)
112
conn, address = self._sock.accept()
113
# On win32, the accepted connection will be non-blocking to start
114
# with because we're using settimeout.
115
conn.setblocking(True)
214
116
while not self.received_bytes.endswith(self._expect_body_tail):
215
117
self.received_bytes += conn.recv(4096)
216
118
conn.sendall('HTTP/1.1 200 OK\r\n')
119
except socket.timeout:
120
# Make sure the client isn't stuck waiting for us to e.g. accept.
218
121
self._sock.close()
219
122
except socket.error:
220
123
# The client may have already closed the socket.
223
def stop_server(self):
225
# Issue a fake connection to wake up the server and allow it to
227
fake_conn = osutils.connect_socket((self.host, self.port))
229
129
except socket.error:
230
130
# We might have already closed it. We don't care.
235
if 'threads' in tests.selftest_debug_flags:
236
sys.stderr.write('Thread joined: %s\n' % (self._thread.ident,))
239
class TestAuthHeader(tests.TestCase):
241
def parse_header(self, header, auth_handler_class=None):
242
if auth_handler_class is None:
243
auth_handler_class = _urllib2_wrappers.AbstractAuthHandler
244
self.auth_handler = auth_handler_class()
245
return self.auth_handler._parse_auth_header(header)
247
def test_empty_header(self):
248
scheme, remainder = self.parse_header('')
249
self.assertEqual('', scheme)
250
self.assertIs(None, remainder)
252
def test_negotiate_header(self):
253
scheme, remainder = self.parse_header('Negotiate')
254
self.assertEqual('negotiate', scheme)
255
self.assertIs(None, remainder)
257
def test_basic_header(self):
258
scheme, remainder = self.parse_header(
259
'Basic realm="Thou should not pass"')
260
self.assertEqual('basic', scheme)
261
self.assertEqual('realm="Thou should not pass"', remainder)
263
def test_basic_extract_realm(self):
264
scheme, remainder = self.parse_header(
265
'Basic realm="Thou should not pass"',
266
_urllib2_wrappers.BasicAuthHandler)
267
match, realm = self.auth_handler.extract_realm(remainder)
268
self.assertTrue(match is not None)
269
self.assertEqual('Thou should not pass', realm)
271
def test_digest_header(self):
272
scheme, remainder = self.parse_header(
273
'Digest realm="Thou should not pass"')
274
self.assertEqual('digest', scheme)
275
self.assertEqual('realm="Thou should not pass"', remainder)
278
class TestHTTPRangeParsing(tests.TestCase):
281
super(TestHTTPRangeParsing, self).setUp()
282
# We focus on range parsing here and ignore everything else
283
class RequestHandler(http_server.TestingHTTPRequestHandler):
284
def setup(self): pass
285
def handle(self): pass
286
def finish(self): pass
288
self.req_handler = RequestHandler(None, None, None)
290
def assertRanges(self, ranges, header, file_size):
291
self.assertEquals(ranges,
292
self.req_handler._parse_ranges(header, file_size))
294
def test_simple_range(self):
295
self.assertRanges([(0,2)], 'bytes=0-2', 12)
298
self.assertRanges([(8, 11)], 'bytes=-4', 12)
300
def test_tail_bigger_than_file(self):
301
self.assertRanges([(0, 11)], 'bytes=-99', 12)
303
def test_range_without_end(self):
304
self.assertRanges([(4, 11)], 'bytes=4-', 12)
306
def test_invalid_ranges(self):
307
self.assertRanges(None, 'bytes=12-22', 12)
308
self.assertRanges(None, 'bytes=1-3,12-22', 12)
309
self.assertRanges(None, 'bytes=-', 12)
312
class TestHTTPServer(tests.TestCase):
313
"""Test the HTTP servers implementations."""
315
def test_invalid_protocol(self):
316
class BogusRequestHandler(http_server.TestingHTTPRequestHandler):
318
protocol_version = 'HTTP/0.1'
320
self.assertRaises(httplib.UnknownProtocol,
321
http_server.HttpServer, BogusRequestHandler)
323
def test_force_invalid_protocol(self):
324
self.assertRaises(httplib.UnknownProtocol,
325
http_server.HttpServer, protocol_version='HTTP/0.1')
327
def test_server_start_and_stop(self):
328
server = http_server.HttpServer()
329
self.addCleanup(server.stop_server)
330
server.start_server()
331
self.assertTrue(server.server is not None)
332
self.assertTrue(server.server.serving is not None)
333
self.assertTrue(server.server.serving)
335
def test_create_http_server_one_zero(self):
336
class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
338
protocol_version = 'HTTP/1.0'
340
server = http_server.HttpServer(RequestHandlerOneZero)
341
self.start_server(server)
342
self.assertIsInstance(server.server, http_server.TestingHTTPServer)
344
def test_create_http_server_one_one(self):
345
class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
347
protocol_version = 'HTTP/1.1'
349
server = http_server.HttpServer(RequestHandlerOneOne)
350
self.start_server(server)
351
self.assertIsInstance(server.server,
352
http_server.TestingThreadingHTTPServer)
354
def test_create_http_server_force_one_one(self):
355
class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
357
protocol_version = 'HTTP/1.0'
359
server = http_server.HttpServer(RequestHandlerOneZero,
360
protocol_version='HTTP/1.1')
361
self.start_server(server)
362
self.assertIsInstance(server.server,
363
http_server.TestingThreadingHTTPServer)
365
def test_create_http_server_force_one_zero(self):
366
class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
368
protocol_version = 'HTTP/1.1'
370
server = http_server.HttpServer(RequestHandlerOneOne,
371
protocol_version='HTTP/1.0')
372
self.start_server(server)
373
self.assertIsInstance(server.server,
374
http_server.TestingHTTPServer)
377
136
class TestWithTransport_pycurl(object):
378
137
"""Test case to inherit from if pycurl is present"""
380
139
def _get_pycurl_maybe(self):
381
self.requireFeature(features.pycurl)
382
return PyCurlTransport
141
from bzrlib.transport.http._pycurl import PyCurlTransport
142
return PyCurlTransport
143
except errors.DependencyNotPresent:
144
raise TestSkipped('pycurl not present')
384
146
_transport = property(_get_pycurl_maybe)
387
class TestHttpTransportUrls(tests.TestCase):
388
"""Test the http urls."""
390
scenarios = vary_by_http_client_implementation()
149
class TestHttpUrls(TestCase):
151
# TODO: This should be moved to authorization tests once they
154
def test_url_parsing(self):
156
url = extract_auth('http://example.com', f)
157
self.assertEquals('http://example.com', url)
158
self.assertEquals(0, len(f.credentials))
159
url = extract_auth('http://user:pass@www.bazaar-vcs.org/bzr/bzr.dev', f)
160
self.assertEquals('http://www.bazaar-vcs.org/bzr/bzr.dev', url)
161
self.assertEquals(1, len(f.credentials))
162
self.assertEquals([None, 'www.bazaar-vcs.org', 'user', 'pass'],
166
class TestHttpTransportUrls(object):
167
"""Test the http urls.
169
This MUST be used by daughter classes that also inherit from
172
We can't inherit directly from TestCase or the
173
test framework will try to create an instance which cannot
174
run, its implementation being incomplete.
392
177
def test_abs_url(self):
393
178
"""Construction of absolute http URLs"""
394
t = self._transport('http://example.com/bzr/bzr.dev/')
179
t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
395
180
eq = self.assertEqualDiff
396
eq(t.abspath('.'), 'http://example.com/bzr/bzr.dev')
397
eq(t.abspath('foo/bar'), 'http://example.com/bzr/bzr.dev/foo/bar')
398
eq(t.abspath('.bzr'), 'http://example.com/bzr/bzr.dev/.bzr')
181
eq(t.abspath('.'), 'http://bazaar-vcs.org/bzr/bzr.dev')
182
eq(t.abspath('foo/bar'), 'http://bazaar-vcs.org/bzr/bzr.dev/foo/bar')
183
eq(t.abspath('.bzr'), 'http://bazaar-vcs.org/bzr/bzr.dev/.bzr')
399
184
eq(t.abspath('.bzr/1//2/./3'),
400
'http://example.com/bzr/bzr.dev/.bzr/1/2/3')
185
'http://bazaar-vcs.org/bzr/bzr.dev/.bzr/1/2/3')
402
187
def test_invalid_http_urls(self):
403
188
"""Trap invalid construction of urls"""
404
self._transport('http://example.com/bzr/bzr.dev/')
405
self.assertRaises(errors.InvalidURL,
407
'http://http://example.com/bzr/bzr.dev/')
189
t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
190
self.assertRaises(ValueError, t.abspath, '.bzr/')
191
t = self._transport('http://http://bazaar-vcs.org/bzr/bzr.dev/')
192
self.assertRaises((errors.InvalidURL, errors.ConnectionError),
409
195
def test_http_root_urls(self):
410
196
"""Construction of URLs from server root"""
411
t = self._transport('http://example.com/')
197
t = self._transport('http://bzr.ozlabs.org/')
412
198
eq = self.assertEqualDiff
413
199
eq(t.abspath('.bzr/tree-version'),
414
'http://example.com/.bzr/tree-version')
200
'http://bzr.ozlabs.org/.bzr/tree-version')
416
202
def test_http_impl_urls(self):
417
203
"""There are servers which ask for particular clients to connect"""
418
204
server = self._server()
419
server.start_server()
421
207
url = server.get_url()
422
self.assertTrue(url.startswith('%s://' % self._url_protocol))
208
self.assertTrue(url.startswith('%s://' % self._qualified_prefix))
427
class TestHttps_pycurl(TestWithTransport_pycurl, tests.TestCase):
213
class TestHttpUrls_urllib(TestHttpTransportUrls, TestCase):
214
"""Test http urls with urllib"""
216
_transport = HttpTransport_urllib
217
_server = HttpServer_urllib
218
_qualified_prefix = 'http+urllib'
221
class TestHttpUrls_pycurl(TestWithTransport_pycurl, TestHttpTransportUrls,
223
"""Test http urls with pycurl"""
225
_server = HttpServer_PyCurl
226
_qualified_prefix = 'http+pycurl'
429
228
# TODO: This should really be moved into another pycurl
430
229
# specific test. When https tests will be implemented, take
815
658
self.assertListRaises((errors.InvalidRange, errors.ShortReadvError,),
816
659
t.readv, 'a', [(12,2)])
818
def test_readv_multiple_get_requests(self):
819
server = self.get_readonly_server()
820
t = self.get_readonly_transport()
821
# force transport to issue multiple requests
822
t._max_readv_combine = 1
823
t._max_get_ranges = 1
824
l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
825
self.assertEqual(l[0], (0, '0'))
826
self.assertEqual(l[1], (1, '1'))
827
self.assertEqual(l[2], (3, '34'))
828
self.assertEqual(l[3], (9, '9'))
829
# The server should have issued 4 requests
830
self.assertEqual(4, server.GET_request_nb)
832
def test_readv_get_max_size(self):
833
server = self.get_readonly_server()
834
t = self.get_readonly_transport()
835
# force transport to issue multiple requests by limiting the number of
836
# bytes by request. Note that this apply to coalesced offsets only, a
837
# single range will keep its size even if bigger than the limit.
839
l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
840
self.assertEqual(l[0], (0, '0'))
841
self.assertEqual(l[1], (1, '1'))
842
self.assertEqual(l[2], (2, '2345'))
843
self.assertEqual(l[3], (6, '6789'))
844
# The server should have issued 3 requests
845
self.assertEqual(3, server.GET_request_nb)
847
def test_complete_readv_leave_pipe_clean(self):
848
server = self.get_readonly_server()
849
t = self.get_readonly_transport()
850
# force transport to issue multiple requests
852
list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
853
# The server should have issued 3 requests
854
self.assertEqual(3, server.GET_request_nb)
855
self.assertEqual('0123456789', t.get_bytes('a'))
856
self.assertEqual(4, server.GET_request_nb)
858
def test_incomplete_readv_leave_pipe_clean(self):
859
server = self.get_readonly_server()
860
t = self.get_readonly_transport()
861
# force transport to issue multiple requests
863
# Don't collapse readv results into a list so that we leave unread
864
# bytes on the socket
865
ireadv = iter(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
866
self.assertEqual((0, '0'), ireadv.next())
867
# The server should have issued one request so far
868
self.assertEqual(1, server.GET_request_nb)
869
self.assertEqual('0123456789', t.get_bytes('a'))
870
# get_bytes issued an additional request, the readv pending ones are
872
self.assertEqual(2, server.GET_request_nb)
875
class SingleRangeRequestHandler(http_server.TestingHTTPRequestHandler):
876
"""Always reply to range request as if they were single.
878
Don't be explicit about it, just to annoy the clients.
881
def get_multiple_ranges(self, file, file_size, ranges):
882
"""Answer as if it was a single range request and ignores the rest"""
883
(start, end) = ranges[0]
884
return self.get_single_range(file, file_size, start, end)
887
662
class TestSingleRangeRequestServer(TestRangeRequestServer):
888
663
"""Test readv against a server which accept only single range requests"""
890
_req_handler_class = SingleRangeRequestHandler
893
class SingleOnlyRangeRequestHandler(http_server.TestingHTTPRequestHandler):
894
"""Only reply to simple range requests, errors out on multiple"""
896
def get_multiple_ranges(self, file, file_size, ranges):
897
"""Refuses the multiple ranges request"""
900
self.send_error(416, "Requested range not satisfiable")
902
(start, end) = ranges[0]
903
return self.get_single_range(file, file_size, start, end)
906
class TestSingleOnlyRangeRequestServer(TestRangeRequestServer):
907
"""Test readv against a server which only accept single range requests"""
909
_req_handler_class = SingleOnlyRangeRequestHandler
912
class NoRangeRequestHandler(http_server.TestingHTTPRequestHandler):
913
"""Ignore range requests without notice"""
916
# Update the statistics
917
self.server.test_case_server.GET_request_nb += 1
918
# Just bypass the range handling done by TestingHTTPRequestHandler
919
return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
665
def create_transport_readonly_server(self):
666
return HttpServer(SingleRangeRequestHandler)
669
class TestSingleRangeRequestServer_urllib(TestSingleRangeRequestServer,
670
TestCaseWithWebserver):
671
"""Tests single range requests accepting server for urllib implementation"""
673
_transport = HttpTransport_urllib
676
class TestSingleRangeRequestServer_pycurl(TestWithTransport_pycurl,
677
TestSingleRangeRequestServer,
678
TestCaseWithWebserver):
679
"""Tests single range requests accepting server for pycurl implementation"""
922
682
class TestNoRangeRequestServer(TestRangeRequestServer):
923
683
"""Test readv against a server which do not accept range requests"""
925
_req_handler_class = NoRangeRequestHandler
928
class MultipleRangeWithoutContentLengthRequestHandler(
929
http_server.TestingHTTPRequestHandler):
930
"""Reply to multiple range requests without content length header."""
932
def get_multiple_ranges(self, file, file_size, ranges):
933
self.send_response(206)
934
self.send_header('Accept-Ranges', 'bytes')
935
# XXX: this is strange; the 'random' name below seems undefined and
936
# yet the tests pass -- mbp 2010-10-11 bug 658773
937
boundary = "%d" % random.randint(0,0x7FFFFFFF)
938
self.send_header("Content-Type",
939
"multipart/byteranges; boundary=%s" % boundary)
941
for (start, end) in ranges:
942
self.wfile.write("--%s\r\n" % boundary)
943
self.send_header("Content-type", 'application/octet-stream')
944
self.send_header("Content-Range", "bytes %d-%d/%d" % (start,
948
self.send_range_content(file, start, end - start + 1)
950
self.wfile.write("--%s\r\n" % boundary)
953
class TestMultipleRangeWithoutContentLengthServer(TestRangeRequestServer):
955
_req_handler_class = MultipleRangeWithoutContentLengthRequestHandler
958
class TruncatedMultipleRangeRequestHandler(
959
http_server.TestingHTTPRequestHandler):
960
"""Reply to multiple range requests truncating the last ones.
962
This server generates responses whose Content-Length describes all the
963
ranges, but fail to include the last ones leading to client short reads.
964
This has been observed randomly with lighttpd (bug #179368).
967
_truncated_ranges = 2
969
def get_multiple_ranges(self, file, file_size, ranges):
970
self.send_response(206)
971
self.send_header('Accept-Ranges', 'bytes')
973
self.send_header('Content-Type',
974
'multipart/byteranges; boundary=%s' % boundary)
975
boundary_line = '--%s\r\n' % boundary
976
# Calculate the Content-Length
978
for (start, end) in ranges:
979
content_length += len(boundary_line)
980
content_length += self._header_line_length(
981
'Content-type', 'application/octet-stream')
982
content_length += self._header_line_length(
983
'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
984
content_length += len('\r\n') # end headers
985
content_length += end - start # + 1
986
content_length += len(boundary_line)
987
self.send_header('Content-length', content_length)
990
# Send the multipart body
992
for (start, end) in ranges:
993
self.wfile.write(boundary_line)
994
self.send_header('Content-type', 'application/octet-stream')
995
self.send_header('Content-Range', 'bytes %d-%d/%d'
996
% (start, end, file_size))
998
if cur + self._truncated_ranges >= len(ranges):
999
# Abruptly ends the response and close the connection
1000
self.close_connection = 1
1002
self.send_range_content(file, start, end - start + 1)
1005
self.wfile.write(boundary_line)
1008
class TestTruncatedMultipleRangeServer(TestSpecificRequestHandler):
1010
_req_handler_class = TruncatedMultipleRangeRequestHandler
1013
super(TestTruncatedMultipleRangeServer, self).setUp()
1014
self.build_tree_contents([('a', '0123456789')],)
1016
def test_readv_with_short_reads(self):
1017
server = self.get_readonly_server()
1018
t = self.get_readonly_transport()
1019
# Force separate ranges for each offset
1020
t._bytes_to_read_before_seek = 0
1021
ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1022
self.assertEqual((0, '0'), ireadv.next())
1023
self.assertEqual((2, '2'), ireadv.next())
1024
if not self._testing_pycurl():
1025
# Only one request have been issued so far (except for pycurl that
1026
# try to read the whole response at once)
1027
self.assertEqual(1, server.GET_request_nb)
1028
self.assertEqual((4, '45'), ireadv.next())
1029
self.assertEqual((9, '9'), ireadv.next())
1030
# Both implementations issue 3 requests but:
1031
# - urllib does two multiple (4 ranges, then 2 ranges) then a single
1033
# - pycurl does two multiple (4 ranges, 4 ranges) then a single range
1034
self.assertEqual(3, server.GET_request_nb)
1035
# Finally the client have tried a single range request and stays in
1037
self.assertEqual('single', t._range_hint)
1040
class TruncatedBeforeBoundaryRequestHandler(
1041
http_server.TestingHTTPRequestHandler):
1042
"""Truncation before a boundary, like in bug 198646"""
1044
_truncated_ranges = 1
1046
def get_multiple_ranges(self, file, file_size, ranges):
1047
self.send_response(206)
1048
self.send_header('Accept-Ranges', 'bytes')
1050
self.send_header('Content-Type',
1051
'multipart/byteranges; boundary=%s' % boundary)
1052
boundary_line = '--%s\r\n' % boundary
1053
# Calculate the Content-Length
1055
for (start, end) in ranges:
1056
content_length += len(boundary_line)
1057
content_length += self._header_line_length(
1058
'Content-type', 'application/octet-stream')
1059
content_length += self._header_line_length(
1060
'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
1061
content_length += len('\r\n') # end headers
1062
content_length += end - start # + 1
1063
content_length += len(boundary_line)
1064
self.send_header('Content-length', content_length)
1067
# Send the multipart body
1069
for (start, end) in ranges:
1070
if cur + self._truncated_ranges >= len(ranges):
1071
# Abruptly ends the response and close the connection
1072
self.close_connection = 1
1074
self.wfile.write(boundary_line)
1075
self.send_header('Content-type', 'application/octet-stream')
1076
self.send_header('Content-Range', 'bytes %d-%d/%d'
1077
% (start, end, file_size))
1079
self.send_range_content(file, start, end - start + 1)
1082
self.wfile.write(boundary_line)
1085
class TestTruncatedBeforeBoundary(TestSpecificRequestHandler):
1086
"""Tests the case of bug 198646, disconnecting before a boundary."""
1088
_req_handler_class = TruncatedBeforeBoundaryRequestHandler
1091
super(TestTruncatedBeforeBoundary, self).setUp()
1092
self.build_tree_contents([('a', '0123456789')],)
1094
def test_readv_with_short_reads(self):
1095
server = self.get_readonly_server()
1096
t = self.get_readonly_transport()
1097
# Force separate ranges for each offset
1098
t._bytes_to_read_before_seek = 0
1099
ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1100
self.assertEqual((0, '0'), ireadv.next())
1101
self.assertEqual((2, '2'), ireadv.next())
1102
self.assertEqual((4, '45'), ireadv.next())
1103
self.assertEqual((9, '9'), ireadv.next())
1106
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1107
"""Errors out when range specifiers exceed the limit"""
1109
def get_multiple_ranges(self, file, file_size, ranges):
1110
"""Refuses the multiple ranges request"""
1111
tcs = self.server.test_case_server
1112
if tcs.range_limit is not None and len(ranges) > tcs.range_limit:
1114
# Emulate apache behavior
1115
self.send_error(400, "Bad Request")
1117
return http_server.TestingHTTPRequestHandler.get_multiple_ranges(
1118
self, file, file_size, ranges)
1121
class LimitedRangeHTTPServer(http_server.HttpServer):
1122
"""An HttpServer erroring out on requests with too much range specifiers"""
1124
def __init__(self, request_handler=LimitedRangeRequestHandler,
1125
protocol_version=None,
1127
http_server.HttpServer.__init__(self, request_handler,
1128
protocol_version=protocol_version)
1129
self.range_limit = range_limit
1132
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1133
"""Tests readv requests against a server erroring out on too much ranges."""
1135
scenarios = multiply_scenarios(
1136
vary_by_http_client_implementation(),
1137
vary_by_http_protocol_version(),
1140
# Requests with more range specifiers will error out
1143
685
def create_transport_readonly_server(self):
1144
return LimitedRangeHTTPServer(range_limit=self.range_limit,
1145
protocol_version=self._protocol_version)
1148
super(TestLimitedRangeRequestServer, self).setUp()
1149
# We need to manipulate ranges that correspond to real chunks in the
1150
# response, so we build a content appropriately.
1151
filler = ''.join(['abcdefghij' for x in range(102)])
1152
content = ''.join(['%04d' % v + filler for v in range(16)])
1153
self.build_tree_contents([('a', content)],)
1155
def test_few_ranges(self):
1156
t = self.get_readonly_transport()
1157
l = list(t.readv('a', ((0, 4), (1024, 4), )))
1158
self.assertEqual(l[0], (0, '0000'))
1159
self.assertEqual(l[1], (1024, '0001'))
1160
self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1162
def test_more_ranges(self):
1163
t = self.get_readonly_transport()
1164
l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1165
self.assertEqual(l[0], (0, '0000'))
1166
self.assertEqual(l[1], (1024, '0001'))
1167
self.assertEqual(l[2], (4096, '0004'))
1168
self.assertEqual(l[3], (8192, '0008'))
1169
# The server will refuse to serve the first request (too much ranges),
1170
# a second request will succeed.
1171
self.assertEqual(2, self.get_readonly_server().GET_request_nb)
1174
class TestHttpProxyWhiteBox(tests.TestCase):
686
return HttpServer(NoRangeRequestHandler)
689
class TestNoRangeRequestServer_urllib(TestNoRangeRequestServer,
690
TestCaseWithWebserver):
691
"""Tests range requests refusing server for urllib implementation"""
693
_transport = HttpTransport_urllib
696
class TestNoRangeRequestServer_pycurl(TestWithTransport_pycurl,
697
TestNoRangeRequestServer,
698
TestCaseWithWebserver):
699
"""Tests range requests refusing server for pycurl implementation"""
702
class TestHttpProxyWhiteBox(TestCase):
1175
703
"""Whitebox test proxy http authorization.
1177
Only the urllib implementation is tested here.
705
These tests concern urllib implementation only.
715
def _set_and_capture_env_var(self, name, new_value):
716
"""Set an environment variable, and reset it when finished."""
717
self._old_env[name] = osutils.set_or_unset_env(name, new_value)
719
def _install_env(self, env):
720
for name, value in env.iteritems():
721
self._set_and_capture_env_var(name, value)
723
def _restore_env(self):
724
for name, value in self._old_env.iteritems():
725
osutils.set_or_unset_env(name, value)
1180
727
def _proxied_request(self):
1181
handler = _urllib2_wrappers.ProxyHandler()
1182
request = _urllib2_wrappers.Request('GET', 'http://baz/buzzle')
728
from bzrlib.transport.http._urllib2_wrappers import (
733
handler = ProxyHandler()
734
request = Request('GET','http://baz/buzzle')
1183
735
handler.set_proxy(request, 'http')
1186
def assertEvaluateProxyBypass(self, expected, host, no_proxy):
1187
handler = _urllib2_wrappers.ProxyHandler()
1188
self.assertEquals(expected,
1189
handler.evaluate_proxy_bypass(host, no_proxy))
1191
738
def test_empty_user(self):
1192
self.overrideEnv('http_proxy', 'http://bar.com')
739
self._install_env({'http_proxy': 'http://bar.com'})
1193
740
request = self._proxied_request()
1194
741
self.assertFalse(request.headers.has_key('Proxy-authorization'))
1196
def test_user_with_at(self):
1197
self.overrideEnv('http_proxy',
1198
'http://username@domain:password@proxy_host:1234')
1199
request = self._proxied_request()
1200
self.assertFalse(request.headers.has_key('Proxy-authorization'))
743
def test_empty_pass(self):
744
self._install_env({'http_proxy': 'http://joe@bar.com'})
745
request = self._proxied_request()
746
self.assertEqual('Basic ' + 'joe:'.encode('base64').strip(),
747
request.headers['Proxy-authorization'])
748
def test_user_pass(self):
749
self._install_env({'http_proxy': 'http://joe:foo@bar.com'})
750
request = self._proxied_request()
751
self.assertEqual('Basic ' + 'joe:foo'.encode('base64').strip(),
752
request.headers['Proxy-authorization'])
1202
754
def test_invalid_proxy(self):
1203
755
"""A proxy env variable without scheme"""
1204
self.overrideEnv('http_proxy', 'host:1234')
756
self._install_env({'http_proxy': 'host:1234'})
1205
757
self.assertRaises(errors.InvalidURL, self._proxied_request)
1207
def test_evaluate_proxy_bypass_true(self):
1208
"""The host is not proxied"""
1209
self.assertEvaluateProxyBypass(True, 'example.com', 'example.com')
1210
self.assertEvaluateProxyBypass(True, 'bzr.example.com', '*example.com')
1212
def test_evaluate_proxy_bypass_false(self):
1213
"""The host is proxied"""
1214
self.assertEvaluateProxyBypass(False, 'bzr.example.com', None)
1216
def test_evaluate_proxy_bypass_unknown(self):
1217
"""The host is not explicitly proxied"""
1218
self.assertEvaluateProxyBypass(None, 'example.com', 'not.example.com')
1219
self.assertEvaluateProxyBypass(None, 'bzr.example.com', 'example.com')
1221
def test_evaluate_proxy_bypass_empty_entries(self):
1222
"""Ignore empty entries"""
1223
self.assertEvaluateProxyBypass(None, 'example.com', '')
1224
self.assertEvaluateProxyBypass(None, 'example.com', ',')
1225
self.assertEvaluateProxyBypass(None, 'example.com', 'foo,,bar')
1228
class TestProxyHttpServer(http_utils.TestCaseWithTwoWebservers):
760
class TestProxyHttpServer(object):
1229
761
"""Tests proxy server.
763
This MUST be used by daughter classes that also inherit from
764
TestCaseWithTwoWebservers.
766
We can't inherit directly from TestCaseWithTwoWebservers or
767
the test framework will try to create an instance which
768
cannot run, its implementation being incomplete.
1231
770
Be aware that we do not setup a real proxy here. Instead, we
1232
771
check that the *connection* goes through the proxy by serving
1233
772
different content (the faked proxy server append '-proxied'
1234
773
to the file names).
1237
scenarios = multiply_scenarios(
1238
vary_by_http_client_implementation(),
1239
vary_by_http_protocol_version(),
1242
776
# FIXME: We don't have an https server available, so we don't
1243
# test https connections. --vila toolongago
777
# test https connections.
779
# FIXME: Once the test suite is better fitted to test
780
# authorization schemes, test proxy authorizations too (see
1245
783
def setUp(self):
1246
super(TestProxyHttpServer, self).setUp()
1247
self.transport_secondary_server = http_utils.ProxyServer
784
TestCaseWithTwoWebservers.setUp(self)
1248
785
self.build_tree_contents([('foo', 'contents of foo\n'),
1249
786
('foo-proxied', 'proxied contents of foo\n')])
1250
787
# Let's setup some attributes for tests
1251
server = self.get_readonly_server()
1252
self.server_host_port = '%s:%d' % (server.host, server.port)
1253
if self._testing_pycurl():
1254
# Oh my ! pycurl does not check for the port as part of
1255
# no_proxy :-( So we just test the host part
1256
self.no_proxy_host = server.host
1258
self.no_proxy_host = self.server_host_port
788
self.server = self.get_readonly_server()
789
# FIXME: We should not rely on 'localhost' being the hostname
790
self.proxy_address = 'localhost:%d' % self.server.port
791
self.no_proxy_host = self.proxy_address
1259
792
# The secondary server is the proxy
1260
self.proxy_url = self.get_secondary_url()
1262
def _testing_pycurl(self):
1263
# TODO: This is duplicated for lots of the classes in this file
1264
return (features.pycurl.available()
1265
and self._transport == PyCurlTransport)
1267
def assertProxied(self):
1268
t = self.get_readonly_transport()
1269
self.assertEqual('proxied contents of foo\n', t.get('foo').read())
1271
def assertNotProxied(self):
1272
t = self.get_readonly_transport()
1273
self.assertEqual('contents of foo\n', t.get('foo').read())
793
self.proxy = self.get_secondary_server()
794
self.proxy_url = self.proxy.get_url()
797
def create_transport_secondary_server(self):
798
"""Creates an http server that will serve files with
799
'-proxied' appended to their names.
801
return HttpServer(FakeProxyRequestHandler)
803
def _set_and_capture_env_var(self, name, new_value):
804
"""Set an environment variable, and reset it when finished."""
805
self._old_env[name] = osutils.set_or_unset_env(name, new_value)
807
def _install_env(self, env):
808
for name, value in env.iteritems():
809
self._set_and_capture_env_var(name, value)
811
def _restore_env(self):
812
for name, value in self._old_env.iteritems():
813
osutils.set_or_unset_env(name, value)
815
def proxied_in_env(self, env):
816
self._install_env(env)
817
url = self.server.get_url()
818
t = self._transport(url)
820
self.assertEqual(t.get('foo').read(), 'proxied contents of foo\n')
824
def not_proxied_in_env(self, env):
825
self._install_env(env)
826
url = self.server.get_url()
827
t = self._transport(url)
829
self.assertEqual(t.get('foo').read(), 'contents of foo\n')
1275
833
def test_http_proxy(self):
1276
self.overrideEnv('http_proxy', self.proxy_url)
1277
self.assertProxied()
834
self.proxied_in_env({'http_proxy': self.proxy_url})
1279
836
def test_HTTP_PROXY(self):
1280
if self._testing_pycurl():
1281
# pycurl does not check HTTP_PROXY for security reasons
1282
# (for use in a CGI context that we do not care
1283
# about. Should we ?)
1284
raise tests.TestNotApplicable(
1285
'pycurl does not check HTTP_PROXY for security reasons')
1286
self.overrideEnv('HTTP_PROXY', self.proxy_url)
1287
self.assertProxied()
837
self.proxied_in_env({'HTTP_PROXY': self.proxy_url})
1289
839
def test_all_proxy(self):
1290
self.overrideEnv('all_proxy', self.proxy_url)
1291
self.assertProxied()
840
self.proxied_in_env({'all_proxy': self.proxy_url})
1293
842
def test_ALL_PROXY(self):
1294
self.overrideEnv('ALL_PROXY', self.proxy_url)
1295
self.assertProxied()
843
self.proxied_in_env({'ALL_PROXY': self.proxy_url})
1297
845
def test_http_proxy_with_no_proxy(self):
1298
self.overrideEnv('no_proxy', self.no_proxy_host)
1299
self.overrideEnv('http_proxy', self.proxy_url)
1300
self.assertNotProxied()
846
self.not_proxied_in_env({'http_proxy': self.proxy_url,
847
'no_proxy': self.no_proxy_host})
1302
849
def test_HTTP_PROXY_with_NO_PROXY(self):
1303
if self._testing_pycurl():
1304
raise tests.TestNotApplicable(
1305
'pycurl does not check HTTP_PROXY for security reasons')
1306
self.overrideEnv('NO_PROXY', self.no_proxy_host)
1307
self.overrideEnv('HTTP_PROXY', self.proxy_url)
1308
self.assertNotProxied()
850
self.not_proxied_in_env({'HTTP_PROXY': self.proxy_url,
851
'NO_PROXY': self.no_proxy_host})
1310
853
def test_all_proxy_with_no_proxy(self):
1311
self.overrideEnv('no_proxy', self.no_proxy_host)
1312
self.overrideEnv('all_proxy', self.proxy_url)
1313
self.assertNotProxied()
854
self.not_proxied_in_env({'all_proxy': self.proxy_url,
855
'no_proxy': self.no_proxy_host})
1315
857
def test_ALL_PROXY_with_NO_PROXY(self):
1316
self.overrideEnv('NO_PROXY', self.no_proxy_host)
1317
self.overrideEnv('ALL_PROXY', self.proxy_url)
1318
self.assertNotProxied()
1320
def test_http_proxy_without_scheme(self):
1321
self.overrideEnv('http_proxy', self.server_host_port)
1322
if self._testing_pycurl():
1323
# pycurl *ignores* invalid proxy env variables. If that ever change
1324
# in the future, this test will fail indicating that pycurl do not
1325
# ignore anymore such variables.
1326
self.assertNotProxied()
1328
self.assertRaises(errors.InvalidURL, self.assertProxied)
1331
class TestRanges(http_utils.TestCaseWithWebserver):
1332
"""Test the Range header in GET methods."""
1334
scenarios = multiply_scenarios(
1335
vary_by_http_client_implementation(),
1336
vary_by_http_protocol_version(),
1340
super(TestRanges, self).setUp()
858
self.not_proxied_in_env({'ALL_PROXY': self.proxy_url,
859
'NO_PROXY': self.no_proxy_host})
861
def test_http_proxy_without_scheme(self):
862
self.assertRaises(errors.InvalidURL,
864
{'http_proxy': self.proxy_address})
867
class TestProxyHttpServer_urllib(TestProxyHttpServer,
868
TestCaseWithTwoWebservers):
869
"""Tests proxy server for urllib implementation"""
871
_transport = HttpTransport_urllib
874
class TestProxyHttpServer_pycurl(TestWithTransport_pycurl,
876
TestCaseWithTwoWebservers):
877
"""Tests proxy server for pycurl implementation"""
880
TestProxyHttpServer.setUp(self)
881
# Oh my ! pycurl does not check for the port as part of
882
# no_proxy :-( So we just test the host part
883
self.no_proxy_host = 'localhost'
885
def test_HTTP_PROXY(self):
886
# pycurl do not check HTTP_PROXY for security reasons
887
# (for use in a CGI context that we do not care
888
# about. Should we ?)
891
def test_HTTP_PROXY_with_NO_PROXY(self):
894
def test_http_proxy_without_scheme(self):
895
# pycurl *ignores* invalid proxy env variables. If that
896
# ever change in the future, this test will fail
897
# indicating that pycurl do not ignore anymore such
899
self.not_proxied_in_env({'http_proxy': self.proxy_address})
902
class TestRanges(object):
903
"""Test the Range header in GET methods..
905
This MUST be used by daughter classes that also inherit from
906
TestCaseWithWebserver.
908
We can't inherit directly from TestCaseWithWebserver or the
909
test framework will try to create an instance which cannot
910
run, its implementation being incomplete.
914
TestCaseWithWebserver.setUp(self)
1341
915
self.build_tree_contents([('a', '0123456789')],)
1343
def create_transport_readonly_server(self):
1344
return http_server.HttpServer(protocol_version=self._protocol_version)
1346
def _file_contents(self, relpath, ranges):
1347
t = self.get_readonly_transport()
1348
offsets = [ (start, end - start + 1) for start, end in ranges]
1349
coalesce = t._coalesce_offsets
1350
coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1351
code, data = t._get(relpath, coalesced)
1352
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1353
for start, end in ranges:
1355
yield data.read(end - start + 1)
916
server = self.get_readonly_server()
917
self.transport = self._transport(server.get_url())
919
def _file_contents(self, relpath, ranges, tail_amount=0):
920
code, data = self.transport._get(relpath, ranges)
921
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
922
for start, end in ranges:
924
yield data.read(end - start + 1)
1357
926
def _file_tail(self, relpath, tail_amount):
1358
t = self.get_readonly_transport()
1359
code, data = t._get(relpath, [], tail_amount)
1360
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1361
data.seek(-tail_amount, 2)
1362
return data.read(tail_amount)
927
code, data = self.transport._get(relpath, [], tail_amount)
928
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
929
data.seek(-tail_amount + 1, 2)
930
return data.read(tail_amount)
1364
932
def test_range_header(self):
1366
934
map(self.assertEqual,['0', '234'],
1367
935
list(self._file_contents('a', [(0,0), (2,4)])),)
1369
def test_range_header_tail(self):
1370
937
self.assertEqual('789', self._file_tail('a', 3))
1372
def test_syntactically_invalid_range_header(self):
1373
self.assertListRaises(errors.InvalidHttpRange,
1374
self._file_contents, 'a', [(4, 3)])
1376
def test_semantically_invalid_range_header(self):
1377
self.assertListRaises(errors.InvalidHttpRange,
1378
self._file_contents, 'a', [(42, 128)])
1381
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1382
"""Test redirection between http servers."""
1384
scenarios = multiply_scenarios(
1385
vary_by_http_client_implementation(),
1386
vary_by_http_protocol_version(),
938
# Syntactically invalid range
939
self.assertRaises(errors.InvalidRange,
940
self.transport._get, 'a', [(4, 3)])
941
# Semantically invalid range
942
self.assertRaises(errors.InvalidRange,
943
self.transport._get, 'a', [(42, 128)])
946
class TestRanges_urllib(TestRanges, TestCaseWithWebserver):
947
"""Test the Range header in GET methods for urllib implementation"""
949
_transport = HttpTransport_urllib
952
class TestRanges_pycurl(TestWithTransport_pycurl,
954
TestCaseWithWebserver):
955
"""Test the Range header in GET methods for pycurl implementation"""
958
class TestHTTPRedirections(object):
959
"""Test redirection between http servers.
961
This MUST be used by daughter classes that also inherit from
962
TestCaseWithRedirectedWebserver.
964
We can't inherit directly from TestCaseWithTwoWebservers or the
965
test framework will try to create an instance which cannot
966
run, its implementation being incomplete.
969
def create_transport_secondary_server(self):
970
"""Create the secondary server redirecting to the primary server"""
971
new = self.get_readonly_server()
973
redirecting = HTTPServerRedirecting()
974
redirecting.redirect_to(new.host, new.port)
1389
977
def setUp(self):
1390
978
super(TestHTTPRedirections, self).setUp()
1554
1144
return self.old_transport.clone(exception.target)
1556
self.assertRaises(errors.TooManyRedirections,
1557
transport.do_catching_redirections,
1146
self.assertRaises(errors.TooManyRedirections, do_catching_redirections,
1558
1147
self.get_a, self.old_transport, redirected)
1561
def _setup_authentication_config(**kwargs):
1562
conf = config.AuthenticationConfig()
1563
conf._get_config().update({'httptest': kwargs})
1567
class TestUrllib2AuthHandler(tests.TestCaseWithTransport):
1568
"""Unit tests for glue by which urllib2 asks us for authentication"""
1570
def test_get_user_password_without_port(self):
1571
"""We cope if urllib2 doesn't tell us the port.
1573
See https://bugs.launchpad.net/bzr/+bug/654684
1577
_setup_authentication_config(scheme='http', host='localhost',
1578
user=user, password=password)
1579
handler = _urllib2_wrappers.HTTPAuthHandler()
1580
got_pass = handler.get_user_password(dict(
1587
self.assertEquals((user, password), got_pass)
1590
class TestAuth(http_utils.TestCaseWithWebserver):
1591
"""Test authentication scheme"""
1593
scenarios = multiply_scenarios(
1594
vary_by_http_client_implementation(),
1595
vary_by_http_protocol_version(),
1596
vary_by_http_auth_scheme(),
1600
super(TestAuth, self).setUp()
1601
self.server = self.get_readonly_server()
1602
self.build_tree_contents([('a', 'contents of a\n'),
1603
('b', 'contents of b\n'),])
1605
def create_transport_readonly_server(self):
1606
server = self._auth_server(protocol_version=self._protocol_version)
1607
server._url_protocol = self._url_protocol
1610
def _testing_pycurl(self):
1611
# TODO: This is duplicated for lots of the classes in this file
1612
return (features.pycurl.available()
1613
and self._transport == PyCurlTransport)
1615
def get_user_url(self, user, password):
1616
"""Build an url embedding user and password"""
1617
url = '%s://' % self.server._url_protocol
1618
if user is not None:
1620
if password is not None:
1621
url += ':' + password
1623
url += '%s:%s/' % (self.server.host, self.server.port)
1626
def get_user_transport(self, user, password):
1627
t = transport.get_transport_from_url(
1628
self.get_user_url(user, password))
1631
def test_no_user(self):
1632
self.server.add_user('joe', 'foo')
1633
t = self.get_user_transport(None, None)
1634
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1635
# Only one 'Authentication Required' error should occur
1636
self.assertEqual(1, self.server.auth_required_errors)
1638
def test_empty_pass(self):
1639
self.server.add_user('joe', '')
1640
t = self.get_user_transport('joe', '')
1641
self.assertEqual('contents of a\n', t.get('a').read())
1642
# Only one 'Authentication Required' error should occur
1643
self.assertEqual(1, self.server.auth_required_errors)
1645
def test_user_pass(self):
1646
self.server.add_user('joe', 'foo')
1647
t = self.get_user_transport('joe', 'foo')
1648
self.assertEqual('contents of a\n', t.get('a').read())
1649
# Only one 'Authentication Required' error should occur
1650
self.assertEqual(1, self.server.auth_required_errors)
1652
def test_unknown_user(self):
1653
self.server.add_user('joe', 'foo')
1654
t = self.get_user_transport('bill', 'foo')
1655
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1656
# Two 'Authentication Required' errors should occur (the
1657
# initial 'who are you' and 'I don't know you, who are
1659
self.assertEqual(2, self.server.auth_required_errors)
1661
def test_wrong_pass(self):
1662
self.server.add_user('joe', 'foo')
1663
t = self.get_user_transport('joe', 'bar')
1664
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1665
# Two 'Authentication Required' errors should occur (the
1666
# initial 'who are you' and 'this is not you, who are you')
1667
self.assertEqual(2, self.server.auth_required_errors)
1669
def test_prompt_for_username(self):
1670
if self._testing_pycurl():
1671
raise tests.TestNotApplicable(
1672
'pycurl cannot prompt, it handles auth by embedding'
1673
' user:pass in urls only')
1675
self.server.add_user('joe', 'foo')
1676
t = self.get_user_transport(None, None)
1677
stdout = tests.StringIOWrapper()
1678
stderr = tests.StringIOWrapper()
1679
ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
1680
stdout=stdout, stderr=stderr)
1681
self.assertEqual('contents of a\n',t.get('a').read())
1682
# stdin should be empty
1683
self.assertEqual('', ui.ui_factory.stdin.readline())
1685
expected_prompt = self._expected_username_prompt(t._unqualified_scheme)
1686
self.assertEqual(expected_prompt, stderr.read(len(expected_prompt)))
1687
self.assertEqual('', stdout.getvalue())
1688
self._check_password_prompt(t._unqualified_scheme, 'joe',
1691
def test_prompt_for_password(self):
1692
if self._testing_pycurl():
1693
raise tests.TestNotApplicable(
1694
'pycurl cannot prompt, it handles auth by embedding'
1695
' user:pass in urls only')
1697
self.server.add_user('joe', 'foo')
1698
t = self.get_user_transport('joe', None)
1699
stdout = tests.StringIOWrapper()
1700
stderr = tests.StringIOWrapper()
1701
ui.ui_factory = tests.TestUIFactory(stdin='foo\n',
1702
stdout=stdout, stderr=stderr)
1703
self.assertEqual('contents of a\n', t.get('a').read())
1704
# stdin should be empty
1705
self.assertEqual('', ui.ui_factory.stdin.readline())
1706
self._check_password_prompt(t._unqualified_scheme, 'joe',
1708
self.assertEqual('', stdout.getvalue())
1709
# And we shouldn't prompt again for a different request
1710
# against the same transport.
1711
self.assertEqual('contents of b\n',t.get('b').read())
1713
# And neither against a clone
1714
self.assertEqual('contents of b\n',t2.get('b').read())
1715
# Only one 'Authentication Required' error should occur
1716
self.assertEqual(1, self.server.auth_required_errors)
1718
def _check_password_prompt(self, scheme, user, actual_prompt):
1719
expected_prompt = (self._password_prompt_prefix
1720
+ ("%s %s@%s:%d, Realm: '%s' password: "
1722
user, self.server.host, self.server.port,
1723
self.server.auth_realm)))
1724
self.assertEqual(expected_prompt, actual_prompt)
1726
def _expected_username_prompt(self, scheme):
1727
return (self._username_prompt_prefix
1728
+ "%s %s:%d, Realm: '%s' username: " % (scheme.upper(),
1729
self.server.host, self.server.port,
1730
self.server.auth_realm))
1732
def test_no_prompt_for_password_when_using_auth_config(self):
1733
if self._testing_pycurl():
1734
raise tests.TestNotApplicable(
1735
'pycurl does not support authentication.conf'
1736
' since it cannot prompt')
1740
stdin_content = 'bar\n' # Not the right password
1741
self.server.add_user(user, password)
1742
t = self.get_user_transport(user, None)
1743
ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1744
stderr=tests.StringIOWrapper())
1745
# Create a minimal config file with the right password
1746
_setup_authentication_config(scheme='http', port=self.server.port,
1747
user=user, password=password)
1748
# Issue a request to the server to connect
1749
self.assertEqual('contents of a\n',t.get('a').read())
1750
# stdin should have been left untouched
1751
self.assertEqual(stdin_content, ui.ui_factory.stdin.readline())
1752
# Only one 'Authentication Required' error should occur
1753
self.assertEqual(1, self.server.auth_required_errors)
1755
def test_changing_nonce(self):
1756
if self._auth_server not in (http_utils.HTTPDigestAuthServer,
1757
http_utils.ProxyDigestAuthServer):
1758
raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1759
if self._testing_pycurl():
1761
'pycurl does not handle a nonce change')
1762
self.server.add_user('joe', 'foo')
1763
t = self.get_user_transport('joe', 'foo')
1764
self.assertEqual('contents of a\n', t.get('a').read())
1765
self.assertEqual('contents of b\n', t.get('b').read())
1766
# Only one 'Authentication Required' error should have
1768
self.assertEqual(1, self.server.auth_required_errors)
1769
# The server invalidates the current nonce
1770
self.server.auth_nonce = self.server.auth_nonce + '. No, now!'
1771
self.assertEqual('contents of a\n', t.get('a').read())
1772
# Two 'Authentication Required' errors should occur (the
1773
# initial 'who are you' and a second 'who are you' with the new nonce)
1774
self.assertEqual(2, self.server.auth_required_errors)
1776
def test_user_from_auth_conf(self):
1777
if self._testing_pycurl():
1778
raise tests.TestNotApplicable(
1779
'pycurl does not support authentication.conf')
1782
self.server.add_user(user, password)
1783
_setup_authentication_config(scheme='http', port=self.server.port,
1784
user=user, password=password)
1785
t = self.get_user_transport(None, None)
1786
# Issue a request to the server to connect
1787
self.assertEqual('contents of a\n', t.get('a').read())
1788
# Only one 'Authentication Required' error should occur
1789
self.assertEqual(1, self.server.auth_required_errors)
1791
def test_no_credential_leaks_in_log(self):
1792
self.overrideAttr(debug, 'debug_flags', set(['http']))
1794
password = 'very-sensitive-password'
1795
self.server.add_user(user, password)
1796
t = self.get_user_transport(user, password)
1797
# Capture the debug calls to mutter
1800
lines = args[0] % args[1:]
1801
# Some calls output multiple lines, just split them now since we
1802
# care about a single one later.
1803
self.mutters.extend(lines.splitlines())
1804
self.overrideAttr(trace, 'mutter', mutter)
1805
# Issue a request to the server to connect
1806
self.assertEqual(True, t.has('a'))
1807
# Only one 'Authentication Required' error should occur
1808
self.assertEqual(1, self.server.auth_required_errors)
1809
# Since the authentification succeeded, there should be a corresponding
1811
sent_auth_headers = [line for line in self.mutters
1812
if line.startswith('> %s' % (self._auth_header,))]
1813
self.assertLength(1, sent_auth_headers)
1814
self.assertStartsWith(sent_auth_headers[0],
1815
'> %s: <masked>' % (self._auth_header,))
1818
class TestProxyAuth(TestAuth):
1819
"""Test proxy authentication schemes.
1821
This inherits from TestAuth to tweak the setUp and filter some failing
1825
scenarios = multiply_scenarios(
1826
vary_by_http_client_implementation(),
1827
vary_by_http_protocol_version(),
1828
vary_by_http_proxy_auth_scheme(),
1832
super(TestProxyAuth, self).setUp()
1833
# Override the contents to avoid false positives
1834
self.build_tree_contents([('a', 'not proxied contents of a\n'),
1835
('b', 'not proxied contents of b\n'),
1836
('a-proxied', 'contents of a\n'),
1837
('b-proxied', 'contents of b\n'),
1840
def get_user_transport(self, user, password):
1841
self.overrideEnv('all_proxy', self.get_user_url(user, password))
1842
return TestAuth.get_user_transport(self, user, password)
1844
def test_empty_pass(self):
1845
if self._testing_pycurl():
1847
if pycurl.version_info()[1] < '7.16.0':
1849
'pycurl < 7.16.0 does not handle empty proxy passwords')
1850
super(TestProxyAuth, self).test_empty_pass()
1853
class SampleSocket(object):
1854
"""A socket-like object for use in testing the HTTP request handler."""
1856
def __init__(self, socket_read_content):
1857
"""Constructs a sample socket.
1859
:param socket_read_content: a byte sequence
1861
# Use plain python StringIO so we can monkey-patch the close method to
1862
# not discard the contents.
1863
from StringIO import StringIO
1864
self.readfile = StringIO(socket_read_content)
1865
self.writefile = StringIO()
1866
self.writefile.close = lambda: None
1867
self.close = lambda: None
1869
def makefile(self, mode='r', bufsize=None):
1871
return self.readfile
1873
return self.writefile
1876
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1878
scenarios = multiply_scenarios(
1879
vary_by_http_client_implementation(),
1880
vary_by_http_protocol_version(),
1884
super(SmartHTTPTunnellingTest, self).setUp()
1885
# We use the VFS layer as part of HTTP tunnelling tests.
1886
self.overrideEnv('BZR_NO_SMART_VFS', None)
1887
self.transport_readonly_server = http_utils.HTTPServerWithSmarts
1888
self.http_server = self.get_readonly_server()
1890
def create_transport_readonly_server(self):
1891
server = http_utils.HTTPServerWithSmarts(
1892
protocol_version=self._protocol_version)
1893
server._url_protocol = self._url_protocol
1896
def test_open_controldir(self):
1897
branch = self.make_branch('relpath')
1898
url = self.http_server.get_url() + 'relpath'
1899
bd = controldir.ControlDir.open(url)
1900
self.addCleanup(bd.transport.disconnect)
1901
self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1903
def test_bulk_data(self):
1904
# We should be able to send and receive bulk data in a single message.
1905
# The 'readv' command in the smart protocol both sends and receives
1906
# bulk data, so we use that.
1907
self.build_tree(['data-file'])
1908
http_transport = transport.get_transport_from_url(
1909
self.http_server.get_url())
1910
medium = http_transport.get_smart_medium()
1911
# Since we provide the medium, the url below will be mostly ignored
1912
# during the test, as long as the path is '/'.
1913
remote_transport = remote.RemoteTransport('bzr://fake_host/',
1916
[(0, "c")], list(remote_transport.readv("data-file", [(0,1)])))
1918
def test_http_send_smart_request(self):
1920
post_body = 'hello\n'
1921
expected_reply_body = 'ok\x012\n'
1923
http_transport = transport.get_transport_from_url(
1924
self.http_server.get_url())
1925
medium = http_transport.get_smart_medium()
1926
response = medium.send_http_smart_request(post_body)
1927
reply_body = response.read()
1928
self.assertEqual(expected_reply_body, reply_body)
1930
def test_smart_http_server_post_request_handler(self):
1931
httpd = self.http_server.server
1933
socket = SampleSocket(
1934
'POST /.bzr/smart %s \r\n' % self._protocol_version
1935
# HTTP/1.1 posts must have a Content-Length (but it doesn't hurt
1937
+ 'Content-Length: 6\r\n'
1940
# Beware: the ('localhost', 80) below is the
1941
# client_address parameter, but we don't have one because
1942
# we have defined a socket which is not bound to an
1943
# address. The test framework never uses this client
1944
# address, so far...
1945
request_handler = http_utils.SmartRequestHandler(socket,
1948
response = socket.writefile.getvalue()
1949
self.assertStartsWith(response, '%s 200 ' % self._protocol_version)
1950
# This includes the end of the HTTP headers, and all the body.
1951
expected_end_of_response = '\r\n\r\nok\x012\n'
1952
self.assertEndsWith(response, expected_end_of_response)
1955
class ForbiddenRequestHandler(http_server.TestingHTTPRequestHandler):
1956
"""No smart server here request handler."""
1959
self.send_error(403, "Forbidden")
1962
class SmartClientAgainstNotSmartServer(TestSpecificRequestHandler):
1963
"""Test smart client behaviour against an http server without smarts."""
1965
_req_handler_class = ForbiddenRequestHandler
1967
def test_probe_smart_server(self):
1968
"""Test error handling against server refusing smart requests."""
1969
t = self.get_readonly_transport()
1970
# No need to build a valid smart request here, the server will not even
1971
# try to interpret it.
1972
self.assertRaises(errors.SmartProtocolError,
1973
t.get_smart_medium().send_http_smart_request,
1977
class Test_redirected_to(tests.TestCase):
1979
scenarios = vary_by_http_client_implementation()
1981
def test_redirected_to_subdir(self):
1982
t = self._transport('http://www.example.com/foo')
1983
r = t._redirected_to('http://www.example.com/foo',
1984
'http://www.example.com/foo/subdir')
1985
self.assertIsInstance(r, type(t))
1986
# Both transports share the some connection
1987
self.assertEqual(t._get_connection(), r._get_connection())
1988
self.assertEquals('http://www.example.com/foo/subdir/', r.base)
1990
def test_redirected_to_self_with_slash(self):
1991
t = self._transport('http://www.example.com/foo')
1992
r = t._redirected_to('http://www.example.com/foo',
1993
'http://www.example.com/foo/')
1994
self.assertIsInstance(r, type(t))
1995
# Both transports share the some connection (one can argue that we
1996
# should return the exact same transport here, but that seems
1998
self.assertEqual(t._get_connection(), r._get_connection())
2000
def test_redirected_to_host(self):
2001
t = self._transport('http://www.example.com/foo')
2002
r = t._redirected_to('http://www.example.com/foo',
2003
'http://foo.example.com/foo/subdir')
2004
self.assertIsInstance(r, type(t))
2005
self.assertEquals('http://foo.example.com/foo/subdir/',
2008
def test_redirected_to_same_host_sibling_protocol(self):
2009
t = self._transport('http://www.example.com/foo')
2010
r = t._redirected_to('http://www.example.com/foo',
2011
'https://www.example.com/foo')
2012
self.assertIsInstance(r, type(t))
2013
self.assertEquals('https://www.example.com/foo/',
2016
def test_redirected_to_same_host_different_protocol(self):
2017
t = self._transport('http://www.example.com/foo')
2018
r = t._redirected_to('http://www.example.com/foo',
2019
'ftp://www.example.com/foo')
2020
self.assertNotEquals(type(r), type(t))
2021
self.assertEquals('ftp://www.example.com/foo/', r.external_url())
2023
def test_redirected_to_same_host_specific_implementation(self):
2024
t = self._transport('http://www.example.com/foo')
2025
r = t._redirected_to('http://www.example.com/foo',
2026
'https+urllib://www.example.com/foo')
2027
self.assertEquals('https://www.example.com/foo/', r.external_url())
2029
def test_redirected_to_different_host_same_user(self):
2030
t = self._transport('http://joe@www.example.com/foo')
2031
r = t._redirected_to('http://www.example.com/foo',
2032
'https://foo.example.com/foo')
2033
self.assertIsInstance(r, type(t))
2034
self.assertEqual(t._parsed_url.user, r._parsed_url.user)
2035
self.assertEquals('https://joe@foo.example.com/foo/', r.external_url())
2038
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
2039
"""Request handler for a unique and pre-defined request.
2041
The only thing we care about here is how many bytes travel on the wire. But
2042
since we want to measure it for a real http client, we have to send it
2045
We expect to receive a *single* request nothing more (and we won't even
2046
check what request it is, we just measure the bytes read until an empty
2050
def _handle_one_request(self):
2051
tcs = self.server.test_case_server
2052
requestline = self.rfile.readline()
2053
headers = self.MessageClass(self.rfile, 0)
2054
# We just read: the request, the headers, an empty line indicating the
2055
# end of the headers.
2056
bytes_read = len(requestline)
2057
for line in headers.headers:
2058
bytes_read += len(line)
2059
bytes_read += len('\r\n')
2060
if requestline.startswith('POST'):
2061
# The body should be a single line (or we don't know where it ends
2062
# and we don't want to issue a blocking read)
2063
body = self.rfile.readline()
2064
bytes_read += len(body)
2065
tcs.bytes_read = bytes_read
2067
# We set the bytes written *before* issuing the write, the client is
2068
# supposed to consume every produced byte *before* checking that value.
2070
# Doing the oppposite may lead to test failure: we may be interrupted
2071
# after the write but before updating the value. The client can then
2072
# continue and read the value *before* we can update it. And yes,
2073
# this has been observed -- vila 20090129
2074
tcs.bytes_written = len(tcs.canned_response)
2075
self.wfile.write(tcs.canned_response)
2078
class ActivityServerMixin(object):
2080
def __init__(self, protocol_version):
2081
super(ActivityServerMixin, self).__init__(
2082
request_handler=PredefinedRequestHandler,
2083
protocol_version=protocol_version)
2084
# Bytes read and written by the server
2086
self.bytes_written = 0
2087
self.canned_response = None
2090
class ActivityHTTPServer(ActivityServerMixin, http_server.HttpServer):
2094
if features.HTTPSServerFeature.available():
2095
from bzrlib.tests import https_server
2096
class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
2100
class TestActivityMixin(object):
2101
"""Test socket activity reporting.
2103
We use a special purpose server to control the bytes sent and received and
2104
be able to predict the activity on the client socket.
2108
self.server = self._activity_server(self._protocol_version)
2109
self.server.start_server()
2110
self.addCleanup(self.server.stop_server)
2111
_activities = {} # Don't close over self and create a cycle
2112
def report_activity(t, bytes, direction):
2113
count = _activities.get(direction, 0)
2115
_activities[direction] = count
2116
self.activities = _activities
2117
# We override at class level because constructors may propagate the
2118
# bound method and render instance overriding ineffective (an
2119
# alternative would be to define a specific ui factory instead...)
2120
self.overrideAttr(self._transport, '_report_activity', report_activity)
2122
def get_transport(self):
2123
t = self._transport(self.server.get_url())
2124
# FIXME: Needs cleanup -- vila 20100611
2127
def assertActivitiesMatch(self):
2128
self.assertEqual(self.server.bytes_read,
2129
self.activities.get('write', 0), 'written bytes')
2130
self.assertEqual(self.server.bytes_written,
2131
self.activities.get('read', 0), 'read bytes')
2134
self.server.canned_response = '''HTTP/1.1 200 OK\r
2135
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
2136
Server: Apache/2.0.54 (Fedora)\r
2137
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
2138
ETag: "56691-23-38e9ae00"\r
2139
Accept-Ranges: bytes\r
2140
Content-Length: 35\r
2142
Content-Type: text/plain; charset=UTF-8\r
2144
Bazaar-NG meta directory, format 1
2146
t = self.get_transport()
2147
self.assertEqual('Bazaar-NG meta directory, format 1\n',
2148
t.get('foo/bar').read())
2149
self.assertActivitiesMatch()
2152
self.server.canned_response = '''HTTP/1.1 200 OK\r
2153
Server: SimpleHTTP/0.6 Python/2.5.2\r
2154
Date: Thu, 29 Jan 2009 20:21:47 GMT\r
2155
Content-type: application/octet-stream\r
2156
Content-Length: 20\r
2157
Last-Modified: Thu, 29 Jan 2009 20:21:47 GMT\r
2160
t = self.get_transport()
2161
self.assertTrue(t.has('foo/bar'))
2162
self.assertActivitiesMatch()
2164
def test_readv(self):
2165
self.server.canned_response = '''HTTP/1.1 206 Partial Content\r
2166
Date: Tue, 11 Jul 2006 04:49:48 GMT\r
2167
Server: Apache/2.0.54 (Fedora)\r
2168
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
2169
ETag: "238a3c-16ec2-805c5540"\r
2170
Accept-Ranges: bytes\r
2171
Content-Length: 1534\r
2173
Content-Type: multipart/byteranges; boundary=418470f848b63279b\r
2176
--418470f848b63279b\r
2177
Content-type: text/plain; charset=UTF-8\r
2178
Content-range: bytes 0-254/93890\r
2180
mbp@sourcefrog.net-20050309040815-13242001617e4a06
2181
mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e7627
2182
mbp@sourcefrog.net-20050309040957-6cad07f466bb0bb8
2183
mbp@sourcefrog.net-20050309041501-c840e09071de3b67
2184
mbp@sourcefrog.net-20050309044615-c24a3250be83220a
2186
--418470f848b63279b\r
2187
Content-type: text/plain; charset=UTF-8\r
2188
Content-range: bytes 1000-2049/93890\r
2191
mbp@sourcefrog.net-20050311063625-07858525021f270b
2192
mbp@sourcefrog.net-20050311231934-aa3776aff5200bb9
2193
mbp@sourcefrog.net-20050311231953-73aeb3a131c3699a
2194
mbp@sourcefrog.net-20050311232353-f5e33da490872c6a
2195
mbp@sourcefrog.net-20050312071639-0a8f59a34a024ff0
2196
mbp@sourcefrog.net-20050312073432-b2c16a55e0d6e9fb
2197
mbp@sourcefrog.net-20050312073831-a47c3335ece1920f
2198
mbp@sourcefrog.net-20050312085412-13373aa129ccbad3
2199
mbp@sourcefrog.net-20050313052251-2bf004cb96b39933
2200
mbp@sourcefrog.net-20050313052856-3edd84094687cb11
2201
mbp@sourcefrog.net-20050313053233-e30a4f28aef48f9d
2202
mbp@sourcefrog.net-20050313053853-7c64085594ff3072
2203
mbp@sourcefrog.net-20050313054757-a86c3f5871069e22
2204
mbp@sourcefrog.net-20050313061422-418f1f73b94879b9
2205
mbp@sourcefrog.net-20050313120651-497bd231b19df600
2206
mbp@sourcefrog.net-20050314024931-eae0170ef25a5d1a
2207
mbp@sourcefrog.net-20050314025438-d52099f915fe65fc
2208
mbp@sourcefrog.net-20050314025539-637a636692c055cf
2209
mbp@sourcefrog.net-20050314025737-55eb441f430ab4ba
2210
mbp@sourcefrog.net-20050314025901-d74aa93bb7ee8f62
2212
--418470f848b63279b--\r
2214
t = self.get_transport()
2215
# Remember that the request is ignored and that the ranges below
2216
# doesn't have to match the canned response.
2217
l = list(t.readv('/foo/bar', ((0, 255), (1000, 1050))))
2218
self.assertEqual(2, len(l))
2219
self.assertActivitiesMatch()
2221
def test_post(self):
2222
self.server.canned_response = '''HTTP/1.1 200 OK\r
2223
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
2224
Server: Apache/2.0.54 (Fedora)\r
2225
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
2226
ETag: "56691-23-38e9ae00"\r
2227
Accept-Ranges: bytes\r
2228
Content-Length: 35\r
2230
Content-Type: text/plain; charset=UTF-8\r
2232
lalala whatever as long as itsssss
2234
t = self.get_transport()
2235
# We must send a single line of body bytes, see
2236
# PredefinedRequestHandler._handle_one_request
2237
code, f = t._post('abc def end-of-body\n')
2238
self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2239
self.assertActivitiesMatch()
2242
class TestActivity(tests.TestCase, TestActivityMixin):
2244
scenarios = multiply_scenarios(
2245
vary_by_http_activity(),
2246
vary_by_http_protocol_version(),
2250
super(TestActivity, self).setUp()
2251
TestActivityMixin.setUp(self)
2254
class TestNoReportActivity(tests.TestCase, TestActivityMixin):
2256
# Unlike TestActivity, we are really testing ReportingFileSocket and
2257
# ReportingSocket, so we don't need all the parametrization. Since
2258
# ReportingFileSocket and ReportingSocket are wrappers, it's easier to
2259
# test them through their use by the transport than directly (that's a
2260
# bit less clean but far more simpler and effective).
2261
_activity_server = ActivityHTTPServer
2262
_protocol_version = 'HTTP/1.1'
2265
super(TestNoReportActivity, self).setUp()
2266
self._transport =_urllib.HttpTransport_urllib
2267
TestActivityMixin.setUp(self)
2269
def assertActivitiesMatch(self):
2270
# Nothing to check here
2274
class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver):
2275
"""Test authentication on the redirected http server."""
2277
scenarios = vary_by_http_protocol_version()
2279
_auth_header = 'Authorization'
2280
_password_prompt_prefix = ''
2281
_username_prompt_prefix = ''
2282
_auth_server = http_utils.HTTPBasicAuthServer
2283
_transport = _urllib.HttpTransport_urllib
2286
super(TestAuthOnRedirected, self).setUp()
2287
self.build_tree_contents([('a','a'),
2289
('1/a', 'redirected once'),
2291
new_prefix = 'http://%s:%s' % (self.new_server.host,
2292
self.new_server.port)
2293
self.old_server.redirections = [
2294
('(.*)', r'%s/1\1' % (new_prefix), 301),]
2295
self.old_transport = self.get_old_transport()
2296
self.new_server.add_user('joe', 'foo')
2297
cleanup_http_redirection_connections(self)
2299
def create_transport_readonly_server(self):
2300
server = self._auth_server(protocol_version=self._protocol_version)
2301
server._url_protocol = self._url_protocol
2307
def test_auth_on_redirected_via_do_catching_redirections(self):
2308
self.redirections = 0
2310
def redirected(t, exception, redirection_notice):
2311
self.redirections += 1
2312
redirected_t = t._redirected_to(exception.source, exception.target)
2313
self.addCleanup(redirected_t.disconnect)
2316
stdout = tests.StringIOWrapper()
2317
stderr = tests.StringIOWrapper()
2318
ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2319
stdout=stdout, stderr=stderr)
2320
self.assertEqual('redirected once',
2321
transport.do_catching_redirections(
2322
self.get_a, self.old_transport, redirected).read())
2323
self.assertEqual(1, self.redirections)
2324
# stdin should be empty
2325
self.assertEqual('', ui.ui_factory.stdin.readline())
2326
# stdout should be empty, stderr will contains the prompts
2327
self.assertEqual('', stdout.getvalue())
2329
def test_auth_on_redirected_via_following_redirections(self):
2330
self.new_server.add_user('joe', 'foo')
2331
stdout = tests.StringIOWrapper()
2332
stderr = tests.StringIOWrapper()
2333
ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2334
stdout=stdout, stderr=stderr)
2335
t = self.old_transport
2336
req = RedirectedRequest('GET', t.abspath('a'))
2337
new_prefix = 'http://%s:%s' % (self.new_server.host,
2338
self.new_server.port)
2339
self.old_server.redirections = [
2340
('(.*)', r'%s/1\1' % (new_prefix), 301),]
2341
self.assertEqual('redirected once', t._perform(req).read())
2342
# stdin should be empty
2343
self.assertEqual('', ui.ui_factory.stdin.readline())
2344
# stdout should be empty, stderr will contains the prompts
2345
self.assertEqual('', stdout.getvalue())
added
removed
bzrlib/tests/test_http.py
