829
651
self.assertListRaises((errors.InvalidRange, errors.ShortReadvError,),
830
652
t.readv, 'a', [(12,2)])
832
def test_readv_multiple_get_requests(self):
833
server = self.get_readonly_server()
834
t = self._transport(server.get_url())
835
# force transport to issue multiple requests
836
t._max_readv_combine = 1
837
t._max_get_ranges = 1
838
l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
839
self.assertEqual(l[0], (0, '0'))
840
self.assertEqual(l[1], (1, '1'))
841
self.assertEqual(l[2], (3, '34'))
842
self.assertEqual(l[3], (9, '9'))
843
# The server should have issued 4 requests
844
self.assertEqual(4, server.GET_request_nb)
846
def test_readv_get_max_size(self):
847
server = self.get_readonly_server()
848
t = self._transport(server.get_url())
849
# force transport to issue multiple requests by limiting the number of
850
# bytes by request. Note that this apply to coalesced offsets only, a
851
# single range will keep its size even if bigger than the limit.
853
l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
854
self.assertEqual(l[0], (0, '0'))
855
self.assertEqual(l[1], (1, '1'))
856
self.assertEqual(l[2], (2, '2345'))
857
self.assertEqual(l[3], (6, '6789'))
858
# The server should have issued 3 requests
859
self.assertEqual(3, server.GET_request_nb)
861
def test_complete_readv_leave_pipe_clean(self):
862
server = self.get_readonly_server()
863
t = self._transport(server.get_url())
864
# force transport to issue multiple requests
866
l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
867
# The server should have issued 3 requests
868
self.assertEqual(3, server.GET_request_nb)
869
self.assertEqual('0123456789', t.get_bytes('a'))
870
self.assertEqual(4, server.GET_request_nb)
872
def test_incomplete_readv_leave_pipe_clean(self):
873
server = self.get_readonly_server()
874
t = self._transport(server.get_url())
875
# force transport to issue multiple requests
877
# Don't collapse readv results into a list so that we leave unread
878
# bytes on the socket
879
ireadv = iter(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
880
self.assertEqual((0, '0'), ireadv.next())
881
# The server should have issued one request so far
882
self.assertEqual(1, server.GET_request_nb)
883
self.assertEqual('0123456789', t.get_bytes('a'))
884
# get_bytes issued an additional request, the readv pending ones are
886
self.assertEqual(2, server.GET_request_nb)
889
class SingleRangeRequestHandler(http_server.TestingHTTPRequestHandler):
890
"""Always reply to range request as if they were single.
892
Don't be explicit about it, just to annoy the clients.
895
def get_multiple_ranges(self, file, file_size, ranges):
896
"""Answer as if it was a single range request and ignores the rest"""
897
(start, end) = ranges[0]
898
return self.get_single_range(file, file_size, start, end)
901
655
class TestSingleRangeRequestServer(TestRangeRequestServer):
902
656
"""Test readv against a server which accept only single range requests"""
904
_req_handler_class = SingleRangeRequestHandler
907
class SingleOnlyRangeRequestHandler(http_server.TestingHTTPRequestHandler):
908
"""Only reply to simple range requests, errors out on multiple"""
910
def get_multiple_ranges(self, file, file_size, ranges):
911
"""Refuses the multiple ranges request"""
914
self.send_error(416, "Requested range not satisfiable")
916
(start, end) = ranges[0]
917
return self.get_single_range(file, file_size, start, end)
920
class TestSingleOnlyRangeRequestServer(TestRangeRequestServer):
921
"""Test readv against a server which only accept single range requests"""
923
_req_handler_class = SingleOnlyRangeRequestHandler
926
class NoRangeRequestHandler(http_server.TestingHTTPRequestHandler):
927
"""Ignore range requests without notice"""
930
# Update the statistics
931
self.server.test_case_server.GET_request_nb += 1
932
# Just bypass the range handling done by TestingHTTPRequestHandler
933
return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
658
def create_transport_readonly_server(self):
659
return HttpServer(SingleRangeRequestHandler)
662
class TestSingleRangeRequestServer_urllib(TestSingleRangeRequestServer,
663
TestCaseWithWebserver):
664
"""Tests single range requests accepting server for urllib implementation"""
666
_transport = HttpTransport_urllib
669
class TestSingleRangeRequestServer_pycurl(TestWithTransport_pycurl,
670
TestSingleRangeRequestServer,
671
TestCaseWithWebserver):
672
"""Tests single range requests accepting server for pycurl implementation"""
936
675
class TestNoRangeRequestServer(TestRangeRequestServer):
937
676
"""Test readv against a server which do not accept range requests"""
939
_req_handler_class = NoRangeRequestHandler
942
class MultipleRangeWithoutContentLengthRequestHandler(
943
http_server.TestingHTTPRequestHandler):
944
"""Reply to multiple range requests without content length header."""
946
def get_multiple_ranges(self, file, file_size, ranges):
947
self.send_response(206)
948
self.send_header('Accept-Ranges', 'bytes')
949
boundary = "%d" % random.randint(0,0x7FFFFFFF)
950
self.send_header("Content-Type",
951
"multipart/byteranges; boundary=%s" % boundary)
953
for (start, end) in ranges:
954
self.wfile.write("--%s\r\n" % boundary)
955
self.send_header("Content-type", 'application/octet-stream')
956
self.send_header("Content-Range", "bytes %d-%d/%d" % (start,
960
self.send_range_content(file, start, end - start + 1)
962
self.wfile.write("--%s\r\n" % boundary)
965
class TestMultipleRangeWithoutContentLengthServer(TestRangeRequestServer):
967
_req_handler_class = MultipleRangeWithoutContentLengthRequestHandler
970
class TruncatedMultipleRangeRequestHandler(
971
http_server.TestingHTTPRequestHandler):
972
"""Reply to multiple range requests truncating the last ones.
974
This server generates responses whose Content-Length describes all the
975
ranges, but fail to include the last ones leading to client short reads.
976
This has been observed randomly with lighttpd (bug #179368).
979
_truncated_ranges = 2
981
def get_multiple_ranges(self, file, file_size, ranges):
982
self.send_response(206)
983
self.send_header('Accept-Ranges', 'bytes')
985
self.send_header('Content-Type',
986
'multipart/byteranges; boundary=%s' % boundary)
987
boundary_line = '--%s\r\n' % boundary
988
# Calculate the Content-Length
990
for (start, end) in ranges:
991
content_length += len(boundary_line)
992
content_length += self._header_line_length(
993
'Content-type', 'application/octet-stream')
994
content_length += self._header_line_length(
995
'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
996
content_length += len('\r\n') # end headers
997
content_length += end - start # + 1
998
content_length += len(boundary_line)
999
self.send_header('Content-length', content_length)
1002
# Send the multipart body
1004
for (start, end) in ranges:
1005
self.wfile.write(boundary_line)
1006
self.send_header('Content-type', 'application/octet-stream')
1007
self.send_header('Content-Range', 'bytes %d-%d/%d'
1008
% (start, end, file_size))
1010
if cur + self._truncated_ranges >= len(ranges):
1011
# Abruptly ends the response and close the connection
1012
self.close_connection = 1
1014
self.send_range_content(file, start, end - start + 1)
1017
self.wfile.write(boundary_line)
1020
class TestTruncatedMultipleRangeServer(TestSpecificRequestHandler):
1022
_req_handler_class = TruncatedMultipleRangeRequestHandler
1025
super(TestTruncatedMultipleRangeServer, self).setUp()
1026
self.build_tree_contents([('a', '0123456789')],)
1028
def test_readv_with_short_reads(self):
1029
server = self.get_readonly_server()
1030
t = self._transport(server.get_url())
1031
# Force separate ranges for each offset
1032
t._bytes_to_read_before_seek = 0
1033
ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1034
self.assertEqual((0, '0'), ireadv.next())
1035
self.assertEqual((2, '2'), ireadv.next())
1036
if not self._testing_pycurl():
1037
# Only one request have been issued so far (except for pycurl that
1038
# try to read the whole response at once)
1039
self.assertEqual(1, server.GET_request_nb)
1040
self.assertEqual((4, '45'), ireadv.next())
1041
self.assertEqual((9, '9'), ireadv.next())
1042
# Both implementations issue 3 requests but:
1043
# - urllib does two multiple (4 ranges, then 2 ranges) then a single
1045
# - pycurl does two multiple (4 ranges, 4 ranges) then a single range
1046
self.assertEqual(3, server.GET_request_nb)
1047
# Finally the client have tried a single range request and stays in
1049
self.assertEqual('single', t._range_hint)
1051
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1052
"""Errors out when range specifiers exceed the limit"""
1054
def get_multiple_ranges(self, file, file_size, ranges):
1055
"""Refuses the multiple ranges request"""
1056
tcs = self.server.test_case_server
1057
if tcs.range_limit is not None and len(ranges) > tcs.range_limit:
1059
# Emulate apache behavior
1060
self.send_error(400, "Bad Request")
1062
return http_server.TestingHTTPRequestHandler.get_multiple_ranges(
1063
self, file, file_size, ranges)
1066
class LimitedRangeHTTPServer(http_server.HttpServer):
1067
"""An HttpServer erroring out on requests with too much range specifiers"""
1069
def __init__(self, request_handler=LimitedRangeRequestHandler,
1070
protocol_version=None,
1072
http_server.HttpServer.__init__(self, request_handler,
1073
protocol_version=protocol_version)
1074
self.range_limit = range_limit
1077
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1078
"""Tests readv requests against a server erroring out on too much ranges."""
1080
# Requests with more range specifiers will error out
1083
678
def create_transport_readonly_server(self):
1084
return LimitedRangeHTTPServer(range_limit=self.range_limit,
1085
protocol_version=self._protocol_version)
1087
def get_transport(self):
1088
return self._transport(self.get_readonly_server().get_url())
1091
http_utils.TestCaseWithWebserver.setUp(self)
1092
# We need to manipulate ranges that correspond to real chunks in the
1093
# response, so we build a content appropriately.
1094
filler = ''.join(['abcdefghij' for x in range(102)])
1095
content = ''.join(['%04d' % v + filler for v in range(16)])
1096
self.build_tree_contents([('a', content)],)
1098
def test_few_ranges(self):
1099
t = self.get_transport()
1100
l = list(t.readv('a', ((0, 4), (1024, 4), )))
1101
self.assertEqual(l[0], (0, '0000'))
1102
self.assertEqual(l[1], (1024, '0001'))
1103
self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1105
def test_more_ranges(self):
1106
t = self.get_transport()
1107
l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1108
self.assertEqual(l[0], (0, '0000'))
1109
self.assertEqual(l[1], (1024, '0001'))
1110
self.assertEqual(l[2], (4096, '0004'))
1111
self.assertEqual(l[3], (8192, '0008'))
1112
# The server will refuse to serve the first request (too much ranges),
1113
# a second request will succeed.
1114
self.assertEqual(2, self.get_readonly_server().GET_request_nb)
1117
class TestHttpProxyWhiteBox(tests.TestCase):
679
return HttpServer(NoRangeRequestHandler)
682
class TestNoRangeRequestServer_urllib(TestNoRangeRequestServer,
683
TestCaseWithWebserver):
684
"""Tests range requests refusing server for urllib implementation"""
686
_transport = HttpTransport_urllib
689
class TestNoRangeRequestServer_pycurl(TestWithTransport_pycurl,
690
TestNoRangeRequestServer,
691
TestCaseWithWebserver):
692
"""Tests range requests refusing server for pycurl implementation"""
695
class TestHttpProxyWhiteBox(TestCase):
1118
696
"""Whitebox test proxy http authorization.
1120
Only the urllib implementation is tested here.
698
These tests concern urllib implementation only.
1123
701
def setUp(self):
1124
tests.TestCase.setUp(self)
1125
703
self._old_env = {}
1127
705
def tearDown(self):
1128
706
self._restore_env()
1129
tests.TestCase.tearDown(self)
708
def _set_and_capture_env_var(self, name, new_value):
709
"""Set an environment variable, and reset it when finished."""
710
self._old_env[name] = osutils.set_or_unset_env(name, new_value)
1131
712
def _install_env(self, env):
1132
713
for name, value in env.iteritems():
1133
self._old_env[name] = osutils.set_or_unset_env(name, value)
714
self._set_and_capture_env_var(name, value)
1135
716
def _restore_env(self):
1136
717
for name, value in self._old_env.iteritems():
1137
718
osutils.set_or_unset_env(name, value)
1139
720
def _proxied_request(self):
1140
handler = _urllib2_wrappers.ProxyHandler()
1141
request = _urllib2_wrappers.Request('GET','http://baz/buzzle')
721
from bzrlib.transport.http._urllib2_wrappers import (
726
handler = ProxyHandler()
727
request = Request('GET','http://baz/buzzle')
1142
728
handler.set_proxy(request, 'http')
1256
852
'NO_PROXY': self.no_proxy_host})
1258
854
def test_http_proxy_without_scheme(self):
1259
if self._testing_pycurl():
1260
# pycurl *ignores* invalid proxy env variables. If that ever change
1261
# in the future, this test will fail indicating that pycurl do not
1262
# ignore anymore such variables.
1263
self.not_proxied_in_env({'http_proxy': self.proxy_address})
1265
self.assertRaises(errors.InvalidURL,
1266
self.proxied_in_env,
1267
{'http_proxy': self.proxy_address})
1270
class TestRanges(http_utils.TestCaseWithWebserver):
1271
"""Test the Range header in GET methods."""
1274
http_utils.TestCaseWithWebserver.setUp(self)
855
self.assertRaises(errors.InvalidURL,
857
{'http_proxy': self.proxy_address})
860
class TestProxyHttpServer_urllib(TestProxyHttpServer,
861
TestCaseWithTwoWebservers):
862
"""Tests proxy server for urllib implementation"""
864
_transport = HttpTransport_urllib
867
class TestProxyHttpServer_pycurl(TestWithTransport_pycurl,
869
TestCaseWithTwoWebservers):
870
"""Tests proxy server for pycurl implementation"""
873
TestProxyHttpServer.setUp(self)
874
# Oh my ! pycurl does not check for the port as part of
875
# no_proxy :-( So we just test the host part
876
self.no_proxy_host = 'localhost'
878
def test_HTTP_PROXY(self):
879
# pycurl do not check HTTP_PROXY for security reasons
880
# (for use in a CGI context that we do not care
881
# about. Should we ?)
884
def test_HTTP_PROXY_with_NO_PROXY(self):
887
def test_http_proxy_without_scheme(self):
888
# pycurl *ignores* invalid proxy env variables. If that
889
# ever change in the future, this test will fail
890
# indicating that pycurl do not ignore anymore such
892
self.not_proxied_in_env({'http_proxy': self.proxy_address})
895
class TestRanges(object):
896
"""Test the Range header in GET methods..
898
This MUST be used by daughter classes that also inherit from
899
TestCaseWithWebserver.
901
We can't inherit directly from TestCaseWithWebserver or the
902
test framework will try to create an instance which cannot
903
run, its implementation being incomplete.
907
TestCaseWithWebserver.setUp(self)
1275
908
self.build_tree_contents([('a', '0123456789')],)
1276
909
server = self.get_readonly_server()
1277
910
self.transport = self._transport(server.get_url())
1279
def create_transport_readonly_server(self):
1280
return http_server.HttpServer(protocol_version=self._protocol_version)
1282
def _file_contents(self, relpath, ranges):
1283
offsets = [ (start, end - start + 1) for start, end in ranges]
1284
coalesce = self.transport._coalesce_offsets
1285
coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1286
code, data = self.transport._get(relpath, coalesced)
1287
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1288
for start, end in ranges:
1290
yield data.read(end - start + 1)
912
def _file_contents(self, relpath, ranges, tail_amount=0):
913
code, data = self.transport._get(relpath, ranges)
914
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
915
for start, end in ranges:
917
yield data.read(end - start + 1)
1292
919
def _file_tail(self, relpath, tail_amount):
1293
code, data = self.transport._get(relpath, [], tail_amount)
1294
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1295
data.seek(-tail_amount, 2)
1296
return data.read(tail_amount)
920
code, data = self.transport._get(relpath, [], tail_amount)
921
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
922
data.seek(-tail_amount + 1, 2)
923
return data.read(tail_amount)
1298
925
def test_range_header(self):
1300
927
map(self.assertEqual,['0', '234'],
1301
928
list(self._file_contents('a', [(0,0), (2,4)])),)
1303
def test_range_header_tail(self):
1304
930
self.assertEqual('789', self._file_tail('a', 3))
1306
def test_syntactically_invalid_range_header(self):
1307
self.assertListRaises(errors.InvalidHttpRange,
1308
self._file_contents, 'a', [(4, 3)])
1310
def test_semantically_invalid_range_header(self):
1311
self.assertListRaises(errors.InvalidHttpRange,
1312
self._file_contents, 'a', [(42, 128)])
1315
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1316
"""Test redirection between http servers."""
1318
def create_transport_secondary_server(self):
1319
"""Create the secondary server redirecting to the primary server"""
1320
new = self.get_readonly_server()
1322
redirecting = http_utils.HTTPServerRedirecting(
1323
protocol_version=self._protocol_version)
1324
redirecting.redirect_to(new.host, new.port)
1328
super(TestHTTPRedirections, self).setUp()
1329
self.build_tree_contents([('a', '0123456789'),
1331
'# Bazaar revision bundle v0.9\n#\n')
1333
# The requests to the old server will be redirected to the new server
1334
self.old_transport = self._transport(self.old_server.get_url())
1336
def test_redirected(self):
1337
self.assertRaises(errors.RedirectRequested, self.old_transport.get, 'a')
1338
t = self._transport(self.new_server.get_url())
1339
self.assertEqual('0123456789', t.get('a').read())
1341
def test_read_redirected_bundle_from_url(self):
1342
from bzrlib.bundle import read_bundle_from_url
1343
url = self.old_transport.abspath('bundle')
1344
bundle = self.applyDeprecated(deprecated_in((1, 12, 0)),
1345
read_bundle_from_url, url)
1346
# If read_bundle_from_url was successful we get an empty bundle
1347
self.assertEqual([], bundle.revisions)
1350
class RedirectedRequest(_urllib2_wrappers.Request):
1351
"""Request following redirections. """
1353
init_orig = _urllib2_wrappers.Request.__init__
1355
def __init__(self, method, url, *args, **kwargs):
1359
# Since the tests using this class will replace
1360
# _urllib2_wrappers.Request, we can't just call the base class __init__
1362
RedirectedRequest.init_orig(self, method, url, *args, **kwargs)
1363
self.follow_redirections = True
1366
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1367
"""Test redirections.
1369
http implementations do not redirect silently anymore (they
1370
do not redirect at all in fact). The mechanism is still in
1371
place at the _urllib2_wrappers.Request level and these tests
1374
For the pycurl implementation
1375
the redirection have been deleted as we may deprecate pycurl
1376
and I have no place to keep a working implementation.
1381
if pycurl_present and self._transport == PyCurlTransport:
1382
raise tests.TestNotApplicable(
1383
"pycurl doesn't redirect silently annymore")
1384
super(TestHTTPSilentRedirections, self).setUp()
1385
self.setup_redirected_request()
1386
self.addCleanup(self.cleanup_redirected_request)
1387
self.build_tree_contents([('a','a'),
1389
('1/a', 'redirected once'),
1391
('2/a', 'redirected twice'),
1393
('3/a', 'redirected thrice'),
1395
('4/a', 'redirected 4 times'),
1397
('5/a', 'redirected 5 times'),
1400
self.old_transport = self._transport(self.old_server.get_url())
1402
def setup_redirected_request(self):
1403
self.original_class = _urllib2_wrappers.Request
1404
_urllib2_wrappers.Request = RedirectedRequest
1406
def cleanup_redirected_request(self):
1407
_urllib2_wrappers.Request = self.original_class
1409
def create_transport_secondary_server(self):
1410
"""Create the secondary server, redirections are defined in the tests"""
1411
return http_utils.HTTPServerRedirecting(
1412
protocol_version=self._protocol_version)
1414
def test_one_redirection(self):
1415
t = self.old_transport
1417
req = RedirectedRequest('GET', t.abspath('a'))
1418
req.follow_redirections = True
1419
new_prefix = 'http://%s:%s' % (self.new_server.host,
1420
self.new_server.port)
1421
self.old_server.redirections = \
1422
[('(.*)', r'%s/1\1' % (new_prefix), 301),]
1423
self.assertEquals('redirected once',t._perform(req).read())
1425
def test_five_redirections(self):
1426
t = self.old_transport
1428
req = RedirectedRequest('GET', t.abspath('a'))
1429
req.follow_redirections = True
1430
old_prefix = 'http://%s:%s' % (self.old_server.host,
1431
self.old_server.port)
1432
new_prefix = 'http://%s:%s' % (self.new_server.host,
1433
self.new_server.port)
1434
self.old_server.redirections = [
1435
('/1(.*)', r'%s/2\1' % (old_prefix), 302),
1436
('/2(.*)', r'%s/3\1' % (old_prefix), 303),
1437
('/3(.*)', r'%s/4\1' % (old_prefix), 307),
1438
('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1439
('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1441
self.assertEquals('redirected 5 times',t._perform(req).read())
1444
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1445
"""Test transport.do_catching_redirections."""
1448
super(TestDoCatchRedirections, self).setUp()
1449
self.build_tree_contents([('a', '0123456789'),],)
1451
self.old_transport = self._transport(self.old_server.get_url())
1453
def get_a(self, transport):
1454
return transport.get('a')
1456
def test_no_redirection(self):
1457
t = self._transport(self.new_server.get_url())
1459
# We use None for redirected so that we fail if redirected
1460
self.assertEquals('0123456789',
1461
transport.do_catching_redirections(
1462
self.get_a, t, None).read())
1464
def test_one_redirection(self):
1465
self.redirections = 0
1467
def redirected(transport, exception, redirection_notice):
1468
self.redirections += 1
1469
dir, file = urlutils.split(exception.target)
1470
return self._transport(dir)
1472
self.assertEquals('0123456789',
1473
transport.do_catching_redirections(
1474
self.get_a, self.old_transport, redirected).read())
1475
self.assertEquals(1, self.redirections)
1477
def test_redirection_loop(self):
1479
def redirected(transport, exception, redirection_notice):
1480
# By using the redirected url as a base dir for the
1481
# *old* transport, we create a loop: a => a/a =>
1483
return self.old_transport.clone(exception.target)
1485
self.assertRaises(errors.TooManyRedirections,
1486
transport.do_catching_redirections,
1487
self.get_a, self.old_transport, redirected)
1490
class TestAuth(http_utils.TestCaseWithWebserver):
1491
"""Test authentication scheme"""
1493
_auth_header = 'Authorization'
1494
_password_prompt_prefix = ''
1495
_username_prompt_prefix = ''
1500
super(TestAuth, self).setUp()
1501
self.server = self.get_readonly_server()
1502
self.build_tree_contents([('a', 'contents of a\n'),
1503
('b', 'contents of b\n'),])
1505
def create_transport_readonly_server(self):
1506
return self._auth_server(protocol_version=self._protocol_version)
1508
def _testing_pycurl(self):
1509
return pycurl_present and self._transport == PyCurlTransport
1511
def get_user_url(self, user, password):
1512
"""Build an url embedding user and password"""
1513
url = '%s://' % self.server._url_protocol
1514
if user is not None:
1516
if password is not None:
1517
url += ':' + password
1519
url += '%s:%s/' % (self.server.host, self.server.port)
1522
def get_user_transport(self, user, password):
1523
return self._transport(self.get_user_url(user, password))
1525
def test_no_user(self):
1526
self.server.add_user('joe', 'foo')
1527
t = self.get_user_transport(None, None)
1528
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1529
# Only one 'Authentication Required' error should occur
1530
self.assertEqual(1, self.server.auth_required_errors)
1532
def test_empty_pass(self):
1533
self.server.add_user('joe', '')
1534
t = self.get_user_transport('joe', '')
1535
self.assertEqual('contents of a\n', t.get('a').read())
1536
# Only one 'Authentication Required' error should occur
1537
self.assertEqual(1, self.server.auth_required_errors)
1539
def test_user_pass(self):
1540
self.server.add_user('joe', 'foo')
1541
t = self.get_user_transport('joe', 'foo')
1542
self.assertEqual('contents of a\n', t.get('a').read())
1543
# Only one 'Authentication Required' error should occur
1544
self.assertEqual(1, self.server.auth_required_errors)
1546
def test_unknown_user(self):
1547
self.server.add_user('joe', 'foo')
1548
t = self.get_user_transport('bill', 'foo')
1549
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1550
# Two 'Authentication Required' errors should occur (the
1551
# initial 'who are you' and 'I don't know you, who are
1553
self.assertEqual(2, self.server.auth_required_errors)
1555
def test_wrong_pass(self):
1556
self.server.add_user('joe', 'foo')
1557
t = self.get_user_transport('joe', 'bar')
1558
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1559
# Two 'Authentication Required' errors should occur (the
1560
# initial 'who are you' and 'this is not you, who are you')
1561
self.assertEqual(2, self.server.auth_required_errors)
1563
def test_prompt_for_username(self):
1564
if self._testing_pycurl():
1565
raise tests.TestNotApplicable(
1566
'pycurl cannot prompt, it handles auth by embedding'
1567
' user:pass in urls only')
1569
self.server.add_user('joe', 'foo')
1570
t = self.get_user_transport(None, None)
1571
stdout = tests.StringIOWrapper()
1572
stderr = tests.StringIOWrapper()
1573
ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
1574
stdout=stdout, stderr=stderr)
1575
self.assertEqual('contents of a\n',t.get('a').read())
1576
# stdin should be empty
1577
self.assertEqual('', ui.ui_factory.stdin.readline())
1579
expected_prompt = self._expected_username_prompt(t._unqualified_scheme)
1580
self.assertEquals(expected_prompt, stderr.read(len(expected_prompt)))
1581
self.assertEquals('', stdout.getvalue())
1582
self._check_password_prompt(t._unqualified_scheme, 'joe',
1585
def test_prompt_for_password(self):
1586
if self._testing_pycurl():
1587
raise tests.TestNotApplicable(
1588
'pycurl cannot prompt, it handles auth by embedding'
1589
' user:pass in urls only')
1591
self.server.add_user('joe', 'foo')
1592
t = self.get_user_transport('joe', None)
1593
stdout = tests.StringIOWrapper()
1594
stderr = tests.StringIOWrapper()
1595
ui.ui_factory = tests.TestUIFactory(stdin='foo\n',
1596
stdout=stdout, stderr=stderr)
1597
self.assertEqual('contents of a\n', t.get('a').read())
1598
# stdin should be empty
1599
self.assertEqual('', ui.ui_factory.stdin.readline())
1600
self._check_password_prompt(t._unqualified_scheme, 'joe',
1602
self.assertEquals('', stdout.getvalue())
1603
# And we shouldn't prompt again for a different request
1604
# against the same transport.
1605
self.assertEqual('contents of b\n',t.get('b').read())
1607
# And neither against a clone
1608
self.assertEqual('contents of b\n',t2.get('b').read())
1609
# Only one 'Authentication Required' error should occur
1610
self.assertEqual(1, self.server.auth_required_errors)
1612
def _check_password_prompt(self, scheme, user, actual_prompt):
1613
expected_prompt = (self._password_prompt_prefix
1614
+ ("%s %s@%s:%d, Realm: '%s' password: "
1616
user, self.server.host, self.server.port,
1617
self.server.auth_realm)))
1618
self.assertEquals(expected_prompt, actual_prompt)
1620
def _expected_username_prompt(self, scheme):
1621
return (self._username_prompt_prefix
1622
+ "%s %s:%d, Realm: '%s' username: " % (scheme.upper(),
1623
self.server.host, self.server.port,
1624
self.server.auth_realm))
1626
def test_no_prompt_for_password_when_using_auth_config(self):
1627
if self._testing_pycurl():
1628
raise tests.TestNotApplicable(
1629
'pycurl does not support authentication.conf'
1630
' since it cannot prompt')
1634
stdin_content = 'bar\n' # Not the right password
1635
self.server.add_user(user, password)
1636
t = self.get_user_transport(user, None)
1637
ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1638
stdout=tests.StringIOWrapper())
1639
# Create a minimal config file with the right password
1640
conf = config.AuthenticationConfig()
1641
conf._get_config().update(
1642
{'httptest': {'scheme': 'http', 'port': self.server.port,
1643
'user': user, 'password': password}})
1645
# Issue a request to the server to connect
1646
self.assertEqual('contents of a\n',t.get('a').read())
1647
# stdin should have been left untouched
1648
self.assertEqual(stdin_content, ui.ui_factory.stdin.readline())
1649
# Only one 'Authentication Required' error should occur
1650
self.assertEqual(1, self.server.auth_required_errors)
1652
def test_user_from_auth_conf(self):
1653
if self._testing_pycurl():
1654
raise tests.TestNotApplicable(
1655
'pycurl does not support authentication.conf')
1658
self.server.add_user(user, password)
1659
# Create a minimal config file with the right password
1660
conf = config.AuthenticationConfig()
1661
conf._get_config().update(
1662
{'httptest': {'scheme': 'http', 'port': self.server.port,
1663
'user': user, 'password': password}})
1665
t = self.get_user_transport(None, None)
1666
# Issue a request to the server to connect
1667
self.assertEqual('contents of a\n', t.get('a').read())
1668
# Only one 'Authentication Required' error should occur
1669
self.assertEqual(1, self.server.auth_required_errors)
1671
def test_changing_nonce(self):
1672
if self._auth_server not in (http_utils.HTTPDigestAuthServer,
1673
http_utils.ProxyDigestAuthServer):
1674
raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1675
if self._testing_pycurl():
1676
raise tests.KnownFailure(
1677
'pycurl does not handle a nonce change')
1678
self.server.add_user('joe', 'foo')
1679
t = self.get_user_transport('joe', 'foo')
1680
self.assertEqual('contents of a\n', t.get('a').read())
1681
self.assertEqual('contents of b\n', t.get('b').read())
1682
# Only one 'Authentication Required' error should have
1684
self.assertEqual(1, self.server.auth_required_errors)
1685
# The server invalidates the current nonce
1686
self.server.auth_nonce = self.server.auth_nonce + '. No, now!'
1687
self.assertEqual('contents of a\n', t.get('a').read())
1688
# Two 'Authentication Required' errors should occur (the
1689
# initial 'who are you' and a second 'who are you' with the new nonce)
1690
self.assertEqual(2, self.server.auth_required_errors)
1694
class TestProxyAuth(TestAuth):
1695
"""Test proxy authentication schemes."""
1697
_auth_header = 'Proxy-authorization'
1698
_password_prompt_prefix = 'Proxy '
1699
_username_prompt_prefix = 'Proxy '
1702
super(TestProxyAuth, self).setUp()
1704
self.addCleanup(self._restore_env)
1705
# Override the contents to avoid false positives
1706
self.build_tree_contents([('a', 'not proxied contents of a\n'),
1707
('b', 'not proxied contents of b\n'),
1708
('a-proxied', 'contents of a\n'),
1709
('b-proxied', 'contents of b\n'),
1712
def get_user_transport(self, user, password):
1713
self._install_env({'all_proxy': self.get_user_url(user, password)})
1714
return self._transport(self.server.get_url())
1716
def _install_env(self, env):
1717
for name, value in env.iteritems():
1718
self._old_env[name] = osutils.set_or_unset_env(name, value)
1720
def _restore_env(self):
1721
for name, value in self._old_env.iteritems():
1722
osutils.set_or_unset_env(name, value)
1724
def test_empty_pass(self):
1725
if self._testing_pycurl():
1727
if pycurl.version_info()[1] < '7.16.0':
1728
raise tests.KnownFailure(
1729
'pycurl < 7.16.0 does not handle empty proxy passwords')
1730
super(TestProxyAuth, self).test_empty_pass()
1733
class SampleSocket(object):
1734
"""A socket-like object for use in testing the HTTP request handler."""
1736
def __init__(self, socket_read_content):
1737
"""Constructs a sample socket.
1739
:param socket_read_content: a byte sequence
1741
# Use plain python StringIO so we can monkey-patch the close method to
1742
# not discard the contents.
1743
from StringIO import StringIO
1744
self.readfile = StringIO(socket_read_content)
1745
self.writefile = StringIO()
1746
self.writefile.close = lambda: None
1748
def makefile(self, mode='r', bufsize=None):
1750
return self.readfile
1752
return self.writefile
1755
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1758
super(SmartHTTPTunnellingTest, self).setUp()
1759
# We use the VFS layer as part of HTTP tunnelling tests.
1760
self._captureVar('BZR_NO_SMART_VFS', None)
1761
self.transport_readonly_server = http_utils.HTTPServerWithSmarts
1763
def create_transport_readonly_server(self):
1764
return http_utils.HTTPServerWithSmarts(
1765
protocol_version=self._protocol_version)
1767
def test_open_bzrdir(self):
1768
branch = self.make_branch('relpath')
1769
http_server = self.get_readonly_server()
1770
url = http_server.get_url() + 'relpath'
1771
bd = bzrdir.BzrDir.open(url)
1772
self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1774
def test_bulk_data(self):
1775
# We should be able to send and receive bulk data in a single message.
1776
# The 'readv' command in the smart protocol both sends and receives
1777
# bulk data, so we use that.
1778
self.build_tree(['data-file'])
1779
http_server = self.get_readonly_server()
1780
http_transport = self._transport(http_server.get_url())
1781
medium = http_transport.get_smart_medium()
1782
# Since we provide the medium, the url below will be mostly ignored
1783
# during the test, as long as the path is '/'.
1784
remote_transport = remote.RemoteTransport('bzr://fake_host/',
1787
[(0, "c")], list(remote_transport.readv("data-file", [(0,1)])))
1789
def test_http_send_smart_request(self):
1791
post_body = 'hello\n'
1792
expected_reply_body = 'ok\x012\n'
1794
http_server = self.get_readonly_server()
1795
http_transport = self._transport(http_server.get_url())
1796
medium = http_transport.get_smart_medium()
1797
response = medium.send_http_smart_request(post_body)
1798
reply_body = response.read()
1799
self.assertEqual(expected_reply_body, reply_body)
1801
def test_smart_http_server_post_request_handler(self):
1802
httpd = self.get_readonly_server()._get_httpd()
1804
socket = SampleSocket(
1805
'POST /.bzr/smart %s \r\n' % self._protocol_version
1806
# HTTP/1.1 posts must have a Content-Length (but it doesn't hurt
1808
+ 'Content-Length: 6\r\n'
1811
# Beware: the ('localhost', 80) below is the
1812
# client_address parameter, but we don't have one because
1813
# we have defined a socket which is not bound to an
1814
# address. The test framework never uses this client
1815
# address, so far...
1816
request_handler = http_utils.SmartRequestHandler(socket,
1819
response = socket.writefile.getvalue()
1820
self.assertStartsWith(response, '%s 200 ' % self._protocol_version)
1821
# This includes the end of the HTTP headers, and all the body.
1822
expected_end_of_response = '\r\n\r\nok\x012\n'
1823
self.assertEndsWith(response, expected_end_of_response)
1826
class ForbiddenRequestHandler(http_server.TestingHTTPRequestHandler):
1827
"""No smart server here request handler."""
1830
self.send_error(403, "Forbidden")
1833
class SmartClientAgainstNotSmartServer(TestSpecificRequestHandler):
1834
"""Test smart client behaviour against an http server without smarts."""
1836
_req_handler_class = ForbiddenRequestHandler
1838
def test_probe_smart_server(self):
1839
"""Test error handling against server refusing smart requests."""
1840
server = self.get_readonly_server()
1841
t = self._transport(server.get_url())
1842
# No need to build a valid smart request here, the server will not even
1843
# try to interpret it.
1844
self.assertRaises(errors.SmartProtocolError,
1845
t.get_smart_medium().send_http_smart_request,
1848
class Test_redirected_to(tests.TestCase):
1850
def test_redirected_to_subdir(self):
1851
t = self._transport('http://www.example.com/foo')
1852
r = t._redirected_to('http://www.example.com/foo',
1853
'http://www.example.com/foo/subdir')
1854
self.assertIsInstance(r, type(t))
1855
# Both transports share the some connection
1856
self.assertEquals(t._get_connection(), r._get_connection())
1858
def test_redirected_to_self_with_slash(self):
1859
t = self._transport('http://www.example.com/foo')
1860
r = t._redirected_to('http://www.example.com/foo',
1861
'http://www.example.com/foo/')
1862
self.assertIsInstance(r, type(t))
1863
# Both transports share the some connection (one can argue that we
1864
# should return the exact same transport here, but that seems
1866
self.assertEquals(t._get_connection(), r._get_connection())
1868
def test_redirected_to_host(self):
1869
t = self._transport('http://www.example.com/foo')
1870
r = t._redirected_to('http://www.example.com/foo',
1871
'http://foo.example.com/foo/subdir')
1872
self.assertIsInstance(r, type(t))
1874
def test_redirected_to_same_host_sibling_protocol(self):
1875
t = self._transport('http://www.example.com/foo')
1876
r = t._redirected_to('http://www.example.com/foo',
1877
'https://www.example.com/foo')
1878
self.assertIsInstance(r, type(t))
1880
def test_redirected_to_same_host_different_protocol(self):
1881
t = self._transport('http://www.example.com/foo')
1882
r = t._redirected_to('http://www.example.com/foo',
1883
'ftp://www.example.com/foo')
1884
self.assertNotEquals(type(r), type(t))
1886
def test_redirected_to_different_host_same_user(self):
1887
t = self._transport('http://joe@www.example.com/foo')
1888
r = t._redirected_to('http://www.example.com/foo',
1889
'https://foo.example.com/foo')
1890
self.assertIsInstance(r, type(t))
1891
self.assertEquals(t._user, r._user)
1894
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
1895
"""Request handler for a unique and pre-defined request.
1897
The only thing we care about here is how many bytes travel on the wire. But
1898
since we want to measure it for a real http client, we have to send it
1901
We expect to receive a *single* request nothing more (and we won't even
1902
check what request it is, we just measure the bytes read until an empty
1906
def handle_one_request(self):
1907
tcs = self.server.test_case_server
1908
requestline = self.rfile.readline()
1909
headers = self.MessageClass(self.rfile, 0)
1910
# We just read: the request, the headers, an empty line indicating the
1911
# end of the headers.
1912
bytes_read = len(requestline)
1913
for line in headers.headers:
1914
bytes_read += len(line)
1915
bytes_read += len('\r\n')
1916
if requestline.startswith('POST'):
1917
# The body should be a single line (or we don't know where it ends
1918
# and we don't want to issue a blocking read)
1919
body = self.rfile.readline()
1920
bytes_read += len(body)
1921
tcs.bytes_read = bytes_read
1923
# We set the bytes written *before* issuing the write, the client is
1924
# supposed to consume every produced byte *before* checking that value.
1926
# Doing the oppposite may lead to test failure: we may be interrupted
1927
# after the write but before updating the value. The client can then
1928
# continue and read the value *before* we can update it. And yes,
1929
# this has been observed -- vila 20090129
1930
tcs.bytes_written = len(tcs.canned_response)
1931
self.wfile.write(tcs.canned_response)
1934
class ActivityServerMixin(object):
1936
def __init__(self, protocol_version):
1937
super(ActivityServerMixin, self).__init__(
1938
request_handler=PredefinedRequestHandler,
1939
protocol_version=protocol_version)
1940
# Bytes read and written by the server
1942
self.bytes_written = 0
1943
self.canned_response = None
1946
class ActivityHTTPServer(ActivityServerMixin, http_server.HttpServer):
1950
if tests.HTTPSServerFeature.available():
1951
from bzrlib.tests import https_server
1952
class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
1956
class TestActivity(tests.TestCase):
1957
"""Test socket activity reporting.
1959
We use a special purpose server to control the bytes sent and received and
1960
be able to predict the activity on the client socket.
1964
tests.TestCase.setUp(self)
1965
self.server = self._activity_server(self._protocol_version)
1967
self.activities = {}
1968
def report_activity(t, bytes, direction):
1969
count = self.activities.get(direction, 0)
1971
self.activities[direction] = count
1973
# We override at class level because constructors may propagate the
1974
# bound method and render instance overriding ineffective (an
1975
# alternative would be to define a specific ui factory instead...)
1976
self.orig_report_activity = self._transport._report_activity
1977
self._transport._report_activity = report_activity
1980
self._transport._report_activity = self.orig_report_activity
1981
self.server.tearDown()
1982
tests.TestCase.tearDown(self)
1984
def get_transport(self):
1985
return self._transport(self.server.get_url())
1987
def assertActivitiesMatch(self):
1988
self.assertEqual(self.server.bytes_read,
1989
self.activities.get('write', 0), 'written bytes')
1990
self.assertEqual(self.server.bytes_written,
1991
self.activities.get('read', 0), 'read bytes')
1994
self.server.canned_response = '''HTTP/1.1 200 OK\r
1995
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
1996
Server: Apache/2.0.54 (Fedora)\r
1997
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
1998
ETag: "56691-23-38e9ae00"\r
1999
Accept-Ranges: bytes\r
2000
Content-Length: 35\r
2002
Content-Type: text/plain; charset=UTF-8\r
2004
Bazaar-NG meta directory, format 1
2006
t = self.get_transport()
2007
self.assertEqual('Bazaar-NG meta directory, format 1\n',
2008
t.get('foo/bar').read())
2009
self.assertActivitiesMatch()
2012
self.server.canned_response = '''HTTP/1.1 200 OK\r
2013
Server: SimpleHTTP/0.6 Python/2.5.2\r
2014
Date: Thu, 29 Jan 2009 20:21:47 GMT\r
2015
Content-type: application/octet-stream\r
2016
Content-Length: 20\r
2017
Last-Modified: Thu, 29 Jan 2009 20:21:47 GMT\r
2020
t = self.get_transport()
2021
self.assertTrue(t.has('foo/bar'))
2022
self.assertActivitiesMatch()
2024
def test_readv(self):
2025
self.server.canned_response = '''HTTP/1.1 206 Partial Content\r
2026
Date: Tue, 11 Jul 2006 04:49:48 GMT\r
2027
Server: Apache/2.0.54 (Fedora)\r
2028
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
2029
ETag: "238a3c-16ec2-805c5540"\r
2030
Accept-Ranges: bytes\r
2031
Content-Length: 1534\r
2033
Content-Type: multipart/byteranges; boundary=418470f848b63279b\r
2036
--418470f848b63279b\r
2037
Content-type: text/plain; charset=UTF-8\r
2038
Content-range: bytes 0-254/93890\r
2040
mbp@sourcefrog.net-20050309040815-13242001617e4a06
2041
mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e7627
2042
mbp@sourcefrog.net-20050309040957-6cad07f466bb0bb8
2043
mbp@sourcefrog.net-20050309041501-c840e09071de3b67
2044
mbp@sourcefrog.net-20050309044615-c24a3250be83220a
2046
--418470f848b63279b\r
2047
Content-type: text/plain; charset=UTF-8\r
2048
Content-range: bytes 1000-2049/93890\r
2051
mbp@sourcefrog.net-20050311063625-07858525021f270b
2052
mbp@sourcefrog.net-20050311231934-aa3776aff5200bb9
2053
mbp@sourcefrog.net-20050311231953-73aeb3a131c3699a
2054
mbp@sourcefrog.net-20050311232353-f5e33da490872c6a
2055
mbp@sourcefrog.net-20050312071639-0a8f59a34a024ff0
2056
mbp@sourcefrog.net-20050312073432-b2c16a55e0d6e9fb
2057
mbp@sourcefrog.net-20050312073831-a47c3335ece1920f
2058
mbp@sourcefrog.net-20050312085412-13373aa129ccbad3
2059
mbp@sourcefrog.net-20050313052251-2bf004cb96b39933
2060
mbp@sourcefrog.net-20050313052856-3edd84094687cb11
2061
mbp@sourcefrog.net-20050313053233-e30a4f28aef48f9d
2062
mbp@sourcefrog.net-20050313053853-7c64085594ff3072
2063
mbp@sourcefrog.net-20050313054757-a86c3f5871069e22
2064
mbp@sourcefrog.net-20050313061422-418f1f73b94879b9
2065
mbp@sourcefrog.net-20050313120651-497bd231b19df600
2066
mbp@sourcefrog.net-20050314024931-eae0170ef25a5d1a
2067
mbp@sourcefrog.net-20050314025438-d52099f915fe65fc
2068
mbp@sourcefrog.net-20050314025539-637a636692c055cf
2069
mbp@sourcefrog.net-20050314025737-55eb441f430ab4ba
2070
mbp@sourcefrog.net-20050314025901-d74aa93bb7ee8f62
2072
--418470f848b63279b--\r
2074
t = self.get_transport()
2075
# Remember that the request is ignored and that the ranges below
2076
# doesn't have to match the canned response.
2077
l = list(t.readv('/foo/bar', ((0, 255), (1000, 1050))))
2078
self.assertEqual(2, len(l))
2079
self.assertActivitiesMatch()
2081
def test_post(self):
2082
self.server.canned_response = '''HTTP/1.1 200 OK\r
2083
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
2084
Server: Apache/2.0.54 (Fedora)\r
2085
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
2086
ETag: "56691-23-38e9ae00"\r
2087
Accept-Ranges: bytes\r
2088
Content-Length: 35\r
2090
Content-Type: text/plain; charset=UTF-8\r
2092
lalala whatever as long as itsssss
2094
t = self.get_transport()
2095
# We must send a single line of body bytes, see
2096
# PredefinedRequestHandler.handle_one_request
2097
code, f = t._post('abc def end-of-body\n')
2098
self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2099
self.assertActivitiesMatch()
931
# Syntactically invalid range
932
self.assertRaises(errors.InvalidRange,
933
self.transport._get, 'a', [(4, 3)])
934
# Semantically invalid range
935
self.assertRaises(errors.InvalidRange,
936
self.transport._get, 'a', [(42, 128)])
939
class TestRanges_urllib(TestRanges, TestCaseWithWebserver):
940
"""Test the Range header in GET methods for urllib implementation"""
942
_transport = HttpTransport_urllib
945
class TestRanges_pycurl(TestWithTransport_pycurl,
947
TestCaseWithWebserver):
948
"""Test the Range header in GET methods for pycurl implementation"""
added
removed
bzrlib/tests/test_http.py
