830
651
self.assertListRaises((errors.InvalidRange, errors.ShortReadvError,),
831
652
t.readv, 'a', [(12,2)])
833
def test_readv_multiple_get_requests(self):
834
server = self.get_readonly_server()
835
t = self.get_readonly_transport()
836
# force transport to issue multiple requests
837
t._max_readv_combine = 1
838
t._max_get_ranges = 1
839
l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
840
self.assertEqual(l[0], (0, '0'))
841
self.assertEqual(l[1], (1, '1'))
842
self.assertEqual(l[2], (3, '34'))
843
self.assertEqual(l[3], (9, '9'))
844
# The server should have issued 4 requests
845
self.assertEqual(4, server.GET_request_nb)
847
def test_readv_get_max_size(self):
848
server = self.get_readonly_server()
849
t = self.get_readonly_transport()
850
# force transport to issue multiple requests by limiting the number of
851
# bytes by request. Note that this apply to coalesced offsets only, a
852
# single range will keep its size even if bigger than the limit.
854
l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
855
self.assertEqual(l[0], (0, '0'))
856
self.assertEqual(l[1], (1, '1'))
857
self.assertEqual(l[2], (2, '2345'))
858
self.assertEqual(l[3], (6, '6789'))
859
# The server should have issued 3 requests
860
self.assertEqual(3, server.GET_request_nb)
862
def test_complete_readv_leave_pipe_clean(self):
863
server = self.get_readonly_server()
864
t = self.get_readonly_transport()
865
# force transport to issue multiple requests
867
list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
868
# The server should have issued 3 requests
869
self.assertEqual(3, server.GET_request_nb)
870
self.assertEqual('0123456789', t.get_bytes('a'))
871
self.assertEqual(4, server.GET_request_nb)
873
def test_incomplete_readv_leave_pipe_clean(self):
874
server = self.get_readonly_server()
875
t = self.get_readonly_transport()
876
# force transport to issue multiple requests
878
# Don't collapse readv results into a list so that we leave unread
879
# bytes on the socket
880
ireadv = iter(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
881
self.assertEqual((0, '0'), ireadv.next())
882
# The server should have issued one request so far
883
self.assertEqual(1, server.GET_request_nb)
884
self.assertEqual('0123456789', t.get_bytes('a'))
885
# get_bytes issued an additional request, the readv pending ones are
887
self.assertEqual(2, server.GET_request_nb)
890
class SingleRangeRequestHandler(http_server.TestingHTTPRequestHandler):
891
"""Always reply to range request as if they were single.
893
Don't be explicit about it, just to annoy the clients.
896
def get_multiple_ranges(self, file, file_size, ranges):
897
"""Answer as if it was a single range request and ignores the rest"""
898
(start, end) = ranges[0]
899
return self.get_single_range(file, file_size, start, end)
902
655
class TestSingleRangeRequestServer(TestRangeRequestServer):
903
656
"""Test readv against a server which accept only single range requests"""
905
_req_handler_class = SingleRangeRequestHandler
908
class SingleOnlyRangeRequestHandler(http_server.TestingHTTPRequestHandler):
909
"""Only reply to simple range requests, errors out on multiple"""
911
def get_multiple_ranges(self, file, file_size, ranges):
912
"""Refuses the multiple ranges request"""
915
self.send_error(416, "Requested range not satisfiable")
917
(start, end) = ranges[0]
918
return self.get_single_range(file, file_size, start, end)
921
class TestSingleOnlyRangeRequestServer(TestRangeRequestServer):
922
"""Test readv against a server which only accept single range requests"""
924
_req_handler_class = SingleOnlyRangeRequestHandler
927
class NoRangeRequestHandler(http_server.TestingHTTPRequestHandler):
928
"""Ignore range requests without notice"""
931
# Update the statistics
932
self.server.test_case_server.GET_request_nb += 1
933
# Just bypass the range handling done by TestingHTTPRequestHandler
934
return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
658
def create_transport_readonly_server(self):
659
return HttpServer(SingleRangeRequestHandler)
662
class TestSingleRangeRequestServer_urllib(TestSingleRangeRequestServer,
663
TestCaseWithWebserver):
664
"""Tests single range requests accepting server for urllib implementation"""
666
_transport = HttpTransport_urllib
669
class TestSingleRangeRequestServer_pycurl(TestWithTransport_pycurl,
670
TestSingleRangeRequestServer,
671
TestCaseWithWebserver):
672
"""Tests single range requests accepting server for pycurl implementation"""
937
675
class TestNoRangeRequestServer(TestRangeRequestServer):
938
676
"""Test readv against a server which do not accept range requests"""
940
_req_handler_class = NoRangeRequestHandler
943
class MultipleRangeWithoutContentLengthRequestHandler(
944
http_server.TestingHTTPRequestHandler):
945
"""Reply to multiple range requests without content length header."""
947
def get_multiple_ranges(self, file, file_size, ranges):
948
self.send_response(206)
949
self.send_header('Accept-Ranges', 'bytes')
950
# XXX: this is strange; the 'random' name below seems undefined and
951
# yet the tests pass -- mbp 2010-10-11 bug 658773
952
boundary = "%d" % random.randint(0,0x7FFFFFFF)
953
self.send_header("Content-Type",
954
"multipart/byteranges; boundary=%s" % boundary)
956
for (start, end) in ranges:
957
self.wfile.write("--%s\r\n" % boundary)
958
self.send_header("Content-type", 'application/octet-stream')
959
self.send_header("Content-Range", "bytes %d-%d/%d" % (start,
963
self.send_range_content(file, start, end - start + 1)
965
self.wfile.write("--%s\r\n" % boundary)
968
class TestMultipleRangeWithoutContentLengthServer(TestRangeRequestServer):
970
_req_handler_class = MultipleRangeWithoutContentLengthRequestHandler
973
class TruncatedMultipleRangeRequestHandler(
974
http_server.TestingHTTPRequestHandler):
975
"""Reply to multiple range requests truncating the last ones.
977
This server generates responses whose Content-Length describes all the
978
ranges, but fail to include the last ones leading to client short reads.
979
This has been observed randomly with lighttpd (bug #179368).
982
_truncated_ranges = 2
984
def get_multiple_ranges(self, file, file_size, ranges):
985
self.send_response(206)
986
self.send_header('Accept-Ranges', 'bytes')
988
self.send_header('Content-Type',
989
'multipart/byteranges; boundary=%s' % boundary)
990
boundary_line = '--%s\r\n' % boundary
991
# Calculate the Content-Length
993
for (start, end) in ranges:
994
content_length += len(boundary_line)
995
content_length += self._header_line_length(
996
'Content-type', 'application/octet-stream')
997
content_length += self._header_line_length(
998
'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
999
content_length += len('\r\n') # end headers
1000
content_length += end - start # + 1
1001
content_length += len(boundary_line)
1002
self.send_header('Content-length', content_length)
1005
# Send the multipart body
1007
for (start, end) in ranges:
1008
self.wfile.write(boundary_line)
1009
self.send_header('Content-type', 'application/octet-stream')
1010
self.send_header('Content-Range', 'bytes %d-%d/%d'
1011
% (start, end, file_size))
1013
if cur + self._truncated_ranges >= len(ranges):
1014
# Abruptly ends the response and close the connection
1015
self.close_connection = 1
1017
self.send_range_content(file, start, end - start + 1)
1020
self.wfile.write(boundary_line)
1023
class TestTruncatedMultipleRangeServer(TestSpecificRequestHandler):
1025
_req_handler_class = TruncatedMultipleRangeRequestHandler
1028
super(TestTruncatedMultipleRangeServer, self).setUp()
1029
self.build_tree_contents([('a', '0123456789')],)
1031
def test_readv_with_short_reads(self):
1032
server = self.get_readonly_server()
1033
t = self.get_readonly_transport()
1034
# Force separate ranges for each offset
1035
t._bytes_to_read_before_seek = 0
1036
ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1037
self.assertEqual((0, '0'), ireadv.next())
1038
self.assertEqual((2, '2'), ireadv.next())
1039
if not self._testing_pycurl():
1040
# Only one request have been issued so far (except for pycurl that
1041
# try to read the whole response at once)
1042
self.assertEqual(1, server.GET_request_nb)
1043
self.assertEqual((4, '45'), ireadv.next())
1044
self.assertEqual((9, '9'), ireadv.next())
1045
# Both implementations issue 3 requests but:
1046
# - urllib does two multiple (4 ranges, then 2 ranges) then a single
1048
# - pycurl does two multiple (4 ranges, 4 ranges) then a single range
1049
self.assertEqual(3, server.GET_request_nb)
1050
# Finally the client have tried a single range request and stays in
1052
self.assertEqual('single', t._range_hint)
1055
class TruncatedBeforeBoundaryRequestHandler(
1056
http_server.TestingHTTPRequestHandler):
1057
"""Truncation before a boundary, like in bug 198646"""
1059
_truncated_ranges = 1
1061
def get_multiple_ranges(self, file, file_size, ranges):
1062
self.send_response(206)
1063
self.send_header('Accept-Ranges', 'bytes')
1065
self.send_header('Content-Type',
1066
'multipart/byteranges; boundary=%s' % boundary)
1067
boundary_line = '--%s\r\n' % boundary
1068
# Calculate the Content-Length
1070
for (start, end) in ranges:
1071
content_length += len(boundary_line)
1072
content_length += self._header_line_length(
1073
'Content-type', 'application/octet-stream')
1074
content_length += self._header_line_length(
1075
'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
1076
content_length += len('\r\n') # end headers
1077
content_length += end - start # + 1
1078
content_length += len(boundary_line)
1079
self.send_header('Content-length', content_length)
1082
# Send the multipart body
1084
for (start, end) in ranges:
1085
if cur + self._truncated_ranges >= len(ranges):
1086
# Abruptly ends the response and close the connection
1087
self.close_connection = 1
1089
self.wfile.write(boundary_line)
1090
self.send_header('Content-type', 'application/octet-stream')
1091
self.send_header('Content-Range', 'bytes %d-%d/%d'
1092
% (start, end, file_size))
1094
self.send_range_content(file, start, end - start + 1)
1097
self.wfile.write(boundary_line)
1100
class TestTruncatedBeforeBoundary(TestSpecificRequestHandler):
1101
"""Tests the case of bug 198646, disconnecting before a boundary."""
1103
_req_handler_class = TruncatedBeforeBoundaryRequestHandler
1106
super(TestTruncatedBeforeBoundary, self).setUp()
1107
self.build_tree_contents([('a', '0123456789')],)
1109
def test_readv_with_short_reads(self):
1110
server = self.get_readonly_server()
1111
t = self.get_readonly_transport()
1112
# Force separate ranges for each offset
1113
t._bytes_to_read_before_seek = 0
1114
ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1115
self.assertEqual((0, '0'), ireadv.next())
1116
self.assertEqual((2, '2'), ireadv.next())
1117
self.assertEqual((4, '45'), ireadv.next())
1118
self.assertEqual((9, '9'), ireadv.next())
1121
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1122
"""Errors out when range specifiers exceed the limit"""
1124
def get_multiple_ranges(self, file, file_size, ranges):
1125
"""Refuses the multiple ranges request"""
1126
tcs = self.server.test_case_server
1127
if tcs.range_limit is not None and len(ranges) > tcs.range_limit:
1129
# Emulate apache behavior
1130
self.send_error(400, "Bad Request")
1132
return http_server.TestingHTTPRequestHandler.get_multiple_ranges(
1133
self, file, file_size, ranges)
1136
class LimitedRangeHTTPServer(http_server.HttpServer):
1137
"""An HttpServer erroring out on requests with too much range specifiers"""
1139
def __init__(self, request_handler=LimitedRangeRequestHandler,
1140
protocol_version=None,
1142
http_server.HttpServer.__init__(self, request_handler,
1143
protocol_version=protocol_version)
1144
self.range_limit = range_limit
1147
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1148
"""Tests readv requests against a server erroring out on too much ranges."""
1150
scenarios = multiply_scenarios(
1151
vary_by_http_client_implementation(),
1152
vary_by_http_protocol_version(),
1155
# Requests with more range specifiers will error out
1158
678
def create_transport_readonly_server(self):
1159
return LimitedRangeHTTPServer(range_limit=self.range_limit,
1160
protocol_version=self._protocol_version)
1163
super(TestLimitedRangeRequestServer, self).setUp()
1164
# We need to manipulate ranges that correspond to real chunks in the
1165
# response, so we build a content appropriately.
1166
filler = ''.join(['abcdefghij' for x in range(102)])
1167
content = ''.join(['%04d' % v + filler for v in range(16)])
1168
self.build_tree_contents([('a', content)],)
1170
def test_few_ranges(self):
1171
t = self.get_readonly_transport()
1172
l = list(t.readv('a', ((0, 4), (1024, 4), )))
1173
self.assertEqual(l[0], (0, '0000'))
1174
self.assertEqual(l[1], (1024, '0001'))
1175
self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1177
def test_more_ranges(self):
1178
t = self.get_readonly_transport()
1179
l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1180
self.assertEqual(l[0], (0, '0000'))
1181
self.assertEqual(l[1], (1024, '0001'))
1182
self.assertEqual(l[2], (4096, '0004'))
1183
self.assertEqual(l[3], (8192, '0008'))
1184
# The server will refuse to serve the first request (too much ranges),
1185
# a second request will succeed.
1186
self.assertEqual(2, self.get_readonly_server().GET_request_nb)
1189
class TestHttpProxyWhiteBox(tests.TestCase):
679
return HttpServer(NoRangeRequestHandler)
682
class TestNoRangeRequestServer_urllib(TestNoRangeRequestServer,
683
TestCaseWithWebserver):
684
"""Tests range requests refusing server for urllib implementation"""
686
_transport = HttpTransport_urllib
689
class TestNoRangeRequestServer_pycurl(TestWithTransport_pycurl,
690
TestNoRangeRequestServer,
691
TestCaseWithWebserver):
692
"""Tests range requests refusing server for pycurl implementation"""
695
class TestHttpProxyWhiteBox(TestCase):
1190
696
"""Whitebox test proxy http authorization.
1192
Only the urllib implementation is tested here.
698
These tests concern urllib implementation only.
708
def _set_and_capture_env_var(self, name, new_value):
709
"""Set an environment variable, and reset it when finished."""
710
self._old_env[name] = osutils.set_or_unset_env(name, new_value)
712
def _install_env(self, env):
713
for name, value in env.iteritems():
714
self._set_and_capture_env_var(name, value)
716
def _restore_env(self):
717
for name, value in self._old_env.iteritems():
718
osutils.set_or_unset_env(name, value)
1195
720
def _proxied_request(self):
1196
handler = _urllib2_wrappers.ProxyHandler()
1197
request = _urllib2_wrappers.Request('GET', 'http://baz/buzzle')
721
from bzrlib.transport.http._urllib2_wrappers import (
726
handler = ProxyHandler()
727
request = Request('GET','http://baz/buzzle')
1198
728
handler.set_proxy(request, 'http')
1201
def assertEvaluateProxyBypass(self, expected, host, no_proxy):
1202
handler = _urllib2_wrappers.ProxyHandler()
1203
self.assertEqual(expected,
1204
handler.evaluate_proxy_bypass(host, no_proxy))
1206
731
def test_empty_user(self):
1207
self.overrideEnv('http_proxy', 'http://bar.com')
732
self._install_env({'http_proxy': 'http://bar.com'})
1208
733
request = self._proxied_request()
1209
734
self.assertFalse(request.headers.has_key('Proxy-authorization'))
1211
def test_user_with_at(self):
1212
self.overrideEnv('http_proxy',
1213
'http://username@domain:password@proxy_host:1234')
1214
request = self._proxied_request()
1215
self.assertFalse(request.headers.has_key('Proxy-authorization'))
736
def test_empty_pass(self):
737
self._install_env({'http_proxy': 'http://joe@bar.com'})
738
request = self._proxied_request()
739
self.assertEqual('Basic ' + 'joe:'.encode('base64').strip(),
740
request.headers['Proxy-authorization'])
741
def test_user_pass(self):
742
self._install_env({'http_proxy': 'http://joe:foo@bar.com'})
743
request = self._proxied_request()
744
self.assertEqual('Basic ' + 'joe:foo'.encode('base64').strip(),
745
request.headers['Proxy-authorization'])
1217
747
def test_invalid_proxy(self):
1218
748
"""A proxy env variable without scheme"""
1219
self.overrideEnv('http_proxy', 'host:1234')
749
self._install_env({'http_proxy': 'host:1234'})
1220
750
self.assertRaises(errors.InvalidURL, self._proxied_request)
1222
def test_evaluate_proxy_bypass_true(self):
1223
"""The host is not proxied"""
1224
self.assertEvaluateProxyBypass(True, 'example.com', 'example.com')
1225
self.assertEvaluateProxyBypass(True, 'bzr.example.com', '*example.com')
1227
def test_evaluate_proxy_bypass_false(self):
1228
"""The host is proxied"""
1229
self.assertEvaluateProxyBypass(False, 'bzr.example.com', None)
1231
def test_evaluate_proxy_bypass_unknown(self):
1232
"""The host is not explicitly proxied"""
1233
self.assertEvaluateProxyBypass(None, 'example.com', 'not.example.com')
1234
self.assertEvaluateProxyBypass(None, 'bzr.example.com', 'example.com')
1236
def test_evaluate_proxy_bypass_empty_entries(self):
1237
"""Ignore empty entries"""
1238
self.assertEvaluateProxyBypass(None, 'example.com', '')
1239
self.assertEvaluateProxyBypass(None, 'example.com', ',')
1240
self.assertEvaluateProxyBypass(None, 'example.com', 'foo,,bar')
1243
class TestProxyHttpServer(http_utils.TestCaseWithTwoWebservers):
753
class TestProxyHttpServer(object):
1244
754
"""Tests proxy server.
756
This MUST be used by daughter classes that also inherit from
757
TestCaseWithTwoWebservers.
759
We can't inherit directly from TestCaseWithTwoWebservers or
760
the test framework will try to create an instance which
761
cannot run, its implementation being incomplete.
1246
763
Be aware that we do not setup a real proxy here. Instead, we
1247
764
check that the *connection* goes through the proxy by serving
1248
765
different content (the faked proxy server append '-proxied'
1249
766
to the file names).
1252
scenarios = multiply_scenarios(
1253
vary_by_http_client_implementation(),
1254
vary_by_http_protocol_version(),
1257
769
# FIXME: We don't have an https server available, so we don't
1258
# test https connections. --vila toolongago
770
# test https connections.
772
# FIXME: Once the test suite is better fitted to test
773
# authorization schemes, test proxy authorizations too (see
1260
776
def setUp(self):
1261
super(TestProxyHttpServer, self).setUp()
1262
self.transport_secondary_server = http_utils.ProxyServer
777
TestCaseWithTwoWebservers.setUp(self)
1263
778
self.build_tree_contents([('foo', 'contents of foo\n'),
1264
779
('foo-proxied', 'proxied contents of foo\n')])
1265
780
# Let's setup some attributes for tests
1266
server = self.get_readonly_server()
1267
self.server_host_port = '%s:%d' % (server.host, server.port)
1268
if self._testing_pycurl():
1269
# Oh my ! pycurl does not check for the port as part of
1270
# no_proxy :-( So we just test the host part
1271
self.no_proxy_host = server.host
1273
self.no_proxy_host = self.server_host_port
781
self.server = self.get_readonly_server()
782
# FIXME: We should not rely on 'localhost' being the hostname
783
self.proxy_address = 'localhost:%d' % self.server.port
784
self.no_proxy_host = self.proxy_address
1274
785
# The secondary server is the proxy
1275
self.proxy_url = self.get_secondary_url()
1277
def _testing_pycurl(self):
1278
# TODO: This is duplicated for lots of the classes in this file
1279
return (features.pycurl.available()
1280
and self._transport == PyCurlTransport)
1282
def assertProxied(self):
1283
t = self.get_readonly_transport()
1284
self.assertEqual('proxied contents of foo\n', t.get('foo').read())
1286
def assertNotProxied(self):
1287
t = self.get_readonly_transport()
1288
self.assertEqual('contents of foo\n', t.get('foo').read())
786
self.proxy = self.get_secondary_server()
787
self.proxy_url = self.proxy.get_url()
790
def create_transport_secondary_server(self):
791
"""Creates an http server that will serve files with
792
'-proxied' appended to their names.
794
return HttpServer(FakeProxyRequestHandler)
796
def _set_and_capture_env_var(self, name, new_value):
797
"""Set an environment variable, and reset it when finished."""
798
self._old_env[name] = osutils.set_or_unset_env(name, new_value)
800
def _install_env(self, env):
801
for name, value in env.iteritems():
802
self._set_and_capture_env_var(name, value)
804
def _restore_env(self):
805
for name, value in self._old_env.iteritems():
806
osutils.set_or_unset_env(name, value)
808
def proxied_in_env(self, env):
809
self._install_env(env)
810
url = self.server.get_url()
811
t = self._transport(url)
813
self.assertEqual(t.get('foo').read(), 'proxied contents of foo\n')
817
def not_proxied_in_env(self, env):
818
self._install_env(env)
819
url = self.server.get_url()
820
t = self._transport(url)
822
self.assertEqual(t.get('foo').read(), 'contents of foo\n')
1290
826
def test_http_proxy(self):
1291
self.overrideEnv('http_proxy', self.proxy_url)
1292
self.assertProxied()
827
self.proxied_in_env({'http_proxy': self.proxy_url})
1294
829
def test_HTTP_PROXY(self):
1295
if self._testing_pycurl():
1296
# pycurl does not check HTTP_PROXY for security reasons
1297
# (for use in a CGI context that we do not care
1298
# about. Should we ?)
1299
raise tests.TestNotApplicable(
1300
'pycurl does not check HTTP_PROXY for security reasons')
1301
self.overrideEnv('HTTP_PROXY', self.proxy_url)
1302
self.assertProxied()
830
self.proxied_in_env({'HTTP_PROXY': self.proxy_url})
1304
832
def test_all_proxy(self):
1305
self.overrideEnv('all_proxy', self.proxy_url)
1306
self.assertProxied()
833
self.proxied_in_env({'all_proxy': self.proxy_url})
1308
835
def test_ALL_PROXY(self):
1309
self.overrideEnv('ALL_PROXY', self.proxy_url)
1310
self.assertProxied()
836
self.proxied_in_env({'ALL_PROXY': self.proxy_url})
1312
838
def test_http_proxy_with_no_proxy(self):
1313
self.overrideEnv('no_proxy', self.no_proxy_host)
1314
self.overrideEnv('http_proxy', self.proxy_url)
1315
self.assertNotProxied()
839
self.not_proxied_in_env({'http_proxy': self.proxy_url,
840
'no_proxy': self.no_proxy_host})
1317
842
def test_HTTP_PROXY_with_NO_PROXY(self):
1318
if self._testing_pycurl():
1319
raise tests.TestNotApplicable(
1320
'pycurl does not check HTTP_PROXY for security reasons')
1321
self.overrideEnv('NO_PROXY', self.no_proxy_host)
1322
self.overrideEnv('HTTP_PROXY', self.proxy_url)
1323
self.assertNotProxied()
843
self.not_proxied_in_env({'HTTP_PROXY': self.proxy_url,
844
'NO_PROXY': self.no_proxy_host})
1325
846
def test_all_proxy_with_no_proxy(self):
1326
self.overrideEnv('no_proxy', self.no_proxy_host)
1327
self.overrideEnv('all_proxy', self.proxy_url)
1328
self.assertNotProxied()
847
self.not_proxied_in_env({'all_proxy': self.proxy_url,
848
'no_proxy': self.no_proxy_host})
1330
850
def test_ALL_PROXY_with_NO_PROXY(self):
1331
self.overrideEnv('NO_PROXY', self.no_proxy_host)
1332
self.overrideEnv('ALL_PROXY', self.proxy_url)
1333
self.assertNotProxied()
1335
def test_http_proxy_without_scheme(self):
1336
self.overrideEnv('http_proxy', self.server_host_port)
1337
if self._testing_pycurl():
1338
# pycurl *ignores* invalid proxy env variables. If that ever change
1339
# in the future, this test will fail indicating that pycurl do not
1340
# ignore anymore such variables.
1341
self.assertNotProxied()
1343
self.assertRaises(errors.InvalidURL, self.assertProxied)
1346
class TestRanges(http_utils.TestCaseWithWebserver):
1347
"""Test the Range header in GET methods."""
1349
scenarios = multiply_scenarios(
1350
vary_by_http_client_implementation(),
1351
vary_by_http_protocol_version(),
1355
super(TestRanges, self).setUp()
851
self.not_proxied_in_env({'ALL_PROXY': self.proxy_url,
852
'NO_PROXY': self.no_proxy_host})
854
def test_http_proxy_without_scheme(self):
855
self.assertRaises(errors.InvalidURL,
857
{'http_proxy': self.proxy_address})
860
class TestProxyHttpServer_urllib(TestProxyHttpServer,
861
TestCaseWithTwoWebservers):
862
"""Tests proxy server for urllib implementation"""
864
_transport = HttpTransport_urllib
867
class TestProxyHttpServer_pycurl(TestWithTransport_pycurl,
869
TestCaseWithTwoWebservers):
870
"""Tests proxy server for pycurl implementation"""
873
TestProxyHttpServer.setUp(self)
874
# Oh my ! pycurl does not check for the port as part of
875
# no_proxy :-( So we just test the host part
876
self.no_proxy_host = 'localhost'
878
def test_HTTP_PROXY(self):
879
# pycurl do not check HTTP_PROXY for security reasons
880
# (for use in a CGI context that we do not care
881
# about. Should we ?)
884
def test_HTTP_PROXY_with_NO_PROXY(self):
887
def test_http_proxy_without_scheme(self):
888
# pycurl *ignores* invalid proxy env variables. If that
889
# ever change in the future, this test will fail
890
# indicating that pycurl do not ignore anymore such
892
self.not_proxied_in_env({'http_proxy': self.proxy_address})
895
class TestRanges(object):
896
"""Test the Range header in GET methods..
898
This MUST be used by daughter classes that also inherit from
899
TestCaseWithWebserver.
901
We can't inherit directly from TestCaseWithWebserver or the
902
test framework will try to create an instance which cannot
903
run, its implementation being incomplete.
907
TestCaseWithWebserver.setUp(self)
1356
908
self.build_tree_contents([('a', '0123456789')],)
1358
def create_transport_readonly_server(self):
1359
return http_server.HttpServer(protocol_version=self._protocol_version)
1361
def _file_contents(self, relpath, ranges):
1362
t = self.get_readonly_transport()
1363
offsets = [ (start, end - start + 1) for start, end in ranges]
1364
coalesce = t._coalesce_offsets
1365
coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1366
code, data = t._get(relpath, coalesced)
1367
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1368
for start, end in ranges:
1370
yield data.read(end - start + 1)
909
server = self.get_readonly_server()
910
self.transport = self._transport(server.get_url())
912
def _file_contents(self, relpath, ranges, tail_amount=0):
913
code, data = self.transport._get(relpath, ranges)
914
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
915
for start, end in ranges:
917
yield data.read(end - start + 1)
1372
919
def _file_tail(self, relpath, tail_amount):
1373
t = self.get_readonly_transport()
1374
code, data = t._get(relpath, [], tail_amount)
1375
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1376
data.seek(-tail_amount, 2)
1377
return data.read(tail_amount)
920
code, data = self.transport._get(relpath, [], tail_amount)
921
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
922
data.seek(-tail_amount + 1, 2)
923
return data.read(tail_amount)
1379
925
def test_range_header(self):
1381
927
map(self.assertEqual,['0', '234'],
1382
928
list(self._file_contents('a', [(0,0), (2,4)])),)
1384
def test_range_header_tail(self):
1385
930
self.assertEqual('789', self._file_tail('a', 3))
1387
def test_syntactically_invalid_range_header(self):
1388
self.assertListRaises(errors.InvalidHttpRange,
1389
self._file_contents, 'a', [(4, 3)])
1391
def test_semantically_invalid_range_header(self):
1392
self.assertListRaises(errors.InvalidHttpRange,
1393
self._file_contents, 'a', [(42, 128)])
1396
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1397
"""Test redirection between http servers."""
1399
scenarios = multiply_scenarios(
1400
vary_by_http_client_implementation(),
1401
vary_by_http_protocol_version(),
1405
super(TestHTTPRedirections, self).setUp()
1406
self.build_tree_contents([('a', '0123456789'),
1408
'# Bazaar revision bundle v0.9\n#\n')
1411
def test_redirected(self):
1412
self.assertRaises(errors.RedirectRequested,
1413
self.get_old_transport().get, 'a')
1414
self.assertEqual('0123456789', self.get_new_transport().get('a').read())
1417
class RedirectedRequest(_urllib2_wrappers.Request):
1418
"""Request following redirections. """
1420
init_orig = _urllib2_wrappers.Request.__init__
1422
def __init__(self, method, url, *args, **kwargs):
1426
# Since the tests using this class will replace
1427
# _urllib2_wrappers.Request, we can't just call the base class __init__
1429
RedirectedRequest.init_orig(self, method, url, *args, **kwargs)
1430
self.follow_redirections = True
1433
def install_redirected_request(test):
1434
test.overrideAttr(_urllib2_wrappers, 'Request', RedirectedRequest)
1437
def cleanup_http_redirection_connections(test):
1438
# Some sockets are opened but never seen by _urllib, so we trap them at
1439
# the _urllib2_wrappers level to be able to clean them up.
1440
def socket_disconnect(sock):
1442
sock.shutdown(socket.SHUT_RDWR)
1444
except socket.error:
1446
def connect(connection):
1447
test.http_connect_orig(connection)
1448
test.addCleanup(socket_disconnect, connection.sock)
1449
test.http_connect_orig = test.overrideAttr(
1450
_urllib2_wrappers.HTTPConnection, 'connect', connect)
1451
def connect(connection):
1452
test.https_connect_orig(connection)
1453
test.addCleanup(socket_disconnect, connection.sock)
1454
test.https_connect_orig = test.overrideAttr(
1455
_urllib2_wrappers.HTTPSConnection, 'connect', connect)
1458
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1459
"""Test redirections.
1461
http implementations do not redirect silently anymore (they
1462
do not redirect at all in fact). The mechanism is still in
1463
place at the _urllib2_wrappers.Request level and these tests
1466
For the pycurl implementation
1467
the redirection have been deleted as we may deprecate pycurl
1468
and I have no place to keep a working implementation.
1472
scenarios = multiply_scenarios(
1473
vary_by_http_client_implementation(),
1474
vary_by_http_protocol_version(),
1478
if (features.pycurl.available()
1479
and self._transport == PyCurlTransport):
1480
raise tests.TestNotApplicable(
1481
"pycurl doesn't redirect silently anymore")
1482
super(TestHTTPSilentRedirections, self).setUp()
1483
install_redirected_request(self)
1484
cleanup_http_redirection_connections(self)
1485
self.build_tree_contents([('a','a'),
1487
('1/a', 'redirected once'),
1489
('2/a', 'redirected twice'),
1491
('3/a', 'redirected thrice'),
1493
('4/a', 'redirected 4 times'),
1495
('5/a', 'redirected 5 times'),
1498
def test_one_redirection(self):
1499
t = self.get_old_transport()
1500
req = RedirectedRequest('GET', t._remote_path('a'))
1501
new_prefix = 'http://%s:%s' % (self.new_server.host,
1502
self.new_server.port)
1503
self.old_server.redirections = \
1504
[('(.*)', r'%s/1\1' % (new_prefix), 301),]
1505
self.assertEqual('redirected once', t._perform(req).read())
1507
def test_five_redirections(self):
1508
t = self.get_old_transport()
1509
req = RedirectedRequest('GET', t._remote_path('a'))
1510
old_prefix = 'http://%s:%s' % (self.old_server.host,
1511
self.old_server.port)
1512
new_prefix = 'http://%s:%s' % (self.new_server.host,
1513
self.new_server.port)
1514
self.old_server.redirections = [
1515
('/1(.*)', r'%s/2\1' % (old_prefix), 302),
1516
('/2(.*)', r'%s/3\1' % (old_prefix), 303),
1517
('/3(.*)', r'%s/4\1' % (old_prefix), 307),
1518
('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1519
('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1521
self.assertEqual('redirected 5 times', t._perform(req).read())
1524
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1525
"""Test transport.do_catching_redirections."""
1527
scenarios = multiply_scenarios(
1528
vary_by_http_client_implementation(),
1529
vary_by_http_protocol_version(),
1533
super(TestDoCatchRedirections, self).setUp()
1534
self.build_tree_contents([('a', '0123456789'),],)
1535
cleanup_http_redirection_connections(self)
1537
self.old_transport = self.get_old_transport()
1542
def test_no_redirection(self):
1543
t = self.get_new_transport()
1545
# We use None for redirected so that we fail if redirected
1546
self.assertEqual('0123456789',
1547
transport.do_catching_redirections(
1548
self.get_a, t, None).read())
1550
def test_one_redirection(self):
1551
self.redirections = 0
1553
def redirected(t, exception, redirection_notice):
1554
self.redirections += 1
1555
redirected_t = t._redirected_to(exception.source, exception.target)
1558
self.assertEqual('0123456789',
1559
transport.do_catching_redirections(
1560
self.get_a, self.old_transport, redirected).read())
1561
self.assertEqual(1, self.redirections)
1563
def test_redirection_loop(self):
1565
def redirected(transport, exception, redirection_notice):
1566
# By using the redirected url as a base dir for the
1567
# *old* transport, we create a loop: a => a/a =>
1569
return self.old_transport.clone(exception.target)
1571
self.assertRaises(errors.TooManyRedirections,
1572
transport.do_catching_redirections,
1573
self.get_a, self.old_transport, redirected)
1576
def _setup_authentication_config(**kwargs):
1577
conf = config.AuthenticationConfig()
1578
conf._get_config().update({'httptest': kwargs})
1582
class TestUrllib2AuthHandler(tests.TestCaseWithTransport):
1583
"""Unit tests for glue by which urllib2 asks us for authentication"""
1585
def test_get_user_password_without_port(self):
1586
"""We cope if urllib2 doesn't tell us the port.
1588
See https://bugs.launchpad.net/bzr/+bug/654684
1592
_setup_authentication_config(scheme='http', host='localhost',
1593
user=user, password=password)
1594
handler = _urllib2_wrappers.HTTPAuthHandler()
1595
got_pass = handler.get_user_password(dict(
1602
self.assertEqual((user, password), got_pass)
1605
class TestAuth(http_utils.TestCaseWithWebserver):
1606
"""Test authentication scheme"""
1608
scenarios = multiply_scenarios(
1609
vary_by_http_client_implementation(),
1610
vary_by_http_protocol_version(),
1611
vary_by_http_auth_scheme(),
1615
super(TestAuth, self).setUp()
1616
self.server = self.get_readonly_server()
1617
self.build_tree_contents([('a', 'contents of a\n'),
1618
('b', 'contents of b\n'),])
1620
def create_transport_readonly_server(self):
1621
server = self._auth_server(protocol_version=self._protocol_version)
1622
server._url_protocol = self._url_protocol
1625
def _testing_pycurl(self):
1626
# TODO: This is duplicated for lots of the classes in this file
1627
return (features.pycurl.available()
1628
and self._transport == PyCurlTransport)
1630
def get_user_url(self, user, password):
1631
"""Build an url embedding user and password"""
1632
url = '%s://' % self.server._url_protocol
1633
if user is not None:
1635
if password is not None:
1636
url += ':' + password
1638
url += '%s:%s/' % (self.server.host, self.server.port)
1641
def get_user_transport(self, user, password):
1642
t = transport.get_transport_from_url(
1643
self.get_user_url(user, password))
1646
def test_no_user(self):
1647
self.server.add_user('joe', 'foo')
1648
t = self.get_user_transport(None, None)
1649
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1650
# Only one 'Authentication Required' error should occur
1651
self.assertEqual(1, self.server.auth_required_errors)
1653
def test_empty_pass(self):
1654
self.server.add_user('joe', '')
1655
t = self.get_user_transport('joe', '')
1656
self.assertEqual('contents of a\n', t.get('a').read())
1657
# Only one 'Authentication Required' error should occur
1658
self.assertEqual(1, self.server.auth_required_errors)
1660
def test_user_pass(self):
1661
self.server.add_user('joe', 'foo')
1662
t = self.get_user_transport('joe', 'foo')
1663
self.assertEqual('contents of a\n', t.get('a').read())
1664
# Only one 'Authentication Required' error should occur
1665
self.assertEqual(1, self.server.auth_required_errors)
1667
def test_unknown_user(self):
1668
self.server.add_user('joe', 'foo')
1669
t = self.get_user_transport('bill', 'foo')
1670
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1671
# Two 'Authentication Required' errors should occur (the
1672
# initial 'who are you' and 'I don't know you, who are
1674
self.assertEqual(2, self.server.auth_required_errors)
1676
def test_wrong_pass(self):
1677
self.server.add_user('joe', 'foo')
1678
t = self.get_user_transport('joe', 'bar')
1679
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1680
# Two 'Authentication Required' errors should occur (the
1681
# initial 'who are you' and 'this is not you, who are you')
1682
self.assertEqual(2, self.server.auth_required_errors)
1684
def test_prompt_for_username(self):
1685
if self._testing_pycurl():
1686
raise tests.TestNotApplicable(
1687
'pycurl cannot prompt, it handles auth by embedding'
1688
' user:pass in urls only')
1690
self.server.add_user('joe', 'foo')
1691
t = self.get_user_transport(None, None)
1692
stdout = tests.StringIOWrapper()
1693
stderr = tests.StringIOWrapper()
1694
ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
1695
stdout=stdout, stderr=stderr)
1696
self.assertEqual('contents of a\n',t.get('a').read())
1697
# stdin should be empty
1698
self.assertEqual('', ui.ui_factory.stdin.readline())
1700
expected_prompt = self._expected_username_prompt(t._unqualified_scheme)
1701
self.assertEqual(expected_prompt, stderr.read(len(expected_prompt)))
1702
self.assertEqual('', stdout.getvalue())
1703
self._check_password_prompt(t._unqualified_scheme, 'joe',
1706
def test_prompt_for_password(self):
1707
if self._testing_pycurl():
1708
raise tests.TestNotApplicable(
1709
'pycurl cannot prompt, it handles auth by embedding'
1710
' user:pass in urls only')
1712
self.server.add_user('joe', 'foo')
1713
t = self.get_user_transport('joe', None)
1714
stdout = tests.StringIOWrapper()
1715
stderr = tests.StringIOWrapper()
1716
ui.ui_factory = tests.TestUIFactory(stdin='foo\n',
1717
stdout=stdout, stderr=stderr)
1718
self.assertEqual('contents of a\n', t.get('a').read())
1719
# stdin should be empty
1720
self.assertEqual('', ui.ui_factory.stdin.readline())
1721
self._check_password_prompt(t._unqualified_scheme, 'joe',
1723
self.assertEqual('', stdout.getvalue())
1724
# And we shouldn't prompt again for a different request
1725
# against the same transport.
1726
self.assertEqual('contents of b\n',t.get('b').read())
1728
# And neither against a clone
1729
self.assertEqual('contents of b\n',t2.get('b').read())
1730
# Only one 'Authentication Required' error should occur
1731
self.assertEqual(1, self.server.auth_required_errors)
1733
def _check_password_prompt(self, scheme, user, actual_prompt):
1734
expected_prompt = (self._password_prompt_prefix
1735
+ ("%s %s@%s:%d, Realm: '%s' password: "
1737
user, self.server.host, self.server.port,
1738
self.server.auth_realm)))
1739
self.assertEqual(expected_prompt, actual_prompt)
1741
def _expected_username_prompt(self, scheme):
1742
return (self._username_prompt_prefix
1743
+ "%s %s:%d, Realm: '%s' username: " % (scheme.upper(),
1744
self.server.host, self.server.port,
1745
self.server.auth_realm))
1747
def test_no_prompt_for_password_when_using_auth_config(self):
1748
if self._testing_pycurl():
1749
raise tests.TestNotApplicable(
1750
'pycurl does not support authentication.conf'
1751
' since it cannot prompt')
1755
stdin_content = 'bar\n' # Not the right password
1756
self.server.add_user(user, password)
1757
t = self.get_user_transport(user, None)
1758
ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1759
stderr=tests.StringIOWrapper())
1760
# Create a minimal config file with the right password
1761
_setup_authentication_config(scheme='http', port=self.server.port,
1762
user=user, password=password)
1763
# Issue a request to the server to connect
1764
self.assertEqual('contents of a\n',t.get('a').read())
1765
# stdin should have been left untouched
1766
self.assertEqual(stdin_content, ui.ui_factory.stdin.readline())
1767
# Only one 'Authentication Required' error should occur
1768
self.assertEqual(1, self.server.auth_required_errors)
1770
def test_changing_nonce(self):
1771
if self._auth_server not in (http_utils.HTTPDigestAuthServer,
1772
http_utils.ProxyDigestAuthServer):
1773
raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1774
if self._testing_pycurl():
1776
'pycurl does not handle a nonce change')
1777
self.server.add_user('joe', 'foo')
1778
t = self.get_user_transport('joe', 'foo')
1779
self.assertEqual('contents of a\n', t.get('a').read())
1780
self.assertEqual('contents of b\n', t.get('b').read())
1781
# Only one 'Authentication Required' error should have
1783
self.assertEqual(1, self.server.auth_required_errors)
1784
# The server invalidates the current nonce
1785
self.server.auth_nonce = self.server.auth_nonce + '. No, now!'
1786
self.assertEqual('contents of a\n', t.get('a').read())
1787
# Two 'Authentication Required' errors should occur (the
1788
# initial 'who are you' and a second 'who are you' with the new nonce)
1789
self.assertEqual(2, self.server.auth_required_errors)
1791
def test_user_from_auth_conf(self):
1792
if self._testing_pycurl():
1793
raise tests.TestNotApplicable(
1794
'pycurl does not support authentication.conf')
1797
self.server.add_user(user, password)
1798
_setup_authentication_config(scheme='http', port=self.server.port,
1799
user=user, password=password)
1800
t = self.get_user_transport(None, None)
1801
# Issue a request to the server to connect
1802
self.assertEqual('contents of a\n', t.get('a').read())
1803
# Only one 'Authentication Required' error should occur
1804
self.assertEqual(1, self.server.auth_required_errors)
1806
def test_no_credential_leaks_in_log(self):
1807
self.overrideAttr(debug, 'debug_flags', set(['http']))
1809
password = 'very-sensitive-password'
1810
self.server.add_user(user, password)
1811
t = self.get_user_transport(user, password)
1812
# Capture the debug calls to mutter
1815
lines = args[0] % args[1:]
1816
# Some calls output multiple lines, just split them now since we
1817
# care about a single one later.
1818
self.mutters.extend(lines.splitlines())
1819
self.overrideAttr(trace, 'mutter', mutter)
1820
# Issue a request to the server to connect
1821
self.assertEqual(True, t.has('a'))
1822
# Only one 'Authentication Required' error should occur
1823
self.assertEqual(1, self.server.auth_required_errors)
1824
# Since the authentification succeeded, there should be a corresponding
1826
sent_auth_headers = [line for line in self.mutters
1827
if line.startswith('> %s' % (self._auth_header,))]
1828
self.assertLength(1, sent_auth_headers)
1829
self.assertStartsWith(sent_auth_headers[0],
1830
'> %s: <masked>' % (self._auth_header,))
1833
class TestProxyAuth(TestAuth):
1834
"""Test proxy authentication schemes.
1836
This inherits from TestAuth to tweak the setUp and filter some failing
1840
scenarios = multiply_scenarios(
1841
vary_by_http_client_implementation(),
1842
vary_by_http_protocol_version(),
1843
vary_by_http_proxy_auth_scheme(),
1847
super(TestProxyAuth, self).setUp()
1848
# Override the contents to avoid false positives
1849
self.build_tree_contents([('a', 'not proxied contents of a\n'),
1850
('b', 'not proxied contents of b\n'),
1851
('a-proxied', 'contents of a\n'),
1852
('b-proxied', 'contents of b\n'),
1855
def get_user_transport(self, user, password):
1856
self.overrideEnv('all_proxy', self.get_user_url(user, password))
1857
return TestAuth.get_user_transport(self, user, password)
1859
def test_empty_pass(self):
1860
if self._testing_pycurl():
1862
if pycurl.version_info()[1] < '7.16.0':
1864
'pycurl < 7.16.0 does not handle empty proxy passwords')
1865
super(TestProxyAuth, self).test_empty_pass()
1868
class SampleSocket(object):
1869
"""A socket-like object for use in testing the HTTP request handler."""
1871
def __init__(self, socket_read_content):
1872
"""Constructs a sample socket.
1874
:param socket_read_content: a byte sequence
1876
# Use plain python StringIO so we can monkey-patch the close method to
1877
# not discard the contents.
1878
from StringIO import StringIO
1879
self.readfile = StringIO(socket_read_content)
1880
self.writefile = StringIO()
1881
self.writefile.close = lambda: None
1882
self.close = lambda: None
1884
def makefile(self, mode='r', bufsize=None):
1886
return self.readfile
1888
return self.writefile
1891
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1893
scenarios = multiply_scenarios(
1894
vary_by_http_client_implementation(),
1895
vary_by_http_protocol_version(),
1899
super(SmartHTTPTunnellingTest, self).setUp()
1900
# We use the VFS layer as part of HTTP tunnelling tests.
1901
self.overrideEnv('BZR_NO_SMART_VFS', None)
1902
self.transport_readonly_server = http_utils.HTTPServerWithSmarts
1903
self.http_server = self.get_readonly_server()
1905
def create_transport_readonly_server(self):
1906
server = http_utils.HTTPServerWithSmarts(
1907
protocol_version=self._protocol_version)
1908
server._url_protocol = self._url_protocol
1911
def test_open_controldir(self):
1912
branch = self.make_branch('relpath')
1913
url = self.http_server.get_url() + 'relpath'
1914
bd = controldir.ControlDir.open(url)
1915
self.addCleanup(bd.transport.disconnect)
1916
self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1918
def test_bulk_data(self):
1919
# We should be able to send and receive bulk data in a single message.
1920
# The 'readv' command in the smart protocol both sends and receives
1921
# bulk data, so we use that.
1922
self.build_tree(['data-file'])
1923
http_transport = transport.get_transport_from_url(
1924
self.http_server.get_url())
1925
medium = http_transport.get_smart_medium()
1926
# Since we provide the medium, the url below will be mostly ignored
1927
# during the test, as long as the path is '/'.
1928
remote_transport = remote.RemoteTransport('bzr://fake_host/',
1931
[(0, "c")], list(remote_transport.readv("data-file", [(0,1)])))
1933
def test_http_send_smart_request(self):
1935
post_body = 'hello\n'
1936
expected_reply_body = 'ok\x012\n'
1938
http_transport = transport.get_transport_from_url(
1939
self.http_server.get_url())
1940
medium = http_transport.get_smart_medium()
1941
response = medium.send_http_smart_request(post_body)
1942
reply_body = response.read()
1943
self.assertEqual(expected_reply_body, reply_body)
1945
def test_smart_http_server_post_request_handler(self):
1946
httpd = self.http_server.server
1948
socket = SampleSocket(
1949
'POST /.bzr/smart %s \r\n' % self._protocol_version
1950
# HTTP/1.1 posts must have a Content-Length (but it doesn't hurt
1952
+ 'Content-Length: 6\r\n'
1955
# Beware: the ('localhost', 80) below is the
1956
# client_address parameter, but we don't have one because
1957
# we have defined a socket which is not bound to an
1958
# address. The test framework never uses this client
1959
# address, so far...
1960
request_handler = http_utils.SmartRequestHandler(socket,
1963
response = socket.writefile.getvalue()
1964
self.assertStartsWith(response, '%s 200 ' % self._protocol_version)
1965
# This includes the end of the HTTP headers, and all the body.
1966
expected_end_of_response = '\r\n\r\nok\x012\n'
1967
self.assertEndsWith(response, expected_end_of_response)
1970
class ForbiddenRequestHandler(http_server.TestingHTTPRequestHandler):
1971
"""No smart server here request handler."""
1974
self.send_error(403, "Forbidden")
1977
class SmartClientAgainstNotSmartServer(TestSpecificRequestHandler):
1978
"""Test smart client behaviour against an http server without smarts."""
1980
_req_handler_class = ForbiddenRequestHandler
1982
def test_probe_smart_server(self):
1983
"""Test error handling against server refusing smart requests."""
1984
t = self.get_readonly_transport()
1985
# No need to build a valid smart request here, the server will not even
1986
# try to interpret it.
1987
self.assertRaises(errors.SmartProtocolError,
1988
t.get_smart_medium().send_http_smart_request,
1992
class Test_redirected_to(tests.TestCase):
1994
scenarios = vary_by_http_client_implementation()
1996
def test_redirected_to_subdir(self):
1997
t = self._transport('http://www.example.com/foo')
1998
r = t._redirected_to('http://www.example.com/foo',
1999
'http://www.example.com/foo/subdir')
2000
self.assertIsInstance(r, type(t))
2001
# Both transports share the some connection
2002
self.assertEqual(t._get_connection(), r._get_connection())
2003
self.assertEqual('http://www.example.com/foo/subdir/', r.base)
2005
def test_redirected_to_self_with_slash(self):
2006
t = self._transport('http://www.example.com/foo')
2007
r = t._redirected_to('http://www.example.com/foo',
2008
'http://www.example.com/foo/')
2009
self.assertIsInstance(r, type(t))
2010
# Both transports share the some connection (one can argue that we
2011
# should return the exact same transport here, but that seems
2013
self.assertEqual(t._get_connection(), r._get_connection())
2015
def test_redirected_to_host(self):
2016
t = self._transport('http://www.example.com/foo')
2017
r = t._redirected_to('http://www.example.com/foo',
2018
'http://foo.example.com/foo/subdir')
2019
self.assertIsInstance(r, type(t))
2020
self.assertEqual('http://foo.example.com/foo/subdir/',
2023
def test_redirected_to_same_host_sibling_protocol(self):
2024
t = self._transport('http://www.example.com/foo')
2025
r = t._redirected_to('http://www.example.com/foo',
2026
'https://www.example.com/foo')
2027
self.assertIsInstance(r, type(t))
2028
self.assertEqual('https://www.example.com/foo/',
2031
def test_redirected_to_same_host_different_protocol(self):
2032
t = self._transport('http://www.example.com/foo')
2033
r = t._redirected_to('http://www.example.com/foo',
2034
'ftp://www.example.com/foo')
2035
self.assertNotEqual(type(r), type(t))
2036
self.assertEqual('ftp://www.example.com/foo/', r.external_url())
2038
def test_redirected_to_same_host_specific_implementation(self):
2039
t = self._transport('http://www.example.com/foo')
2040
r = t._redirected_to('http://www.example.com/foo',
2041
'https+urllib://www.example.com/foo')
2042
self.assertEqual('https://www.example.com/foo/', r.external_url())
2044
def test_redirected_to_different_host_same_user(self):
2045
t = self._transport('http://joe@www.example.com/foo')
2046
r = t._redirected_to('http://www.example.com/foo',
2047
'https://foo.example.com/foo')
2048
self.assertIsInstance(r, type(t))
2049
self.assertEqual(t._parsed_url.user, r._parsed_url.user)
2050
self.assertEqual('https://joe@foo.example.com/foo/', r.external_url())
2053
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
2054
"""Request handler for a unique and pre-defined request.
2056
The only thing we care about here is how many bytes travel on the wire. But
2057
since we want to measure it for a real http client, we have to send it
2060
We expect to receive a *single* request nothing more (and we won't even
2061
check what request it is, we just measure the bytes read until an empty
2065
def _handle_one_request(self):
2066
tcs = self.server.test_case_server
2067
requestline = self.rfile.readline()
2068
headers = self.MessageClass(self.rfile, 0)
2069
# We just read: the request, the headers, an empty line indicating the
2070
# end of the headers.
2071
bytes_read = len(requestline)
2072
for line in headers.headers:
2073
bytes_read += len(line)
2074
bytes_read += len('\r\n')
2075
if requestline.startswith('POST'):
2076
# The body should be a single line (or we don't know where it ends
2077
# and we don't want to issue a blocking read)
2078
body = self.rfile.readline()
2079
bytes_read += len(body)
2080
tcs.bytes_read = bytes_read
2082
# We set the bytes written *before* issuing the write, the client is
2083
# supposed to consume every produced byte *before* checking that value.
2085
# Doing the oppposite may lead to test failure: we may be interrupted
2086
# after the write but before updating the value. The client can then
2087
# continue and read the value *before* we can update it. And yes,
2088
# this has been observed -- vila 20090129
2089
tcs.bytes_written = len(tcs.canned_response)
2090
self.wfile.write(tcs.canned_response)
2093
class ActivityServerMixin(object):
2095
def __init__(self, protocol_version):
2096
super(ActivityServerMixin, self).__init__(
2097
request_handler=PredefinedRequestHandler,
2098
protocol_version=protocol_version)
2099
# Bytes read and written by the server
2101
self.bytes_written = 0
2102
self.canned_response = None
2105
class ActivityHTTPServer(ActivityServerMixin, http_server.HttpServer):
2109
if features.HTTPSServerFeature.available():
2110
from bzrlib.tests import https_server
2111
class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
2115
class TestActivityMixin(object):
2116
"""Test socket activity reporting.
2118
We use a special purpose server to control the bytes sent and received and
2119
be able to predict the activity on the client socket.
2123
self.server = self._activity_server(self._protocol_version)
2124
self.server.start_server()
2125
self.addCleanup(self.server.stop_server)
2126
_activities = {} # Don't close over self and create a cycle
2127
def report_activity(t, bytes, direction):
2128
count = _activities.get(direction, 0)
2130
_activities[direction] = count
2131
self.activities = _activities
2132
# We override at class level because constructors may propagate the
2133
# bound method and render instance overriding ineffective (an
2134
# alternative would be to define a specific ui factory instead...)
2135
self.overrideAttr(self._transport, '_report_activity', report_activity)
2137
def get_transport(self):
2138
t = self._transport(self.server.get_url())
2139
# FIXME: Needs cleanup -- vila 20100611
2142
def assertActivitiesMatch(self):
2143
self.assertEqual(self.server.bytes_read,
2144
self.activities.get('write', 0), 'written bytes')
2145
self.assertEqual(self.server.bytes_written,
2146
self.activities.get('read', 0), 'read bytes')
2149
self.server.canned_response = '''HTTP/1.1 200 OK\r
2150
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
2151
Server: Apache/2.0.54 (Fedora)\r
2152
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
2153
ETag: "56691-23-38e9ae00"\r
2154
Accept-Ranges: bytes\r
2155
Content-Length: 35\r
2157
Content-Type: text/plain; charset=UTF-8\r
2159
Bazaar-NG meta directory, format 1
2161
t = self.get_transport()
2162
self.assertEqual('Bazaar-NG meta directory, format 1\n',
2163
t.get('foo/bar').read())
2164
self.assertActivitiesMatch()
2167
self.server.canned_response = '''HTTP/1.1 200 OK\r
2168
Server: SimpleHTTP/0.6 Python/2.5.2\r
2169
Date: Thu, 29 Jan 2009 20:21:47 GMT\r
2170
Content-type: application/octet-stream\r
2171
Content-Length: 20\r
2172
Last-Modified: Thu, 29 Jan 2009 20:21:47 GMT\r
2175
t = self.get_transport()
2176
self.assertTrue(t.has('foo/bar'))
2177
self.assertActivitiesMatch()
2179
def test_readv(self):
2180
self.server.canned_response = '''HTTP/1.1 206 Partial Content\r
2181
Date: Tue, 11 Jul 2006 04:49:48 GMT\r
2182
Server: Apache/2.0.54 (Fedora)\r
2183
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
2184
ETag: "238a3c-16ec2-805c5540"\r
2185
Accept-Ranges: bytes\r
2186
Content-Length: 1534\r
2188
Content-Type: multipart/byteranges; boundary=418470f848b63279b\r
2191
--418470f848b63279b\r
2192
Content-type: text/plain; charset=UTF-8\r
2193
Content-range: bytes 0-254/93890\r
2195
mbp@sourcefrog.net-20050309040815-13242001617e4a06
2196
mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e7627
2197
mbp@sourcefrog.net-20050309040957-6cad07f466bb0bb8
2198
mbp@sourcefrog.net-20050309041501-c840e09071de3b67
2199
mbp@sourcefrog.net-20050309044615-c24a3250be83220a
2201
--418470f848b63279b\r
2202
Content-type: text/plain; charset=UTF-8\r
2203
Content-range: bytes 1000-2049/93890\r
2206
mbp@sourcefrog.net-20050311063625-07858525021f270b
2207
mbp@sourcefrog.net-20050311231934-aa3776aff5200bb9
2208
mbp@sourcefrog.net-20050311231953-73aeb3a131c3699a
2209
mbp@sourcefrog.net-20050311232353-f5e33da490872c6a
2210
mbp@sourcefrog.net-20050312071639-0a8f59a34a024ff0
2211
mbp@sourcefrog.net-20050312073432-b2c16a55e0d6e9fb
2212
mbp@sourcefrog.net-20050312073831-a47c3335ece1920f
2213
mbp@sourcefrog.net-20050312085412-13373aa129ccbad3
2214
mbp@sourcefrog.net-20050313052251-2bf004cb96b39933
2215
mbp@sourcefrog.net-20050313052856-3edd84094687cb11
2216
mbp@sourcefrog.net-20050313053233-e30a4f28aef48f9d
2217
mbp@sourcefrog.net-20050313053853-7c64085594ff3072
2218
mbp@sourcefrog.net-20050313054757-a86c3f5871069e22
2219
mbp@sourcefrog.net-20050313061422-418f1f73b94879b9
2220
mbp@sourcefrog.net-20050313120651-497bd231b19df600
2221
mbp@sourcefrog.net-20050314024931-eae0170ef25a5d1a
2222
mbp@sourcefrog.net-20050314025438-d52099f915fe65fc
2223
mbp@sourcefrog.net-20050314025539-637a636692c055cf
2224
mbp@sourcefrog.net-20050314025737-55eb441f430ab4ba
2225
mbp@sourcefrog.net-20050314025901-d74aa93bb7ee8f62
2227
--418470f848b63279b--\r
2229
t = self.get_transport()
2230
# Remember that the request is ignored and that the ranges below
2231
# doesn't have to match the canned response.
2232
l = list(t.readv('/foo/bar', ((0, 255), (1000, 1050))))
2233
self.assertEqual(2, len(l))
2234
self.assertActivitiesMatch()
2236
def test_post(self):
2237
self.server.canned_response = '''HTTP/1.1 200 OK\r
2238
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
2239
Server: Apache/2.0.54 (Fedora)\r
2240
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
2241
ETag: "56691-23-38e9ae00"\r
2242
Accept-Ranges: bytes\r
2243
Content-Length: 35\r
2245
Content-Type: text/plain; charset=UTF-8\r
2247
lalala whatever as long as itsssss
2249
t = self.get_transport()
2250
# We must send a single line of body bytes, see
2251
# PredefinedRequestHandler._handle_one_request
2252
code, f = t._post('abc def end-of-body\n')
2253
self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2254
self.assertActivitiesMatch()
2257
class TestActivity(tests.TestCase, TestActivityMixin):
2259
scenarios = multiply_scenarios(
2260
vary_by_http_activity(),
2261
vary_by_http_protocol_version(),
2265
super(TestActivity, self).setUp()
2266
TestActivityMixin.setUp(self)
2269
class TestNoReportActivity(tests.TestCase, TestActivityMixin):
2271
# Unlike TestActivity, we are really testing ReportingFileSocket and
2272
# ReportingSocket, so we don't need all the parametrization. Since
2273
# ReportingFileSocket and ReportingSocket are wrappers, it's easier to
2274
# test them through their use by the transport than directly (that's a
2275
# bit less clean but far more simpler and effective).
2276
_activity_server = ActivityHTTPServer
2277
_protocol_version = 'HTTP/1.1'
2280
super(TestNoReportActivity, self).setUp()
2281
self._transport =_urllib.HttpTransport_urllib
2282
TestActivityMixin.setUp(self)
2284
def assertActivitiesMatch(self):
2285
# Nothing to check here
2289
class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver):
2290
"""Test authentication on the redirected http server."""
2292
scenarios = vary_by_http_protocol_version()
2294
_auth_header = 'Authorization'
2295
_password_prompt_prefix = ''
2296
_username_prompt_prefix = ''
2297
_auth_server = http_utils.HTTPBasicAuthServer
2298
_transport = _urllib.HttpTransport_urllib
2301
super(TestAuthOnRedirected, self).setUp()
2302
self.build_tree_contents([('a','a'),
2304
('1/a', 'redirected once'),
2306
new_prefix = 'http://%s:%s' % (self.new_server.host,
2307
self.new_server.port)
2308
self.old_server.redirections = [
2309
('(.*)', r'%s/1\1' % (new_prefix), 301),]
2310
self.old_transport = self.get_old_transport()
2311
self.new_server.add_user('joe', 'foo')
2312
cleanup_http_redirection_connections(self)
2314
def create_transport_readonly_server(self):
2315
server = self._auth_server(protocol_version=self._protocol_version)
2316
server._url_protocol = self._url_protocol
2322
def test_auth_on_redirected_via_do_catching_redirections(self):
2323
self.redirections = 0
2325
def redirected(t, exception, redirection_notice):
2326
self.redirections += 1
2327
redirected_t = t._redirected_to(exception.source, exception.target)
2328
self.addCleanup(redirected_t.disconnect)
2331
stdout = tests.StringIOWrapper()
2332
stderr = tests.StringIOWrapper()
2333
ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2334
stdout=stdout, stderr=stderr)
2335
self.assertEqual('redirected once',
2336
transport.do_catching_redirections(
2337
self.get_a, self.old_transport, redirected).read())
2338
self.assertEqual(1, self.redirections)
2339
# stdin should be empty
2340
self.assertEqual('', ui.ui_factory.stdin.readline())
2341
# stdout should be empty, stderr will contains the prompts
2342
self.assertEqual('', stdout.getvalue())
2344
def test_auth_on_redirected_via_following_redirections(self):
2345
self.new_server.add_user('joe', 'foo')
2346
stdout = tests.StringIOWrapper()
2347
stderr = tests.StringIOWrapper()
2348
ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2349
stdout=stdout, stderr=stderr)
2350
t = self.old_transport
2351
req = RedirectedRequest('GET', t.abspath('a'))
2352
new_prefix = 'http://%s:%s' % (self.new_server.host,
2353
self.new_server.port)
2354
self.old_server.redirections = [
2355
('(.*)', r'%s/1\1' % (new_prefix), 301),]
2356
self.assertEqual('redirected once', t._perform(req).read())
2357
# stdin should be empty
2358
self.assertEqual('', ui.ui_factory.stdin.readline())
2359
# stdout should be empty, stderr will contains the prompts
2360
self.assertEqual('', stdout.getvalue())
931
# Syntactically invalid range
932
self.assertRaises(errors.InvalidRange,
933
self.transport._get, 'a', [(4, 3)])
934
# Semantically invalid range
935
self.assertRaises(errors.InvalidRange,
936
self.transport._get, 'a', [(42, 128)])
939
class TestRanges_urllib(TestRanges, TestCaseWithWebserver):
940
"""Test the Range header in GET methods for urllib implementation"""
942
_transport = HttpTransport_urllib
945
class TestRanges_pycurl(TestWithTransport_pycurl,
947
TestCaseWithWebserver):
948
"""Test the Range header in GET methods for pycurl implementation"""
added
removed
bzrlib/tests/test_http.py
