651
815
self.assertListRaises((errors.InvalidRange, errors.ShortReadvError,),
652
816
t.readv, 'a', [(12,2)])
818
def test_readv_multiple_get_requests(self):
819
server = self.get_readonly_server()
820
t = self.get_readonly_transport()
821
# force transport to issue multiple requests
822
t._max_readv_combine = 1
823
t._max_get_ranges = 1
824
l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
825
self.assertEqual(l[0], (0, '0'))
826
self.assertEqual(l[1], (1, '1'))
827
self.assertEqual(l[2], (3, '34'))
828
self.assertEqual(l[3], (9, '9'))
829
# The server should have issued 4 requests
830
self.assertEqual(4, server.GET_request_nb)
832
def test_readv_get_max_size(self):
833
server = self.get_readonly_server()
834
t = self.get_readonly_transport()
835
# force transport to issue multiple requests by limiting the number of
836
# bytes by request. Note that this apply to coalesced offsets only, a
837
# single range will keep its size even if bigger than the limit.
839
l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
840
self.assertEqual(l[0], (0, '0'))
841
self.assertEqual(l[1], (1, '1'))
842
self.assertEqual(l[2], (2, '2345'))
843
self.assertEqual(l[3], (6, '6789'))
844
# The server should have issued 3 requests
845
self.assertEqual(3, server.GET_request_nb)
847
def test_complete_readv_leave_pipe_clean(self):
848
server = self.get_readonly_server()
849
t = self.get_readonly_transport()
850
# force transport to issue multiple requests
852
list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
853
# The server should have issued 3 requests
854
self.assertEqual(3, server.GET_request_nb)
855
self.assertEqual('0123456789', t.get_bytes('a'))
856
self.assertEqual(4, server.GET_request_nb)
858
def test_incomplete_readv_leave_pipe_clean(self):
859
server = self.get_readonly_server()
860
t = self.get_readonly_transport()
861
# force transport to issue multiple requests
863
# Don't collapse readv results into a list so that we leave unread
864
# bytes on the socket
865
ireadv = iter(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
866
self.assertEqual((0, '0'), ireadv.next())
867
# The server should have issued one request so far
868
self.assertEqual(1, server.GET_request_nb)
869
self.assertEqual('0123456789', t.get_bytes('a'))
870
# get_bytes issued an additional request, the readv pending ones are
872
self.assertEqual(2, server.GET_request_nb)
875
class SingleRangeRequestHandler(http_server.TestingHTTPRequestHandler):
876
"""Always reply to range request as if they were single.
878
Don't be explicit about it, just to annoy the clients.
881
def get_multiple_ranges(self, file, file_size, ranges):
882
"""Answer as if it was a single range request and ignores the rest"""
883
(start, end) = ranges[0]
884
return self.get_single_range(file, file_size, start, end)
655
887
class TestSingleRangeRequestServer(TestRangeRequestServer):
656
888
"""Test readv against a server which accept only single range requests"""
658
def create_transport_readonly_server(self):
659
return HttpServer(SingleRangeRequestHandler)
662
class TestSingleRangeRequestServer_urllib(TestSingleRangeRequestServer,
663
TestCaseWithWebserver):
664
"""Tests single range requests accepting server for urllib implementation"""
666
_transport = HttpTransport_urllib
669
class TestSingleRangeRequestServer_pycurl(TestWithTransport_pycurl,
670
TestSingleRangeRequestServer,
671
TestCaseWithWebserver):
672
"""Tests single range requests accepting server for pycurl implementation"""
890
_req_handler_class = SingleRangeRequestHandler
893
class SingleOnlyRangeRequestHandler(http_server.TestingHTTPRequestHandler):
894
"""Only reply to simple range requests, errors out on multiple"""
896
def get_multiple_ranges(self, file, file_size, ranges):
897
"""Refuses the multiple ranges request"""
900
self.send_error(416, "Requested range not satisfiable")
902
(start, end) = ranges[0]
903
return self.get_single_range(file, file_size, start, end)
906
class TestSingleOnlyRangeRequestServer(TestRangeRequestServer):
907
"""Test readv against a server which only accept single range requests"""
909
_req_handler_class = SingleOnlyRangeRequestHandler
912
class NoRangeRequestHandler(http_server.TestingHTTPRequestHandler):
913
"""Ignore range requests without notice"""
916
# Update the statistics
917
self.server.test_case_server.GET_request_nb += 1
918
# Just bypass the range handling done by TestingHTTPRequestHandler
919
return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
675
922
class TestNoRangeRequestServer(TestRangeRequestServer):
676
923
"""Test readv against a server which do not accept range requests"""
925
_req_handler_class = NoRangeRequestHandler
928
class MultipleRangeWithoutContentLengthRequestHandler(
929
http_server.TestingHTTPRequestHandler):
930
"""Reply to multiple range requests without content length header."""
932
def get_multiple_ranges(self, file, file_size, ranges):
933
self.send_response(206)
934
self.send_header('Accept-Ranges', 'bytes')
935
# XXX: this is strange; the 'random' name below seems undefined and
936
# yet the tests pass -- mbp 2010-10-11 bug 658773
937
boundary = "%d" % random.randint(0,0x7FFFFFFF)
938
self.send_header("Content-Type",
939
"multipart/byteranges; boundary=%s" % boundary)
941
for (start, end) in ranges:
942
self.wfile.write("--%s\r\n" % boundary)
943
self.send_header("Content-type", 'application/octet-stream')
944
self.send_header("Content-Range", "bytes %d-%d/%d" % (start,
948
self.send_range_content(file, start, end - start + 1)
950
self.wfile.write("--%s\r\n" % boundary)
953
class TestMultipleRangeWithoutContentLengthServer(TestRangeRequestServer):
955
_req_handler_class = MultipleRangeWithoutContentLengthRequestHandler
958
class TruncatedMultipleRangeRequestHandler(
959
http_server.TestingHTTPRequestHandler):
960
"""Reply to multiple range requests truncating the last ones.
962
This server generates responses whose Content-Length describes all the
963
ranges, but fail to include the last ones leading to client short reads.
964
This has been observed randomly with lighttpd (bug #179368).
967
_truncated_ranges = 2
969
def get_multiple_ranges(self, file, file_size, ranges):
970
self.send_response(206)
971
self.send_header('Accept-Ranges', 'bytes')
973
self.send_header('Content-Type',
974
'multipart/byteranges; boundary=%s' % boundary)
975
boundary_line = '--%s\r\n' % boundary
976
# Calculate the Content-Length
978
for (start, end) in ranges:
979
content_length += len(boundary_line)
980
content_length += self._header_line_length(
981
'Content-type', 'application/octet-stream')
982
content_length += self._header_line_length(
983
'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
984
content_length += len('\r\n') # end headers
985
content_length += end - start # + 1
986
content_length += len(boundary_line)
987
self.send_header('Content-length', content_length)
990
# Send the multipart body
992
for (start, end) in ranges:
993
self.wfile.write(boundary_line)
994
self.send_header('Content-type', 'application/octet-stream')
995
self.send_header('Content-Range', 'bytes %d-%d/%d'
996
% (start, end, file_size))
998
if cur + self._truncated_ranges >= len(ranges):
999
# Abruptly ends the response and close the connection
1000
self.close_connection = 1
1002
self.send_range_content(file, start, end - start + 1)
1005
self.wfile.write(boundary_line)
1008
class TestTruncatedMultipleRangeServer(TestSpecificRequestHandler):
1010
_req_handler_class = TruncatedMultipleRangeRequestHandler
1013
super(TestTruncatedMultipleRangeServer, self).setUp()
1014
self.build_tree_contents([('a', '0123456789')],)
1016
def test_readv_with_short_reads(self):
1017
server = self.get_readonly_server()
1018
t = self.get_readonly_transport()
1019
# Force separate ranges for each offset
1020
t._bytes_to_read_before_seek = 0
1021
ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1022
self.assertEqual((0, '0'), ireadv.next())
1023
self.assertEqual((2, '2'), ireadv.next())
1024
if not self._testing_pycurl():
1025
# Only one request have been issued so far (except for pycurl that
1026
# try to read the whole response at once)
1027
self.assertEqual(1, server.GET_request_nb)
1028
self.assertEqual((4, '45'), ireadv.next())
1029
self.assertEqual((9, '9'), ireadv.next())
1030
# Both implementations issue 3 requests but:
1031
# - urllib does two multiple (4 ranges, then 2 ranges) then a single
1033
# - pycurl does two multiple (4 ranges, 4 ranges) then a single range
1034
self.assertEqual(3, server.GET_request_nb)
1035
# Finally the client have tried a single range request and stays in
1037
self.assertEqual('single', t._range_hint)
1040
class TruncatedBeforeBoundaryRequestHandler(
1041
http_server.TestingHTTPRequestHandler):
1042
"""Truncation before a boundary, like in bug 198646"""
1044
_truncated_ranges = 1
1046
def get_multiple_ranges(self, file, file_size, ranges):
1047
self.send_response(206)
1048
self.send_header('Accept-Ranges', 'bytes')
1050
self.send_header('Content-Type',
1051
'multipart/byteranges; boundary=%s' % boundary)
1052
boundary_line = '--%s\r\n' % boundary
1053
# Calculate the Content-Length
1055
for (start, end) in ranges:
1056
content_length += len(boundary_line)
1057
content_length += self._header_line_length(
1058
'Content-type', 'application/octet-stream')
1059
content_length += self._header_line_length(
1060
'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
1061
content_length += len('\r\n') # end headers
1062
content_length += end - start # + 1
1063
content_length += len(boundary_line)
1064
self.send_header('Content-length', content_length)
1067
# Send the multipart body
1069
for (start, end) in ranges:
1070
if cur + self._truncated_ranges >= len(ranges):
1071
# Abruptly ends the response and close the connection
1072
self.close_connection = 1
1074
self.wfile.write(boundary_line)
1075
self.send_header('Content-type', 'application/octet-stream')
1076
self.send_header('Content-Range', 'bytes %d-%d/%d'
1077
% (start, end, file_size))
1079
self.send_range_content(file, start, end - start + 1)
1082
self.wfile.write(boundary_line)
1085
class TestTruncatedBeforeBoundary(TestSpecificRequestHandler):
1086
"""Tests the case of bug 198646, disconnecting before a boundary."""
1088
_req_handler_class = TruncatedBeforeBoundaryRequestHandler
1091
super(TestTruncatedBeforeBoundary, self).setUp()
1092
self.build_tree_contents([('a', '0123456789')],)
1094
def test_readv_with_short_reads(self):
1095
server = self.get_readonly_server()
1096
t = self.get_readonly_transport()
1097
# Force separate ranges for each offset
1098
t._bytes_to_read_before_seek = 0
1099
ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1100
self.assertEqual((0, '0'), ireadv.next())
1101
self.assertEqual((2, '2'), ireadv.next())
1102
self.assertEqual((4, '45'), ireadv.next())
1103
self.assertEqual((9, '9'), ireadv.next())
1106
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1107
"""Errors out when range specifiers exceed the limit"""
1109
def get_multiple_ranges(self, file, file_size, ranges):
1110
"""Refuses the multiple ranges request"""
1111
tcs = self.server.test_case_server
1112
if tcs.range_limit is not None and len(ranges) > tcs.range_limit:
1114
# Emulate apache behavior
1115
self.send_error(400, "Bad Request")
1117
return http_server.TestingHTTPRequestHandler.get_multiple_ranges(
1118
self, file, file_size, ranges)
1121
class LimitedRangeHTTPServer(http_server.HttpServer):
1122
"""An HttpServer erroring out on requests with too much range specifiers"""
1124
def __init__(self, request_handler=LimitedRangeRequestHandler,
1125
protocol_version=None,
1127
http_server.HttpServer.__init__(self, request_handler,
1128
protocol_version=protocol_version)
1129
self.range_limit = range_limit
1132
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1133
"""Tests readv requests against a server erroring out on too much ranges."""
1135
scenarios = multiply_scenarios(
1136
vary_by_http_client_implementation(),
1137
vary_by_http_protocol_version(),
1140
# Requests with more range specifiers will error out
678
1143
def create_transport_readonly_server(self):
679
return HttpServer(NoRangeRequestHandler)
682
class TestNoRangeRequestServer_urllib(TestNoRangeRequestServer,
683
TestCaseWithWebserver):
684
"""Tests range requests refusing server for urllib implementation"""
686
_transport = HttpTransport_urllib
689
class TestNoRangeRequestServer_pycurl(TestWithTransport_pycurl,
690
TestNoRangeRequestServer,
691
TestCaseWithWebserver):
692
"""Tests range requests refusing server for pycurl implementation"""
695
class TestHttpProxyWhiteBox(TestCase):
1144
return LimitedRangeHTTPServer(range_limit=self.range_limit,
1145
protocol_version=self._protocol_version)
1148
super(TestLimitedRangeRequestServer, self).setUp()
1149
# We need to manipulate ranges that correspond to real chunks in the
1150
# response, so we build a content appropriately.
1151
filler = ''.join(['abcdefghij' for x in range(102)])
1152
content = ''.join(['%04d' % v + filler for v in range(16)])
1153
self.build_tree_contents([('a', content)],)
1155
def test_few_ranges(self):
1156
t = self.get_readonly_transport()
1157
l = list(t.readv('a', ((0, 4), (1024, 4), )))
1158
self.assertEqual(l[0], (0, '0000'))
1159
self.assertEqual(l[1], (1024, '0001'))
1160
self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1162
def test_more_ranges(self):
1163
t = self.get_readonly_transport()
1164
l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1165
self.assertEqual(l[0], (0, '0000'))
1166
self.assertEqual(l[1], (1024, '0001'))
1167
self.assertEqual(l[2], (4096, '0004'))
1168
self.assertEqual(l[3], (8192, '0008'))
1169
# The server will refuse to serve the first request (too much ranges),
1170
# a second request will succeed.
1171
self.assertEqual(2, self.get_readonly_server().GET_request_nb)
1174
class TestHttpProxyWhiteBox(tests.TestCase):
696
1175
"""Whitebox test proxy http authorization.
698
These tests concern urllib implementation only.
1177
Only the urllib implementation is tested here.
708
def _set_and_capture_env_var(self, name, new_value):
709
"""Set an environment variable, and reset it when finished."""
710
self._old_env[name] = osutils.set_or_unset_env(name, new_value)
712
def _install_env(self, env):
713
for name, value in env.iteritems():
714
self._set_and_capture_env_var(name, value)
716
def _restore_env(self):
717
for name, value in self._old_env.iteritems():
718
osutils.set_or_unset_env(name, value)
720
1180
def _proxied_request(self):
721
from bzrlib.transport.http._urllib2_wrappers import (
726
handler = ProxyHandler()
727
request = Request('GET','http://baz/buzzle')
1181
handler = _urllib2_wrappers.ProxyHandler()
1182
request = _urllib2_wrappers.Request('GET', 'http://baz/buzzle')
728
1183
handler.set_proxy(request, 'http')
1186
def assertEvaluateProxyBypass(self, expected, host, no_proxy):
1187
handler = _urllib2_wrappers.ProxyHandler()
1188
self.assertEquals(expected,
1189
handler.evaluate_proxy_bypass(host, no_proxy))
731
1191
def test_empty_user(self):
732
self._install_env({'http_proxy': 'http://bar.com'})
1192
self.overrideEnv('http_proxy', 'http://bar.com')
733
1193
request = self._proxied_request()
734
1194
self.assertFalse(request.headers.has_key('Proxy-authorization'))
736
def test_empty_pass(self):
737
self._install_env({'http_proxy': 'http://joe@bar.com'})
738
request = self._proxied_request()
739
self.assertEqual('Basic ' + 'joe:'.encode('base64').strip(),
740
request.headers['Proxy-authorization'])
741
def test_user_pass(self):
742
self._install_env({'http_proxy': 'http://joe:foo@bar.com'})
743
request = self._proxied_request()
744
self.assertEqual('Basic ' + 'joe:foo'.encode('base64').strip(),
745
request.headers['Proxy-authorization'])
1196
def test_user_with_at(self):
1197
self.overrideEnv('http_proxy',
1198
'http://username@domain:password@proxy_host:1234')
1199
request = self._proxied_request()
1200
self.assertFalse(request.headers.has_key('Proxy-authorization'))
747
1202
def test_invalid_proxy(self):
748
1203
"""A proxy env variable without scheme"""
749
self._install_env({'http_proxy': 'host:1234'})
1204
self.overrideEnv('http_proxy', 'host:1234')
750
1205
self.assertRaises(errors.InvalidURL, self._proxied_request)
753
class TestProxyHttpServer(object):
1207
def test_evaluate_proxy_bypass_true(self):
1208
"""The host is not proxied"""
1209
self.assertEvaluateProxyBypass(True, 'example.com', 'example.com')
1210
self.assertEvaluateProxyBypass(True, 'bzr.example.com', '*example.com')
1212
def test_evaluate_proxy_bypass_false(self):
1213
"""The host is proxied"""
1214
self.assertEvaluateProxyBypass(False, 'bzr.example.com', None)
1216
def test_evaluate_proxy_bypass_unknown(self):
1217
"""The host is not explicitly proxied"""
1218
self.assertEvaluateProxyBypass(None, 'example.com', 'not.example.com')
1219
self.assertEvaluateProxyBypass(None, 'bzr.example.com', 'example.com')
1221
def test_evaluate_proxy_bypass_empty_entries(self):
1222
"""Ignore empty entries"""
1223
self.assertEvaluateProxyBypass(None, 'example.com', '')
1224
self.assertEvaluateProxyBypass(None, 'example.com', ',')
1225
self.assertEvaluateProxyBypass(None, 'example.com', 'foo,,bar')
1228
class TestProxyHttpServer(http_utils.TestCaseWithTwoWebservers):
754
1229
"""Tests proxy server.
756
This MUST be used by daughter classes that also inherit from
757
TestCaseWithTwoWebservers.
759
We can't inherit directly from TestCaseWithTwoWebservers or
760
the test framework will try to create an instance which
761
cannot run, its implementation being incomplete.
763
1231
Be aware that we do not setup a real proxy here. Instead, we
764
1232
check that the *connection* goes through the proxy by serving
765
1233
different content (the faked proxy server append '-proxied'
766
1234
to the file names).
1237
scenarios = multiply_scenarios(
1238
vary_by_http_client_implementation(),
1239
vary_by_http_protocol_version(),
769
1242
# FIXME: We don't have an https server available, so we don't
770
# test https connections.
772
# FIXME: Once the test suite is better fitted to test
773
# authorization schemes, test proxy authorizations too (see
1243
# test https connections. --vila toolongago
776
1245
def setUp(self):
777
TestCaseWithTwoWebservers.setUp(self)
1246
super(TestProxyHttpServer, self).setUp()
1247
self.transport_secondary_server = http_utils.ProxyServer
778
1248
self.build_tree_contents([('foo', 'contents of foo\n'),
779
1249
('foo-proxied', 'proxied contents of foo\n')])
780
1250
# Let's setup some attributes for tests
781
self.server = self.get_readonly_server()
782
# FIXME: We should not rely on 'localhost' being the hostname
783
self.proxy_address = 'localhost:%d' % self.server.port
784
self.no_proxy_host = self.proxy_address
1251
server = self.get_readonly_server()
1252
self.server_host_port = '%s:%d' % (server.host, server.port)
1253
if self._testing_pycurl():
1254
# Oh my ! pycurl does not check for the port as part of
1255
# no_proxy :-( So we just test the host part
1256
self.no_proxy_host = server.host
1258
self.no_proxy_host = self.server_host_port
785
1259
# The secondary server is the proxy
786
self.proxy = self.get_secondary_server()
787
self.proxy_url = self.proxy.get_url()
790
def create_transport_secondary_server(self):
791
"""Creates an http server that will serve files with
792
'-proxied' appended to their names.
794
return HttpServer(FakeProxyRequestHandler)
796
def _set_and_capture_env_var(self, name, new_value):
797
"""Set an environment variable, and reset it when finished."""
798
self._old_env[name] = osutils.set_or_unset_env(name, new_value)
800
def _install_env(self, env):
801
for name, value in env.iteritems():
802
self._set_and_capture_env_var(name, value)
804
def _restore_env(self):
805
for name, value in self._old_env.iteritems():
806
osutils.set_or_unset_env(name, value)
808
def proxied_in_env(self, env):
809
self._install_env(env)
810
url = self.server.get_url()
811
t = self._transport(url)
813
self.assertEqual(t.get('foo').read(), 'proxied contents of foo\n')
817
def not_proxied_in_env(self, env):
818
self._install_env(env)
819
url = self.server.get_url()
820
t = self._transport(url)
822
self.assertEqual(t.get('foo').read(), 'contents of foo\n')
1260
self.proxy_url = self.get_secondary_url()
1262
def _testing_pycurl(self):
1263
# TODO: This is duplicated for lots of the classes in this file
1264
return (features.pycurl.available()
1265
and self._transport == PyCurlTransport)
1267
def assertProxied(self):
1268
t = self.get_readonly_transport()
1269
self.assertEqual('proxied contents of foo\n', t.get('foo').read())
1271
def assertNotProxied(self):
1272
t = self.get_readonly_transport()
1273
self.assertEqual('contents of foo\n', t.get('foo').read())
826
1275
def test_http_proxy(self):
827
self.proxied_in_env({'http_proxy': self.proxy_url})
1276
self.overrideEnv('http_proxy', self.proxy_url)
1277
self.assertProxied()
829
1279
def test_HTTP_PROXY(self):
830
self.proxied_in_env({'HTTP_PROXY': self.proxy_url})
1280
if self._testing_pycurl():
1281
# pycurl does not check HTTP_PROXY for security reasons
1282
# (for use in a CGI context that we do not care
1283
# about. Should we ?)
1284
raise tests.TestNotApplicable(
1285
'pycurl does not check HTTP_PROXY for security reasons')
1286
self.overrideEnv('HTTP_PROXY', self.proxy_url)
1287
self.assertProxied()
832
1289
def test_all_proxy(self):
833
self.proxied_in_env({'all_proxy': self.proxy_url})
1290
self.overrideEnv('all_proxy', self.proxy_url)
1291
self.assertProxied()
835
1293
def test_ALL_PROXY(self):
836
self.proxied_in_env({'ALL_PROXY': self.proxy_url})
1294
self.overrideEnv('ALL_PROXY', self.proxy_url)
1295
self.assertProxied()
838
1297
def test_http_proxy_with_no_proxy(self):
839
self.not_proxied_in_env({'http_proxy': self.proxy_url,
840
'no_proxy': self.no_proxy_host})
1298
self.overrideEnv('no_proxy', self.no_proxy_host)
1299
self.overrideEnv('http_proxy', self.proxy_url)
1300
self.assertNotProxied()
842
1302
def test_HTTP_PROXY_with_NO_PROXY(self):
843
self.not_proxied_in_env({'HTTP_PROXY': self.proxy_url,
844
'NO_PROXY': self.no_proxy_host})
1303
if self._testing_pycurl():
1304
raise tests.TestNotApplicable(
1305
'pycurl does not check HTTP_PROXY for security reasons')
1306
self.overrideEnv('NO_PROXY', self.no_proxy_host)
1307
self.overrideEnv('HTTP_PROXY', self.proxy_url)
1308
self.assertNotProxied()
846
1310
def test_all_proxy_with_no_proxy(self):
847
self.not_proxied_in_env({'all_proxy': self.proxy_url,
848
'no_proxy': self.no_proxy_host})
1311
self.overrideEnv('no_proxy', self.no_proxy_host)
1312
self.overrideEnv('all_proxy', self.proxy_url)
1313
self.assertNotProxied()
850
1315
def test_ALL_PROXY_with_NO_PROXY(self):
851
self.not_proxied_in_env({'ALL_PROXY': self.proxy_url,
852
'NO_PROXY': self.no_proxy_host})
854
def test_http_proxy_without_scheme(self):
855
self.assertRaises(errors.InvalidURL,
857
{'http_proxy': self.proxy_address})
860
class TestProxyHttpServer_urllib(TestProxyHttpServer,
861
TestCaseWithTwoWebservers):
862
"""Tests proxy server for urllib implementation"""
864
_transport = HttpTransport_urllib
867
class TestProxyHttpServer_pycurl(TestWithTransport_pycurl,
869
TestCaseWithTwoWebservers):
870
"""Tests proxy server for pycurl implementation"""
873
TestProxyHttpServer.setUp(self)
874
# Oh my ! pycurl does not check for the port as part of
875
# no_proxy :-( So we just test the host part
876
self.no_proxy_host = 'localhost'
878
def test_HTTP_PROXY(self):
879
# pycurl do not check HTTP_PROXY for security reasons
880
# (for use in a CGI context that we do not care
881
# about. Should we ?)
884
def test_HTTP_PROXY_with_NO_PROXY(self):
887
def test_http_proxy_without_scheme(self):
888
# pycurl *ignores* invalid proxy env variables. If that
889
# ever change in the future, this test will fail
890
# indicating that pycurl do not ignore anymore such
892
self.not_proxied_in_env({'http_proxy': self.proxy_address})
895
class TestRanges(object):
896
"""Test the Range header in GET methods..
898
This MUST be used by daughter classes that also inherit from
899
TestCaseWithWebserver.
901
We can't inherit directly from TestCaseWithWebserver or the
902
test framework will try to create an instance which cannot
903
run, its implementation being incomplete.
907
TestCaseWithWebserver.setUp(self)
1316
self.overrideEnv('NO_PROXY', self.no_proxy_host)
1317
self.overrideEnv('ALL_PROXY', self.proxy_url)
1318
self.assertNotProxied()
1320
def test_http_proxy_without_scheme(self):
1321
self.overrideEnv('http_proxy', self.server_host_port)
1322
if self._testing_pycurl():
1323
# pycurl *ignores* invalid proxy env variables. If that ever change
1324
# in the future, this test will fail indicating that pycurl do not
1325
# ignore anymore such variables.
1326
self.assertNotProxied()
1328
self.assertRaises(errors.InvalidURL, self.assertProxied)
1331
class TestRanges(http_utils.TestCaseWithWebserver):
1332
"""Test the Range header in GET methods."""
1334
scenarios = multiply_scenarios(
1335
vary_by_http_client_implementation(),
1336
vary_by_http_protocol_version(),
1340
super(TestRanges, self).setUp()
908
1341
self.build_tree_contents([('a', '0123456789')],)
909
server = self.get_readonly_server()
910
self.transport = self._transport(server.get_url())
912
def _file_contents(self, relpath, ranges, tail_amount=0):
913
code, data = self.transport._get(relpath, ranges)
914
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
915
for start, end in ranges:
917
yield data.read(end - start + 1)
1343
def create_transport_readonly_server(self):
1344
return http_server.HttpServer(protocol_version=self._protocol_version)
1346
def _file_contents(self, relpath, ranges):
1347
t = self.get_readonly_transport()
1348
offsets = [ (start, end - start + 1) for start, end in ranges]
1349
coalesce = t._coalesce_offsets
1350
coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1351
code, data = t._get(relpath, coalesced)
1352
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1353
for start, end in ranges:
1355
yield data.read(end - start + 1)
919
1357
def _file_tail(self, relpath, tail_amount):
920
code, data = self.transport._get(relpath, [], tail_amount)
921
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
922
data.seek(-tail_amount + 1, 2)
923
return data.read(tail_amount)
1358
t = self.get_readonly_transport()
1359
code, data = t._get(relpath, [], tail_amount)
1360
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1361
data.seek(-tail_amount, 2)
1362
return data.read(tail_amount)
925
1364
def test_range_header(self):
927
1366
map(self.assertEqual,['0', '234'],
928
1367
list(self._file_contents('a', [(0,0), (2,4)])),)
1369
def test_range_header_tail(self):
930
1370
self.assertEqual('789', self._file_tail('a', 3))
931
# Syntactically invalid range
932
self.assertRaises(errors.InvalidRange,
933
self.transport._get, 'a', [(4, 3)])
934
# Semantically invalid range
935
self.assertRaises(errors.InvalidRange,
936
self.transport._get, 'a', [(42, 128)])
939
class TestRanges_urllib(TestRanges, TestCaseWithWebserver):
940
"""Test the Range header in GET methods for urllib implementation"""
942
_transport = HttpTransport_urllib
945
class TestRanges_pycurl(TestWithTransport_pycurl,
947
TestCaseWithWebserver):
948
"""Test the Range header in GET methods for pycurl implementation"""
1372
def test_syntactically_invalid_range_header(self):
1373
self.assertListRaises(errors.InvalidHttpRange,
1374
self._file_contents, 'a', [(4, 3)])
1376
def test_semantically_invalid_range_header(self):
1377
self.assertListRaises(errors.InvalidHttpRange,
1378
self._file_contents, 'a', [(42, 128)])
1381
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1382
"""Test redirection between http servers."""
1384
scenarios = multiply_scenarios(
1385
vary_by_http_client_implementation(),
1386
vary_by_http_protocol_version(),
1390
super(TestHTTPRedirections, self).setUp()
1391
self.build_tree_contents([('a', '0123456789'),
1393
'# Bazaar revision bundle v0.9\n#\n')
1396
def test_redirected(self):
1397
self.assertRaises(errors.RedirectRequested,
1398
self.get_old_transport().get, 'a')
1399
self.assertEqual('0123456789', self.get_new_transport().get('a').read())
1402
class RedirectedRequest(_urllib2_wrappers.Request):
1403
"""Request following redirections. """
1405
init_orig = _urllib2_wrappers.Request.__init__
1407
def __init__(self, method, url, *args, **kwargs):
1411
# Since the tests using this class will replace
1412
# _urllib2_wrappers.Request, we can't just call the base class __init__
1414
RedirectedRequest.init_orig(self, method, url, *args, **kwargs)
1415
self.follow_redirections = True
1418
def install_redirected_request(test):
1419
test.overrideAttr(_urllib2_wrappers, 'Request', RedirectedRequest)
1422
def cleanup_http_redirection_connections(test):
1423
# Some sockets are opened but never seen by _urllib, so we trap them at
1424
# the _urllib2_wrappers level to be able to clean them up.
1425
def socket_disconnect(sock):
1427
sock.shutdown(socket.SHUT_RDWR)
1429
except socket.error:
1431
def connect(connection):
1432
test.http_connect_orig(connection)
1433
test.addCleanup(socket_disconnect, connection.sock)
1434
test.http_connect_orig = test.overrideAttr(
1435
_urllib2_wrappers.HTTPConnection, 'connect', connect)
1436
def connect(connection):
1437
test.https_connect_orig(connection)
1438
test.addCleanup(socket_disconnect, connection.sock)
1439
test.https_connect_orig = test.overrideAttr(
1440
_urllib2_wrappers.HTTPSConnection, 'connect', connect)
1443
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1444
"""Test redirections.
1446
http implementations do not redirect silently anymore (they
1447
do not redirect at all in fact). The mechanism is still in
1448
place at the _urllib2_wrappers.Request level and these tests
1451
For the pycurl implementation
1452
the redirection have been deleted as we may deprecate pycurl
1453
and I have no place to keep a working implementation.
1457
scenarios = multiply_scenarios(
1458
vary_by_http_client_implementation(),
1459
vary_by_http_protocol_version(),
1463
if (features.pycurl.available()
1464
and self._transport == PyCurlTransport):
1465
raise tests.TestNotApplicable(
1466
"pycurl doesn't redirect silently anymore")
1467
super(TestHTTPSilentRedirections, self).setUp()
1468
install_redirected_request(self)
1469
cleanup_http_redirection_connections(self)
1470
self.build_tree_contents([('a','a'),
1472
('1/a', 'redirected once'),
1474
('2/a', 'redirected twice'),
1476
('3/a', 'redirected thrice'),
1478
('4/a', 'redirected 4 times'),
1480
('5/a', 'redirected 5 times'),
1483
def test_one_redirection(self):
1484
t = self.get_old_transport()
1485
req = RedirectedRequest('GET', t._remote_path('a'))
1486
new_prefix = 'http://%s:%s' % (self.new_server.host,
1487
self.new_server.port)
1488
self.old_server.redirections = \
1489
[('(.*)', r'%s/1\1' % (new_prefix), 301),]
1490
self.assertEqual('redirected once', t._perform(req).read())
1492
def test_five_redirections(self):
1493
t = self.get_old_transport()
1494
req = RedirectedRequest('GET', t._remote_path('a'))
1495
old_prefix = 'http://%s:%s' % (self.old_server.host,
1496
self.old_server.port)
1497
new_prefix = 'http://%s:%s' % (self.new_server.host,
1498
self.new_server.port)
1499
self.old_server.redirections = [
1500
('/1(.*)', r'%s/2\1' % (old_prefix), 302),
1501
('/2(.*)', r'%s/3\1' % (old_prefix), 303),
1502
('/3(.*)', r'%s/4\1' % (old_prefix), 307),
1503
('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1504
('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1506
self.assertEqual('redirected 5 times', t._perform(req).read())
1509
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1510
"""Test transport.do_catching_redirections."""
1512
scenarios = multiply_scenarios(
1513
vary_by_http_client_implementation(),
1514
vary_by_http_protocol_version(),
1518
super(TestDoCatchRedirections, self).setUp()
1519
self.build_tree_contents([('a', '0123456789'),],)
1520
cleanup_http_redirection_connections(self)
1522
self.old_transport = self.get_old_transport()
1527
def test_no_redirection(self):
1528
t = self.get_new_transport()
1530
# We use None for redirected so that we fail if redirected
1531
self.assertEqual('0123456789',
1532
transport.do_catching_redirections(
1533
self.get_a, t, None).read())
1535
def test_one_redirection(self):
1536
self.redirections = 0
1538
def redirected(t, exception, redirection_notice):
1539
self.redirections += 1
1540
redirected_t = t._redirected_to(exception.source, exception.target)
1543
self.assertEqual('0123456789',
1544
transport.do_catching_redirections(
1545
self.get_a, self.old_transport, redirected).read())
1546
self.assertEqual(1, self.redirections)
1548
def test_redirection_loop(self):
1550
def redirected(transport, exception, redirection_notice):
1551
# By using the redirected url as a base dir for the
1552
# *old* transport, we create a loop: a => a/a =>
1554
return self.old_transport.clone(exception.target)
1556
self.assertRaises(errors.TooManyRedirections,
1557
transport.do_catching_redirections,
1558
self.get_a, self.old_transport, redirected)
1561
def _setup_authentication_config(**kwargs):
1562
conf = config.AuthenticationConfig()
1563
conf._get_config().update({'httptest': kwargs})
1567
class TestUrllib2AuthHandler(tests.TestCaseWithTransport):
1568
"""Unit tests for glue by which urllib2 asks us for authentication"""
1570
def test_get_user_password_without_port(self):
1571
"""We cope if urllib2 doesn't tell us the port.
1573
See https://bugs.launchpad.net/bzr/+bug/654684
1577
_setup_authentication_config(scheme='http', host='localhost',
1578
user=user, password=password)
1579
handler = _urllib2_wrappers.HTTPAuthHandler()
1580
got_pass = handler.get_user_password(dict(
1587
self.assertEquals((user, password), got_pass)
1590
class TestAuth(http_utils.TestCaseWithWebserver):
1591
"""Test authentication scheme"""
1593
scenarios = multiply_scenarios(
1594
vary_by_http_client_implementation(),
1595
vary_by_http_protocol_version(),
1596
vary_by_http_auth_scheme(),
1600
super(TestAuth, self).setUp()
1601
self.server = self.get_readonly_server()
1602
self.build_tree_contents([('a', 'contents of a\n'),
1603
('b', 'contents of b\n'),])
1605
def create_transport_readonly_server(self):
1606
server = self._auth_server(protocol_version=self._protocol_version)
1607
server._url_protocol = self._url_protocol
1610
def _testing_pycurl(self):
1611
# TODO: This is duplicated for lots of the classes in this file
1612
return (features.pycurl.available()
1613
and self._transport == PyCurlTransport)
1615
def get_user_url(self, user, password):
1616
"""Build an url embedding user and password"""
1617
url = '%s://' % self.server._url_protocol
1618
if user is not None:
1620
if password is not None:
1621
url += ':' + password
1623
url += '%s:%s/' % (self.server.host, self.server.port)
1626
def get_user_transport(self, user, password):
1627
t = transport.get_transport_from_url(
1628
self.get_user_url(user, password))
1631
def test_no_user(self):
1632
self.server.add_user('joe', 'foo')
1633
t = self.get_user_transport(None, None)
1634
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1635
# Only one 'Authentication Required' error should occur
1636
self.assertEqual(1, self.server.auth_required_errors)
1638
def test_empty_pass(self):
1639
self.server.add_user('joe', '')
1640
t = self.get_user_transport('joe', '')
1641
self.assertEqual('contents of a\n', t.get('a').read())
1642
# Only one 'Authentication Required' error should occur
1643
self.assertEqual(1, self.server.auth_required_errors)
1645
def test_user_pass(self):
1646
self.server.add_user('joe', 'foo')
1647
t = self.get_user_transport('joe', 'foo')
1648
self.assertEqual('contents of a\n', t.get('a').read())
1649
# Only one 'Authentication Required' error should occur
1650
self.assertEqual(1, self.server.auth_required_errors)
1652
def test_unknown_user(self):
1653
self.server.add_user('joe', 'foo')
1654
t = self.get_user_transport('bill', 'foo')
1655
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1656
# Two 'Authentication Required' errors should occur (the
1657
# initial 'who are you' and 'I don't know you, who are
1659
self.assertEqual(2, self.server.auth_required_errors)
1661
def test_wrong_pass(self):
1662
self.server.add_user('joe', 'foo')
1663
t = self.get_user_transport('joe', 'bar')
1664
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1665
# Two 'Authentication Required' errors should occur (the
1666
# initial 'who are you' and 'this is not you, who are you')
1667
self.assertEqual(2, self.server.auth_required_errors)
1669
def test_prompt_for_username(self):
1670
if self._testing_pycurl():
1671
raise tests.TestNotApplicable(
1672
'pycurl cannot prompt, it handles auth by embedding'
1673
' user:pass in urls only')
1675
self.server.add_user('joe', 'foo')
1676
t = self.get_user_transport(None, None)
1677
stdout = tests.StringIOWrapper()
1678
stderr = tests.StringIOWrapper()
1679
ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
1680
stdout=stdout, stderr=stderr)
1681
self.assertEqual('contents of a\n',t.get('a').read())
1682
# stdin should be empty
1683
self.assertEqual('', ui.ui_factory.stdin.readline())
1685
expected_prompt = self._expected_username_prompt(t._unqualified_scheme)
1686
self.assertEqual(expected_prompt, stderr.read(len(expected_prompt)))
1687
self.assertEqual('', stdout.getvalue())
1688
self._check_password_prompt(t._unqualified_scheme, 'joe',
1691
def test_prompt_for_password(self):
1692
if self._testing_pycurl():
1693
raise tests.TestNotApplicable(
1694
'pycurl cannot prompt, it handles auth by embedding'
1695
' user:pass in urls only')
1697
self.server.add_user('joe', 'foo')
1698
t = self.get_user_transport('joe', None)
1699
stdout = tests.StringIOWrapper()
1700
stderr = tests.StringIOWrapper()
1701
ui.ui_factory = tests.TestUIFactory(stdin='foo\n',
1702
stdout=stdout, stderr=stderr)
1703
self.assertEqual('contents of a\n', t.get('a').read())
1704
# stdin should be empty
1705
self.assertEqual('', ui.ui_factory.stdin.readline())
1706
self._check_password_prompt(t._unqualified_scheme, 'joe',
1708
self.assertEqual('', stdout.getvalue())
1709
# And we shouldn't prompt again for a different request
1710
# against the same transport.
1711
self.assertEqual('contents of b\n',t.get('b').read())
1713
# And neither against a clone
1714
self.assertEqual('contents of b\n',t2.get('b').read())
1715
# Only one 'Authentication Required' error should occur
1716
self.assertEqual(1, self.server.auth_required_errors)
1718
def _check_password_prompt(self, scheme, user, actual_prompt):
1719
expected_prompt = (self._password_prompt_prefix
1720
+ ("%s %s@%s:%d, Realm: '%s' password: "
1722
user, self.server.host, self.server.port,
1723
self.server.auth_realm)))
1724
self.assertEqual(expected_prompt, actual_prompt)
1726
def _expected_username_prompt(self, scheme):
1727
return (self._username_prompt_prefix
1728
+ "%s %s:%d, Realm: '%s' username: " % (scheme.upper(),
1729
self.server.host, self.server.port,
1730
self.server.auth_realm))
1732
def test_no_prompt_for_password_when_using_auth_config(self):
1733
if self._testing_pycurl():
1734
raise tests.TestNotApplicable(
1735
'pycurl does not support authentication.conf'
1736
' since it cannot prompt')
1740
stdin_content = 'bar\n' # Not the right password
1741
self.server.add_user(user, password)
1742
t = self.get_user_transport(user, None)
1743
ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1744
stderr=tests.StringIOWrapper())
1745
# Create a minimal config file with the right password
1746
_setup_authentication_config(scheme='http', port=self.server.port,
1747
user=user, password=password)
1748
# Issue a request to the server to connect
1749
self.assertEqual('contents of a\n',t.get('a').read())
1750
# stdin should have been left untouched
1751
self.assertEqual(stdin_content, ui.ui_factory.stdin.readline())
1752
# Only one 'Authentication Required' error should occur
1753
self.assertEqual(1, self.server.auth_required_errors)
1755
def test_changing_nonce(self):
1756
if self._auth_server not in (http_utils.HTTPDigestAuthServer,
1757
http_utils.ProxyDigestAuthServer):
1758
raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1759
if self._testing_pycurl():
1761
'pycurl does not handle a nonce change')
1762
self.server.add_user('joe', 'foo')
1763
t = self.get_user_transport('joe', 'foo')
1764
self.assertEqual('contents of a\n', t.get('a').read())
1765
self.assertEqual('contents of b\n', t.get('b').read())
1766
# Only one 'Authentication Required' error should have
1768
self.assertEqual(1, self.server.auth_required_errors)
1769
# The server invalidates the current nonce
1770
self.server.auth_nonce = self.server.auth_nonce + '. No, now!'
1771
self.assertEqual('contents of a\n', t.get('a').read())
1772
# Two 'Authentication Required' errors should occur (the
1773
# initial 'who are you' and a second 'who are you' with the new nonce)
1774
self.assertEqual(2, self.server.auth_required_errors)
1776
def test_user_from_auth_conf(self):
1777
if self._testing_pycurl():
1778
raise tests.TestNotApplicable(
1779
'pycurl does not support authentication.conf')
1782
self.server.add_user(user, password)
1783
_setup_authentication_config(scheme='http', port=self.server.port,
1784
user=user, password=password)
1785
t = self.get_user_transport(None, None)
1786
# Issue a request to the server to connect
1787
self.assertEqual('contents of a\n', t.get('a').read())
1788
# Only one 'Authentication Required' error should occur
1789
self.assertEqual(1, self.server.auth_required_errors)
1791
def test_no_credential_leaks_in_log(self):
1792
self.overrideAttr(debug, 'debug_flags', set(['http']))
1794
password = 'very-sensitive-password'
1795
self.server.add_user(user, password)
1796
t = self.get_user_transport(user, password)
1797
# Capture the debug calls to mutter
1800
lines = args[0] % args[1:]
1801
# Some calls output multiple lines, just split them now since we
1802
# care about a single one later.
1803
self.mutters.extend(lines.splitlines())
1804
self.overrideAttr(trace, 'mutter', mutter)
1805
# Issue a request to the server to connect
1806
self.assertEqual(True, t.has('a'))
1807
# Only one 'Authentication Required' error should occur
1808
self.assertEqual(1, self.server.auth_required_errors)
1809
# Since the authentification succeeded, there should be a corresponding
1811
sent_auth_headers = [line for line in self.mutters
1812
if line.startswith('> %s' % (self._auth_header,))]
1813
self.assertLength(1, sent_auth_headers)
1814
self.assertStartsWith(sent_auth_headers[0],
1815
'> %s: <masked>' % (self._auth_header,))
1818
class TestProxyAuth(TestAuth):
1819
"""Test proxy authentication schemes.
1821
This inherits from TestAuth to tweak the setUp and filter some failing
1825
scenarios = multiply_scenarios(
1826
vary_by_http_client_implementation(),
1827
vary_by_http_protocol_version(),
1828
vary_by_http_proxy_auth_scheme(),
1832
super(TestProxyAuth, self).setUp()
1833
# Override the contents to avoid false positives
1834
self.build_tree_contents([('a', 'not proxied contents of a\n'),
1835
('b', 'not proxied contents of b\n'),
1836
('a-proxied', 'contents of a\n'),
1837
('b-proxied', 'contents of b\n'),
1840
def get_user_transport(self, user, password):
1841
self.overrideEnv('all_proxy', self.get_user_url(user, password))
1842
return TestAuth.get_user_transport(self, user, password)
1844
def test_empty_pass(self):
1845
if self._testing_pycurl():
1847
if pycurl.version_info()[1] < '7.16.0':
1849
'pycurl < 7.16.0 does not handle empty proxy passwords')
1850
super(TestProxyAuth, self).test_empty_pass()
1853
class SampleSocket(object):
1854
"""A socket-like object for use in testing the HTTP request handler."""
1856
def __init__(self, socket_read_content):
1857
"""Constructs a sample socket.
1859
:param socket_read_content: a byte sequence
1861
# Use plain python StringIO so we can monkey-patch the close method to
1862
# not discard the contents.
1863
from StringIO import StringIO
1864
self.readfile = StringIO(socket_read_content)
1865
self.writefile = StringIO()
1866
self.writefile.close = lambda: None
1867
self.close = lambda: None
1869
def makefile(self, mode='r', bufsize=None):
1871
return self.readfile
1873
return self.writefile
1876
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1878
scenarios = multiply_scenarios(
1879
vary_by_http_client_implementation(),
1880
vary_by_http_protocol_version(),
1884
super(SmartHTTPTunnellingTest, self).setUp()
1885
# We use the VFS layer as part of HTTP tunnelling tests.
1886
self.overrideEnv('BZR_NO_SMART_VFS', None)
1887
self.transport_readonly_server = http_utils.HTTPServerWithSmarts
1888
self.http_server = self.get_readonly_server()
1890
def create_transport_readonly_server(self):
1891
server = http_utils.HTTPServerWithSmarts(
1892
protocol_version=self._protocol_version)
1893
server._url_protocol = self._url_protocol
1896
def test_open_controldir(self):
1897
branch = self.make_branch('relpath')
1898
url = self.http_server.get_url() + 'relpath'
1899
bd = controldir.ControlDir.open(url)
1900
self.addCleanup(bd.transport.disconnect)
1901
self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1903
def test_bulk_data(self):
1904
# We should be able to send and receive bulk data in a single message.
1905
# The 'readv' command in the smart protocol both sends and receives
1906
# bulk data, so we use that.
1907
self.build_tree(['data-file'])
1908
http_transport = transport.get_transport_from_url(
1909
self.http_server.get_url())
1910
medium = http_transport.get_smart_medium()
1911
# Since we provide the medium, the url below will be mostly ignored
1912
# during the test, as long as the path is '/'.
1913
remote_transport = remote.RemoteTransport('bzr://fake_host/',
1916
[(0, "c")], list(remote_transport.readv("data-file", [(0,1)])))
1918
def test_http_send_smart_request(self):
1920
post_body = 'hello\n'
1921
expected_reply_body = 'ok\x012\n'
1923
http_transport = transport.get_transport_from_url(
1924
self.http_server.get_url())
1925
medium = http_transport.get_smart_medium()
1926
response = medium.send_http_smart_request(post_body)
1927
reply_body = response.read()
1928
self.assertEqual(expected_reply_body, reply_body)
1930
def test_smart_http_server_post_request_handler(self):
1931
httpd = self.http_server.server
1933
socket = SampleSocket(
1934
'POST /.bzr/smart %s \r\n' % self._protocol_version
1935
# HTTP/1.1 posts must have a Content-Length (but it doesn't hurt
1937
+ 'Content-Length: 6\r\n'
1940
# Beware: the ('localhost', 80) below is the
1941
# client_address parameter, but we don't have one because
1942
# we have defined a socket which is not bound to an
1943
# address. The test framework never uses this client
1944
# address, so far...
1945
request_handler = http_utils.SmartRequestHandler(socket,
1948
response = socket.writefile.getvalue()
1949
self.assertStartsWith(response, '%s 200 ' % self._protocol_version)
1950
# This includes the end of the HTTP headers, and all the body.
1951
expected_end_of_response = '\r\n\r\nok\x012\n'
1952
self.assertEndsWith(response, expected_end_of_response)
1955
class ForbiddenRequestHandler(http_server.TestingHTTPRequestHandler):
1956
"""No smart server here request handler."""
1959
self.send_error(403, "Forbidden")
1962
class SmartClientAgainstNotSmartServer(TestSpecificRequestHandler):
1963
"""Test smart client behaviour against an http server without smarts."""
1965
_req_handler_class = ForbiddenRequestHandler
1967
def test_probe_smart_server(self):
1968
"""Test error handling against server refusing smart requests."""
1969
t = self.get_readonly_transport()
1970
# No need to build a valid smart request here, the server will not even
1971
# try to interpret it.
1972
self.assertRaises(errors.SmartProtocolError,
1973
t.get_smart_medium().send_http_smart_request,
1977
class Test_redirected_to(tests.TestCase):
1979
scenarios = vary_by_http_client_implementation()
1981
def test_redirected_to_subdir(self):
1982
t = self._transport('http://www.example.com/foo')
1983
r = t._redirected_to('http://www.example.com/foo',
1984
'http://www.example.com/foo/subdir')
1985
self.assertIsInstance(r, type(t))
1986
# Both transports share the some connection
1987
self.assertEqual(t._get_connection(), r._get_connection())
1988
self.assertEquals('http://www.example.com/foo/subdir/', r.base)
1990
def test_redirected_to_self_with_slash(self):
1991
t = self._transport('http://www.example.com/foo')
1992
r = t._redirected_to('http://www.example.com/foo',
1993
'http://www.example.com/foo/')
1994
self.assertIsInstance(r, type(t))
1995
# Both transports share the some connection (one can argue that we
1996
# should return the exact same transport here, but that seems
1998
self.assertEqual(t._get_connection(), r._get_connection())
2000
def test_redirected_to_host(self):
2001
t = self._transport('http://www.example.com/foo')
2002
r = t._redirected_to('http://www.example.com/foo',
2003
'http://foo.example.com/foo/subdir')
2004
self.assertIsInstance(r, type(t))
2005
self.assertEquals('http://foo.example.com/foo/subdir/',
2008
def test_redirected_to_same_host_sibling_protocol(self):
2009
t = self._transport('http://www.example.com/foo')
2010
r = t._redirected_to('http://www.example.com/foo',
2011
'https://www.example.com/foo')
2012
self.assertIsInstance(r, type(t))
2013
self.assertEquals('https://www.example.com/foo/',
2016
def test_redirected_to_same_host_different_protocol(self):
2017
t = self._transport('http://www.example.com/foo')
2018
r = t._redirected_to('http://www.example.com/foo',
2019
'ftp://www.example.com/foo')
2020
self.assertNotEquals(type(r), type(t))
2021
self.assertEquals('ftp://www.example.com/foo/', r.external_url())
2023
def test_redirected_to_same_host_specific_implementation(self):
2024
t = self._transport('http://www.example.com/foo')
2025
r = t._redirected_to('http://www.example.com/foo',
2026
'https+urllib://www.example.com/foo')
2027
self.assertEquals('https://www.example.com/foo/', r.external_url())
2029
def test_redirected_to_different_host_same_user(self):
2030
t = self._transport('http://joe@www.example.com/foo')
2031
r = t._redirected_to('http://www.example.com/foo',
2032
'https://foo.example.com/foo')
2033
self.assertIsInstance(r, type(t))
2034
self.assertEqual(t._parsed_url.user, r._parsed_url.user)
2035
self.assertEquals('https://joe@foo.example.com/foo/', r.external_url())
2038
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
2039
"""Request handler for a unique and pre-defined request.
2041
The only thing we care about here is how many bytes travel on the wire. But
2042
since we want to measure it for a real http client, we have to send it
2045
We expect to receive a *single* request nothing more (and we won't even
2046
check what request it is, we just measure the bytes read until an empty
2050
def _handle_one_request(self):
2051
tcs = self.server.test_case_server
2052
requestline = self.rfile.readline()
2053
headers = self.MessageClass(self.rfile, 0)
2054
# We just read: the request, the headers, an empty line indicating the
2055
# end of the headers.
2056
bytes_read = len(requestline)
2057
for line in headers.headers:
2058
bytes_read += len(line)
2059
bytes_read += len('\r\n')
2060
if requestline.startswith('POST'):
2061
# The body should be a single line (or we don't know where it ends
2062
# and we don't want to issue a blocking read)
2063
body = self.rfile.readline()
2064
bytes_read += len(body)
2065
tcs.bytes_read = bytes_read
2067
# We set the bytes written *before* issuing the write, the client is
2068
# supposed to consume every produced byte *before* checking that value.
2070
# Doing the oppposite may lead to test failure: we may be interrupted
2071
# after the write but before updating the value. The client can then
2072
# continue and read the value *before* we can update it. And yes,
2073
# this has been observed -- vila 20090129
2074
tcs.bytes_written = len(tcs.canned_response)
2075
self.wfile.write(tcs.canned_response)
2078
class ActivityServerMixin(object):
2080
def __init__(self, protocol_version):
2081
super(ActivityServerMixin, self).__init__(
2082
request_handler=PredefinedRequestHandler,
2083
protocol_version=protocol_version)
2084
# Bytes read and written by the server
2086
self.bytes_written = 0
2087
self.canned_response = None
2090
class ActivityHTTPServer(ActivityServerMixin, http_server.HttpServer):
2094
if features.HTTPSServerFeature.available():
2095
from bzrlib.tests import https_server
2096
class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
2100
class TestActivityMixin(object):
2101
"""Test socket activity reporting.
2103
We use a special purpose server to control the bytes sent and received and
2104
be able to predict the activity on the client socket.
2108
self.server = self._activity_server(self._protocol_version)
2109
self.server.start_server()
2110
self.addCleanup(self.server.stop_server)
2111
_activities = {} # Don't close over self and create a cycle
2112
def report_activity(t, bytes, direction):
2113
count = _activities.get(direction, 0)
2115
_activities[direction] = count
2116
self.activities = _activities
2117
# We override at class level because constructors may propagate the
2118
# bound method and render instance overriding ineffective (an
2119
# alternative would be to define a specific ui factory instead...)
2120
self.overrideAttr(self._transport, '_report_activity', report_activity)
2122
def get_transport(self):
2123
t = self._transport(self.server.get_url())
2124
# FIXME: Needs cleanup -- vila 20100611
2127
def assertActivitiesMatch(self):
2128
self.assertEqual(self.server.bytes_read,
2129
self.activities.get('write', 0), 'written bytes')
2130
self.assertEqual(self.server.bytes_written,
2131
self.activities.get('read', 0), 'read bytes')
2134
self.server.canned_response = '''HTTP/1.1 200 OK\r
2135
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
2136
Server: Apache/2.0.54 (Fedora)\r
2137
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
2138
ETag: "56691-23-38e9ae00"\r
2139
Accept-Ranges: bytes\r
2140
Content-Length: 35\r
2142
Content-Type: text/plain; charset=UTF-8\r
2144
Bazaar-NG meta directory, format 1
2146
t = self.get_transport()
2147
self.assertEqual('Bazaar-NG meta directory, format 1\n',
2148
t.get('foo/bar').read())
2149
self.assertActivitiesMatch()
2152
self.server.canned_response = '''HTTP/1.1 200 OK\r
2153
Server: SimpleHTTP/0.6 Python/2.5.2\r
2154
Date: Thu, 29 Jan 2009 20:21:47 GMT\r
2155
Content-type: application/octet-stream\r
2156
Content-Length: 20\r
2157
Last-Modified: Thu, 29 Jan 2009 20:21:47 GMT\r
2160
t = self.get_transport()
2161
self.assertTrue(t.has('foo/bar'))
2162
self.assertActivitiesMatch()
2164
def test_readv(self):
2165
self.server.canned_response = '''HTTP/1.1 206 Partial Content\r
2166
Date: Tue, 11 Jul 2006 04:49:48 GMT\r
2167
Server: Apache/2.0.54 (Fedora)\r
2168
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
2169
ETag: "238a3c-16ec2-805c5540"\r
2170
Accept-Ranges: bytes\r
2171
Content-Length: 1534\r
2173
Content-Type: multipart/byteranges; boundary=418470f848b63279b\r
2176
--418470f848b63279b\r
2177
Content-type: text/plain; charset=UTF-8\r
2178
Content-range: bytes 0-254/93890\r
2180
mbp@sourcefrog.net-20050309040815-13242001617e4a06
2181
mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e7627
2182
mbp@sourcefrog.net-20050309040957-6cad07f466bb0bb8
2183
mbp@sourcefrog.net-20050309041501-c840e09071de3b67
2184
mbp@sourcefrog.net-20050309044615-c24a3250be83220a
2186
--418470f848b63279b\r
2187
Content-type: text/plain; charset=UTF-8\r
2188
Content-range: bytes 1000-2049/93890\r
2191
mbp@sourcefrog.net-20050311063625-07858525021f270b
2192
mbp@sourcefrog.net-20050311231934-aa3776aff5200bb9
2193
mbp@sourcefrog.net-20050311231953-73aeb3a131c3699a
2194
mbp@sourcefrog.net-20050311232353-f5e33da490872c6a
2195
mbp@sourcefrog.net-20050312071639-0a8f59a34a024ff0
2196
mbp@sourcefrog.net-20050312073432-b2c16a55e0d6e9fb
2197
mbp@sourcefrog.net-20050312073831-a47c3335ece1920f
2198
mbp@sourcefrog.net-20050312085412-13373aa129ccbad3
2199
mbp@sourcefrog.net-20050313052251-2bf004cb96b39933
2200
mbp@sourcefrog.net-20050313052856-3edd84094687cb11
2201
mbp@sourcefrog.net-20050313053233-e30a4f28aef48f9d
2202
mbp@sourcefrog.net-20050313053853-7c64085594ff3072
2203
mbp@sourcefrog.net-20050313054757-a86c3f5871069e22
2204
mbp@sourcefrog.net-20050313061422-418f1f73b94879b9
2205
mbp@sourcefrog.net-20050313120651-497bd231b19df600
2206
mbp@sourcefrog.net-20050314024931-eae0170ef25a5d1a
2207
mbp@sourcefrog.net-20050314025438-d52099f915fe65fc
2208
mbp@sourcefrog.net-20050314025539-637a636692c055cf
2209
mbp@sourcefrog.net-20050314025737-55eb441f430ab4ba
2210
mbp@sourcefrog.net-20050314025901-d74aa93bb7ee8f62
2212
--418470f848b63279b--\r
2214
t = self.get_transport()
2215
# Remember that the request is ignored and that the ranges below
2216
# doesn't have to match the canned response.
2217
l = list(t.readv('/foo/bar', ((0, 255), (1000, 1050))))
2218
self.assertEqual(2, len(l))
2219
self.assertActivitiesMatch()
2221
def test_post(self):
2222
self.server.canned_response = '''HTTP/1.1 200 OK\r
2223
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
2224
Server: Apache/2.0.54 (Fedora)\r
2225
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
2226
ETag: "56691-23-38e9ae00"\r
2227
Accept-Ranges: bytes\r
2228
Content-Length: 35\r
2230
Content-Type: text/plain; charset=UTF-8\r
2232
lalala whatever as long as itsssss
2234
t = self.get_transport()
2235
# We must send a single line of body bytes, see
2236
# PredefinedRequestHandler._handle_one_request
2237
code, f = t._post('abc def end-of-body\n')
2238
self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2239
self.assertActivitiesMatch()
2242
class TestActivity(tests.TestCase, TestActivityMixin):
2244
scenarios = multiply_scenarios(
2245
vary_by_http_activity(),
2246
vary_by_http_protocol_version(),
2250
super(TestActivity, self).setUp()
2251
TestActivityMixin.setUp(self)
2254
class TestNoReportActivity(tests.TestCase, TestActivityMixin):
2256
# Unlike TestActivity, we are really testing ReportingFileSocket and
2257
# ReportingSocket, so we don't need all the parametrization. Since
2258
# ReportingFileSocket and ReportingSocket are wrappers, it's easier to
2259
# test them through their use by the transport than directly (that's a
2260
# bit less clean but far more simpler and effective).
2261
_activity_server = ActivityHTTPServer
2262
_protocol_version = 'HTTP/1.1'
2265
super(TestNoReportActivity, self).setUp()
2266
self._transport =_urllib.HttpTransport_urllib
2267
TestActivityMixin.setUp(self)
2269
def assertActivitiesMatch(self):
2270
# Nothing to check here
2274
class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver):
2275
"""Test authentication on the redirected http server."""
2277
scenarios = vary_by_http_protocol_version()
2279
_auth_header = 'Authorization'
2280
_password_prompt_prefix = ''
2281
_username_prompt_prefix = ''
2282
_auth_server = http_utils.HTTPBasicAuthServer
2283
_transport = _urllib.HttpTransport_urllib
2286
super(TestAuthOnRedirected, self).setUp()
2287
self.build_tree_contents([('a','a'),
2289
('1/a', 'redirected once'),
2291
new_prefix = 'http://%s:%s' % (self.new_server.host,
2292
self.new_server.port)
2293
self.old_server.redirections = [
2294
('(.*)', r'%s/1\1' % (new_prefix), 301),]
2295
self.old_transport = self.get_old_transport()
2296
self.new_server.add_user('joe', 'foo')
2297
cleanup_http_redirection_connections(self)
2299
def create_transport_readonly_server(self):
2300
server = self._auth_server(protocol_version=self._protocol_version)
2301
server._url_protocol = self._url_protocol
2307
def test_auth_on_redirected_via_do_catching_redirections(self):
2308
self.redirections = 0
2310
def redirected(t, exception, redirection_notice):
2311
self.redirections += 1
2312
redirected_t = t._redirected_to(exception.source, exception.target)
2313
self.addCleanup(redirected_t.disconnect)
2316
stdout = tests.StringIOWrapper()
2317
stderr = tests.StringIOWrapper()
2318
ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2319
stdout=stdout, stderr=stderr)
2320
self.assertEqual('redirected once',
2321
transport.do_catching_redirections(
2322
self.get_a, self.old_transport, redirected).read())
2323
self.assertEqual(1, self.redirections)
2324
# stdin should be empty
2325
self.assertEqual('', ui.ui_factory.stdin.readline())
2326
# stdout should be empty, stderr will contains the prompts
2327
self.assertEqual('', stdout.getvalue())
2329
def test_auth_on_redirected_via_following_redirections(self):
2330
self.new_server.add_user('joe', 'foo')
2331
stdout = tests.StringIOWrapper()
2332
stderr = tests.StringIOWrapper()
2333
ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2334
stdout=stdout, stderr=stderr)
2335
t = self.old_transport
2336
req = RedirectedRequest('GET', t.abspath('a'))
2337
new_prefix = 'http://%s:%s' % (self.new_server.host,
2338
self.new_server.port)
2339
self.old_server.redirections = [
2340
('(.*)', r'%s/1\1' % (new_prefix), 301),]
2341
self.assertEqual('redirected once', t._perform(req).read())
2342
# stdin should be empty
2343
self.assertEqual('', ui.ui_factory.stdin.readline())
2344
# stdout should be empty, stderr will contains the prompts
2345
self.assertEqual('', stdout.getvalue())
added
removed
bzrlib/tests/test_http.py
