13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
# FIXME: This test should be repeated for each available http client
18
# implementation; at the moment we have urllib and pycurl.
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Tests for HTTP implementations.
19
This module defines a load_tests() method that parametrize tests classes for
20
transport implementation, http protocol versions and authentication schemes.
20
23
# TODO: Should be renamed to bzrlib.transport.http.tests?
21
24
# TODO: What about renaming to bzrlib.tests.transport.http ?
26
from cStringIO import StringIO
30
import SimpleHTTPServer
29
36
from bzrlib import (
41
remote as _mod_remote,
47
from bzrlib.symbol_versioning import (
34
50
from bzrlib.tests import (
38
from bzrlib.tests.HttpServer import (
43
from bzrlib.tests.HTTPTestUtil import (
44
BadProtocolRequestHandler,
45
BadStatusRequestHandler,
46
FakeProxyRequestHandler,
47
ForbiddenRequestHandler,
48
HTTPServerRedirecting,
49
InvalidStatusRequestHandler,
50
NoRangeRequestHandler,
51
SingleRangeRequestHandler,
52
TestCaseWithRedirectedWebserver,
53
TestCaseWithTwoWebservers,
54
TestCaseWithWebserver,
57
54
from bzrlib.transport import (
58
do_catching_redirections,
62
58
from bzrlib.transport.http import (
67
from bzrlib.transport.http._urllib import HttpTransport_urllib
65
from bzrlib.transport.http._pycurl import PyCurlTransport
67
except errors.DependencyNotPresent:
68
pycurl_present = False
71
def load_tests(standard_tests, module, loader):
72
"""Multiply tests for http clients and protocol versions."""
73
result = loader.suiteClass()
75
# one for each transport implementation
76
t_tests, remaining_tests = tests.split_suite_by_condition(
77
standard_tests, tests.condition_isinstance((
78
TestHttpTransportRegistration,
79
TestHttpTransportUrls,
82
transport_scenarios = [
83
('urllib', dict(_transport=_urllib.HttpTransport_urllib,
84
_server=http_server.HttpServer_urllib,
85
_qualified_prefix='http+urllib',)),
88
transport_scenarios.append(
89
('pycurl', dict(_transport=PyCurlTransport,
90
_server=http_server.HttpServer_PyCurl,
91
_qualified_prefix='http+pycurl',)))
92
tests.multiply_tests(t_tests, transport_scenarios, result)
94
# each implementation tested with each HTTP version
95
tp_tests, remaining_tests = tests.split_suite_by_condition(
96
remaining_tests, tests.condition_isinstance((
97
SmartHTTPTunnellingTest,
98
TestDoCatchRedirections,
100
TestHTTPRedirections,
101
TestHTTPSilentRedirections,
102
TestLimitedRangeRequestServer,
106
TestSpecificRequestHandler,
108
protocol_scenarios = [
109
('HTTP/1.0', dict(_protocol_version='HTTP/1.0')),
110
('HTTP/1.1', dict(_protocol_version='HTTP/1.1')),
112
tp_scenarios = tests.multiply_scenarios(transport_scenarios,
114
tests.multiply_tests(tp_tests, tp_scenarios, result)
116
# auth: each auth scheme on all http versions on all implementations.
117
tpa_tests, remaining_tests = tests.split_suite_by_condition(
118
remaining_tests, tests.condition_isinstance((
121
auth_scheme_scenarios = [
122
('basic', dict(_auth_scheme='basic')),
123
('digest', dict(_auth_scheme='digest')),
125
tpa_scenarios = tests.multiply_scenarios(tp_scenarios,
126
auth_scheme_scenarios)
127
tests.multiply_tests(tpa_tests, tpa_scenarios, result)
129
# activity: activity on all http versions on all implementations
130
tpact_tests, remaining_tests = tests.split_suite_by_condition(
131
remaining_tests, tests.condition_isinstance((
134
activity_scenarios = [
135
('http', dict(_activity_server=ActivityHTTPServer)),
137
if tests.HTTPSServerFeature.available():
138
activity_scenarios.append(
139
('https', dict(_activity_server=ActivityHTTPSServer)))
140
tpact_scenarios = tests.multiply_scenarios(tp_scenarios,
142
tests.multiply_tests(tpact_tests, tpact_scenarios, result)
144
# No parametrization for the remaining tests
145
result.addTests(remaining_tests)
70
150
class FakeManager(object):
658
777
self.assertListRaises((errors.InvalidRange, errors.ShortReadvError,),
659
778
t.readv, 'a', [(12,2)])
780
def test_readv_multiple_get_requests(self):
781
server = self.get_readonly_server()
782
t = self._transport(server.get_url())
783
# force transport to issue multiple requests
784
t._max_readv_combine = 1
785
t._max_get_ranges = 1
786
l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
787
self.assertEqual(l[0], (0, '0'))
788
self.assertEqual(l[1], (1, '1'))
789
self.assertEqual(l[2], (3, '34'))
790
self.assertEqual(l[3], (9, '9'))
791
# The server should have issued 4 requests
792
self.assertEqual(4, server.GET_request_nb)
794
def test_readv_get_max_size(self):
795
server = self.get_readonly_server()
796
t = self._transport(server.get_url())
797
# force transport to issue multiple requests by limiting the number of
798
# bytes by request. Note that this apply to coalesced offsets only, a
799
# single range will keep its size even if bigger than the limit.
801
l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
802
self.assertEqual(l[0], (0, '0'))
803
self.assertEqual(l[1], (1, '1'))
804
self.assertEqual(l[2], (2, '2345'))
805
self.assertEqual(l[3], (6, '6789'))
806
# The server should have issued 3 requests
807
self.assertEqual(3, server.GET_request_nb)
809
def test_complete_readv_leave_pipe_clean(self):
810
server = self.get_readonly_server()
811
t = self._transport(server.get_url())
812
# force transport to issue multiple requests
814
l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
815
# The server should have issued 3 requests
816
self.assertEqual(3, server.GET_request_nb)
817
self.assertEqual('0123456789', t.get_bytes('a'))
818
self.assertEqual(4, server.GET_request_nb)
820
def test_incomplete_readv_leave_pipe_clean(self):
821
server = self.get_readonly_server()
822
t = self._transport(server.get_url())
823
# force transport to issue multiple requests
825
# Don't collapse readv results into a list so that we leave unread
826
# bytes on the socket
827
ireadv = iter(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
828
self.assertEqual((0, '0'), ireadv.next())
829
# The server should have issued one request so far
830
self.assertEqual(1, server.GET_request_nb)
831
self.assertEqual('0123456789', t.get_bytes('a'))
832
# get_bytes issued an additional request, the readv pending ones are
834
self.assertEqual(2, server.GET_request_nb)
837
class SingleRangeRequestHandler(http_server.TestingHTTPRequestHandler):
838
"""Always reply to range request as if they were single.
840
Don't be explicit about it, just to annoy the clients.
843
def get_multiple_ranges(self, file, file_size, ranges):
844
"""Answer as if it was a single range request and ignores the rest"""
845
(start, end) = ranges[0]
846
return self.get_single_range(file, file_size, start, end)
662
849
class TestSingleRangeRequestServer(TestRangeRequestServer):
663
850
"""Test readv against a server which accept only single range requests"""
665
def create_transport_readonly_server(self):
666
return HttpServer(SingleRangeRequestHandler)
669
class TestSingleRangeRequestServer_urllib(TestSingleRangeRequestServer,
670
TestCaseWithWebserver):
671
"""Tests single range requests accepting server for urllib implementation"""
673
_transport = HttpTransport_urllib
676
class TestSingleRangeRequestServer_pycurl(TestWithTransport_pycurl,
677
TestSingleRangeRequestServer,
678
TestCaseWithWebserver):
679
"""Tests single range requests accepting server for pycurl implementation"""
852
_req_handler_class = SingleRangeRequestHandler
855
class SingleOnlyRangeRequestHandler(http_server.TestingHTTPRequestHandler):
856
"""Only reply to simple range requests, errors out on multiple"""
858
def get_multiple_ranges(self, file, file_size, ranges):
859
"""Refuses the multiple ranges request"""
862
self.send_error(416, "Requested range not satisfiable")
864
(start, end) = ranges[0]
865
return self.get_single_range(file, file_size, start, end)
868
class TestSingleOnlyRangeRequestServer(TestRangeRequestServer):
869
"""Test readv against a server which only accept single range requests"""
871
_req_handler_class = SingleOnlyRangeRequestHandler
874
class NoRangeRequestHandler(http_server.TestingHTTPRequestHandler):
875
"""Ignore range requests without notice"""
878
# Update the statistics
879
self.server.test_case_server.GET_request_nb += 1
880
# Just bypass the range handling done by TestingHTTPRequestHandler
881
return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
682
884
class TestNoRangeRequestServer(TestRangeRequestServer):
683
885
"""Test readv against a server which do not accept range requests"""
887
_req_handler_class = NoRangeRequestHandler
890
class MultipleRangeWithoutContentLengthRequestHandler(
891
http_server.TestingHTTPRequestHandler):
892
"""Reply to multiple range requests without content length header."""
894
def get_multiple_ranges(self, file, file_size, ranges):
895
self.send_response(206)
896
self.send_header('Accept-Ranges', 'bytes')
897
boundary = "%d" % random.randint(0,0x7FFFFFFF)
898
self.send_header("Content-Type",
899
"multipart/byteranges; boundary=%s" % boundary)
901
for (start, end) in ranges:
902
self.wfile.write("--%s\r\n" % boundary)
903
self.send_header("Content-type", 'application/octet-stream')
904
self.send_header("Content-Range", "bytes %d-%d/%d" % (start,
908
self.send_range_content(file, start, end - start + 1)
910
self.wfile.write("--%s\r\n" % boundary)
913
class TestMultipleRangeWithoutContentLengthServer(TestRangeRequestServer):
915
_req_handler_class = MultipleRangeWithoutContentLengthRequestHandler
918
class TruncatedMultipleRangeRequestHandler(
919
http_server.TestingHTTPRequestHandler):
920
"""Reply to multiple range requests truncating the last ones.
922
This server generates responses whose Content-Length describes all the
923
ranges, but fail to include the last ones leading to client short reads.
924
This has been observed randomly with lighttpd (bug #179368).
927
_truncated_ranges = 2
929
def get_multiple_ranges(self, file, file_size, ranges):
930
self.send_response(206)
931
self.send_header('Accept-Ranges', 'bytes')
933
self.send_header('Content-Type',
934
'multipart/byteranges; boundary=%s' % boundary)
935
boundary_line = '--%s\r\n' % boundary
936
# Calculate the Content-Length
938
for (start, end) in ranges:
939
content_length += len(boundary_line)
940
content_length += self._header_line_length(
941
'Content-type', 'application/octet-stream')
942
content_length += self._header_line_length(
943
'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
944
content_length += len('\r\n') # end headers
945
content_length += end - start # + 1
946
content_length += len(boundary_line)
947
self.send_header('Content-length', content_length)
950
# Send the multipart body
952
for (start, end) in ranges:
953
self.wfile.write(boundary_line)
954
self.send_header('Content-type', 'application/octet-stream')
955
self.send_header('Content-Range', 'bytes %d-%d/%d'
956
% (start, end, file_size))
958
if cur + self._truncated_ranges >= len(ranges):
959
# Abruptly ends the response and close the connection
960
self.close_connection = 1
962
self.send_range_content(file, start, end - start + 1)
965
self.wfile.write(boundary_line)
968
class TestTruncatedMultipleRangeServer(TestSpecificRequestHandler):
970
_req_handler_class = TruncatedMultipleRangeRequestHandler
973
super(TestTruncatedMultipleRangeServer, self).setUp()
974
self.build_tree_contents([('a', '0123456789')],)
976
def test_readv_with_short_reads(self):
977
server = self.get_readonly_server()
978
t = self._transport(server.get_url())
979
# Force separate ranges for each offset
980
t._bytes_to_read_before_seek = 0
981
ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
982
self.assertEqual((0, '0'), ireadv.next())
983
self.assertEqual((2, '2'), ireadv.next())
984
if not self._testing_pycurl():
985
# Only one request have been issued so far (except for pycurl that
986
# try to read the whole response at once)
987
self.assertEqual(1, server.GET_request_nb)
988
self.assertEqual((4, '45'), ireadv.next())
989
self.assertEqual((9, '9'), ireadv.next())
990
# Both implementations issue 3 requests but:
991
# - urllib does two multiple (4 ranges, then 2 ranges) then a single
993
# - pycurl does two multiple (4 ranges, 4 ranges) then a single range
994
self.assertEqual(3, server.GET_request_nb)
995
# Finally the client have tried a single range request and stays in
997
self.assertEqual('single', t._range_hint)
999
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1000
"""Errors out when range specifiers exceed the limit"""
1002
def get_multiple_ranges(self, file, file_size, ranges):
1003
"""Refuses the multiple ranges request"""
1004
tcs = self.server.test_case_server
1005
if tcs.range_limit is not None and len(ranges) > tcs.range_limit:
1007
# Emulate apache behavior
1008
self.send_error(400, "Bad Request")
1010
return http_server.TestingHTTPRequestHandler.get_multiple_ranges(
1011
self, file, file_size, ranges)
1014
class LimitedRangeHTTPServer(http_server.HttpServer):
1015
"""An HttpServer erroring out on requests with too much range specifiers"""
1017
def __init__(self, request_handler=LimitedRangeRequestHandler,
1018
protocol_version=None,
1020
http_server.HttpServer.__init__(self, request_handler,
1021
protocol_version=protocol_version)
1022
self.range_limit = range_limit
1025
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1026
"""Tests readv requests against a server erroring out on too much ranges."""
1028
# Requests with more range specifiers will error out
685
1031
def create_transport_readonly_server(self):
686
return HttpServer(NoRangeRequestHandler)
689
class TestNoRangeRequestServer_urllib(TestNoRangeRequestServer,
690
TestCaseWithWebserver):
691
"""Tests range requests refusing server for urllib implementation"""
693
_transport = HttpTransport_urllib
696
class TestNoRangeRequestServer_pycurl(TestWithTransport_pycurl,
697
TestNoRangeRequestServer,
698
TestCaseWithWebserver):
699
"""Tests range requests refusing server for pycurl implementation"""
702
class TestHttpProxyWhiteBox(TestCase):
1032
return LimitedRangeHTTPServer(range_limit=self.range_limit,
1033
protocol_version=self._protocol_version)
1035
def get_transport(self):
1036
return self._transport(self.get_readonly_server().get_url())
1039
http_utils.TestCaseWithWebserver.setUp(self)
1040
# We need to manipulate ranges that correspond to real chunks in the
1041
# response, so we build a content appropriately.
1042
filler = ''.join(['abcdefghij' for x in range(102)])
1043
content = ''.join(['%04d' % v + filler for v in range(16)])
1044
self.build_tree_contents([('a', content)],)
1046
def test_few_ranges(self):
1047
t = self.get_transport()
1048
l = list(t.readv('a', ((0, 4), (1024, 4), )))
1049
self.assertEqual(l[0], (0, '0000'))
1050
self.assertEqual(l[1], (1024, '0001'))
1051
self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1053
def test_more_ranges(self):
1054
t = self.get_transport()
1055
l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1056
self.assertEqual(l[0], (0, '0000'))
1057
self.assertEqual(l[1], (1024, '0001'))
1058
self.assertEqual(l[2], (4096, '0004'))
1059
self.assertEqual(l[3], (8192, '0008'))
1060
# The server will refuse to serve the first request (too much ranges),
1061
# a second request will succeed.
1062
self.assertEqual(2, self.get_readonly_server().GET_request_nb)
1065
class TestHttpProxyWhiteBox(tests.TestCase):
703
1066
"""Whitebox test proxy http authorization.
705
These tests concern urllib implementation only.
1068
Only the urllib implementation is tested here.
708
1071
def setUp(self):
1072
tests.TestCase.setUp(self)
710
1073
self._old_env = {}
712
1075
def tearDown(self):
713
1076
self._restore_env()
715
def _set_and_capture_env_var(self, name, new_value):
716
"""Set an environment variable, and reset it when finished."""
717
self._old_env[name] = osutils.set_or_unset_env(name, new_value)
1077
tests.TestCase.tearDown(self)
719
1079
def _install_env(self, env):
720
1080
for name, value in env.iteritems():
721
self._set_and_capture_env_var(name, value)
1081
self._old_env[name] = osutils.set_or_unset_env(name, value)
723
1083
def _restore_env(self):
724
1084
for name, value in self._old_env.iteritems():
725
1085
osutils.set_or_unset_env(name, value)
727
1087
def _proxied_request(self):
728
from bzrlib.transport.http._urllib2_wrappers import (
733
handler = ProxyHandler()
734
request = Request('GET','http://baz/buzzle')
1088
handler = _urllib2_wrappers.ProxyHandler()
1089
request = _urllib2_wrappers.Request('GET','http://baz/buzzle')
735
1090
handler.set_proxy(request, 'http')
859
1204
'NO_PROXY': self.no_proxy_host})
861
1206
def test_http_proxy_without_scheme(self):
862
self.assertRaises(errors.InvalidURL,
864
{'http_proxy': self.proxy_address})
867
class TestProxyHttpServer_urllib(TestProxyHttpServer,
868
TestCaseWithTwoWebservers):
869
"""Tests proxy server for urllib implementation"""
871
_transport = HttpTransport_urllib
874
class TestProxyHttpServer_pycurl(TestWithTransport_pycurl,
876
TestCaseWithTwoWebservers):
877
"""Tests proxy server for pycurl implementation"""
880
TestProxyHttpServer.setUp(self)
881
# Oh my ! pycurl does not check for the port as part of
882
# no_proxy :-( So we just test the host part
883
self.no_proxy_host = 'localhost'
885
def test_HTTP_PROXY(self):
886
# pycurl do not check HTTP_PROXY for security reasons
887
# (for use in a CGI context that we do not care
888
# about. Should we ?)
891
def test_HTTP_PROXY_with_NO_PROXY(self):
894
def test_http_proxy_without_scheme(self):
895
# pycurl *ignores* invalid proxy env variables. If that
896
# ever change in the future, this test will fail
897
# indicating that pycurl do not ignore anymore such
899
self.not_proxied_in_env({'http_proxy': self.proxy_address})
902
class TestRanges(object):
903
"""Test the Range header in GET methods..
905
This MUST be used by daughter classes that also inherit from
906
TestCaseWithWebserver.
908
We can't inherit directly from TestCaseWithWebserver or the
909
test framework will try to create an instance which cannot
910
run, its implementation being incomplete.
914
TestCaseWithWebserver.setUp(self)
1207
if self._testing_pycurl():
1208
# pycurl *ignores* invalid proxy env variables. If that ever change
1209
# in the future, this test will fail indicating that pycurl do not
1210
# ignore anymore such variables.
1211
self.not_proxied_in_env({'http_proxy': self.proxy_address})
1213
self.assertRaises(errors.InvalidURL,
1214
self.proxied_in_env,
1215
{'http_proxy': self.proxy_address})
1218
class TestRanges(http_utils.TestCaseWithWebserver):
1219
"""Test the Range header in GET methods."""
1222
http_utils.TestCaseWithWebserver.setUp(self)
915
1223
self.build_tree_contents([('a', '0123456789')],)
916
1224
server = self.get_readonly_server()
917
1225
self.transport = self._transport(server.get_url())
919
def _file_contents(self, relpath, ranges, tail_amount=0):
920
code, data = self.transport._get(relpath, ranges)
921
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
922
for start, end in ranges:
924
yield data.read(end - start + 1)
1227
def create_transport_readonly_server(self):
1228
return http_server.HttpServer(protocol_version=self._protocol_version)
1230
def _file_contents(self, relpath, ranges):
1231
offsets = [ (start, end - start + 1) for start, end in ranges]
1232
coalesce = self.transport._coalesce_offsets
1233
coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1234
code, data = self.transport._get(relpath, coalesced)
1235
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1236
for start, end in ranges:
1238
yield data.read(end - start + 1)
926
1240
def _file_tail(self, relpath, tail_amount):
927
code, data = self.transport._get(relpath, [], tail_amount)
928
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
929
data.seek(-tail_amount + 1, 2)
930
return data.read(tail_amount)
1241
code, data = self.transport._get(relpath, [], tail_amount)
1242
self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1243
data.seek(-tail_amount, 2)
1244
return data.read(tail_amount)
932
1246
def test_range_header(self):
934
1248
map(self.assertEqual,['0', '234'],
935
1249
list(self._file_contents('a', [(0,0), (2,4)])),)
1251
def test_range_header_tail(self):
937
1252
self.assertEqual('789', self._file_tail('a', 3))
938
# Syntactically invalid range
939
self.assertRaises(errors.InvalidRange,
940
self.transport._get, 'a', [(4, 3)])
941
# Semantically invalid range
942
self.assertRaises(errors.InvalidRange,
943
self.transport._get, 'a', [(42, 128)])
946
class TestRanges_urllib(TestRanges, TestCaseWithWebserver):
947
"""Test the Range header in GET methods for urllib implementation"""
949
_transport = HttpTransport_urllib
952
class TestRanges_pycurl(TestWithTransport_pycurl,
954
TestCaseWithWebserver):
955
"""Test the Range header in GET methods for pycurl implementation"""
958
class TestHTTPRedirections(object):
959
"""Test redirection between http servers.
961
This MUST be used by daughter classes that also inherit from
962
TestCaseWithRedirectedWebserver.
964
We can't inherit directly from TestCaseWithTwoWebservers or the
965
test framework will try to create an instance which cannot
966
run, its implementation being incomplete.
1254
def test_syntactically_invalid_range_header(self):
1255
self.assertListRaises(errors.InvalidHttpRange,
1256
self._file_contents, 'a', [(4, 3)])
1258
def test_semantically_invalid_range_header(self):
1259
self.assertListRaises(errors.InvalidHttpRange,
1260
self._file_contents, 'a', [(42, 128)])
1263
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1264
"""Test redirection between http servers."""
969
1266
def create_transport_secondary_server(self):
970
1267
"""Create the secondary server redirecting to the primary server"""
971
1268
new = self.get_readonly_server()
973
redirecting = HTTPServerRedirecting()
1270
redirecting = http_utils.HTTPServerRedirecting(
1271
protocol_version=self._protocol_version)
974
1272
redirecting.redirect_to(new.host, new.port)
975
1273
return redirecting
1144
1431
return self.old_transport.clone(exception.target)
1146
self.assertRaises(errors.TooManyRedirections, do_catching_redirections,
1433
self.assertRaises(errors.TooManyRedirections,
1434
transport.do_catching_redirections,
1147
1435
self.get_a, self.old_transport, redirected)
1438
class TestAuth(http_utils.TestCaseWithWebserver):
1439
"""Test authentication scheme"""
1441
_auth_header = 'Authorization'
1442
_password_prompt_prefix = ''
1445
super(TestAuth, self).setUp()
1446
self.server = self.get_readonly_server()
1447
self.build_tree_contents([('a', 'contents of a\n'),
1448
('b', 'contents of b\n'),])
1450
def create_transport_readonly_server(self):
1451
if self._auth_scheme == 'basic':
1452
server = http_utils.HTTPBasicAuthServer(
1453
protocol_version=self._protocol_version)
1455
if self._auth_scheme != 'digest':
1456
raise AssertionError('Unknown auth scheme: %r'
1457
% self._auth_scheme)
1458
server = http_utils.HTTPDigestAuthServer(
1459
protocol_version=self._protocol_version)
1462
def _testing_pycurl(self):
1463
return pycurl_present and self._transport == PyCurlTransport
1465
def get_user_url(self, user, password):
1466
"""Build an url embedding user and password"""
1467
url = '%s://' % self.server._url_protocol
1468
if user is not None:
1470
if password is not None:
1471
url += ':' + password
1473
url += '%s:%s/' % (self.server.host, self.server.port)
1476
def get_user_transport(self, user, password):
1477
return self._transport(self.get_user_url(user, password))
1479
def test_no_user(self):
1480
self.server.add_user('joe', 'foo')
1481
t = self.get_user_transport(None, None)
1482
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1483
# Only one 'Authentication Required' error should occur
1484
self.assertEqual(1, self.server.auth_required_errors)
1486
def test_empty_pass(self):
1487
self.server.add_user('joe', '')
1488
t = self.get_user_transport('joe', '')
1489
self.assertEqual('contents of a\n', t.get('a').read())
1490
# Only one 'Authentication Required' error should occur
1491
self.assertEqual(1, self.server.auth_required_errors)
1493
def test_user_pass(self):
1494
self.server.add_user('joe', 'foo')
1495
t = self.get_user_transport('joe', 'foo')
1496
self.assertEqual('contents of a\n', t.get('a').read())
1497
# Only one 'Authentication Required' error should occur
1498
self.assertEqual(1, self.server.auth_required_errors)
1500
def test_unknown_user(self):
1501
self.server.add_user('joe', 'foo')
1502
t = self.get_user_transport('bill', 'foo')
1503
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1504
# Two 'Authentication Required' errors should occur (the
1505
# initial 'who are you' and 'I don't know you, who are
1507
self.assertEqual(2, self.server.auth_required_errors)
1509
def test_wrong_pass(self):
1510
self.server.add_user('joe', 'foo')
1511
t = self.get_user_transport('joe', 'bar')
1512
self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1513
# Two 'Authentication Required' errors should occur (the
1514
# initial 'who are you' and 'this is not you, who are you')
1515
self.assertEqual(2, self.server.auth_required_errors)
1517
def test_prompt_for_password(self):
1518
if self._testing_pycurl():
1519
raise tests.TestNotApplicable(
1520
'pycurl cannot prompt, it handles auth by embedding'
1521
' user:pass in urls only')
1523
self.server.add_user('joe', 'foo')
1524
t = self.get_user_transport('joe', None)
1525
stdout = tests.StringIOWrapper()
1526
ui.ui_factory = tests.TestUIFactory(stdin='foo\n', stdout=stdout)
1527
self.assertEqual('contents of a\n',t.get('a').read())
1528
# stdin should be empty
1529
self.assertEqual('', ui.ui_factory.stdin.readline())
1530
self._check_password_prompt(t._unqualified_scheme, 'joe',
1532
# And we shouldn't prompt again for a different request
1533
# against the same transport.
1534
self.assertEqual('contents of b\n',t.get('b').read())
1536
# And neither against a clone
1537
self.assertEqual('contents of b\n',t2.get('b').read())
1538
# Only one 'Authentication Required' error should occur
1539
self.assertEqual(1, self.server.auth_required_errors)
1541
def _check_password_prompt(self, scheme, user, actual_prompt):
1542
expected_prompt = (self._password_prompt_prefix
1543
+ ("%s %s@%s:%d, Realm: '%s' password: "
1545
user, self.server.host, self.server.port,
1546
self.server.auth_realm)))
1547
self.assertEquals(expected_prompt, actual_prompt)
1549
def test_no_prompt_for_password_when_using_auth_config(self):
1550
if self._testing_pycurl():
1551
raise tests.TestNotApplicable(
1552
'pycurl does not support authentication.conf'
1553
' since it cannot prompt')
1557
stdin_content = 'bar\n' # Not the right password
1558
self.server.add_user(user, password)
1559
t = self.get_user_transport(user, None)
1560
ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1561
stdout=tests.StringIOWrapper())
1562
# Create a minimal config file with the right password
1563
conf = config.AuthenticationConfig()
1564
conf._get_config().update(
1565
{'httptest': {'scheme': 'http', 'port': self.server.port,
1566
'user': user, 'password': password}})
1568
# Issue a request to the server to connect
1569
self.assertEqual('contents of a\n',t.get('a').read())
1570
# stdin should have been left untouched
1571
self.assertEqual(stdin_content, ui.ui_factory.stdin.readline())
1572
# Only one 'Authentication Required' error should occur
1573
self.assertEqual(1, self.server.auth_required_errors)
1575
def test_user_from_auth_conf(self):
1576
if self._testing_pycurl():
1577
raise tests.TestNotApplicable(
1578
'pycurl does not support authentication.conf')
1581
self.server.add_user(user, password)
1582
# Create a minimal config file with the right password
1583
conf = config.AuthenticationConfig()
1584
conf._get_config().update(
1585
{'httptest': {'scheme': 'http', 'port': self.server.port,
1586
'user': user, 'password': password}})
1588
t = self.get_user_transport(None, None)
1589
# Issue a request to the server to connect
1590
self.assertEqual('contents of a\n', t.get('a').read())
1591
# Only one 'Authentication Required' error should occur
1592
self.assertEqual(1, self.server.auth_required_errors)
1594
def test_changing_nonce(self):
1595
if self._auth_scheme != 'digest':
1596
raise tests.TestNotApplicable('HTTP auth digest only test')
1597
if self._testing_pycurl():
1598
raise tests.KnownFailure(
1599
'pycurl does not handle a nonce change')
1600
self.server.add_user('joe', 'foo')
1601
t = self.get_user_transport('joe', 'foo')
1602
self.assertEqual('contents of a\n', t.get('a').read())
1603
self.assertEqual('contents of b\n', t.get('b').read())
1604
# Only one 'Authentication Required' error should have
1606
self.assertEqual(1, self.server.auth_required_errors)
1607
# The server invalidates the current nonce
1608
self.server.auth_nonce = self.server.auth_nonce + '. No, now!'
1609
self.assertEqual('contents of a\n', t.get('a').read())
1610
# Two 'Authentication Required' errors should occur (the
1611
# initial 'who are you' and a second 'who are you' with the new nonce)
1612
self.assertEqual(2, self.server.auth_required_errors)
1616
class TestProxyAuth(TestAuth):
1617
"""Test proxy authentication schemes."""
1619
_auth_header = 'Proxy-authorization'
1620
_password_prompt_prefix='Proxy '
1623
super(TestProxyAuth, self).setUp()
1625
self.addCleanup(self._restore_env)
1626
# Override the contents to avoid false positives
1627
self.build_tree_contents([('a', 'not proxied contents of a\n'),
1628
('b', 'not proxied contents of b\n'),
1629
('a-proxied', 'contents of a\n'),
1630
('b-proxied', 'contents of b\n'),
1633
def create_transport_readonly_server(self):
1634
if self._auth_scheme == 'basic':
1635
server = http_utils.ProxyBasicAuthServer(
1636
protocol_version=self._protocol_version)
1638
if self._auth_scheme != 'digest':
1639
raise AssertionError('Unknown auth scheme: %r'
1640
% self._auth_scheme)
1641
server = http_utils.ProxyDigestAuthServer(
1642
protocol_version=self._protocol_version)
1645
def get_user_transport(self, user, password):
1646
self._install_env({'all_proxy': self.get_user_url(user, password)})
1647
return self._transport(self.server.get_url())
1649
def _install_env(self, env):
1650
for name, value in env.iteritems():
1651
self._old_env[name] = osutils.set_or_unset_env(name, value)
1653
def _restore_env(self):
1654
for name, value in self._old_env.iteritems():
1655
osutils.set_or_unset_env(name, value)
1657
def test_empty_pass(self):
1658
if self._testing_pycurl():
1660
if pycurl.version_info()[1] < '7.16.0':
1661
raise tests.KnownFailure(
1662
'pycurl < 7.16.0 does not handle empty proxy passwords')
1663
super(TestProxyAuth, self).test_empty_pass()
1666
class SampleSocket(object):
1667
"""A socket-like object for use in testing the HTTP request handler."""
1669
def __init__(self, socket_read_content):
1670
"""Constructs a sample socket.
1672
:param socket_read_content: a byte sequence
1674
# Use plain python StringIO so we can monkey-patch the close method to
1675
# not discard the contents.
1676
from StringIO import StringIO
1677
self.readfile = StringIO(socket_read_content)
1678
self.writefile = StringIO()
1679
self.writefile.close = lambda: None
1681
def makefile(self, mode='r', bufsize=None):
1683
return self.readfile
1685
return self.writefile
1688
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1691
super(SmartHTTPTunnellingTest, self).setUp()
1692
# We use the VFS layer as part of HTTP tunnelling tests.
1693
self._captureVar('BZR_NO_SMART_VFS', None)
1694
self.transport_readonly_server = http_utils.HTTPServerWithSmarts
1696
def create_transport_readonly_server(self):
1697
return http_utils.HTTPServerWithSmarts(
1698
protocol_version=self._protocol_version)
1700
def test_open_bzrdir(self):
1701
branch = self.make_branch('relpath')
1702
http_server = self.get_readonly_server()
1703
url = http_server.get_url() + 'relpath'
1704
bd = bzrdir.BzrDir.open(url)
1705
self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1707
def test_bulk_data(self):
1708
# We should be able to send and receive bulk data in a single message.
1709
# The 'readv' command in the smart protocol both sends and receives
1710
# bulk data, so we use that.
1711
self.build_tree(['data-file'])
1712
http_server = self.get_readonly_server()
1713
http_transport = self._transport(http_server.get_url())
1714
medium = http_transport.get_smart_medium()
1715
# Since we provide the medium, the url below will be mostly ignored
1716
# during the test, as long as the path is '/'.
1717
remote_transport = remote.RemoteTransport('bzr://fake_host/',
1720
[(0, "c")], list(remote_transport.readv("data-file", [(0,1)])))
1722
def test_http_send_smart_request(self):
1724
post_body = 'hello\n'
1725
expected_reply_body = 'ok\x012\n'
1727
http_server = self.get_readonly_server()
1728
http_transport = self._transport(http_server.get_url())
1729
medium = http_transport.get_smart_medium()
1730
response = medium.send_http_smart_request(post_body)
1731
reply_body = response.read()
1732
self.assertEqual(expected_reply_body, reply_body)
1734
def test_smart_http_server_post_request_handler(self):
1735
httpd = self.get_readonly_server()._get_httpd()
1737
socket = SampleSocket(
1738
'POST /.bzr/smart %s \r\n' % self._protocol_version
1739
# HTTP/1.1 posts must have a Content-Length (but it doesn't hurt
1741
+ 'Content-Length: 6\r\n'
1744
# Beware: the ('localhost', 80) below is the
1745
# client_address parameter, but we don't have one because
1746
# we have defined a socket which is not bound to an
1747
# address. The test framework never uses this client
1748
# address, so far...
1749
request_handler = http_utils.SmartRequestHandler(socket,
1752
response = socket.writefile.getvalue()
1753
self.assertStartsWith(response, '%s 200 ' % self._protocol_version)
1754
# This includes the end of the HTTP headers, and all the body.
1755
expected_end_of_response = '\r\n\r\nok\x012\n'
1756
self.assertEndsWith(response, expected_end_of_response)
1759
class ForbiddenRequestHandler(http_server.TestingHTTPRequestHandler):
1760
"""No smart server here request handler."""
1763
self.send_error(403, "Forbidden")
1766
class SmartClientAgainstNotSmartServer(TestSpecificRequestHandler):
1767
"""Test smart client behaviour against an http server without smarts."""
1769
_req_handler_class = ForbiddenRequestHandler
1771
def test_probe_smart_server(self):
1772
"""Test error handling against server refusing smart requests."""
1773
server = self.get_readonly_server()
1774
t = self._transport(server.get_url())
1775
# No need to build a valid smart request here, the server will not even
1776
# try to interpret it.
1777
self.assertRaises(errors.SmartProtocolError,
1778
t.get_smart_medium().send_http_smart_request,
1781
class Test_redirected_to(tests.TestCase):
1783
def test_redirected_to_subdir(self):
1784
t = self._transport('http://www.example.com/foo')
1785
r = t._redirected_to('http://www.example.com/foo',
1786
'http://www.example.com/foo/subdir')
1787
self.assertIsInstance(r, type(t))
1788
# Both transports share the some connection
1789
self.assertEquals(t._get_connection(), r._get_connection())
1791
def test_redirected_to_self_with_slash(self):
1792
t = self._transport('http://www.example.com/foo')
1793
r = t._redirected_to('http://www.example.com/foo',
1794
'http://www.example.com/foo/')
1795
self.assertIsInstance(r, type(t))
1796
# Both transports share the some connection (one can argue that we
1797
# should return the exact same transport here, but that seems
1799
self.assertEquals(t._get_connection(), r._get_connection())
1801
def test_redirected_to_host(self):
1802
t = self._transport('http://www.example.com/foo')
1803
r = t._redirected_to('http://www.example.com/foo',
1804
'http://foo.example.com/foo/subdir')
1805
self.assertIsInstance(r, type(t))
1807
def test_redirected_to_same_host_sibling_protocol(self):
1808
t = self._transport('http://www.example.com/foo')
1809
r = t._redirected_to('http://www.example.com/foo',
1810
'https://www.example.com/foo')
1811
self.assertIsInstance(r, type(t))
1813
def test_redirected_to_same_host_different_protocol(self):
1814
t = self._transport('http://www.example.com/foo')
1815
r = t._redirected_to('http://www.example.com/foo',
1816
'ftp://www.example.com/foo')
1817
self.assertNotEquals(type(r), type(t))
1819
def test_redirected_to_different_host_same_user(self):
1820
t = self._transport('http://joe@www.example.com/foo')
1821
r = t._redirected_to('http://www.example.com/foo',
1822
'https://foo.example.com/foo')
1823
self.assertIsInstance(r, type(t))
1824
self.assertEquals(t._user, r._user)
1827
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
1828
"""Request handler for a unique and pre-defined request.
1830
The only thing we care about here is how many bytes travel on the wire. But
1831
since we want to measure it for a real http client, we have to send it
1834
We expect to receive a *single* request nothing more (and we won't even
1835
check what request it is, we just measure the bytes read until an empty
1839
def handle_one_request(self):
1840
tcs = self.server.test_case_server
1841
requestline = self.rfile.readline()
1842
headers = self.MessageClass(self.rfile, 0)
1843
# We just read: the request, the headers, an empty line indicating the
1844
# end of the headers.
1845
bytes_read = len(requestline)
1846
for line in headers.headers:
1847
bytes_read += len(line)
1848
bytes_read += len('\r\n')
1849
if requestline.startswith('POST'):
1850
# The body should be a single line (or we don't know where it ends
1851
# and we don't want to issue a blocking read)
1852
body = self.rfile.readline()
1853
bytes_read += len(body)
1854
tcs.bytes_read = bytes_read
1856
# We set the bytes written *before* issuing the write, the client is
1857
# supposed to consume every produced byte *before* checking that value.
1859
# Doing the oppposite may lead to test failure: we may be interrupted
1860
# after the write but before updating the value. The client can then
1861
# continue and read the value *before* we can update it. And yes,
1862
# this has been observed -- vila 20090129
1863
tcs.bytes_written = len(tcs.canned_response)
1864
self.wfile.write(tcs.canned_response)
1867
class ActivityServerMixin(object):
1869
def __init__(self, protocol_version):
1870
super(ActivityServerMixin, self).__init__(
1871
request_handler=PredefinedRequestHandler,
1872
protocol_version=protocol_version)
1873
# Bytes read and written by the server
1875
self.bytes_written = 0
1876
self.canned_response = None
1879
class ActivityHTTPServer(ActivityServerMixin, http_server.HttpServer):
1883
if tests.HTTPSServerFeature.available():
1884
from bzrlib.tests import https_server
1885
class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
1889
class TestActivity(tests.TestCase):
1890
"""Test socket activity reporting.
1892
We use a special purpose server to control the bytes sent and received and
1893
be able to predict the activity on the client socket.
1897
tests.TestCase.setUp(self)
1898
self.server = self._activity_server(self._protocol_version)
1900
self.activities = {}
1901
def report_activity(t, bytes, direction):
1902
count = self.activities.get(direction, 0)
1904
self.activities[direction] = count
1906
# We override at class level because constructors may propagate the
1907
# bound method and render instance overriding ineffective (an
1908
# alternative would be to define a specific ui factory instead...)
1909
self.orig_report_activity = self._transport._report_activity
1910
self._transport._report_activity = report_activity
1913
self._transport._report_activity = self.orig_report_activity
1914
self.server.tearDown()
1915
tests.TestCase.tearDown(self)
1917
def get_transport(self):
1918
return self._transport(self.server.get_url())
1920
def assertActivitiesMatch(self):
1921
self.assertEqual(self.server.bytes_read,
1922
self.activities.get('write', 0), 'written bytes')
1923
self.assertEqual(self.server.bytes_written,
1924
self.activities.get('read', 0), 'read bytes')
1927
self.server.canned_response = '''HTTP/1.1 200 OK\r
1928
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
1929
Server: Apache/2.0.54 (Fedora)\r
1930
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
1931
ETag: "56691-23-38e9ae00"\r
1932
Accept-Ranges: bytes\r
1933
Content-Length: 35\r
1935
Content-Type: text/plain; charset=UTF-8\r
1937
Bazaar-NG meta directory, format 1
1939
t = self.get_transport()
1940
self.assertEqual('Bazaar-NG meta directory, format 1\n',
1941
t.get('foo/bar').read())
1942
self.assertActivitiesMatch()
1945
self.server.canned_response = '''HTTP/1.1 200 OK\r
1946
Server: SimpleHTTP/0.6 Python/2.5.2\r
1947
Date: Thu, 29 Jan 2009 20:21:47 GMT\r
1948
Content-type: application/octet-stream\r
1949
Content-Length: 20\r
1950
Last-Modified: Thu, 29 Jan 2009 20:21:47 GMT\r
1953
t = self.get_transport()
1954
self.assertTrue(t.has('foo/bar'))
1955
self.assertActivitiesMatch()
1957
def test_readv(self):
1958
self.server.canned_response = '''HTTP/1.1 206 Partial Content\r
1959
Date: Tue, 11 Jul 2006 04:49:48 GMT\r
1960
Server: Apache/2.0.54 (Fedora)\r
1961
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
1962
ETag: "238a3c-16ec2-805c5540"\r
1963
Accept-Ranges: bytes\r
1964
Content-Length: 1534\r
1966
Content-Type: multipart/byteranges; boundary=418470f848b63279b\r
1969
--418470f848b63279b\r
1970
Content-type: text/plain; charset=UTF-8\r
1971
Content-range: bytes 0-254/93890\r
1973
mbp@sourcefrog.net-20050309040815-13242001617e4a06
1974
mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e7627
1975
mbp@sourcefrog.net-20050309040957-6cad07f466bb0bb8
1976
mbp@sourcefrog.net-20050309041501-c840e09071de3b67
1977
mbp@sourcefrog.net-20050309044615-c24a3250be83220a
1979
--418470f848b63279b\r
1980
Content-type: text/plain; charset=UTF-8\r
1981
Content-range: bytes 1000-2049/93890\r
1984
mbp@sourcefrog.net-20050311063625-07858525021f270b
1985
mbp@sourcefrog.net-20050311231934-aa3776aff5200bb9
1986
mbp@sourcefrog.net-20050311231953-73aeb3a131c3699a
1987
mbp@sourcefrog.net-20050311232353-f5e33da490872c6a
1988
mbp@sourcefrog.net-20050312071639-0a8f59a34a024ff0
1989
mbp@sourcefrog.net-20050312073432-b2c16a55e0d6e9fb
1990
mbp@sourcefrog.net-20050312073831-a47c3335ece1920f
1991
mbp@sourcefrog.net-20050312085412-13373aa129ccbad3
1992
mbp@sourcefrog.net-20050313052251-2bf004cb96b39933
1993
mbp@sourcefrog.net-20050313052856-3edd84094687cb11
1994
mbp@sourcefrog.net-20050313053233-e30a4f28aef48f9d
1995
mbp@sourcefrog.net-20050313053853-7c64085594ff3072
1996
mbp@sourcefrog.net-20050313054757-a86c3f5871069e22
1997
mbp@sourcefrog.net-20050313061422-418f1f73b94879b9
1998
mbp@sourcefrog.net-20050313120651-497bd231b19df600
1999
mbp@sourcefrog.net-20050314024931-eae0170ef25a5d1a
2000
mbp@sourcefrog.net-20050314025438-d52099f915fe65fc
2001
mbp@sourcefrog.net-20050314025539-637a636692c055cf
2002
mbp@sourcefrog.net-20050314025737-55eb441f430ab4ba
2003
mbp@sourcefrog.net-20050314025901-d74aa93bb7ee8f62
2005
--418470f848b63279b--\r
2007
t = self.get_transport()
2008
# Remember that the request is ignored and that the ranges below
2009
# doesn't have to match the canned response.
2010
l = list(t.readv('/foo/bar', ((0, 255), (1000, 1050))))
2011
self.assertEqual(2, len(l))
2012
self.assertActivitiesMatch()
2014
def test_post(self):
2015
self.server.canned_response = '''HTTP/1.1 200 OK\r
2016
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
2017
Server: Apache/2.0.54 (Fedora)\r
2018
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
2019
ETag: "56691-23-38e9ae00"\r
2020
Accept-Ranges: bytes\r
2021
Content-Length: 35\r
2023
Content-Type: text/plain; charset=UTF-8\r
2025
lalala whatever as long as itsssss
2027
t = self.get_transport()
2028
# We must send a single line of body bytes, see
2029
# PredefinedRequestHandler.handle_one_request
2030
code, f = t._post('abc def end-of-body\n')
2031
self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2032
self.assertActivitiesMatch()
added
removed
bzrlib/tests/test_http.py
