← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_http.py

  • Committer: Tarmac
  • Author(s): Vincent Ladeuil, Patch Queue Manager, Jelmer Vernooij
  • Date: 2017-01-17 16:20:41 UTC
  • mfrom: (6619.1.2 trunk)
  • Revision ID: tarmac-20170117162041-oo62uk1qsmgc9j31
Merge 2.7 into trunk including fixes for bugs #1622039, #1644003, #1579093 and #1645017. [r=vila]

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_http.py
1
 
# Copyright (C) 2005-2011 Canonical Ltd
 
1
# Copyright (C) 2005-2012, 2015, 2016, 2017 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
31
31
 
32
32
import bzrlib
33
33
from bzrlib import (
34
 
    bzrdir,
35
 
    cethread,
36
34
    config,
 
35
    controldir,
37
36
    debug,
38
37
    errors,
39
38
    osutils,
128
127
        ('urllib,http', dict(_activity_server=ActivityHTTPServer,
129
128
                            _transport=_urllib.HttpTransport_urllib,)),
130
129
        ]
131
 
    if tests.HTTPSServerFeature.available():
132
 
        activity_scenarios.append(
133
 
            ('urllib,https', dict(_activity_server=ActivityHTTPSServer,
134
 
                                _transport=_urllib.HttpTransport_urllib,)),)
135
130
    if features.pycurl.available():
136
131
        activity_scenarios.append(
137
132
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
138
133
                                _transport=PyCurlTransport,)),)
139
 
        if tests.HTTPSServerFeature.available():
140
 
            from bzrlib.tests import (
141
 
                ssl_certs,
142
 
                )
143
 
            # FIXME: Until we have a better way to handle self-signed
144
 
            # certificates (like allowing them in a test specific
145
 
            # authentication.conf for example), we need some specialized pycurl
146
 
            # transport for tests.
 
134
    if features.HTTPSServerFeature.available():
 
135
        # FIXME: Until we have a better way to handle self-signed certificates
 
136
        # (like allowing them in a test specific authentication.conf for
 
137
        # example), we need some specialized pycurl/urllib transport for tests.
 
138
        # -- vila 2012-01-20
 
139
        from bzrlib.tests import (
 
140
            ssl_certs,
 
141
            )
 
142
        class HTTPS_urllib_transport(_urllib.HttpTransport_urllib):
 
143
 
 
144
            def __init__(self, base, _from_transport=None):
 
145
                super(HTTPS_urllib_transport, self).__init__(
 
146
                    base, _from_transport=_from_transport,
 
147
                    ca_certs=ssl_certs.build_path('ca.crt'))
 
148
 
 
149
        activity_scenarios.append(
 
150
            ('urllib,https', dict(_activity_server=ActivityHTTPSServer,
 
151
                                  _transport=HTTPS_urllib_transport,)),)
 
152
        if features.pycurl.available():
147
153
            class HTTPS_pycurl_transport(PyCurlTransport):
148
154
 
149
155
                def __init__(self, base, _from_transport=None):
254
260
        self.assertEqual('basic', scheme)
255
261
        self.assertEqual('realm="Thou should not pass"', remainder)
256
262
 
 
263
    def test_build_basic_header_with_long_creds(self):
 
264
        handler = _urllib2_wrappers.BasicAuthHandler()
 
265
        user = 'user' * 10  # length 40
 
266
        password = 'password' * 5  # length 40
 
267
        header = handler.build_auth_header(
 
268
            dict(user=user, password=password), None)
 
269
        # https://bugs.launchpad.net/bzr/+bug/1606203 was caused by incorrectly
 
270
        # creating a header value with an embedded '\n'
 
271
        self.assertFalse('\n' in header)
 
272
 
257
273
    def test_basic_extract_realm(self):
258
274
        scheme, remainder = self.parse_header(
259
275
            'Basic realm="Thou should not pass"',
282
298
        self.req_handler = RequestHandler(None, None, None)
283
299
 
284
300
    def assertRanges(self, ranges, header, file_size):
285
 
        self.assertEquals(ranges,
 
301
        self.assertEqual(ranges,
286
302
                          self.req_handler._parse_ranges(header, file_size))
287
303
 
288
304
    def test_simple_range(self):
378
394
    _transport = property(_get_pycurl_maybe)
379
395
 
380
396
 
381
 
class TestHttpUrls(tests.TestCase):
382
 
 
383
 
    # TODO: This should be moved to authorization tests once they
384
 
    # are written.
385
 
 
386
 
    def test_url_parsing(self):
387
 
        f = FakeManager()
388
 
        url = http.extract_auth('http://example.com', f)
389
 
        self.assertEqual('http://example.com', url)
390
 
        self.assertEqual(0, len(f.credentials))
391
 
        url = http.extract_auth(
392
 
            'http://user:pass@example.com/bzr/bzr.dev', f)
393
 
        self.assertEqual('http://example.com/bzr/bzr.dev', url)
394
 
        self.assertEqual(1, len(f.credentials))
395
 
        self.assertEqual([None, 'example.com', 'user', 'pass'],
396
 
                         f.credentials[0])
397
 
 
398
 
 
399
397
class TestHttpTransportUrls(tests.TestCase):
400
398
    """Test the http urls."""
401
399
 
481
479
        )
482
480
 
483
481
    def setUp(self):
484
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
482
        super(TestHTTPConnections, self).setUp()
485
483
        self.build_tree(['foo/', 'foo/bar'], line_endings='binary',
486
484
                        transport=self.get_transport())
487
485
 
533
531
    scenarios = vary_by_http_client_implementation()
534
532
 
535
533
    def test_http_registered(self):
536
 
        t = transport.get_transport('%s://foo.com/' % self._url_protocol)
 
534
        t = transport.get_transport_from_url(
 
535
            '%s://foo.com/' % self._url_protocol)
537
536
        self.assertIsInstance(t, transport.Transport)
538
537
        self.assertIsInstance(t, self._transport)
539
538
 
551
550
        self.start_server(server)
552
551
        url = server.get_url()
553
552
        # FIXME: needs a cleanup -- vila 20100611
554
 
        http_transport = transport.get_transport(url)
 
553
        http_transport = transport.get_transport_from_url(url)
555
554
        code, response = http_transport._post('abc def end-of-body')
556
555
        self.assertTrue(
557
556
            server.received_bytes.startswith('POST /.bzr/smart HTTP/1.'))
667
666
 
668
667
    _req_handler_class = BadStatusRequestHandler
669
668
 
 
669
    def setUp(self):
 
670
        super(TestBadStatusServer, self).setUp()
 
671
        # See https://bugs.launchpad.net/bzr/+bug/1451448 for details.
 
672
        # TD;LR: Running both a TCP client and server in the same process and
 
673
        # thread uncovers a race in python. The fix is to run the server in a
 
674
        # different process. Trying to fix yet another race here is not worth
 
675
        # the effort. -- vila 2015-09-06
 
676
        if 'HTTP/1.0' in self.id():
 
677
            raise tests.TestSkipped(
 
678
                'Client/Server in the same process and thread can hang')
 
679
 
670
680
    def test_http_has(self):
671
681
        t = self.get_readonly_transport()
672
 
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
 
682
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
 
683
                           errors.InvalidHttpResponse),
 
684
                          t.has, 'foo/bar')
673
685
 
674
686
    def test_http_get(self):
675
687
        t = self.get_readonly_transport()
676
 
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
 
688
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
 
689
                           errors.InvalidHttpResponse),
 
690
                          t.get, 'foo/bar')
677
691
 
678
692
 
679
693
class InvalidStatusRequestHandler(http_server.TestingHTTPRequestHandler):
1048
1062
        self.assertEqual('single', t._range_hint)
1049
1063
 
1050
1064
 
 
1065
class TruncatedBeforeBoundaryRequestHandler(
 
1066
    http_server.TestingHTTPRequestHandler):
 
1067
    """Truncation before a boundary, like in bug 198646"""
 
1068
 
 
1069
    _truncated_ranges = 1
 
1070
 
 
1071
    def get_multiple_ranges(self, file, file_size, ranges):
 
1072
        self.send_response(206)
 
1073
        self.send_header('Accept-Ranges', 'bytes')
 
1074
        boundary = 'tagada'
 
1075
        self.send_header('Content-Type',
 
1076
                         'multipart/byteranges; boundary=%s' % boundary)
 
1077
        boundary_line = '--%s\r\n' % boundary
 
1078
        # Calculate the Content-Length
 
1079
        content_length = 0
 
1080
        for (start, end) in ranges:
 
1081
            content_length += len(boundary_line)
 
1082
            content_length += self._header_line_length(
 
1083
                'Content-type', 'application/octet-stream')
 
1084
            content_length += self._header_line_length(
 
1085
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
 
1086
            content_length += len('\r\n') # end headers
 
1087
            content_length += end - start # + 1
 
1088
        content_length += len(boundary_line)
 
1089
        self.send_header('Content-length', content_length)
 
1090
        self.end_headers()
 
1091
 
 
1092
        # Send the multipart body
 
1093
        cur = 0
 
1094
        for (start, end) in ranges:
 
1095
            if cur + self._truncated_ranges >= len(ranges):
 
1096
                # Abruptly ends the response and close the connection
 
1097
                self.close_connection = 1
 
1098
                return
 
1099
            self.wfile.write(boundary_line)
 
1100
            self.send_header('Content-type', 'application/octet-stream')
 
1101
            self.send_header('Content-Range', 'bytes %d-%d/%d'
 
1102
                             % (start, end, file_size))
 
1103
            self.end_headers()
 
1104
            self.send_range_content(file, start, end - start + 1)
 
1105
            cur += 1
 
1106
        # Final boundary
 
1107
        self.wfile.write(boundary_line)
 
1108
 
 
1109
 
 
1110
class TestTruncatedBeforeBoundary(TestSpecificRequestHandler):
 
1111
    """Tests the case of bug 198646, disconnecting before a boundary."""
 
1112
 
 
1113
    _req_handler_class = TruncatedBeforeBoundaryRequestHandler
 
1114
 
 
1115
    def setUp(self):
 
1116
        super(TestTruncatedBeforeBoundary, self).setUp()
 
1117
        self.build_tree_contents([('a', '0123456789')],)
 
1118
 
 
1119
    def test_readv_with_short_reads(self):
 
1120
        server = self.get_readonly_server()
 
1121
        t = self.get_readonly_transport()
 
1122
        # Force separate ranges for each offset
 
1123
        t._bytes_to_read_before_seek = 0
 
1124
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
 
1125
        self.assertEqual((0, '0'), ireadv.next())
 
1126
        self.assertEqual((2, '2'), ireadv.next())
 
1127
        self.assertEqual((4, '45'), ireadv.next())
 
1128
        self.assertEqual((9, '9'), ireadv.next())
 
1129
 
 
1130
 
1051
1131
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1052
1132
    """Errors out when range specifiers exceed the limit"""
1053
1133
 
1090
1170
                                      protocol_version=self._protocol_version)
1091
1171
 
1092
1172
    def setUp(self):
1093
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
1173
        super(TestLimitedRangeRequestServer, self).setUp()
1094
1174
        # We need to manipulate ranges that correspond to real chunks in the
1095
1175
        # response, so we build a content appropriately.
1096
1176
        filler = ''.join(['abcdefghij' for x in range(102)])
1130
1210
 
1131
1211
    def assertEvaluateProxyBypass(self, expected, host, no_proxy):
1132
1212
        handler = _urllib2_wrappers.ProxyHandler()
1133
 
        self.assertEquals(expected,
 
1213
        self.assertEqual(expected,
1134
1214
                          handler.evaluate_proxy_bypass(host, no_proxy))
1135
1215
 
1136
1216
    def test_empty_user(self):
1203
1283
            self.no_proxy_host = self.server_host_port
1204
1284
        # The secondary server is the proxy
1205
1285
        self.proxy_url = self.get_secondary_url()
 
1286
        if self._testing_pycurl():
 
1287
            self.proxy_url = self.proxy_url.replace('+pycurl', '')
1206
1288
 
1207
1289
    def _testing_pycurl(self):
1208
1290
        # TODO: This is duplicated for lots of the classes in this file
1282
1364
        )
1283
1365
 
1284
1366
    def setUp(self):
1285
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
1367
        super(TestRanges, self).setUp()
1286
1368
        self.build_tree_contents([('a', '0123456789')],)
1287
1369
 
1288
1370
    def create_transport_readonly_server(self):
1529
1611
            path='/',
1530
1612
            realm='Realm',
1531
1613
            ))
1532
 
        self.assertEquals((user, password), got_pass)
 
1614
        self.assertEqual((user, password), got_pass)
1533
1615
 
1534
1616
 
1535
1617
class TestAuth(http_utils.TestCaseWithWebserver):
1569
1651
        return url
1570
1652
 
1571
1653
    def get_user_transport(self, user, password):
1572
 
        t = transport.get_transport(self.get_user_url(user, password))
 
1654
        t = transport.get_transport_from_url(
 
1655
            self.get_user_url(user, password))
1573
1656
        return t
1574
1657
 
1575
1658
    def test_no_user(self):
1701
1784
                                     http_utils.ProxyDigestAuthServer):
1702
1785
            raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1703
1786
        if self._testing_pycurl():
1704
 
            raise tests.KnownFailure(
 
1787
            self.knownFailure(
1705
1788
                'pycurl does not handle a nonce change')
1706
1789
        self.server.add_user('joe', 'foo')
1707
1790
        t = self.get_user_transport('joe', 'foo')
1782
1865
                                  ])
1783
1866
 
1784
1867
    def get_user_transport(self, user, password):
1785
 
        self.overrideEnv('all_proxy', self.get_user_url(user, password))
 
1868
        proxy_url = self.get_user_url(user, password)
 
1869
        if self._testing_pycurl():
 
1870
            proxy_url = proxy_url.replace('+pycurl', '')
 
1871
        self.overrideEnv('all_proxy', proxy_url)
1786
1872
        return TestAuth.get_user_transport(self, user, password)
1787
1873
 
1788
1874
    def test_empty_pass(self):
1789
1875
        if self._testing_pycurl():
1790
1876
            import pycurl
1791
1877
            if pycurl.version_info()[1] < '7.16.0':
1792
 
                raise tests.KnownFailure(
 
1878
                self.knownFailure(
1793
1879
                    'pycurl < 7.16.0 does not handle empty proxy passwords')
1794
1880
        super(TestProxyAuth, self).test_empty_pass()
1795
1881
 
1837
1923
        server._url_protocol = self._url_protocol
1838
1924
        return server
1839
1925
 
1840
 
    def test_open_bzrdir(self):
 
1926
    def test_open_controldir(self):
1841
1927
        branch = self.make_branch('relpath')
1842
1928
        url = self.http_server.get_url() + 'relpath'
1843
 
        bd = bzrdir.BzrDir.open(url)
 
1929
        bd = controldir.ControlDir.open(url)
1844
1930
        self.addCleanup(bd.transport.disconnect)
1845
1931
        self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1846
1932
 
1849
1935
        # The 'readv' command in the smart protocol both sends and receives
1850
1936
        # bulk data, so we use that.
1851
1937
        self.build_tree(['data-file'])
1852
 
        http_transport = transport.get_transport(self.http_server.get_url())
 
1938
        http_transport = transport.get_transport_from_url(
 
1939
            self.http_server.get_url())
1853
1940
        medium = http_transport.get_smart_medium()
1854
1941
        # Since we provide the medium, the url below will be mostly ignored
1855
1942
        # during the test, as long as the path is '/'.
1863
1950
        post_body = 'hello\n'
1864
1951
        expected_reply_body = 'ok\x012\n'
1865
1952
 
1866
 
        http_transport = transport.get_transport(self.http_server.get_url())
 
1953
        http_transport = transport.get_transport_from_url(
 
1954
            self.http_server.get_url())
1867
1955
        medium = http_transport.get_smart_medium()
1868
1956
        response = medium.send_http_smart_request(post_body)
1869
1957
        reply_body = response.read()
1927
2015
        self.assertIsInstance(r, type(t))
1928
2016
        # Both transports share the some connection
1929
2017
        self.assertEqual(t._get_connection(), r._get_connection())
 
2018
        self.assertEqual('http://www.example.com/foo/subdir/', r.base)
1930
2019
 
1931
2020
    def test_redirected_to_self_with_slash(self):
1932
2021
        t = self._transport('http://www.example.com/foo')
1943
2032
        r = t._redirected_to('http://www.example.com/foo',
1944
2033
                             'http://foo.example.com/foo/subdir')
1945
2034
        self.assertIsInstance(r, type(t))
 
2035
        self.assertEqual('http://foo.example.com/foo/subdir/',
 
2036
            r.external_url())
1946
2037
 
1947
2038
    def test_redirected_to_same_host_sibling_protocol(self):
1948
2039
        t = self._transport('http://www.example.com/foo')
1949
2040
        r = t._redirected_to('http://www.example.com/foo',
1950
2041
                             'https://www.example.com/foo')
1951
2042
        self.assertIsInstance(r, type(t))
 
2043
        self.assertEqual('https://www.example.com/foo/',
 
2044
            r.external_url())
1952
2045
 
1953
2046
    def test_redirected_to_same_host_different_protocol(self):
1954
2047
        t = self._transport('http://www.example.com/foo')
1955
2048
        r = t._redirected_to('http://www.example.com/foo',
1956
2049
                             'ftp://www.example.com/foo')
1957
 
        self.assertNotEquals(type(r), type(t))
 
2050
        self.assertNotEqual(type(r), type(t))
 
2051
        self.assertEqual('ftp://www.example.com/foo/', r.external_url())
 
2052
 
 
2053
    def test_redirected_to_same_host_specific_implementation(self):
 
2054
        t = self._transport('http://www.example.com/foo')
 
2055
        r = t._redirected_to('http://www.example.com/foo',
 
2056
                             'https+urllib://www.example.com/foo')
 
2057
        self.assertEqual('https://www.example.com/foo/', r.external_url())
1958
2058
 
1959
2059
    def test_redirected_to_different_host_same_user(self):
1960
2060
        t = self._transport('http://joe@www.example.com/foo')
1961
2061
        r = t._redirected_to('http://www.example.com/foo',
1962
2062
                             'https://foo.example.com/foo')
1963
2063
        self.assertIsInstance(r, type(t))
1964
 
        self.assertEqual(t._user, r._user)
 
2064
        self.assertEqual(t._parsed_url.user, r._parsed_url.user)
 
2065
        self.assertEqual('https://joe@foo.example.com/foo/', r.external_url())
1965
2066
 
1966
2067
 
1967
2068
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
2020
2121
    pass
2021
2122
 
2022
2123
 
2023
 
if tests.HTTPSServerFeature.available():
 
2124
if features.HTTPSServerFeature.available():
2024
2125
    from bzrlib.tests import https_server
2025
2126
    class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
2026
2127
        pass
2034
2135
    """
2035
2136
 
2036
2137
    def setUp(self):
2037
 
        tests.TestCase.setUp(self)
2038
2138
        self.server = self._activity_server(self._protocol_version)
2039
2139
        self.server.start_server()
 
2140
        self.addCleanup(self.server.stop_server)
2040
2141
        _activities = {} # Don't close over self and create a cycle
2041
2142
        def report_activity(t, bytes, direction):
2042
2143
            count = _activities.get(direction, 0)
2043
2144
            count += bytes
2044
2145
            _activities[direction] = count
2045
2146
        self.activities = _activities
2046
 
 
2047
2147
        # We override at class level because constructors may propagate the
2048
2148
        # bound method and render instance overriding ineffective (an
2049
2149
        # alternative would be to define a specific ui factory instead...)
2050
2150
        self.overrideAttr(self._transport, '_report_activity', report_activity)
2051
 
        self.addCleanup(self.server.stop_server)
2052
2151
 
2053
2152
    def get_transport(self):
2054
2153
        t = self._transport(self.server.get_url())
2178
2277
        )
2179
2278
 
2180
2279
    def setUp(self):
 
2280
        super(TestActivity, self).setUp()
2181
2281
        TestActivityMixin.setUp(self)
2182
2282
 
2183
2283
 
2192
2292
    _protocol_version = 'HTTP/1.1'
2193
2293
 
2194
2294
    def setUp(self):
 
2295
        super(TestNoReportActivity, self).setUp()
2195
2296
        self._transport =_urllib.HttpTransport_urllib
2196
2297
        TestActivityMixin.setUp(self)
2197
2298