~bzr-pqm/bzr/bzr.dev

6615.8.2 by Vincent Ladeuil
Merge 2.7
1
# Copyright (C) 2005-2012, 2015, 2016, 2017 Canonical Ltd
1540.3.24 by Martin Pool
Add new protocol 'http+pycurl' that always uses PyCurl.
2
#
1540.3.15 by Martin Pool
[merge] large merge to sync with bzr.dev
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1540.3.24 by Martin Pool
Add new protocol 'http+pycurl' that always uses PyCurl.
7
#
1540.3.15 by Martin Pool
[merge] large merge to sync with bzr.dev
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1540.3.24 by Martin Pool
Add new protocol 'http+pycurl' that always uses PyCurl.
12
#
1540.3.15 by Martin Pool
[merge] large merge to sync with bzr.dev
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
1185.16.68 by Martin Pool
- http url fixes suggested by Robey Pointer, and tests
16
3111.1.30 by Vincent Ladeuil
Update NEWS. Some cosmetic changes.
17
"""Tests for HTTP implementations.
3111.1.10 by Vincent Ladeuil
Finish http parameterization, 24 auth tests failing for pycurl (not
18
3111.1.30 by Vincent Ladeuil
Update NEWS. Some cosmetic changes.
19
This module defines a load_tests() method that parametrize tests classes for
20
transport implementation, http protocol versions and authentication schemes.
3111.1.10 by Vincent Ladeuil
Finish http parameterization, 24 auth tests failing for pycurl (not
21
"""
1540.3.3 by Martin Pool
Review updates of pycurl transport
22
1540.3.22 by Martin Pool
[patch] Add TestCase.assertIsInstance
23
# TODO: Should be renamed to bzrlib.transport.http.tests?
2167.3.5 by v.ladeuil+lp at free
Tests for proxies, covering #74759.
24
# TODO: What about renaming to bzrlib.tests.transport.http ?
1540.3.22 by Martin Pool
[patch] Add TestCase.assertIsInstance
25
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
26
import httplib
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
27
import SimpleHTTPServer
2000.2.2 by John Arbash Meinel
Update the urllib.has test.
28
import socket
2420.1.20 by Vincent Ladeuil
Fix test failure on pqm.
29
import sys
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
30
import threading
2000.2.2 by John Arbash Meinel
Update the urllib.has test.
31
1553.1.2 by James Henstridge
Add a test to make sure the user-agent header is being sent correctly.
32
import bzrlib
2164.2.22 by Vincent Ladeuil
Take Aaron's review comments into account.
33
from bzrlib import (
2900.2.6 by Vincent Ladeuil
Make http aware of authentication config.
34
    config,
6472.2.3 by Jelmer Vernooij
More control dir.
35
    controldir,
5957.2.3 by Vincent Ladeuil
Mask credentials in the -Dhttp logging
36
    debug,
2164.2.22 by Vincent Ladeuil
Take Aaron's review comments into account.
37
    errors,
38
    osutils,
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
39
    remote as _mod_remote,
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
40
    tests,
5957.2.3 by Vincent Ladeuil
Mask credentials in the -Dhttp logging
41
    trace,
3111.1.10 by Vincent Ladeuil
Finish http parameterization, 24 auth tests failing for pycurl (not
42
    transport,
2363.4.12 by Vincent Ladeuil
Take jam's review comments into account. Fix typos, give better
43
    ui,
3995.2.2 by Martin Pool
Cope with read_bundle_from_url deprecation in test_http
44
    )
3102.1.1 by Vincent Ladeuil
Rename bzrlib/test/HTTPTestUtils.py to bzrlib/tests/http_utils.py and fix
45
from bzrlib.tests import (
4913.2.11 by John Arbash Meinel
Convert a bunch more features over to ModuleAvailableFeature
46
    features,
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
47
    http_server,
3111.1.7 by Vincent Ladeuil
Further refactoring.
48
    http_utils,
5247.2.5 by Vincent Ladeuil
Some cleanups.
49
    test_server,
3102.1.1 by Vincent Ladeuil
Rename bzrlib/test/HTTPTestUtils.py to bzrlib/tests/http_utils.py and fix
50
    )
5462.3.15 by Martin Pool
Turn variations into scenario lists
51
from bzrlib.tests.scenarios import (
5462.3.21 by Martin Pool
Rename to load_tests_apply_scenarios
52
    load_tests_apply_scenarios,
5462.3.15 by Martin Pool
Turn variations into scenario lists
53
    multiply_scenarios,
5462.3.2 by Martin Pool
Split variations code to bzrlib.tests.variations
54
    )
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
55
from bzrlib.transport import (
56
    http,
57
    remote,
58
    )
2004.3.3 by vila
Better (but still incomplete) design for bogus servers.
59
from bzrlib.transport.http import (
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
60
    _urllib,
2164.2.29 by Vincent Ladeuil
Test the http redirection at the request level even if it's not
61
    _urllib2_wrappers,
2004.3.3 by vila
Better (but still incomplete) design for bogus servers.
62
    )
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
63
64
4913.2.11 by John Arbash Meinel
Convert a bunch more features over to ModuleAvailableFeature
65
if features.pycurl.available():
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
66
    from bzrlib.transport.http._pycurl import PyCurlTransport
67
68
5462.3.21 by Martin Pool
Rename to load_tests_apply_scenarios
69
load_tests = load_tests_apply_scenarios
5462.3.15 by Martin Pool
Turn variations into scenario lists
70
71
72
def vary_by_http_client_implementation():
5462.3.1 by Martin Pool
Split out generation of scenario lists into TestVariations
73
    """Test the two libraries we can use, pycurl and urllib."""
5462.3.15 by Martin Pool
Turn variations into scenario lists
74
    transport_scenarios = [
75
        ('urllib', dict(_transport=_urllib.HttpTransport_urllib,
76
                        _server=http_server.HttpServer_urllib,
77
                        _url_protocol='http+urllib',)),
78
        ]
79
    if features.pycurl.available():
80
        transport_scenarios.append(
81
            ('pycurl', dict(_transport=PyCurlTransport,
82
                            _server=http_server.HttpServer_PyCurl,
83
                            _url_protocol='http+pycurl',)))
84
    return transport_scenarios
85
86
87
def vary_by_http_protocol_version():
5462.3.1 by Martin Pool
Split out generation of scenario lists into TestVariations
88
    """Test on http/1.0 and 1.1"""
5462.3.15 by Martin Pool
Turn variations into scenario lists
89
    return [
90
        ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
91
        ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
92
        ]
93
94
5957.2.1 by Vincent Ladeuil
Tweak the http Authorization tests to make some parts easier to reuse (parametrization mostly).
95
def vary_by_http_auth_scheme():
96
    scenarios = [
97
        ('basic', dict(_auth_server=http_utils.HTTPBasicAuthServer)),
98
        ('digest', dict(_auth_server=http_utils.HTTPDigestAuthServer)),
99
        ('basicdigest',
100
            dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
101
        ]
102
    # Add some attributes common to all scenarios
103
    for scenario_id, scenario_dict in scenarios:
5957.2.3 by Vincent Ladeuil
Mask credentials in the -Dhttp logging
104
        scenario_dict.update(_auth_header='Authorization',
105
                             _username_prompt_prefix='',
5957.2.1 by Vincent Ladeuil
Tweak the http Authorization tests to make some parts easier to reuse (parametrization mostly).
106
                             _password_prompt_prefix='')
107
    return scenarios
108
109
5462.3.15 by Martin Pool
Turn variations into scenario lists
110
def vary_by_http_proxy_auth_scheme():
5957.2.1 by Vincent Ladeuil
Tweak the http Authorization tests to make some parts easier to reuse (parametrization mostly).
111
    scenarios = [
5957.2.3 by Vincent Ladeuil
Mask credentials in the -Dhttp logging
112
        ('proxy-basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
113
        ('proxy-digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
114
        ('proxy-basicdigest',
5462.3.15 by Martin Pool
Turn variations into scenario lists
115
            dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
116
        ]
5957.2.1 by Vincent Ladeuil
Tweak the http Authorization tests to make some parts easier to reuse (parametrization mostly).
117
    # Add some attributes common to all scenarios
118
    for scenario_id, scenario_dict in scenarios:
5957.2.3 by Vincent Ladeuil
Mask credentials in the -Dhttp logging
119
        scenario_dict.update(_auth_header='Proxy-Authorization',
120
                             _username_prompt_prefix='Proxy ',
5957.2.1 by Vincent Ladeuil
Tweak the http Authorization tests to make some parts easier to reuse (parametrization mostly).
121
                             _password_prompt_prefix='Proxy ')
122
    return scenarios
5462.3.15 by Martin Pool
Turn variations into scenario lists
123
124
125
def vary_by_http_activity():
126
    activity_scenarios = [
127
        ('urllib,http', dict(_activity_server=ActivityHTTPServer,
128
                            _transport=_urllib.HttpTransport_urllib,)),
129
        ]
6238.2.23 by Vincent Ladeuil
Using the right transport classes, creating a dedicated config store is not needed anymore.
130
    if features.pycurl.available():
131
        activity_scenarios.append(
132
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
133
                                _transport=PyCurlTransport,)),)
5967.12.1 by Martin Pool
Move all test features into bzrlib.tests.features
134
    if features.HTTPSServerFeature.available():
6238.2.23 by Vincent Ladeuil
Using the right transport classes, creating a dedicated config store is not needed anymore.
135
        # FIXME: Until we have a better way to handle self-signed certificates
136
        # (like allowing them in a test specific authentication.conf for
137
        # example), we need some specialized pycurl/urllib transport for tests.
138
        # -- vila 2012-01-20
139
        from bzrlib.tests import (
140
            ssl_certs,
141
            )
142
        class HTTPS_urllib_transport(_urllib.HttpTransport_urllib):
143
144
            def __init__(self, base, _from_transport=None):
145
                super(HTTPS_urllib_transport, self).__init__(
146
                    base, _from_transport=_from_transport,
147
                    ca_certs=ssl_certs.build_path('ca.crt'))
148
5462.3.15 by Martin Pool
Turn variations into scenario lists
149
        activity_scenarios.append(
150
            ('urllib,https', dict(_activity_server=ActivityHTTPSServer,
6238.2.23 by Vincent Ladeuil
Using the right transport classes, creating a dedicated config store is not needed anymore.
151
                                  _transport=HTTPS_urllib_transport,)),)
152
        if features.pycurl.available():
5462.3.15 by Martin Pool
Turn variations into scenario lists
153
            class HTTPS_pycurl_transport(PyCurlTransport):
154
155
                def __init__(self, base, _from_transport=None):
156
                    super(HTTPS_pycurl_transport, self).__init__(
157
                        base, _from_transport)
158
                    self.cabundle = str(ssl_certs.build_path('ca.crt'))
159
160
            activity_scenarios.append(
161
                ('pycurl,https', dict(_activity_server=ActivityHTTPSServer,
162
                                    _transport=HTTPS_pycurl_transport,)),)
163
    return activity_scenarios
5462.3.1 by Martin Pool
Split out generation of scenario lists into TestVariations
164
165
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
166
class FakeManager(object):
1786.1.8 by John Arbash Meinel
[merge] Johan Rydberg test updates
167
1185.40.20 by Robey Pointer
allow user:pass@ info in http urls to be used for auth; this should be easily expandable later to use auth config files
168
    def __init__(self):
169
        self.credentials = []
2004.3.1 by vila
Test ConnectionError exceptions.
170
1185.40.20 by Robey Pointer
allow user:pass@ info in http urls to be used for auth; this should be easily expandable later to use auth config files
171
    def add_password(self, realm, host, username, password):
172
        self.credentials.append([realm, host, username, password])
173
1553.1.2 by James Henstridge
Add a test to make sure the user-agent header is being sent correctly.
174
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
175
class RecordingServer(object):
176
    """A fake HTTP server.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
177
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
178
    It records the bytes sent to it, and replies with a 200.
179
    """
180
4691.2.1 by Robert Collins
Add stronger test isolation by interception BzrDir.open and checking the thing being opened is known to the test suite.
181
    def __init__(self, expect_body_tail=None, scheme=''):
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
182
        """Constructor.
183
184
        :type expect_body_tail: str
185
        :param expect_body_tail: a reply won't be sent until this string is
186
            received.
187
        """
188
        self._expect_body_tail = expect_body_tail
189
        self.host = None
190
        self.port = None
191
        self.received_bytes = ''
4691.2.1 by Robert Collins
Add stronger test isolation by interception BzrDir.open and checking the thing being opened is known to the test suite.
192
        self.scheme = scheme
193
194
    def get_url(self):
195
        return '%s://%s:%s/' % (self.scheme, self.host, self.port)
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
196
4934.3.3 by Martin Pool
Rename Server.setUp to Server.start_server
197
    def start_server(self):
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
198
        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
199
        self._sock.bind(('127.0.0.1', 0))
200
        self.host, self.port = self._sock.getsockname()
201
        self._ready = threading.Event()
5652.1.1 by Vincent Ladeuil
Split ThreadWithException out of the tests hierarchy.
202
        self._thread = test_server.TestThread(
5652.1.2 by Vincent Ladeuil
Use clearer names.
203
            sync_event=self._ready, target=self._accept_read_and_reply)
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
204
        self._thread.start()
4731.2.9 by Vincent Ladeuil
Implement a new -Ethreads to better track the leaks.
205
        if 'threads' in tests.selftest_debug_flags:
5247.5.29 by Vincent Ladeuil
Fixed as per jam's review.
206
            sys.stderr.write('Thread started: %s\n' % (self._thread.ident,))
4731.2.4 by Vincent Ladeuil
No more leaks in http tests.
207
        self._ready.wait()
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
208
209
    def _accept_read_and_reply(self):
210
        self._sock.listen(1)
211
        self._ready.set()
5247.2.5 by Vincent Ladeuil
Some cleanups.
212
        conn, address = self._sock.accept()
213
        if self._expect_body_tail is not None:
214
            while not self.received_bytes.endswith(self._expect_body_tail):
215
                self.received_bytes += conn.recv(4096)
216
            conn.sendall('HTTP/1.1 200 OK\r\n')
4731.2.4 by Vincent Ladeuil
No more leaks in http tests.
217
        try:
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
218
            self._sock.close()
219
        except socket.error:
220
            # The client may have already closed the socket.
221
            pass
222
4934.3.1 by Martin Pool
Rename Server.tearDown to .stop_server
223
    def stop_server(self):
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
224
        try:
4731.2.4 by Vincent Ladeuil
No more leaks in http tests.
225
            # Issue a fake connection to wake up the server and allow it to
226
            # finish quickly
5247.3.7 by Vincent Ladeuil
Provide connect_socket (socket.create_connection) for pythons older than 2.6.
227
            fake_conn = osutils.connect_socket((self.host, self.port))
4731.2.4 by Vincent Ladeuil
No more leaks in http tests.
228
            fake_conn.close()
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
229
        except socket.error:
230
            # We might have already closed it.  We don't care.
231
            pass
232
        self.host = None
233
        self.port = None
4731.2.4 by Vincent Ladeuil
No more leaks in http tests.
234
        self._thread.join()
4731.2.9 by Vincent Ladeuil
Implement a new -Ethreads to better track the leaks.
235
        if 'threads' in tests.selftest_debug_flags:
5247.5.29 by Vincent Ladeuil
Fixed as per jam's review.
236
            sys.stderr.write('Thread  joined: %s\n' % (self._thread.ident,))
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
237
238
4050.2.2 by Vincent Ladeuil
Ensures all auth handlers correctly parse all auth headers.
239
class TestAuthHeader(tests.TestCase):
240
4284.1.1 by Vincent Ladeuil
Fix wrong realm extraction in http basic authentication (reported
241
    def parse_header(self, header, auth_handler_class=None):
242
        if auth_handler_class is None:
243
            auth_handler_class = _urllib2_wrappers.AbstractAuthHandler
244
        self.auth_handler =  auth_handler_class()
245
        return self.auth_handler._parse_auth_header(header)
4050.2.2 by Vincent Ladeuil
Ensures all auth handlers correctly parse all auth headers.
246
247
    def test_empty_header(self):
248
        scheme, remainder = self.parse_header('')
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
249
        self.assertEqual('', scheme)
4050.2.2 by Vincent Ladeuil
Ensures all auth handlers correctly parse all auth headers.
250
        self.assertIs(None, remainder)
251
252
    def test_negotiate_header(self):
253
        scheme, remainder = self.parse_header('Negotiate')
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
254
        self.assertEqual('negotiate', scheme)
4050.2.2 by Vincent Ladeuil
Ensures all auth handlers correctly parse all auth headers.
255
        self.assertIs(None, remainder)
256
257
    def test_basic_header(self):
258
        scheme, remainder = self.parse_header(
259
            'Basic realm="Thou should not pass"')
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
260
        self.assertEqual('basic', scheme)
261
        self.assertEqual('realm="Thou should not pass"', remainder)
4050.2.2 by Vincent Ladeuil
Ensures all auth handlers correctly parse all auth headers.
262
6615.4.1 by Vincent Ladeuil
Fix long user and password header generation for Basic http authentification
263
    def test_build_basic_header_with_long_creds(self):
264
        handler = _urllib2_wrappers.BasicAuthHandler()
265
        user = 'user' * 10  # length 40
266
        password = 'password' * 5  # length 40
267
        header = handler.build_auth_header(
268
            dict(user=user, password=password), None)
269
        # https://bugs.launchpad.net/bzr/+bug/1606203 was caused by incorrectly
270
        # creating a header value with an embedded '\n'
271
        self.assertFalse('\n' in header)
272
4284.1.1 by Vincent Ladeuil
Fix wrong realm extraction in http basic authentication (reported
273
    def test_basic_extract_realm(self):
274
        scheme, remainder = self.parse_header(
275
            'Basic realm="Thou should not pass"',
276
            _urllib2_wrappers.BasicAuthHandler)
277
        match, realm = self.auth_handler.extract_realm(remainder)
278
        self.assertTrue(match is not None)
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
279
        self.assertEqual('Thou should not pass', realm)
4284.1.1 by Vincent Ladeuil
Fix wrong realm extraction in http basic authentication (reported
280
4050.2.2 by Vincent Ladeuil
Ensures all auth handlers correctly parse all auth headers.
281
    def test_digest_header(self):
282
        scheme, remainder = self.parse_header(
283
            'Digest realm="Thou should not pass"')
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
284
        self.assertEqual('digest', scheme)
285
        self.assertEqual('realm="Thou should not pass"', remainder)
4050.2.2 by Vincent Ladeuil
Ensures all auth handlers correctly parse all auth headers.
286
287
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
288
class TestHTTPRangeParsing(tests.TestCase):
289
290
    def setUp(self):
291
        super(TestHTTPRangeParsing, self).setUp()
292
        # We focus on range  parsing here and ignore everything else
293
        class RequestHandler(http_server.TestingHTTPRequestHandler):
294
            def setup(self): pass
295
            def handle(self): pass
296
            def finish(self): pass
297
5705.1.2 by Vincent Ladeuil
Fix spurious space and add a test.
298
        self.req_handler = RequestHandler(None, None, None)
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
299
300
    def assertRanges(self, ranges, header, file_size):
6614.1.3 by Vincent Ladeuil
Fix assertEquals being deprecated by using assertEqual.
301
        self.assertEqual(ranges,
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
302
                          self.req_handler._parse_ranges(header, file_size))
303
304
    def test_simple_range(self):
305
        self.assertRanges([(0,2)], 'bytes=0-2', 12)
306
307
    def test_tail(self):
308
        self.assertRanges([(8, 11)], 'bytes=-4', 12)
309
310
    def test_tail_bigger_than_file(self):
311
        self.assertRanges([(0, 11)], 'bytes=-99', 12)
312
313
    def test_range_without_end(self):
314
        self.assertRanges([(4, 11)], 'bytes=4-', 12)
315
316
    def test_invalid_ranges(self):
317
        self.assertRanges(None, 'bytes=12-22', 12)
318
        self.assertRanges(None, 'bytes=1-3,12-22', 12)
5705.1.2 by Vincent Ladeuil
Fix spurious space and add a test.
319
        self.assertRanges(None, 'bytes=-', 12)
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
320
321
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
322
class TestHTTPServer(tests.TestCase):
323
    """Test the HTTP servers implementations."""
324
325
    def test_invalid_protocol(self):
326
        class BogusRequestHandler(http_server.TestingHTTPRequestHandler):
327
328
            protocol_version = 'HTTP/0.1'
329
5247.3.15 by Vincent Ladeuil
All http tests passing, https failing.
330
        self.assertRaises(httplib.UnknownProtocol,
331
                          http_server.HttpServer, BogusRequestHandler)
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
332
3111.1.17 by Vincent Ladeuil
Add tests for the protocol version parameter.
333
    def test_force_invalid_protocol(self):
5247.3.15 by Vincent Ladeuil
All http tests passing, https failing.
334
        self.assertRaises(httplib.UnknownProtocol,
335
                          http_server.HttpServer, protocol_version='HTTP/0.1')
3111.1.17 by Vincent Ladeuil
Add tests for the protocol version parameter.
336
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
337
    def test_server_start_and_stop(self):
338
        server = http_server.HttpServer()
5247.2.4 by Vincent Ladeuil
Add an event to ThreadWithException that can be shared with the calling thread.
339
        self.addCleanup(server.stop_server)
4934.3.3 by Martin Pool
Rename Server.setUp to Server.start_server
340
        server.start_server()
5247.3.15 by Vincent Ladeuil
All http tests passing, https failing.
341
        self.assertTrue(server.server is not None)
342
        self.assertTrue(server.server.serving is not None)
5247.6.7 by Vincent Ladeuil
Merge propagate-exceptions into http-leaks resolving conflicts
343
        self.assertTrue(server.server.serving)
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
344
345
    def test_create_http_server_one_zero(self):
346
        class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
347
348
            protocol_version = 'HTTP/1.0'
349
350
        server = http_server.HttpServer(RequestHandlerOneZero)
4659.1.2 by Robert Collins
Refactor creation and shutdown of test servers to use a common helper,
351
        self.start_server(server)
5247.3.15 by Vincent Ladeuil
All http tests passing, https failing.
352
        self.assertIsInstance(server.server, http_server.TestingHTTPServer)
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
353
354
    def test_create_http_server_one_one(self):
355
        class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
356
357
            protocol_version = 'HTTP/1.1'
358
359
        server = http_server.HttpServer(RequestHandlerOneOne)
4659.1.2 by Robert Collins
Refactor creation and shutdown of test servers to use a common helper,
360
        self.start_server(server)
5247.3.15 by Vincent Ladeuil
All http tests passing, https failing.
361
        self.assertIsInstance(server.server,
3111.1.17 by Vincent Ladeuil
Add tests for the protocol version parameter.
362
                              http_server.TestingThreadingHTTPServer)
363
364
    def test_create_http_server_force_one_one(self):
365
        class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
366
367
            protocol_version = 'HTTP/1.0'
368
369
        server = http_server.HttpServer(RequestHandlerOneZero,
370
                                        protocol_version='HTTP/1.1')
4659.1.2 by Robert Collins
Refactor creation and shutdown of test servers to use a common helper,
371
        self.start_server(server)
5247.3.15 by Vincent Ladeuil
All http tests passing, https failing.
372
        self.assertIsInstance(server.server,
3111.1.17 by Vincent Ladeuil
Add tests for the protocol version parameter.
373
                              http_server.TestingThreadingHTTPServer)
374
375
    def test_create_http_server_force_one_zero(self):
376
        class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
377
378
            protocol_version = 'HTTP/1.1'
379
380
        server = http_server.HttpServer(RequestHandlerOneOne,
381
                                        protocol_version='HTTP/1.0')
4659.1.2 by Robert Collins
Refactor creation and shutdown of test servers to use a common helper,
382
        self.start_server(server)
5247.3.15 by Vincent Ladeuil
All http tests passing, https failing.
383
        self.assertIsInstance(server.server,
3111.1.17 by Vincent Ladeuil
Add tests for the protocol version parameter.
384
                              http_server.TestingHTTPServer)
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
385
386
2294.3.1 by Vincent Ladeuil
Fix #85305 by issuing an exception instead of a traceback.
387
class TestWithTransport_pycurl(object):
388
    """Test case to inherit from if pycurl is present"""
389
390
    def _get_pycurl_maybe(self):
4913.2.11 by John Arbash Meinel
Convert a bunch more features over to ModuleAvailableFeature
391
        self.requireFeature(features.pycurl)
392
        return PyCurlTransport
2294.3.1 by Vincent Ladeuil
Fix #85305 by issuing an exception instead of a traceback.
393
394
    _transport = property(_get_pycurl_maybe)
395
396
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
397
class TestHttpTransportUrls(tests.TestCase):
398
    """Test the http urls."""
399
5462.3.15 by Martin Pool
Turn variations into scenario lists
400
    scenarios = vary_by_http_client_implementation()
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
401
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
402
    def test_abs_url(self):
403
        """Construction of absolute http URLs"""
5560.2.1 by Vincent Ladeuil
Fix the remaining references to http://bazaar-vcs.org (except the explicitly historical ones).
404
        t = self._transport('http://example.com/bzr/bzr.dev/')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
405
        eq = self.assertEqualDiff
5560.2.1 by Vincent Ladeuil
Fix the remaining references to http://bazaar-vcs.org (except the explicitly historical ones).
406
        eq(t.abspath('.'), 'http://example.com/bzr/bzr.dev')
407
        eq(t.abspath('foo/bar'), 'http://example.com/bzr/bzr.dev/foo/bar')
408
        eq(t.abspath('.bzr'), 'http://example.com/bzr/bzr.dev/.bzr')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
409
        eq(t.abspath('.bzr/1//2/./3'),
5560.2.1 by Vincent Ladeuil
Fix the remaining references to http://bazaar-vcs.org (except the explicitly historical ones).
410
           'http://example.com/bzr/bzr.dev/.bzr/1/2/3')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
411
412
    def test_invalid_http_urls(self):
413
        """Trap invalid construction of urls"""
5560.2.1 by Vincent Ladeuil
Fix the remaining references to http://bazaar-vcs.org (except the explicitly historical ones).
414
        self._transport('http://example.com/bzr/bzr.dev/')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
415
        self.assertRaises(errors.InvalidURL,
416
                          self._transport,
5560.2.1 by Vincent Ladeuil
Fix the remaining references to http://bazaar-vcs.org (except the explicitly historical ones).
417
                          'http://http://example.com/bzr/bzr.dev/')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
418
419
    def test_http_root_urls(self):
420
        """Construction of URLs from server root"""
5560.2.1 by Vincent Ladeuil
Fix the remaining references to http://bazaar-vcs.org (except the explicitly historical ones).
421
        t = self._transport('http://example.com/')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
422
        eq = self.assertEqualDiff
423
        eq(t.abspath('.bzr/tree-version'),
5560.2.1 by Vincent Ladeuil
Fix the remaining references to http://bazaar-vcs.org (except the explicitly historical ones).
424
           'http://example.com/.bzr/tree-version')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
425
426
    def test_http_impl_urls(self):
427
        """There are servers which ask for particular clients to connect"""
428
        server = self._server()
4934.3.3 by Martin Pool
Rename Server.setUp to Server.start_server
429
        server.start_server()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
430
        try:
431
            url = server.get_url()
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
432
            self.assertTrue(url.startswith('%s://' % self._url_protocol))
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
433
        finally:
4934.3.1 by Martin Pool
Rename Server.tearDown to .stop_server
434
            server.stop_server()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
435
436
3111.1.9 by Vincent Ladeuil
Most refactoring regarding parameterization for urllib/pycurl and custom
437
class TestHttps_pycurl(TestWithTransport_pycurl, tests.TestCase):
2294.3.1 by Vincent Ladeuil
Fix #85305 by issuing an exception instead of a traceback.
438
439
    # TODO: This should really be moved into another pycurl
440
    # specific test. When https tests will be implemented, take
441
    # this one into account.
442
    def test_pycurl_without_https_support(self):
443
        """Test that pycurl without SSL do not fail with a traceback.
444
445
        For the purpose of the test, we force pycurl to ignore
446
        https by supplying a fake version_info that do not
447
        support it.
448
        """
4913.2.13 by John Arbash Meinel
Finish the pycurl feature.
449
        self.requireFeature(features.pycurl)
4926.1.1 by Vincent Ladeuil
Fix ModuleFeature() side-effect.
450
        # Import the module locally now that we now it's available.
451
        pycurl = features.pycurl.module
3111.1.14 by Vincent Ladeuil
Fix test leakage.
452
4985.1.5 by Vincent Ladeuil
Deploying the new overrideAttr facility further reduces the complexity
453
        self.overrideAttr(pycurl, 'version_info',
454
                          # Fake the pycurl version_info This was taken from
455
                          # a windows pycurl without SSL (thanks to bialix)
456
                          lambda : (2,
457
                                    '7.13.2',
458
                                    462082,
459
                                    'i386-pc-win32',
460
                                    2576,
461
                                    None,
462
                                    0,
463
                                    None,
464
                                    ('ftp', 'gopher', 'telnet',
465
                                     'dict', 'ldap', 'http', 'file'),
466
                                    None,
467
                                    0,
468
                                    None))
4926.1.1 by Vincent Ladeuil
Fix ModuleFeature() side-effect.
469
        self.assertRaises(errors.DependencyNotPresent, self._transport,
470
                          'https://launchpad.net')
2294.3.1 by Vincent Ladeuil
Fix #85305 by issuing an exception instead of a traceback.
471
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
472
473
class TestHTTPConnections(http_utils.TestCaseWithWebserver):
474
    """Test the http connections."""
475
5462.3.15 by Martin Pool
Turn variations into scenario lists
476
    scenarios = multiply_scenarios(
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
477
        vary_by_http_client_implementation(),
5462.3.15 by Martin Pool
Turn variations into scenario lists
478
        vary_by_http_protocol_version(),
479
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
480
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
481
    def setUp(self):
6552.1.4 by Vincent Ladeuil
Remaining tests matching setup(self) that can be rewritten with super().
482
        super(TestHTTPConnections, self).setUp()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
483
        self.build_tree(['foo/', 'foo/bar'], line_endings='binary',
484
                        transport=self.get_transport())
485
486
    def test_http_has(self):
487
        server = self.get_readonly_server()
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
488
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
489
        self.assertEqual(t.has('foo/bar'), True)
490
        self.assertEqual(len(server.logs), 1)
491
        self.assertContainsRe(server.logs[0],
492
            r'"HEAD /foo/bar HTTP/1.." (200|302) - "-" "bzr/')
493
494
    def test_http_has_not_found(self):
495
        server = self.get_readonly_server()
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
496
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
497
        self.assertEqual(t.has('not-found'), False)
498
        self.assertContainsRe(server.logs[1],
499
            r'"HEAD /not-found HTTP/1.." 404 - "-" "bzr/')
500
501
    def test_http_get(self):
502
        server = self.get_readonly_server()
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
503
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
504
        fp = t.get('foo/bar')
505
        self.assertEqualDiff(
506
            fp.read(),
507
            'contents of foo/bar\n')
508
        self.assertEqual(len(server.logs), 1)
509
        self.assertTrue(server.logs[0].find(
510
            '"GET /foo/bar HTTP/1.1" 200 - "-" "bzr/%s'
511
            % bzrlib.__version__) > -1)
512
513
    def test_has_on_bogus_host(self):
514
        # Get a free address and don't 'accept' on it, so that we
515
        # can be sure there is no http handler there, but set a
516
        # reasonable timeout to not slow down tests too much.
517
        default_timeout = socket.getdefaulttimeout()
518
        try:
519
            socket.setdefaulttimeout(2)
520
            s = socket.socket()
521
            s.bind(('localhost', 0))
522
            t = self._transport('http://%s:%s/' % s.getsockname())
523
            self.assertRaises(errors.ConnectionError, t.has, 'foo/bar')
524
        finally:
525
            socket.setdefaulttimeout(default_timeout)
526
527
528
class TestHttpTransportRegistration(tests.TestCase):
529
    """Test registrations of various http implementations"""
530
5462.3.15 by Martin Pool
Turn variations into scenario lists
531
    scenarios = vary_by_http_client_implementation()
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
532
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
533
    def test_http_registered(self):
6083.1.1 by Jelmer Vernooij
Use get_transport_from_{url,path} in more places.
534
        t = transport.get_transport_from_url(
535
            '%s://foo.com/' % self._url_protocol)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
536
        self.assertIsInstance(t, transport.Transport)
537
        self.assertIsInstance(t, self._transport)
538
539
540
class TestPost(tests.TestCase):
541
5462.3.15 by Martin Pool
Turn variations into scenario lists
542
    scenarios = multiply_scenarios(
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
543
        vary_by_http_client_implementation(),
5462.3.15 by Martin Pool
Turn variations into scenario lists
544
        vary_by_http_protocol_version(),
545
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
546
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
547
    def test_post_body_is_received(self):
4691.2.1 by Robert Collins
Add stronger test isolation by interception BzrDir.open and checking the thing being opened is known to the test suite.
548
        server = RecordingServer(expect_body_tail='end-of-body',
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
549
                                 scheme=self._url_protocol)
4659.1.2 by Robert Collins
Refactor creation and shutdown of test servers to use a common helper,
550
        self.start_server(server)
4691.2.1 by Robert Collins
Add stronger test isolation by interception BzrDir.open and checking the thing being opened is known to the test suite.
551
        url = server.get_url()
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
552
        # FIXME: needs a cleanup -- vila 20100611
6083.1.1 by Jelmer Vernooij
Use get_transport_from_{url,path} in more places.
553
        http_transport = transport.get_transport_from_url(url)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
554
        code, response = http_transport._post('abc def end-of-body')
555
        self.assertTrue(
556
            server.received_bytes.startswith('POST /.bzr/smart HTTP/1.'))
557
        self.assertTrue('content-length: 19\r' in server.received_bytes.lower())
5514.1.1 by Vincent Ladeuil
Correctly set the Content-Type header when POSTing.
558
        self.assertTrue('content-type: application/octet-stream\r'
559
                        in server.received_bytes.lower())
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
560
        # The transport should not be assuming that the server can accept
561
        # chunked encoding the first time it connects, because HTTP/1.1, so we
562
        # check for the literal string.
563
        self.assertTrue(
564
            server.received_bytes.endswith('\r\n\r\nabc def end-of-body'))
565
566
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
567
class TestRangeHeader(tests.TestCase):
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
568
    """Test range_header method"""
569
570
    def check_header(self, value, ranges=[], tail=0):
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
571
        offsets = [ (start, end - start + 1) for start, end in ranges]
3111.1.10 by Vincent Ladeuil
Finish http parameterization, 24 auth tests failing for pycurl (not
572
        coalesce = transport.Transport._coalesce_offsets
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
573
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
574
        range_header = http.HttpTransportBase._range_header
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
575
        self.assertEqual(value, range_header(coalesced, tail))
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
576
577
    def test_range_header_single(self):
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
578
        self.check_header('0-9', ranges=[(0,9)])
579
        self.check_header('100-109', ranges=[(100,109)])
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
580
581
    def test_range_header_tail(self):
1786.1.36 by John Arbash Meinel
pycurl expects us to just set the range of bytes, not including bytes=
582
        self.check_header('-10', tail=10)
583
        self.check_header('-50', tail=50)
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
584
585
    def test_range_header_multi(self):
1786.1.36 by John Arbash Meinel
pycurl expects us to just set the range of bytes, not including bytes=
586
        self.check_header('0-9,100-200,300-5000',
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
587
                          ranges=[(0,9), (100, 200), (300,5000)])
588
589
    def test_range_header_mixed(self):
1786.1.36 by John Arbash Meinel
pycurl expects us to just set the range of bytes, not including bytes=
590
        self.check_header('0-9,300-5000,-50',
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
591
                          ranges=[(0,9), (300,5000)],
592
                          tail=50)
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
593
2004.1.15 by v.ladeuil+lp at free
Better design for bogus servers. Both urllib and pycurl pass tests.
594
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
595
class TestSpecificRequestHandler(http_utils.TestCaseWithWebserver):
596
    """Tests a specific request handler.
597
3111.1.31 by Vincent Ladeuil
Review feeback.
598
    Daughter classes are expected to override _req_handler_class
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
599
    """
600
5462.3.15 by Martin Pool
Turn variations into scenario lists
601
    scenarios = multiply_scenarios(
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
602
        vary_by_http_client_implementation(),
5462.3.15 by Martin Pool
Turn variations into scenario lists
603
        vary_by_http_protocol_version(),
604
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
605
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
606
    # Provide a useful default
607
    _req_handler_class = http_server.TestingHTTPRequestHandler
608
609
    def create_transport_readonly_server(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
610
        server = http_server.HttpServer(self._req_handler_class,
611
                                        protocol_version=self._protocol_version)
612
        server._url_protocol = self._url_protocol
613
        return server
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
614
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
615
    def _testing_pycurl(self):
4913.2.11 by John Arbash Meinel
Convert a bunch more features over to ModuleAvailableFeature
616
        # TODO: This is duplicated for lots of the classes in this file
617
        return (features.pycurl.available()
618
                and self._transport == PyCurlTransport)
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
619
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
620
621
class WallRequestHandler(http_server.TestingHTTPRequestHandler):
622
    """Whatever request comes in, close the connection"""
623
4731.2.3 by Vincent Ladeuil
Reduce the leaking http tests from ~200 to ~5.
624
    def _handle_one_request(self):
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
625
        """Handle a single HTTP request, by abruptly closing the connection"""
626
        self.close_connection = 1
627
628
629
class TestWallServer(TestSpecificRequestHandler):
630
    """Tests exceptions during the connection phase"""
631
632
    _req_handler_class = WallRequestHandler
633
634
    def test_http_has(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
635
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
636
        # Unfortunately httplib (see HTTPResponse._read_status
637
        # for details) make no distinction between a closed
638
        # socket and badly formatted status line, so we can't
639
        # just test for ConnectionError, we have to test
4628.1.2 by Vincent Ladeuil
More complete fix.
640
        # InvalidHttpResponse too. And pycurl may raise ConnectionReset
641
        # instead of ConnectionError too.
642
        self.assertRaises(( errors.ConnectionError, errors.ConnectionReset,
643
                            errors.InvalidHttpResponse),
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
644
                          t.has, 'foo/bar')
645
646
    def test_http_get(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
647
        t = self.get_readonly_transport()
4628.1.2 by Vincent Ladeuil
More complete fix.
648
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
649
                           errors.InvalidHttpResponse),
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
650
                          t.get, 'foo/bar')
651
652
653
class BadStatusRequestHandler(http_server.TestingHTTPRequestHandler):
654
    """Whatever request comes in, returns a bad status"""
655
656
    def parse_request(self):
657
        """Fakes handling a single HTTP request, returns a bad status"""
658
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
659
        self.send_response(0, "Bad status")
660
        self.close_connection = 1
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
661
        return False
662
663
664
class TestBadStatusServer(TestSpecificRequestHandler):
665
    """Tests bad status from server."""
666
667
    _req_handler_class = BadStatusRequestHandler
668
6603.1.1 by Vincent Ladeuil
Fix the http racy tests (hanging with python 2.7.9).
669
    def setUp(self):
670
        super(TestBadStatusServer, self).setUp()
671
        # See https://bugs.launchpad.net/bzr/+bug/1451448 for details.
672
        # TD;LR: Running both a TCP client and server in the same process and
673
        # thread uncovers a race in python. The fix is to run the server in a
674
        # different process. Trying to fix yet another race here is not worth
675
        # the effort. -- vila 2015-09-06
676
        if 'HTTP/1.0' in self.id():
677
            raise tests.TestSkipped(
678
                'Client/Server in the same process and thread can hang')
679
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
680
    def test_http_has(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
681
        t = self.get_readonly_transport()
6603.1.1 by Vincent Ladeuil
Fix the http racy tests (hanging with python 2.7.9).
682
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
683
                           errors.InvalidHttpResponse),
684
                          t.has, 'foo/bar')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
685
686
    def test_http_get(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
687
        t = self.get_readonly_transport()
6603.1.1 by Vincent Ladeuil
Fix the http racy tests (hanging with python 2.7.9).
688
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
689
                           errors.InvalidHttpResponse),
690
                          t.get, 'foo/bar')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
691
692
693
class InvalidStatusRequestHandler(http_server.TestingHTTPRequestHandler):
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
694
    """Whatever request comes in, returns an invalid status"""
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
695
696
    def parse_request(self):
697
        """Fakes handling a single HTTP request, returns a bad status"""
698
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
699
        self.wfile.write("Invalid status line\r\n")
5247.6.6 by Vincent Ladeuil
Closing the connection is what pycurl was waiting for.
700
        # If we don't close the connection pycurl will hang. Since this is a
701
        # stress test we don't *have* to respect the protocol, but we don't
702
        # have to sabotage it too much either.
703
        self.close_connection = True
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
704
        return False
705
706
707
class TestInvalidStatusServer(TestBadStatusServer):
708
    """Tests invalid status from server.
709
710
    Both implementations raises the same error as for a bad status.
711
    """
712
713
    _req_handler_class = InvalidStatusRequestHandler
714
715
716
class BadProtocolRequestHandler(http_server.TestingHTTPRequestHandler):
717
    """Whatever request comes in, returns a bad protocol version"""
718
719
    def parse_request(self):
720
        """Fakes handling a single HTTP request, returns a bad status"""
721
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
722
        # Returns an invalid protocol version, but curl just
723
        # ignores it and those cannot be tested.
724
        self.wfile.write("%s %d %s\r\n" % ('HTTP/0.0',
725
                                           404,
726
                                           'Look at my protocol version'))
727
        return False
728
729
730
class TestBadProtocolServer(TestSpecificRequestHandler):
731
    """Tests bad protocol from server."""
732
733
    _req_handler_class = BadProtocolRequestHandler
734
735
    def setUp(self):
4913.2.11 by John Arbash Meinel
Convert a bunch more features over to ModuleAvailableFeature
736
        if self._testing_pycurl():
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
737
            raise tests.TestNotApplicable(
738
                "pycurl doesn't check the protocol version")
739
        super(TestBadProtocolServer, self).setUp()
740
741
    def test_http_has(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
742
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
743
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
744
745
    def test_http_get(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
746
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
747
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
748
749
750
class ForbiddenRequestHandler(http_server.TestingHTTPRequestHandler):
751
    """Whatever request comes in, returns a 403 code"""
752
753
    def parse_request(self):
754
        """Handle a single HTTP request, by replying we cannot handle it"""
755
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
756
        self.send_error(403)
757
        return False
758
759
760
class TestForbiddenServer(TestSpecificRequestHandler):
761
    """Tests forbidden server"""
762
763
    _req_handler_class = ForbiddenRequestHandler
764
765
    def test_http_has(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
766
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
767
        self.assertRaises(errors.TransportError, t.has, 'foo/bar')
768
769
    def test_http_get(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
770
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
771
        self.assertRaises(errors.TransportError, t.get, 'foo/bar')
772
773
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
774
class TestRecordingServer(tests.TestCase):
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
775
776
    def test_create(self):
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
777
        server = RecordingServer(expect_body_tail=None)
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
778
        self.assertEqual('', server.received_bytes)
779
        self.assertEqual(None, server.host)
780
        self.assertEqual(None, server.port)
781
4934.3.1 by Martin Pool
Rename Server.tearDown to .stop_server
782
    def test_setUp_and_stop(self):
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
783
        server = RecordingServer(expect_body_tail=None)
4934.3.3 by Martin Pool
Rename Server.setUp to Server.start_server
784
        server.start_server()
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
785
        try:
786
            self.assertNotEqual(None, server.host)
787
            self.assertNotEqual(None, server.port)
788
        finally:
4934.3.1 by Martin Pool
Rename Server.tearDown to .stop_server
789
            server.stop_server()
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
790
        self.assertEqual(None, server.host)
791
        self.assertEqual(None, server.port)
792
793
    def test_send_receive_bytes(self):
4691.2.1 by Robert Collins
Add stronger test isolation by interception BzrDir.open and checking the thing being opened is known to the test suite.
794
        server = RecordingServer(expect_body_tail='c', scheme='http')
4659.1.2 by Robert Collins
Refactor creation and shutdown of test servers to use a common helper,
795
        self.start_server(server)
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
796
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
797
        sock.connect((server.host, server.port))
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
798
        sock.sendall('abc')
799
        self.assertEqual('HTTP/1.1 200 OK\r\n',
800
                         osutils.recv_all(sock, 4096))
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
801
        self.assertEqual('abc', server.received_bytes)
2004.1.29 by v.ladeuil+lp at free
New tests for http range requests handling.
802
803
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
804
class TestRangeRequestServer(TestSpecificRequestHandler):
805
    """Tests readv requests against server.
806
807
    We test against default "normal" server.
808
    """
809
810
    def setUp(self):
811
        super(TestRangeRequestServer, self).setUp()
812
        self.build_tree_contents([('a', '0123456789')],)
813
814
    def test_readv(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
815
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
816
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
817
        self.assertEqual(l[0], (0, '0'))
818
        self.assertEqual(l[1], (1, '1'))
819
        self.assertEqual(l[2], (3, '34'))
820
        self.assertEqual(l[3], (9, '9'))
821
822
    def test_readv_out_of_order(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
823
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
824
        l = list(t.readv('a', ((1, 1), (9, 1), (0, 1), (3, 2))))
825
        self.assertEqual(l[0], (1, '1'))
826
        self.assertEqual(l[1], (9, '9'))
827
        self.assertEqual(l[2], (0, '0'))
828
        self.assertEqual(l[3], (3, '34'))
829
830
    def test_readv_invalid_ranges(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
831
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
832
833
        # This is intentionally reading off the end of the file
834
        # since we are sure that it cannot get there
835
        self.assertListRaises((errors.InvalidRange, errors.ShortReadvError,),
836
                              t.readv, 'a', [(1,1), (8,10)])
837
838
        # This is trying to seek past the end of the file, it should
839
        # also raise a special error
840
        self.assertListRaises((errors.InvalidRange, errors.ShortReadvError,),
841
                              t.readv, 'a', [(12,2)])
842
843
    def test_readv_multiple_get_requests(self):
844
        server = self.get_readonly_server()
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
845
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
846
        # force transport to issue multiple requests
847
        t._max_readv_combine = 1
848
        t._max_get_ranges = 1
849
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
850
        self.assertEqual(l[0], (0, '0'))
851
        self.assertEqual(l[1], (1, '1'))
852
        self.assertEqual(l[2], (3, '34'))
853
        self.assertEqual(l[3], (9, '9'))
854
        # The server should have issued 4 requests
855
        self.assertEqual(4, server.GET_request_nb)
856
857
    def test_readv_get_max_size(self):
858
        server = self.get_readonly_server()
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
859
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
860
        # force transport to issue multiple requests by limiting the number of
861
        # bytes by request. Note that this apply to coalesced offsets only, a
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
862
        # single range will keep its size even if bigger than the limit.
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
863
        t._get_max_size = 2
864
        l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
865
        self.assertEqual(l[0], (0, '0'))
866
        self.assertEqual(l[1], (1, '1'))
867
        self.assertEqual(l[2], (2, '2345'))
868
        self.assertEqual(l[3], (6, '6789'))
869
        # The server should have issued 3 requests
870
        self.assertEqual(3, server.GET_request_nb)
871
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
872
    def test_complete_readv_leave_pipe_clean(self):
873
        server = self.get_readonly_server()
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
874
        t = self.get_readonly_transport()
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
875
        # force transport to issue multiple requests
876
        t._get_max_size = 2
5462.3.16 by Martin Pool
pyflakes cleanups
877
        list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
878
        # The server should have issued 3 requests
879
        self.assertEqual(3, server.GET_request_nb)
880
        self.assertEqual('0123456789', t.get_bytes('a'))
881
        self.assertEqual(4, server.GET_request_nb)
882
883
    def test_incomplete_readv_leave_pipe_clean(self):
884
        server = self.get_readonly_server()
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
885
        t = self.get_readonly_transport()
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
886
        # force transport to issue multiple requests
887
        t._get_max_size = 2
888
        # Don't collapse readv results into a list so that we leave unread
889
        # bytes on the socket
890
        ireadv = iter(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
891
        self.assertEqual((0, '0'), ireadv.next())
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
892
        # The server should have issued one request so far
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
893
        self.assertEqual(1, server.GET_request_nb)
894
        self.assertEqual('0123456789', t.get_bytes('a'))
895
        # get_bytes issued an additional request, the readv pending ones are
896
        # lost
897
        self.assertEqual(2, server.GET_request_nb)
898
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
899
900
class SingleRangeRequestHandler(http_server.TestingHTTPRequestHandler):
901
    """Always reply to range request as if they were single.
902
903
    Don't be explicit about it, just to annoy the clients.
904
    """
905
906
    def get_multiple_ranges(self, file, file_size, ranges):
907
        """Answer as if it was a single range request and ignores the rest"""
908
        (start, end) = ranges[0]
909
        return self.get_single_range(file, file_size, start, end)
910
911
912
class TestSingleRangeRequestServer(TestRangeRequestServer):
913
    """Test readv against a server which accept only single range requests"""
914
915
    _req_handler_class = SingleRangeRequestHandler
916
917
918
class SingleOnlyRangeRequestHandler(http_server.TestingHTTPRequestHandler):
919
    """Only reply to simple range requests, errors out on multiple"""
920
921
    def get_multiple_ranges(self, file, file_size, ranges):
922
        """Refuses the multiple ranges request"""
923
        if len(ranges) > 1:
924
            file.close()
925
            self.send_error(416, "Requested range not satisfiable")
926
            return
927
        (start, end) = ranges[0]
928
        return self.get_single_range(file, file_size, start, end)
929
930
931
class TestSingleOnlyRangeRequestServer(TestRangeRequestServer):
932
    """Test readv against a server which only accept single range requests"""
933
934
    _req_handler_class = SingleOnlyRangeRequestHandler
935
936
937
class NoRangeRequestHandler(http_server.TestingHTTPRequestHandler):
938
    """Ignore range requests without notice"""
939
940
    def do_GET(self):
941
        # Update the statistics
942
        self.server.test_case_server.GET_request_nb += 1
943
        # Just bypass the range handling done by TestingHTTPRequestHandler
944
        return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
945
946
947
class TestNoRangeRequestServer(TestRangeRequestServer):
948
    """Test readv against a server which do not accept range requests"""
949
950
    _req_handler_class = NoRangeRequestHandler
951
952
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
953
class MultipleRangeWithoutContentLengthRequestHandler(
954
    http_server.TestingHTTPRequestHandler):
955
    """Reply to multiple range requests without content length header."""
956
957
    def get_multiple_ranges(self, file, file_size, ranges):
958
        self.send_response(206)
959
        self.send_header('Accept-Ranges', 'bytes')
5462.3.16 by Martin Pool
pyflakes cleanups
960
        # XXX: this is strange; the 'random' name below seems undefined and
5462.3.19 by Martin Pool
Mention bug 658773 in comment
961
        # yet the tests pass -- mbp 2010-10-11 bug 658773
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
962
        boundary = "%d" % random.randint(0,0x7FFFFFFF)
963
        self.send_header("Content-Type",
964
                         "multipart/byteranges; boundary=%s" % boundary)
965
        self.end_headers()
966
        for (start, end) in ranges:
967
            self.wfile.write("--%s\r\n" % boundary)
968
            self.send_header("Content-type", 'application/octet-stream')
969
            self.send_header("Content-Range", "bytes %d-%d/%d" % (start,
970
                                                                  end,
971
                                                                  file_size))
972
            self.end_headers()
973
            self.send_range_content(file, start, end - start + 1)
974
        # Final boundary
975
        self.wfile.write("--%s\r\n" % boundary)
976
977
978
class TestMultipleRangeWithoutContentLengthServer(TestRangeRequestServer):
979
980
    _req_handler_class = MultipleRangeWithoutContentLengthRequestHandler
981
3146.3.2 by Vincent Ladeuil
Fix #179368 by keeping the current range hint on ShortReadvErrors.
982
983
class TruncatedMultipleRangeRequestHandler(
984
    http_server.TestingHTTPRequestHandler):
985
    """Reply to multiple range requests truncating the last ones.
986
987
    This server generates responses whose Content-Length describes all the
988
    ranges, but fail to include the last ones leading to client short reads.
989
    This has been observed randomly with lighttpd (bug #179368).
990
    """
991
992
    _truncated_ranges = 2
993
994
    def get_multiple_ranges(self, file, file_size, ranges):
995
        self.send_response(206)
996
        self.send_header('Accept-Ranges', 'bytes')
997
        boundary = 'tagada'
998
        self.send_header('Content-Type',
999
                         'multipart/byteranges; boundary=%s' % boundary)
1000
        boundary_line = '--%s\r\n' % boundary
1001
        # Calculate the Content-Length
1002
        content_length = 0
1003
        for (start, end) in ranges:
1004
            content_length += len(boundary_line)
1005
            content_length += self._header_line_length(
1006
                'Content-type', 'application/octet-stream')
1007
            content_length += self._header_line_length(
1008
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
1009
            content_length += len('\r\n') # end headers
1010
            content_length += end - start # + 1
1011
        content_length += len(boundary_line)
1012
        self.send_header('Content-length', content_length)
1013
        self.end_headers()
1014
1015
        # Send the multipart body
1016
        cur = 0
1017
        for (start, end) in ranges:
1018
            self.wfile.write(boundary_line)
1019
            self.send_header('Content-type', 'application/octet-stream')
1020
            self.send_header('Content-Range', 'bytes %d-%d/%d'
1021
                             % (start, end, file_size))
1022
            self.end_headers()
1023
            if cur + self._truncated_ranges >= len(ranges):
1024
                # Abruptly ends the response and close the connection
1025
                self.close_connection = 1
1026
                return
1027
            self.send_range_content(file, start, end - start + 1)
1028
            cur += 1
5504.4.1 by Vincent Ladeuil
Fix http test spurious failures and get rid of some useless messages in log.
1029
        # Final boundary
3146.3.2 by Vincent Ladeuil
Fix #179368 by keeping the current range hint on ShortReadvErrors.
1030
        self.wfile.write(boundary_line)
1031
1032
1033
class TestTruncatedMultipleRangeServer(TestSpecificRequestHandler):
1034
1035
    _req_handler_class = TruncatedMultipleRangeRequestHandler
1036
1037
    def setUp(self):
1038
        super(TestTruncatedMultipleRangeServer, self).setUp()
1039
        self.build_tree_contents([('a', '0123456789')],)
1040
1041
    def test_readv_with_short_reads(self):
1042
        server = self.get_readonly_server()
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1043
        t = self.get_readonly_transport()
3146.3.2 by Vincent Ladeuil
Fix #179368 by keeping the current range hint on ShortReadvErrors.
1044
        # Force separate ranges for each offset
1045
        t._bytes_to_read_before_seek = 0
1046
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1047
        self.assertEqual((0, '0'), ireadv.next())
1048
        self.assertEqual((2, '2'), ireadv.next())
1049
        if not self._testing_pycurl():
1050
            # Only one request have been issued so far (except for pycurl that
1051
            # try to read the whole response at once)
1052
            self.assertEqual(1, server.GET_request_nb)
1053
        self.assertEqual((4, '45'), ireadv.next())
1054
        self.assertEqual((9, '9'), ireadv.next())
1055
        # Both implementations issue 3 requests but:
1056
        # - urllib does two multiple (4 ranges, then 2 ranges) then a single
1057
        #   range,
1058
        # - pycurl does two multiple (4 ranges, 4 ranges) then a single range
1059
        self.assertEqual(3, server.GET_request_nb)
1060
        # Finally the client have tried a single range request and stays in
1061
        # that mode
1062
        self.assertEqual('single', t._range_hint)
1063
5504.4.1 by Vincent Ladeuil
Fix http test spurious failures and get rid of some useless messages in log.
1064
5609.52.1 by Martin Pool
Cope with buggy squids interrupting the response before a mime multipart boundary
1065
class TruncatedBeforeBoundaryRequestHandler(
1066
    http_server.TestingHTTPRequestHandler):
1067
    """Truncation before a boundary, like in bug 198646"""
1068
1069
    _truncated_ranges = 1
1070
1071
    def get_multiple_ranges(self, file, file_size, ranges):
1072
        self.send_response(206)
1073
        self.send_header('Accept-Ranges', 'bytes')
1074
        boundary = 'tagada'
1075
        self.send_header('Content-Type',
1076
                         'multipart/byteranges; boundary=%s' % boundary)
1077
        boundary_line = '--%s\r\n' % boundary
1078
        # Calculate the Content-Length
1079
        content_length = 0
1080
        for (start, end) in ranges:
1081
            content_length += len(boundary_line)
1082
            content_length += self._header_line_length(
1083
                'Content-type', 'application/octet-stream')
1084
            content_length += self._header_line_length(
1085
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
1086
            content_length += len('\r\n') # end headers
1087
            content_length += end - start # + 1
1088
        content_length += len(boundary_line)
1089
        self.send_header('Content-length', content_length)
1090
        self.end_headers()
1091
1092
        # Send the multipart body
1093
        cur = 0
1094
        for (start, end) in ranges:
1095
            if cur + self._truncated_ranges >= len(ranges):
1096
                # Abruptly ends the response and close the connection
1097
                self.close_connection = 1
1098
                return
1099
            self.wfile.write(boundary_line)
1100
            self.send_header('Content-type', 'application/octet-stream')
1101
            self.send_header('Content-Range', 'bytes %d-%d/%d'
1102
                             % (start, end, file_size))
1103
            self.end_headers()
1104
            self.send_range_content(file, start, end - start + 1)
1105
            cur += 1
1106
        # Final boundary
1107
        self.wfile.write(boundary_line)
1108
1109
1110
class TestTruncatedBeforeBoundary(TestSpecificRequestHandler):
1111
    """Tests the case of bug 198646, disconnecting before a boundary."""
1112
1113
    _req_handler_class = TruncatedBeforeBoundaryRequestHandler
1114
1115
    def setUp(self):
1116
        super(TestTruncatedBeforeBoundary, self).setUp()
1117
        self.build_tree_contents([('a', '0123456789')],)
1118
1119
    def test_readv_with_short_reads(self):
1120
        server = self.get_readonly_server()
1121
        t = self.get_readonly_transport()
1122
        # Force separate ranges for each offset
1123
        t._bytes_to_read_before_seek = 0
1124
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1125
        self.assertEqual((0, '0'), ireadv.next())
1126
        self.assertEqual((2, '2'), ireadv.next())
1127
        self.assertEqual((4, '45'), ireadv.next())
1128
        self.assertEqual((9, '9'), ireadv.next())
1129
1130
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1131
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1132
    """Errors out when range specifiers exceed the limit"""
1133
1134
    def get_multiple_ranges(self, file, file_size, ranges):
1135
        """Refuses the multiple ranges request"""
1136
        tcs = self.server.test_case_server
1137
        if tcs.range_limit is not None and len(ranges) > tcs.range_limit:
1138
            file.close()
1139
            # Emulate apache behavior
1140
            self.send_error(400, "Bad Request")
1141
            return
1142
        return http_server.TestingHTTPRequestHandler.get_multiple_ranges(
1143
            self, file, file_size, ranges)
1144
1145
1146
class LimitedRangeHTTPServer(http_server.HttpServer):
1147
    """An HttpServer erroring out on requests with too much range specifiers"""
1148
1149
    def __init__(self, request_handler=LimitedRangeRequestHandler,
1150
                 protocol_version=None,
1151
                 range_limit=None):
1152
        http_server.HttpServer.__init__(self, request_handler,
1153
                                        protocol_version=protocol_version)
1154
        self.range_limit = range_limit
1155
1156
1157
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1158
    """Tests readv requests against a server erroring out on too much ranges."""
1159
5462.3.15 by Martin Pool
Turn variations into scenario lists
1160
    scenarios = multiply_scenarios(
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
1161
        vary_by_http_client_implementation(),
5462.3.15 by Martin Pool
Turn variations into scenario lists
1162
        vary_by_http_protocol_version(),
1163
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
1164
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
1165
    # Requests with more range specifiers will error out
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1166
    range_limit = 3
1167
1168
    def create_transport_readonly_server(self):
1169
        return LimitedRangeHTTPServer(range_limit=self.range_limit,
1170
                                      protocol_version=self._protocol_version)
1171
1172
    def setUp(self):
6552.1.4 by Vincent Ladeuil
Remaining tests matching setup(self) that can be rewritten with super().
1173
        super(TestLimitedRangeRequestServer, self).setUp()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1174
        # We need to manipulate ranges that correspond to real chunks in the
1175
        # response, so we build a content appropriately.
1176
        filler = ''.join(['abcdefghij' for x in range(102)])
1177
        content = ''.join(['%04d' % v + filler for v in range(16)])
1178
        self.build_tree_contents([('a', content)],)
1179
1180
    def test_few_ranges(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1181
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1182
        l = list(t.readv('a', ((0, 4), (1024, 4), )))
1183
        self.assertEqual(l[0], (0, '0000'))
1184
        self.assertEqual(l[1], (1024, '0001'))
1185
        self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1186
1187
    def test_more_ranges(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1188
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1189
        l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1190
        self.assertEqual(l[0], (0, '0000'))
1191
        self.assertEqual(l[1], (1024, '0001'))
1192
        self.assertEqual(l[2], (4096, '0004'))
1193
        self.assertEqual(l[3], (8192, '0008'))
1194
        # The server will refuse to serve the first request (too much ranges),
3199.1.2 by Vincent Ladeuil
Fix two more leaked log files.
1195
        # a second request will succeed.
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1196
        self.assertEqual(2, self.get_readonly_server().GET_request_nb)
1197
1198
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
1199
class TestHttpProxyWhiteBox(tests.TestCase):
2298.7.1 by Vincent Ladeuil
Fix bug #87765: proxy env variables without scheme should cause
1200
    """Whitebox test proxy http authorization.
1201
2420.1.3 by Vincent Ladeuil
Implement http proxy basic authentication.
1202
    Only the urllib implementation is tested here.
2298.7.1 by Vincent Ladeuil
Fix bug #87765: proxy env variables without scheme should cause
1203
    """
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1204
1205
    def _proxied_request(self):
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
1206
        handler = _urllib2_wrappers.ProxyHandler()
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
1207
        request = _urllib2_wrappers.Request('GET', 'http://baz/buzzle')
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1208
        handler.set_proxy(request, 'http')
1209
        return request
1210
5639.2.2 by Vincent Ladeuil
Add tests and comments to clarify the feature.
1211
    def assertEvaluateProxyBypass(self, expected, host, no_proxy):
1212
        handler = _urllib2_wrappers.ProxyHandler()
6614.1.3 by Vincent Ladeuil
Fix assertEquals being deprecated by using assertEqual.
1213
        self.assertEqual(expected,
5639.2.2 by Vincent Ladeuil
Add tests and comments to clarify the feature.
1214
                          handler.evaluate_proxy_bypass(host, no_proxy))
1215
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1216
    def test_empty_user(self):
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1217
        self.overrideEnv('http_proxy', 'http://bar.com')
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1218
        request = self._proxied_request()
1219
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
1220
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
1221
    def test_user_with_at(self):
1222
        self.overrideEnv('http_proxy',
1223
                         'http://username@domain:password@proxy_host:1234')
1224
        request = self._proxied_request()
1225
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
1226
2298.7.1 by Vincent Ladeuil
Fix bug #87765: proxy env variables without scheme should cause
1227
    def test_invalid_proxy(self):
1228
        """A proxy env variable without scheme"""
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1229
        self.overrideEnv('http_proxy', 'host:1234')
2298.7.1 by Vincent Ladeuil
Fix bug #87765: proxy env variables without scheme should cause
1230
        self.assertRaises(errors.InvalidURL, self._proxied_request)
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1231
5639.2.2 by Vincent Ladeuil
Add tests and comments to clarify the feature.
1232
    def test_evaluate_proxy_bypass_true(self):
1233
        """The host is not proxied"""
1234
        self.assertEvaluateProxyBypass(True, 'example.com', 'example.com')
1235
        self.assertEvaluateProxyBypass(True, 'bzr.example.com', '*example.com')
1236
1237
    def test_evaluate_proxy_bypass_false(self):
1238
        """The host is proxied"""
1239
        self.assertEvaluateProxyBypass(False, 'bzr.example.com', None)
1240
1241
    def test_evaluate_proxy_bypass_unknown(self):
1242
        """The host is not explicitly proxied"""
1243
        self.assertEvaluateProxyBypass(None, 'example.com', 'not.example.com')
1244
        self.assertEvaluateProxyBypass(None, 'bzr.example.com', 'example.com')
1245
1246
    def test_evaluate_proxy_bypass_empty_entries(self):
1247
        """Ignore empty entries"""
1248
        self.assertEvaluateProxyBypass(None, 'example.com', '')
1249
        self.assertEvaluateProxyBypass(None, 'example.com', ',')
1250
        self.assertEvaluateProxyBypass(None, 'example.com', 'foo,,bar')
5639.2.1 by Martin Pool
Empty entries in the ``NO_PROXY`` variable are no longer treated as matching every host.
1251
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1252
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1253
class TestProxyHttpServer(http_utils.TestCaseWithTwoWebservers):
1254
    """Tests proxy server.
1255
1256
    Be aware that we do not setup a real proxy here. Instead, we
1257
    check that the *connection* goes through the proxy by serving
1258
    different content (the faked proxy server append '-proxied'
1259
    to the file names).
1260
    """
1261
5462.3.15 by Martin Pool
Turn variations into scenario lists
1262
    scenarios = multiply_scenarios(
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
1263
        vary_by_http_client_implementation(),
5462.3.15 by Martin Pool
Turn variations into scenario lists
1264
        vary_by_http_protocol_version(),
1265
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
1266
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1267
    # FIXME: We don't have an https server available, so we don't
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1268
    # test https connections. --vila toolongago
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1269
1270
    def setUp(self):
1271
        super(TestProxyHttpServer, self).setUp()
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1272
        self.transport_secondary_server = http_utils.ProxyServer
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1273
        self.build_tree_contents([('foo', 'contents of foo\n'),
1274
                                  ('foo-proxied', 'proxied contents of foo\n')])
1275
        # Let's setup some attributes for tests
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1276
        server = self.get_readonly_server()
1277
        self.server_host_port = '%s:%d' % (server.host, server.port)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1278
        if self._testing_pycurl():
1279
            # Oh my ! pycurl does not check for the port as part of
1280
            # no_proxy :-( So we just test the host part
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1281
            self.no_proxy_host = server.host
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1282
        else:
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1283
            self.no_proxy_host = self.server_host_port
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1284
        # The secondary server is the proxy
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1285
        self.proxy_url = self.get_secondary_url()
6615.7.1 by Vincent Ladeuil
Fix pycurl proxy tests for newer and stricter pycurl versions
1286
        if self._testing_pycurl():
1287
            self.proxy_url = self.proxy_url.replace('+pycurl', '')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1288
1289
    def _testing_pycurl(self):
4913.2.11 by John Arbash Meinel
Convert a bunch more features over to ModuleAvailableFeature
1290
        # TODO: This is duplicated for lots of the classes in this file
1291
        return (features.pycurl.available()
1292
                and self._transport == PyCurlTransport)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1293
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1294
    def assertProxied(self):
1295
        t = self.get_readonly_transport()
1296
        self.assertEqual('proxied contents of foo\n', t.get('foo').read())
1297
1298
    def assertNotProxied(self):
1299
        t = self.get_readonly_transport()
1300
        self.assertEqual('contents of foo\n', t.get('foo').read())
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1301
1302
    def test_http_proxy(self):
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1303
        self.overrideEnv('http_proxy', self.proxy_url)
1304
        self.assertProxied()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1305
1306
    def test_HTTP_PROXY(self):
1307
        if self._testing_pycurl():
1308
            # pycurl does not check HTTP_PROXY for security reasons
1309
            # (for use in a CGI context that we do not care
1310
            # about. Should we ?)
1311
            raise tests.TestNotApplicable(
1312
                'pycurl does not check HTTP_PROXY for security reasons')
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1313
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
1314
        self.assertProxied()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1315
1316
    def test_all_proxy(self):
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1317
        self.overrideEnv('all_proxy', self.proxy_url)
1318
        self.assertProxied()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1319
1320
    def test_ALL_PROXY(self):
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1321
        self.overrideEnv('ALL_PROXY', self.proxy_url)
1322
        self.assertProxied()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1323
1324
    def test_http_proxy_with_no_proxy(self):
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1325
        self.overrideEnv('no_proxy', self.no_proxy_host)
1326
        self.overrideEnv('http_proxy', self.proxy_url)
1327
        self.assertNotProxied()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1328
1329
    def test_HTTP_PROXY_with_NO_PROXY(self):
1330
        if self._testing_pycurl():
1331
            raise tests.TestNotApplicable(
1332
                'pycurl does not check HTTP_PROXY for security reasons')
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1333
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
1334
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
1335
        self.assertNotProxied()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1336
1337
    def test_all_proxy_with_no_proxy(self):
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1338
        self.overrideEnv('no_proxy', self.no_proxy_host)
1339
        self.overrideEnv('all_proxy', self.proxy_url)
1340
        self.assertNotProxied()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1341
1342
    def test_ALL_PROXY_with_NO_PROXY(self):
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1343
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
1344
        self.overrideEnv('ALL_PROXY', self.proxy_url)
1345
        self.assertNotProxied()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1346
1347
    def test_http_proxy_without_scheme(self):
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1348
        self.overrideEnv('http_proxy', self.server_host_port)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1349
        if self._testing_pycurl():
1350
            # pycurl *ignores* invalid proxy env variables. If that ever change
1351
            # in the future, this test will fail indicating that pycurl do not
1352
            # ignore anymore such variables.
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1353
            self.assertNotProxied()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1354
        else:
5570.3.10 by Vincent Ladeuil
Rework the http tests with overrideEnv.
1355
            self.assertRaises(errors.InvalidURL, self.assertProxied)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1356
1357
1358
class TestRanges(http_utils.TestCaseWithWebserver):
1359
    """Test the Range header in GET methods."""
1360
5462.3.15 by Martin Pool
Turn variations into scenario lists
1361
    scenarios = multiply_scenarios(
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
1362
        vary_by_http_client_implementation(),
5462.3.15 by Martin Pool
Turn variations into scenario lists
1363
        vary_by_http_protocol_version(),
1364
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
1365
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1366
    def setUp(self):
6552.1.4 by Vincent Ladeuil
Remaining tests matching setup(self) that can be rewritten with super().
1367
        super(TestRanges, self).setUp()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1368
        self.build_tree_contents([('a', '0123456789')],)
1369
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
1370
    def create_transport_readonly_server(self):
1371
        return http_server.HttpServer(protocol_version=self._protocol_version)
1372
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1373
    def _file_contents(self, relpath, ranges):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1374
        t = self.get_readonly_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1375
        offsets = [ (start, end - start + 1) for start, end in ranges]
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1376
        coalesce = t._coalesce_offsets
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1377
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1378
        code, data = t._get(relpath, coalesced)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1379
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1380
        for start, end in ranges:
1381
            data.seek(start)
1382
            yield data.read(end - start + 1)
1383
1384
    def _file_tail(self, relpath, tail_amount):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1385
        t = self.get_readonly_transport()
1386
        code, data = t._get(relpath, [], tail_amount)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1387
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1388
        data.seek(-tail_amount, 2)
1389
        return data.read(tail_amount)
1390
1391
    def test_range_header(self):
1392
        # Valid ranges
1393
        map(self.assertEqual,['0', '234'],
1394
            list(self._file_contents('a', [(0,0), (2,4)])),)
1395
1396
    def test_range_header_tail(self):
1397
        self.assertEqual('789', self._file_tail('a', 3))
1398
1399
    def test_syntactically_invalid_range_header(self):
1400
        self.assertListRaises(errors.InvalidHttpRange,
1401
                          self._file_contents, 'a', [(4, 3)])
1402
1403
    def test_semantically_invalid_range_header(self):
1404
        self.assertListRaises(errors.InvalidHttpRange,
1405
                          self._file_contents, 'a', [(42, 128)])
1406
1407
1408
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1409
    """Test redirection between http servers."""
1410
5462.3.15 by Martin Pool
Turn variations into scenario lists
1411
    scenarios = multiply_scenarios(
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
1412
        vary_by_http_client_implementation(),
5462.3.15 by Martin Pool
Turn variations into scenario lists
1413
        vary_by_http_protocol_version(),
1414
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
1415
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1416
    def setUp(self):
1417
        super(TestHTTPRedirections, self).setUp()
1418
        self.build_tree_contents([('a', '0123456789'),
1419
                                  ('bundle',
1420
                                  '# Bazaar revision bundle v0.9\n#\n')
1421
                                  ],)
1422
1423
    def test_redirected(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1424
        self.assertRaises(errors.RedirectRequested,
1425
                          self.get_old_transport().get, 'a')
1426
        self.assertEqual('0123456789', self.get_new_transport().get('a').read())
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1427
1428
1429
class RedirectedRequest(_urllib2_wrappers.Request):
1430
    """Request following redirections. """
1431
1432
    init_orig = _urllib2_wrappers.Request.__init__
1433
1434
    def __init__(self, method, url, *args, **kwargs):
1435
        """Constructor.
1436
1437
        """
1438
        # Since the tests using this class will replace
1439
        # _urllib2_wrappers.Request, we can't just call the base class __init__
1440
        # or we'll loop.
4208.3.2 by Andrew Bennetts
Fix one test failure in test_http under Python 2.7a0.
1441
        RedirectedRequest.init_orig(self, method, url, *args, **kwargs)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1442
        self.follow_redirections = True
1443
1444
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
1445
def install_redirected_request(test):
4985.1.5 by Vincent Ladeuil
Deploying the new overrideAttr facility further reduces the complexity
1446
    test.overrideAttr(_urllib2_wrappers, 'Request', RedirectedRequest)
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
1447
1448
5247.2.26 by Vincent Ladeuil
Fix http redirection socket leaks.
1449
def cleanup_http_redirection_connections(test):
1450
    # Some sockets are opened but never seen by _urllib, so we trap them at
1451
    # the _urllib2_wrappers level to be able to clean them up.
1452
    def socket_disconnect(sock):
1453
        try:
1454
            sock.shutdown(socket.SHUT_RDWR)
1455
            sock.close()
1456
        except socket.error:
1457
            pass
1458
    def connect(connection):
1459
        test.http_connect_orig(connection)
1460
        test.addCleanup(socket_disconnect, connection.sock)
1461
    test.http_connect_orig = test.overrideAttr(
1462
        _urllib2_wrappers.HTTPConnection, 'connect', connect)
1463
    def connect(connection):
1464
        test.https_connect_orig(connection)
1465
        test.addCleanup(socket_disconnect, connection.sock)
1466
    test.https_connect_orig = test.overrideAttr(
1467
        _urllib2_wrappers.HTTPSConnection, 'connect', connect)
1468
1469
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1470
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1471
    """Test redirections.
1472
1473
    http implementations do not redirect silently anymore (they
1474
    do not redirect at all in fact). The mechanism is still in
1475
    place at the _urllib2_wrappers.Request level and these tests
1476
    exercise it.
1477
1478
    For the pycurl implementation
1479
    the redirection have been deleted as we may deprecate pycurl
1480
    and I have no place to keep a working implementation.
1481
    -- vila 20070212
1482
    """
1483
5462.3.15 by Martin Pool
Turn variations into scenario lists
1484
    scenarios = multiply_scenarios(
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
1485
        vary_by_http_client_implementation(),
5462.3.15 by Martin Pool
Turn variations into scenario lists
1486
        vary_by_http_protocol_version(),
1487
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
1488
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1489
    def setUp(self):
4913.2.11 by John Arbash Meinel
Convert a bunch more features over to ModuleAvailableFeature
1490
        if (features.pycurl.available()
1491
            and self._transport == PyCurlTransport):
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1492
            raise tests.TestNotApplicable(
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1493
                "pycurl doesn't redirect silently anymore")
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1494
        super(TestHTTPSilentRedirections, self).setUp()
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
1495
        install_redirected_request(self)
5247.2.26 by Vincent Ladeuil
Fix http redirection socket leaks.
1496
        cleanup_http_redirection_connections(self)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1497
        self.build_tree_contents([('a','a'),
1498
                                  ('1/',),
1499
                                  ('1/a', 'redirected once'),
1500
                                  ('2/',),
1501
                                  ('2/a', 'redirected twice'),
1502
                                  ('3/',),
1503
                                  ('3/a', 'redirected thrice'),
1504
                                  ('4/',),
1505
                                  ('4/a', 'redirected 4 times'),
1506
                                  ('5/',),
1507
                                  ('5/a', 'redirected 5 times'),
1508
                                  ],)
1509
1510
    def test_one_redirection(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1511
        t = self.get_old_transport()
1512
        req = RedirectedRequest('GET', t._remote_path('a'))
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1513
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1514
                                       self.new_server.port)
1515
        self.old_server.redirections = \
1516
            [('(.*)', r'%s/1\1' % (new_prefix), 301),]
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1517
        self.assertEqual('redirected once', t._perform(req).read())
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1518
1519
    def test_five_redirections(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1520
        t = self.get_old_transport()
1521
        req = RedirectedRequest('GET', t._remote_path('a'))
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1522
        old_prefix = 'http://%s:%s' % (self.old_server.host,
1523
                                       self.old_server.port)
1524
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1525
                                       self.new_server.port)
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
1526
        self.old_server.redirections = [
1527
            ('/1(.*)', r'%s/2\1' % (old_prefix), 302),
1528
            ('/2(.*)', r'%s/3\1' % (old_prefix), 303),
1529
            ('/3(.*)', r'%s/4\1' % (old_prefix), 307),
1530
            ('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1531
            ('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1532
            ]
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1533
        self.assertEqual('redirected 5 times', t._perform(req).read())
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1534
1535
1536
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1537
    """Test transport.do_catching_redirections."""
1538
5462.3.15 by Martin Pool
Turn variations into scenario lists
1539
    scenarios = multiply_scenarios(
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
1540
        vary_by_http_client_implementation(),
5462.3.15 by Martin Pool
Turn variations into scenario lists
1541
        vary_by_http_protocol_version(),
1542
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
1543
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1544
    def setUp(self):
1545
        super(TestDoCatchRedirections, self).setUp()
1546
        self.build_tree_contents([('a', '0123456789'),],)
5247.2.26 by Vincent Ladeuil
Fix http redirection socket leaks.
1547
        cleanup_http_redirection_connections(self)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1548
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1549
        self.old_transport = self.get_old_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1550
5247.3.31 by Vincent Ladeuil
Cleanup some imports in http_utils.py.
1551
    def get_a(self, t):
1552
        return t.get('a')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1553
1554
    def test_no_redirection(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1555
        t = self.get_new_transport()
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1556
1557
        # We use None for redirected so that we fail if redirected
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
1558
        self.assertEqual('0123456789',
1559
                         transport.do_catching_redirections(
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1560
                self.get_a, t, None).read())
1561
1562
    def test_one_redirection(self):
1563
        self.redirections = 0
1564
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1565
        def redirected(t, exception, redirection_notice):
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1566
            self.redirections += 1
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1567
            redirected_t = t._redirected_to(exception.source, exception.target)
1568
            return redirected_t
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1569
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
1570
        self.assertEqual('0123456789',
1571
                         transport.do_catching_redirections(
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1572
                self.get_a, self.old_transport, redirected).read())
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
1573
        self.assertEqual(1, self.redirections)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1574
1575
    def test_redirection_loop(self):
1576
1577
        def redirected(transport, exception, redirection_notice):
1578
            # By using the redirected url as a base dir for the
1579
            # *old* transport, we create a loop: a => a/a =>
1580
            # a/a/a
1581
            return self.old_transport.clone(exception.target)
1582
1583
        self.assertRaises(errors.TooManyRedirections,
1584
                          transport.do_catching_redirections,
1585
                          self.get_a, self.old_transport, redirected)
1586
1587
5957.2.1 by Vincent Ladeuil
Tweak the http Authorization tests to make some parts easier to reuse (parametrization mostly).
1588
def _setup_authentication_config(**kwargs):
1589
    conf = config.AuthenticationConfig()
1590
    conf._get_config().update({'httptest': kwargs})
1591
    conf._save()
1592
1593
1594
class TestUrllib2AuthHandler(tests.TestCaseWithTransport):
1595
    """Unit tests for glue by which urllib2 asks us for authentication"""
1596
1597
    def test_get_user_password_without_port(self):
1598
        """We cope if urllib2 doesn't tell us the port.
1599
1600
        See https://bugs.launchpad.net/bzr/+bug/654684
1601
        """
1602
        user = 'joe'
1603
        password = 'foo'
1604
        _setup_authentication_config(scheme='http', host='localhost',
1605
                                     user=user, password=password)
1606
        handler = _urllib2_wrappers.HTTPAuthHandler()
1607
        got_pass = handler.get_user_password(dict(
1608
            user='joe',
1609
            protocol='http',
1610
            host='localhost',
1611
            path='/',
1612
            realm='Realm',
1613
            ))
6614.1.3 by Vincent Ladeuil
Fix assertEquals being deprecated by using assertEqual.
1614
        self.assertEqual((user, password), got_pass)
5957.2.1 by Vincent Ladeuil
Tweak the http Authorization tests to make some parts easier to reuse (parametrization mostly).
1615
1616
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1617
class TestAuth(http_utils.TestCaseWithWebserver):
1618
    """Test authentication scheme"""
1619
5462.3.15 by Martin Pool
Turn variations into scenario lists
1620
    scenarios = multiply_scenarios(
1621
        vary_by_http_client_implementation(),
1622
        vary_by_http_protocol_version(),
1623
        vary_by_http_auth_scheme(),
1624
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
1625
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1626
    def setUp(self):
1627
        super(TestAuth, self).setUp()
1628
        self.server = self.get_readonly_server()
1629
        self.build_tree_contents([('a', 'contents of a\n'),
1630
                                  ('b', 'contents of b\n'),])
1631
1632
    def create_transport_readonly_server(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1633
        server = self._auth_server(protocol_version=self._protocol_version)
1634
        server._url_protocol = self._url_protocol
1635
        return server
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1636
1637
    def _testing_pycurl(self):
4913.2.11 by John Arbash Meinel
Convert a bunch more features over to ModuleAvailableFeature
1638
        # TODO: This is duplicated for lots of the classes in this file
1639
        return (features.pycurl.available()
1640
                and self._transport == PyCurlTransport)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1641
3910.2.4 by Vincent Ladeuil
Fixed as per John's review.
1642
    def get_user_url(self, user, password):
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1643
        """Build an url embedding user and password"""
1644
        url = '%s://' % self.server._url_protocol
1645
        if user is not None:
1646
            url += user
1647
            if password is not None:
1648
                url += ':' + password
1649
            url += '@'
1650
        url += '%s:%s/' % (self.server.host, self.server.port)
1651
        return url
1652
3910.2.4 by Vincent Ladeuil
Fixed as per John's review.
1653
    def get_user_transport(self, user, password):
6083.1.1 by Jelmer Vernooij
Use get_transport_from_{url,path} in more places.
1654
        t = transport.get_transport_from_url(
1655
            self.get_user_url(user, password))
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1656
        return t
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1657
1658
    def test_no_user(self):
1659
        self.server.add_user('joe', 'foo')
3910.2.4 by Vincent Ladeuil
Fixed as per John's review.
1660
        t = self.get_user_transport(None, None)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1661
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1662
        # Only one 'Authentication Required' error should occur
1663
        self.assertEqual(1, self.server.auth_required_errors)
1664
1665
    def test_empty_pass(self):
1666
        self.server.add_user('joe', '')
1667
        t = self.get_user_transport('joe', '')
1668
        self.assertEqual('contents of a\n', t.get('a').read())
1669
        # Only one 'Authentication Required' error should occur
1670
        self.assertEqual(1, self.server.auth_required_errors)
1671
1672
    def test_user_pass(self):
1673
        self.server.add_user('joe', 'foo')
1674
        t = self.get_user_transport('joe', 'foo')
1675
        self.assertEqual('contents of a\n', t.get('a').read())
1676
        # Only one 'Authentication Required' error should occur
1677
        self.assertEqual(1, self.server.auth_required_errors)
1678
1679
    def test_unknown_user(self):
1680
        self.server.add_user('joe', 'foo')
1681
        t = self.get_user_transport('bill', 'foo')
1682
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1683
        # Two 'Authentication Required' errors should occur (the
1684
        # initial 'who are you' and 'I don't know you, who are
1685
        # you').
1686
        self.assertEqual(2, self.server.auth_required_errors)
1687
1688
    def test_wrong_pass(self):
1689
        self.server.add_user('joe', 'foo')
1690
        t = self.get_user_transport('joe', 'bar')
1691
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1692
        # Two 'Authentication Required' errors should occur (the
1693
        # initial 'who are you' and 'this is not you, who are you')
1694
        self.assertEqual(2, self.server.auth_required_errors)
1695
4222.3.12 by Jelmer Vernooij
Check that the HTTP transport prompts for usernames.
1696
    def test_prompt_for_username(self):
1697
        if self._testing_pycurl():
1698
            raise tests.TestNotApplicable(
1699
                'pycurl cannot prompt, it handles auth by embedding'
1700
                ' user:pass in urls only')
1701
1702
        self.server.add_user('joe', 'foo')
1703
        t = self.get_user_transport(None, None)
1704
        stdout = tests.StringIOWrapper()
4368.3.1 by Vincent Ladeuil
Use stderr for UI prompt to address bug #376582.
1705
        stderr = tests.StringIOWrapper()
1706
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
1707
                                            stdout=stdout, stderr=stderr)
4222.3.12 by Jelmer Vernooij
Check that the HTTP transport prompts for usernames.
1708
        self.assertEqual('contents of a\n',t.get('a').read())
1709
        # stdin should be empty
1710
        self.assertEqual('', ui.ui_factory.stdin.readline())
4368.3.1 by Vincent Ladeuil
Use stderr for UI prompt to address bug #376582.
1711
        stderr.seek(0)
4222.3.12 by Jelmer Vernooij
Check that the HTTP transport prompts for usernames.
1712
        expected_prompt = self._expected_username_prompt(t._unqualified_scheme)
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
1713
        self.assertEqual(expected_prompt, stderr.read(len(expected_prompt)))
1714
        self.assertEqual('', stdout.getvalue())
4222.3.12 by Jelmer Vernooij
Check that the HTTP transport prompts for usernames.
1715
        self._check_password_prompt(t._unqualified_scheme, 'joe',
4368.3.1 by Vincent Ladeuil
Use stderr for UI prompt to address bug #376582.
1716
                                    stderr.readline())
4284.1.2 by Vincent Ladeuil
Delete spurious space.
1717
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1718
    def test_prompt_for_password(self):
1719
        if self._testing_pycurl():
1720
            raise tests.TestNotApplicable(
1721
                'pycurl cannot prompt, it handles auth by embedding'
1722
                ' user:pass in urls only')
1723
1724
        self.server.add_user('joe', 'foo')
1725
        t = self.get_user_transport('joe', None)
1726
        stdout = tests.StringIOWrapper()
4368.3.1 by Vincent Ladeuil
Use stderr for UI prompt to address bug #376582.
1727
        stderr = tests.StringIOWrapper()
1728
        ui.ui_factory = tests.TestUIFactory(stdin='foo\n',
1729
                                            stdout=stdout, stderr=stderr)
1730
        self.assertEqual('contents of a\n', t.get('a').read())
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1731
        # stdin should be empty
1732
        self.assertEqual('', ui.ui_factory.stdin.readline())
1733
        self._check_password_prompt(t._unqualified_scheme, 'joe',
4368.3.1 by Vincent Ladeuil
Use stderr for UI prompt to address bug #376582.
1734
                                    stderr.getvalue())
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
1735
        self.assertEqual('', stdout.getvalue())
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1736
        # And we shouldn't prompt again for a different request
1737
        # against the same transport.
1738
        self.assertEqual('contents of b\n',t.get('b').read())
1739
        t2 = t.clone()
1740
        # And neither against a clone
1741
        self.assertEqual('contents of b\n',t2.get('b').read())
1742
        # Only one 'Authentication Required' error should occur
1743
        self.assertEqual(1, self.server.auth_required_errors)
1744
1745
    def _check_password_prompt(self, scheme, user, actual_prompt):
1746
        expected_prompt = (self._password_prompt_prefix
1747
                           + ("%s %s@%s:%d, Realm: '%s' password: "
1748
                              % (scheme.upper(),
1749
                                 user, self.server.host, self.server.port,
1750
                                 self.server.auth_realm)))
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
1751
        self.assertEqual(expected_prompt, actual_prompt)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1752
4222.3.12 by Jelmer Vernooij
Check that the HTTP transport prompts for usernames.
1753
    def _expected_username_prompt(self, scheme):
1754
        return (self._username_prompt_prefix
1755
                + "%s %s:%d, Realm: '%s' username: " % (scheme.upper(),
1756
                                 self.server.host, self.server.port,
1757
                                 self.server.auth_realm))
1758
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1759
    def test_no_prompt_for_password_when_using_auth_config(self):
1760
        if self._testing_pycurl():
1761
            raise tests.TestNotApplicable(
1762
                'pycurl does not support authentication.conf'
1763
                ' since it cannot prompt')
1764
1765
        user =' joe'
1766
        password = 'foo'
1767
        stdin_content = 'bar\n'  # Not the right password
1768
        self.server.add_user(user, password)
1769
        t = self.get_user_transport(user, None)
1770
        ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
1771
                                            stderr=tests.StringIOWrapper())
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1772
        # Create a minimal config file with the right password
5957.2.1 by Vincent Ladeuil
Tweak the http Authorization tests to make some parts easier to reuse (parametrization mostly).
1773
        _setup_authentication_config(scheme='http', port=self.server.port,
1774
                                     user=user, password=password)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1775
        # Issue a request to the server to connect
1776
        self.assertEqual('contents of a\n',t.get('a').read())
1777
        # stdin should have  been left untouched
1778
        self.assertEqual(stdin_content, ui.ui_factory.stdin.readline())
1779
        # Only one 'Authentication Required' error should occur
1780
        self.assertEqual(1, self.server.auth_required_errors)
1781
3111.1.26 by Vincent Ladeuil
Re-add a test lost in refactoring.
1782
    def test_changing_nonce(self):
4307.4.2 by Vincent Ladeuil
Handle servers proposing several authentication schemes.
1783
        if self._auth_server not in (http_utils.HTTPDigestAuthServer,
1784
                                     http_utils.ProxyDigestAuthServer):
1785
            raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
3111.1.26 by Vincent Ladeuil
Re-add a test lost in refactoring.
1786
        if self._testing_pycurl():
6050.1.2 by Martin
Make tests raising KnownFailure use the knownFailure method instead
1787
            self.knownFailure(
3111.1.26 by Vincent Ladeuil
Re-add a test lost in refactoring.
1788
                'pycurl does not handle a nonce change')
1789
        self.server.add_user('joe', 'foo')
1790
        t = self.get_user_transport('joe', 'foo')
1791
        self.assertEqual('contents of a\n', t.get('a').read())
1792
        self.assertEqual('contents of b\n', t.get('b').read())
1793
        # Only one 'Authentication Required' error should have
1794
        # occured so far
1795
        self.assertEqual(1, self.server.auth_required_errors)
1796
        # The server invalidates the current nonce
1797
        self.server.auth_nonce = self.server.auth_nonce + '. No, now!'
1798
        self.assertEqual('contents of a\n', t.get('a').read())
1799
        # Two 'Authentication Required' errors should occur (the
1800
        # initial 'who are you' and a second 'who are you' with the new nonce)
1801
        self.assertEqual(2, self.server.auth_required_errors)
1802
5484.2.1 by Martin Pool
Add failing test for AbstractAuthHandler and bug 654684
1803
    def test_user_from_auth_conf(self):
1804
        if self._testing_pycurl():
1805
            raise tests.TestNotApplicable(
1806
                'pycurl does not support authentication.conf')
1807
        user = 'joe'
1808
        password = 'foo'
1809
        self.server.add_user(user, password)
5957.2.1 by Vincent Ladeuil
Tweak the http Authorization tests to make some parts easier to reuse (parametrization mostly).
1810
        _setup_authentication_config(scheme='http', port=self.server.port,
1811
                                     user=user, password=password)
5484.2.1 by Martin Pool
Add failing test for AbstractAuthHandler and bug 654684
1812
        t = self.get_user_transport(None, None)
1813
        # Issue a request to the server to connect
1814
        self.assertEqual('contents of a\n', t.get('a').read())
1815
        # Only one 'Authentication Required' error should occur
1816
        self.assertEqual(1, self.server.auth_required_errors)
1817
5957.2.3 by Vincent Ladeuil
Mask credentials in the -Dhttp logging
1818
    def test_no_credential_leaks_in_log(self):
1819
        self.overrideAttr(debug, 'debug_flags', set(['http']))
1820
        user = 'joe'
1821
        password = 'very-sensitive-password'
1822
        self.server.add_user(user, password)
1823
        t = self.get_user_transport(user, password)
1824
        # Capture the debug calls to mutter
1825
        self.mutters = []
1826
        def mutter(*args):
1827
            lines = args[0] % args[1:]
1828
            # Some calls output multiple lines, just split them now since we
1829
            # care about a single one later.
1830
            self.mutters.extend(lines.splitlines())
1831
        self.overrideAttr(trace, 'mutter', mutter)
1832
        # Issue a request to the server to connect
1833
        self.assertEqual(True, t.has('a'))
1834
        # Only one 'Authentication Required' error should occur
1835
        self.assertEqual(1, self.server.auth_required_errors)
1836
        # Since the authentification succeeded, there should be a corresponding
1837
        # debug line
1838
        sent_auth_headers = [line for line in self.mutters
1839
                             if line.startswith('> %s' % (self._auth_header,))]
1840
        self.assertLength(1, sent_auth_headers)
1841
        self.assertStartsWith(sent_auth_headers[0],
1842
                              '> %s: <masked>' % (self._auth_header,))
1843
5484.2.1 by Martin Pool
Add failing test for AbstractAuthHandler and bug 654684
1844
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1845
class TestProxyAuth(TestAuth):
5957.2.1 by Vincent Ladeuil
Tweak the http Authorization tests to make some parts easier to reuse (parametrization mostly).
1846
    """Test proxy authentication schemes.
1847
1848
    This inherits from TestAuth to tweak the setUp and filter some failing
1849
    tests.
1850
    """
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1851
5462.3.15 by Martin Pool
Turn variations into scenario lists
1852
    scenarios = multiply_scenarios(
1853
        vary_by_http_client_implementation(),
1854
        vary_by_http_protocol_version(),
1855
        vary_by_http_proxy_auth_scheme(),
1856
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
1857
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1858
    def setUp(self):
1859
        super(TestProxyAuth, self).setUp()
1860
        # Override the contents to avoid false positives
1861
        self.build_tree_contents([('a', 'not proxied contents of a\n'),
1862
                                  ('b', 'not proxied contents of b\n'),
1863
                                  ('a-proxied', 'contents of a\n'),
1864
                                  ('b-proxied', 'contents of b\n'),
1865
                                  ])
1866
3910.2.4 by Vincent Ladeuil
Fixed as per John's review.
1867
    def get_user_transport(self, user, password):
6615.7.1 by Vincent Ladeuil
Fix pycurl proxy tests for newer and stricter pycurl versions
1868
        proxy_url = self.get_user_url(user, password)
1869
        if self._testing_pycurl():
1870
            proxy_url = proxy_url.replace('+pycurl', '')
1871
        self.overrideEnv('all_proxy', proxy_url)
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1872
        return TestAuth.get_user_transport(self, user, password)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1873
1874
    def test_empty_pass(self):
1875
        if self._testing_pycurl():
1876
            import pycurl
1877
            if pycurl.version_info()[1] < '7.16.0':
6050.1.2 by Martin
Make tests raising KnownFailure use the knownFailure method instead
1878
                self.knownFailure(
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1879
                    'pycurl < 7.16.0 does not handle empty proxy passwords')
1880
        super(TestProxyAuth, self).test_empty_pass()
1881
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1882
1883
class SampleSocket(object):
1884
    """A socket-like object for use in testing the HTTP request handler."""
1885
1886
    def __init__(self, socket_read_content):
1887
        """Constructs a sample socket.
1888
1889
        :param socket_read_content: a byte sequence
1890
        """
1891
        # Use plain python StringIO so we can monkey-patch the close method to
1892
        # not discard the contents.
1893
        from StringIO import StringIO
1894
        self.readfile = StringIO(socket_read_content)
1895
        self.writefile = StringIO()
1896
        self.writefile.close = lambda: None
5247.3.18 by Vincent Ladeuil
Fix some fallouts from previous fixes, all tests passing (no more http leaks).
1897
        self.close = lambda: None
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1898
1899
    def makefile(self, mode='r', bufsize=None):
1900
        if 'r' in mode:
1901
            return self.readfile
1902
        else:
1903
            return self.writefile
1904
1905
1906
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1907
5462.3.15 by Martin Pool
Turn variations into scenario lists
1908
    scenarios = multiply_scenarios(
5705.1.1 by Vincent Ladeuil
Correctly parse partial range specifiers in the HTTP test server
1909
        vary_by_http_client_implementation(),
5462.3.15 by Martin Pool
Turn variations into scenario lists
1910
        vary_by_http_protocol_version(),
1911
        )
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
1912
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1913
    def setUp(self):
1914
        super(SmartHTTPTunnellingTest, self).setUp()
1915
        # We use the VFS layer as part of HTTP tunnelling tests.
5570.3.6 by Vincent Ladeuil
Get rid of all _captureVar() calls, no test failures, pfew.
1916
        self.overrideEnv('BZR_NO_SMART_VFS', None)
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1917
        self.transport_readonly_server = http_utils.HTTPServerWithSmarts
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1918
        self.http_server = self.get_readonly_server()
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1919
1920
    def create_transport_readonly_server(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1921
        server = http_utils.HTTPServerWithSmarts(
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1922
            protocol_version=self._protocol_version)
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1923
        server._url_protocol = self._url_protocol
1924
        return server
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1925
6472.2.3 by Jelmer Vernooij
More control dir.
1926
    def test_open_controldir(self):
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
1927
        branch = self.make_branch('relpath')
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1928
        url = self.http_server.get_url() + 'relpath'
6472.2.3 by Jelmer Vernooij
More control dir.
1929
        bd = controldir.ControlDir.open(url)
5247.2.28 by Vincent Ladeuil
Some more cleanups spotted on windows.
1930
        self.addCleanup(bd.transport.disconnect)
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
1931
        self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1932
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1933
    def test_bulk_data(self):
1934
        # We should be able to send and receive bulk data in a single message.
1935
        # The 'readv' command in the smart protocol both sends and receives
1936
        # bulk data, so we use that.
1937
        self.build_tree(['data-file'])
6083.1.1 by Jelmer Vernooij
Use get_transport_from_{url,path} in more places.
1938
        http_transport = transport.get_transport_from_url(
1939
            self.http_server.get_url())
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1940
        medium = http_transport.get_smart_medium()
1941
        # Since we provide the medium, the url below will be mostly ignored
1942
        # during the test, as long as the path is '/'.
1943
        remote_transport = remote.RemoteTransport('bzr://fake_host/',
1944
                                                  medium=medium)
1945
        self.assertEqual(
1946
            [(0, "c")], list(remote_transport.readv("data-file", [(0,1)])))
1947
1948
    def test_http_send_smart_request(self):
1949
1950
        post_body = 'hello\n'
3245.4.59 by Andrew Bennetts
Various tweaks in response to Martin's review.
1951
        expected_reply_body = 'ok\x012\n'
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1952
6083.1.1 by Jelmer Vernooij
Use get_transport_from_{url,path} in more places.
1953
        http_transport = transport.get_transport_from_url(
1954
            self.http_server.get_url())
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1955
        medium = http_transport.get_smart_medium()
1956
        response = medium.send_http_smart_request(post_body)
1957
        reply_body = response.read()
1958
        self.assertEqual(expected_reply_body, reply_body)
1959
1960
    def test_smart_http_server_post_request_handler(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1961
        httpd = self.http_server.server
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1962
1963
        socket = SampleSocket(
1964
            'POST /.bzr/smart %s \r\n' % self._protocol_version
1965
            # HTTP/1.1 posts must have a Content-Length (but it doesn't hurt
1966
            # for 1.0)
1967
            + 'Content-Length: 6\r\n'
1968
            '\r\n'
1969
            'hello\n')
1970
        # Beware: the ('localhost', 80) below is the
1971
        # client_address parameter, but we don't have one because
1972
        # we have defined a socket which is not bound to an
1973
        # address. The test framework never uses this client
1974
        # address, so far...
1975
        request_handler = http_utils.SmartRequestHandler(socket,
1976
                                                         ('localhost', 80),
1977
                                                         httpd)
1978
        response = socket.writefile.getvalue()
1979
        self.assertStartsWith(response, '%s 200 ' % self._protocol_version)
1980
        # This includes the end of the HTTP headers, and all the body.
3245.4.59 by Andrew Bennetts
Various tweaks in response to Martin's review.
1981
        expected_end_of_response = '\r\n\r\nok\x012\n'
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1982
        self.assertEndsWith(response, expected_end_of_response)
1983
1984
3430.3.4 by Vincent Ladeuil
Of course we can write tests !
1985
class ForbiddenRequestHandler(http_server.TestingHTTPRequestHandler):
1986
    """No smart server here request handler."""
1987
1988
    def do_POST(self):
1989
        self.send_error(403, "Forbidden")
1990
1991
1992
class SmartClientAgainstNotSmartServer(TestSpecificRequestHandler):
1993
    """Test smart client behaviour against an http server without smarts."""
1994
1995
    _req_handler_class = ForbiddenRequestHandler
1996
1997
    def test_probe_smart_server(self):
1998
        """Test error handling against server refusing smart requests."""
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
1999
        t = self.get_readonly_transport()
3430.3.4 by Vincent Ladeuil
Of course we can write tests !
2000
        # No need to build a valid smart request here, the server will not even
2001
        # try to interpret it.
2002
        self.assertRaises(errors.SmartProtocolError,
5599.3.6 by John Arbash Meinel
Revert the medium check, since it isn't necessary, and vila is worried about confusing people.
2003
                          t.get_smart_medium().send_http_smart_request,
2004
                          'whatever')
3430.3.4 by Vincent Ladeuil
Of course we can write tests !
2005
5247.3.15 by Vincent Ladeuil
All http tests passing, https failing.
2006
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2007
class Test_redirected_to(tests.TestCase):
2008
5462.3.15 by Martin Pool
Turn variations into scenario lists
2009
    scenarios = vary_by_http_client_implementation()
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
2010
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2011
    def test_redirected_to_subdir(self):
2012
        t = self._transport('http://www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
2013
        r = t._redirected_to('http://www.example.com/foo',
2014
                             'http://www.example.com/foo/subdir')
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2015
        self.assertIsInstance(r, type(t))
2016
        # Both transports share the some connection
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
2017
        self.assertEqual(t._get_connection(), r._get_connection())
6614.1.3 by Vincent Ladeuil
Fix assertEquals being deprecated by using assertEqual.
2018
        self.assertEqual('http://www.example.com/foo/subdir/', r.base)
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2019
3878.4.3 by Vincent Ladeuil
Fix bug #303959 by returning a transport based on the same url
2020
    def test_redirected_to_self_with_slash(self):
2021
        t = self._transport('http://www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
2022
        r = t._redirected_to('http://www.example.com/foo',
2023
                             'http://www.example.com/foo/')
3878.4.3 by Vincent Ladeuil
Fix bug #303959 by returning a transport based on the same url
2024
        self.assertIsInstance(r, type(t))
2025
        # Both transports share the some connection (one can argue that we
2026
        # should return the exact same transport here, but that seems
2027
        # overkill).
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
2028
        self.assertEqual(t._get_connection(), r._get_connection())
3878.4.3 by Vincent Ladeuil
Fix bug #303959 by returning a transport based on the same url
2029
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2030
    def test_redirected_to_host(self):
2031
        t = self._transport('http://www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
2032
        r = t._redirected_to('http://www.example.com/foo',
2033
                             'http://foo.example.com/foo/subdir')
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2034
        self.assertIsInstance(r, type(t))
6614.1.3 by Vincent Ladeuil
Fix assertEquals being deprecated by using assertEqual.
2035
        self.assertEqual('http://foo.example.com/foo/subdir/',
6145.1.1 by Jelmer Vernooij
Add some tests.
2036
            r.external_url())
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2037
2038
    def test_redirected_to_same_host_sibling_protocol(self):
2039
        t = self._transport('http://www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
2040
        r = t._redirected_to('http://www.example.com/foo',
2041
                             'https://www.example.com/foo')
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2042
        self.assertIsInstance(r, type(t))
6614.1.3 by Vincent Ladeuil
Fix assertEquals being deprecated by using assertEqual.
2043
        self.assertEqual('https://www.example.com/foo/',
6145.1.1 by Jelmer Vernooij
Add some tests.
2044
            r.external_url())
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2045
2046
    def test_redirected_to_same_host_different_protocol(self):
2047
        t = self._transport('http://www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
2048
        r = t._redirected_to('http://www.example.com/foo',
2049
                             'ftp://www.example.com/foo')
6614.1.2 by Vincent Ladeuil
Fix assertNotEquals being deprecated by using assertNotEqual.
2050
        self.assertNotEqual(type(r), type(t))
6614.1.3 by Vincent Ladeuil
Fix assertEquals being deprecated by using assertEqual.
2051
        self.assertEqual('ftp://www.example.com/foo/', r.external_url())
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2052
6145.1.3 by Jelmer Vernooij
Fix redirecting to other transports.
2053
    def test_redirected_to_same_host_specific_implementation(self):
2054
        t = self._transport('http://www.example.com/foo')
2055
        r = t._redirected_to('http://www.example.com/foo',
2056
                             'https+urllib://www.example.com/foo')
6614.1.3 by Vincent Ladeuil
Fix assertEquals being deprecated by using assertEqual.
2057
        self.assertEqual('https://www.example.com/foo/', r.external_url())
6145.1.3 by Jelmer Vernooij
Fix redirecting to other transports.
2058
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2059
    def test_redirected_to_different_host_same_user(self):
2060
        t = self._transport('http://joe@www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
2061
        r = t._redirected_to('http://www.example.com/foo',
2062
                             'https://foo.example.com/foo')
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
2063
        self.assertIsInstance(r, type(t))
6055.2.1 by Jelmer Vernooij
Add UnparsedUrl.
2064
        self.assertEqual(t._parsed_url.user, r._parsed_url.user)
6614.1.3 by Vincent Ladeuil
Fix assertEquals being deprecated by using assertEqual.
2065
        self.assertEqual('https://joe@foo.example.com/foo/', r.external_url())
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
2066
2067
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
2068
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
2069
    """Request handler for a unique and pre-defined request.
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
2070
2071
    The only thing we care about here is how many bytes travel on the wire. But
2072
    since we want to measure it for a real http client, we have to send it
2073
    correct responses.
2074
2075
    We expect to receive a *single* request nothing more (and we won't even
2076
    check what request it is, we just measure the bytes read until an empty
2077
    line.
2078
    """
2079
4731.2.3 by Vincent Ladeuil
Reduce the leaking http tests from ~200 to ~5.
2080
    def _handle_one_request(self):
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
2081
        tcs = self.server.test_case_server
2082
        requestline = self.rfile.readline()
2083
        headers = self.MessageClass(self.rfile, 0)
2084
        # We just read: the request, the headers, an empty line indicating the
2085
        # end of the headers.
2086
        bytes_read = len(requestline)
2087
        for line in headers.headers:
2088
            bytes_read += len(line)
2089
        bytes_read += len('\r\n')
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
2090
        if requestline.startswith('POST'):
2091
            # The body should be a single line (or we don't know where it ends
2092
            # and we don't want to issue a blocking read)
2093
            body = self.rfile.readline()
2094
            bytes_read += len(body)
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
2095
        tcs.bytes_read = bytes_read
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
2096
2097
        # We set the bytes written *before* issuing the write, the client is
2098
        # supposed to consume every produced byte *before* checking that value.
3945.1.7 by Vincent Ladeuil
Test against https.
2099
2100
        # Doing the oppposite may lead to test failure: we may be interrupted
2101
        # after the write but before updating the value. The client can then
2102
        # continue and read the value *before* we can update it. And yes,
2103
        # this has been observed -- vila 20090129
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
2104
        tcs.bytes_written = len(tcs.canned_response)
2105
        self.wfile.write(tcs.canned_response)
2106
2107
2108
class ActivityServerMixin(object):
2109
2110
    def __init__(self, protocol_version):
2111
        super(ActivityServerMixin, self).__init__(
2112
            request_handler=PredefinedRequestHandler,
2113
            protocol_version=protocol_version)
2114
        # Bytes read and written by the server
2115
        self.bytes_read = 0
2116
        self.bytes_written = 0
2117
        self.canned_response = None
2118
2119
2120
class ActivityHTTPServer(ActivityServerMixin, http_server.HttpServer):
2121
    pass
2122
2123
5967.12.1 by Martin Pool
Move all test features into bzrlib.tests.features
2124
if features.HTTPSServerFeature.available():
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
2125
    from bzrlib.tests import https_server
2126
    class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
2127
        pass
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
2128
2129
4776.2.1 by Vincent Ladeuil
Support no activity report on http sockets.
2130
class TestActivityMixin(object):
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
2131
    """Test socket activity reporting.
2132
2133
    We use a special purpose server to control the bytes sent and received and
2134
    be able to predict the activity on the client socket.
2135
    """
2136
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
2137
    def setUp(self):
2138
        self.server = self._activity_server(self._protocol_version)
4934.3.3 by Martin Pool
Rename Server.setUp to Server.start_server
2139
        self.server.start_server()
6238.2.20 by Vincent Ladeuil
Call the proper base class setUp() and set the inheritance order correctly now that get_transport exists in the base class.
2140
        self.addCleanup(self.server.stop_server)
5340.15.1 by John Arbash Meinel
supersede exc-info branch
2141
        _activities = {} # Don't close over self and create a cycle
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
2142
        def report_activity(t, bytes, direction):
5340.15.1 by John Arbash Meinel
supersede exc-info branch
2143
            count = _activities.get(direction, 0)
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
2144
            count += bytes
5340.15.1 by John Arbash Meinel
supersede exc-info branch
2145
            _activities[direction] = count
2146
        self.activities = _activities
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
2147
        # We override at class level because constructors may propagate the
2148
        # bound method and render instance overriding ineffective (an
4031.3.1 by Frank Aspell
Fixing various typos
2149
        # alternative would be to define a specific ui factory instead...)
4986.2.6 by Martin Pool
Clean up test_http setUp methods
2150
        self.overrideAttr(self._transport, '_report_activity', report_activity)
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
2151
2152
    def get_transport(self):
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
2153
        t = self._transport(self.server.get_url())
2154
        # FIXME: Needs cleanup -- vila 20100611
2155
        return t
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
2156
2157
    def assertActivitiesMatch(self):
2158
        self.assertEqual(self.server.bytes_read,
2159
                         self.activities.get('write', 0), 'written bytes')
2160
        self.assertEqual(self.server.bytes_written,
2161
                         self.activities.get('read', 0), 'read bytes')
2162
2163
    def test_get(self):
2164
        self.server.canned_response = '''HTTP/1.1 200 OK\r
2165
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
2166
Server: Apache/2.0.54 (Fedora)\r
2167
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
2168
ETag: "56691-23-38e9ae00"\r
2169
Accept-Ranges: bytes\r
2170
Content-Length: 35\r
2171
Connection: close\r
2172
Content-Type: text/plain; charset=UTF-8\r
2173
\r
2174
Bazaar-NG meta directory, format 1
2175
'''
2176
        t = self.get_transport()
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
2177
        self.assertEqual('Bazaar-NG meta directory, format 1\n',
2178
                         t.get('foo/bar').read())
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
2179
        self.assertActivitiesMatch()
2180
2181
    def test_has(self):
2182
        self.server.canned_response = '''HTTP/1.1 200 OK\r
2183
Server: SimpleHTTP/0.6 Python/2.5.2\r
2184
Date: Thu, 29 Jan 2009 20:21:47 GMT\r
2185
Content-type: application/octet-stream\r
2186
Content-Length: 20\r
2187
Last-Modified: Thu, 29 Jan 2009 20:21:47 GMT\r
2188
\r
2189
'''
2190
        t = self.get_transport()
2191
        self.assertTrue(t.has('foo/bar'))
2192
        self.assertActivitiesMatch()
2193
2194
    def test_readv(self):
2195
        self.server.canned_response = '''HTTP/1.1 206 Partial Content\r
2196
Date: Tue, 11 Jul 2006 04:49:48 GMT\r
2197
Server: Apache/2.0.54 (Fedora)\r
2198
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
2199
ETag: "238a3c-16ec2-805c5540"\r
2200
Accept-Ranges: bytes\r
2201
Content-Length: 1534\r
2202
Connection: close\r
2203
Content-Type: multipart/byteranges; boundary=418470f848b63279b\r
2204
\r
2205
\r
2206
--418470f848b63279b\r
2207
Content-type: text/plain; charset=UTF-8\r
2208
Content-range: bytes 0-254/93890\r
2209
\r
2210
mbp@sourcefrog.net-20050309040815-13242001617e4a06
2211
mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e7627
2212
mbp@sourcefrog.net-20050309040957-6cad07f466bb0bb8
2213
mbp@sourcefrog.net-20050309041501-c840e09071de3b67
2214
mbp@sourcefrog.net-20050309044615-c24a3250be83220a
2215
\r
2216
--418470f848b63279b\r
2217
Content-type: text/plain; charset=UTF-8\r
2218
Content-range: bytes 1000-2049/93890\r
2219
\r
2220
40-fd4ec249b6b139ab
2221
mbp@sourcefrog.net-20050311063625-07858525021f270b
2222
mbp@sourcefrog.net-20050311231934-aa3776aff5200bb9
2223
mbp@sourcefrog.net-20050311231953-73aeb3a131c3699a
2224
mbp@sourcefrog.net-20050311232353-f5e33da490872c6a
2225
mbp@sourcefrog.net-20050312071639-0a8f59a34a024ff0
2226
mbp@sourcefrog.net-20050312073432-b2c16a55e0d6e9fb
2227
mbp@sourcefrog.net-20050312073831-a47c3335ece1920f
2228
mbp@sourcefrog.net-20050312085412-13373aa129ccbad3
2229
mbp@sourcefrog.net-20050313052251-2bf004cb96b39933
2230
mbp@sourcefrog.net-20050313052856-3edd84094687cb11
2231
mbp@sourcefrog.net-20050313053233-e30a4f28aef48f9d
2232
mbp@sourcefrog.net-20050313053853-7c64085594ff3072
2233
mbp@sourcefrog.net-20050313054757-a86c3f5871069e22
2234
mbp@sourcefrog.net-20050313061422-418f1f73b94879b9
2235
mbp@sourcefrog.net-20050313120651-497bd231b19df600
2236
mbp@sourcefrog.net-20050314024931-eae0170ef25a5d1a
2237
mbp@sourcefrog.net-20050314025438-d52099f915fe65fc
2238
mbp@sourcefrog.net-20050314025539-637a636692c055cf
2239
mbp@sourcefrog.net-20050314025737-55eb441f430ab4ba
2240
mbp@sourcefrog.net-20050314025901-d74aa93bb7ee8f62
2241
mbp@source\r
2242
--418470f848b63279b--\r
2243
'''
2244
        t = self.get_transport()
2245
        # Remember that the request is ignored and that the ranges below
2246
        # doesn't have to match the canned response.
2247
        l = list(t.readv('/foo/bar', ((0, 255), (1000, 1050))))
2248
        self.assertEqual(2, len(l))
2249
        self.assertActivitiesMatch()
2250
2251
    def test_post(self):
2252
        self.server.canned_response = '''HTTP/1.1 200 OK\r
2253
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
2254
Server: Apache/2.0.54 (Fedora)\r
2255
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
2256
ETag: "56691-23-38e9ae00"\r
2257
Accept-Ranges: bytes\r
2258
Content-Length: 35\r
2259
Connection: close\r
2260
Content-Type: text/plain; charset=UTF-8\r
2261
\r
2262
lalala whatever as long as itsssss
2263
'''
2264
        t = self.get_transport()
2265
        # We must send a single line of body bytes, see
4731.2.3 by Vincent Ladeuil
Reduce the leaking http tests from ~200 to ~5.
2266
        # PredefinedRequestHandler._handle_one_request
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
2267
        code, f = t._post('abc def end-of-body\n')
2268
        self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2269
        self.assertActivitiesMatch()
4776.2.1 by Vincent Ladeuil
Support no activity report on http sockets.
2270
2271
6238.2.23 by Vincent Ladeuil
Using the right transport classes, creating a dedicated config store is not needed anymore.
2272
class TestActivity(tests.TestCase, TestActivityMixin):
4776.2.1 by Vincent Ladeuil
Support no activity report on http sockets.
2273
5462.3.15 by Martin Pool
Turn variations into scenario lists
2274
    scenarios = multiply_scenarios(
2275
        vary_by_http_activity(),
2276
        vary_by_http_protocol_version(),
2277
        )
5462.3.11 by Martin Pool
Delete test_http.load_tests, and just specify variations per test
2278
4776.2.1 by Vincent Ladeuil
Support no activity report on http sockets.
2279
    def setUp(self):
6552.1.2 by Vincent Ladeuil
Stop assuming that classes deriving from TestActivityMixin will also derive directly from tests.TestCase.
2280
        super(TestActivity, self).setUp()
4986.2.6 by Martin Pool
Clean up test_http setUp methods
2281
        TestActivityMixin.setUp(self)
4776.2.1 by Vincent Ladeuil
Support no activity report on http sockets.
2282
2283
6238.2.23 by Vincent Ladeuil
Using the right transport classes, creating a dedicated config store is not needed anymore.
2284
class TestNoReportActivity(tests.TestCase, TestActivityMixin):
4776.2.1 by Vincent Ladeuil
Support no activity report on http sockets.
2285
4986.2.6 by Martin Pool
Clean up test_http setUp methods
2286
    # Unlike TestActivity, we are really testing ReportingFileSocket and
2287
    # ReportingSocket, so we don't need all the parametrization. Since
2288
    # ReportingFileSocket and ReportingSocket are wrappers, it's easier to
2289
    # test them through their use by the transport than directly (that's a
2290
    # bit less clean but far more simpler and effective).
2291
    _activity_server = ActivityHTTPServer
2292
    _protocol_version = 'HTTP/1.1'
2293
4776.2.1 by Vincent Ladeuil
Support no activity report on http sockets.
2294
    def setUp(self):
6552.1.2 by Vincent Ladeuil
Stop assuming that classes deriving from TestActivityMixin will also derive directly from tests.TestCase.
2295
        super(TestNoReportActivity, self).setUp()
4986.2.6 by Martin Pool
Clean up test_http setUp methods
2296
        self._transport =_urllib.HttpTransport_urllib
2297
        TestActivityMixin.setUp(self)
4776.2.1 by Vincent Ladeuil
Support no activity report on http sockets.
2298
2299
    def assertActivitiesMatch(self):
2300
        # Nothing to check here
2301
        pass
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2302
2303
2304
class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver):
2305
    """Test authentication on the redirected http server."""
2306
5462.3.15 by Martin Pool
Turn variations into scenario lists
2307
    scenarios = vary_by_http_protocol_version()
5462.3.9 by Martin Pool
List variations within the test classes, rather than in load_tests
2308
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2309
    _auth_header = 'Authorization'
2310
    _password_prompt_prefix = ''
2311
    _username_prompt_prefix = ''
2312
    _auth_server = http_utils.HTTPBasicAuthServer
2313
    _transport = _urllib.HttpTransport_urllib
2314
2315
    def setUp(self):
2316
        super(TestAuthOnRedirected, self).setUp()
2317
        self.build_tree_contents([('a','a'),
2318
                                  ('1/',),
2319
                                  ('1/a', 'redirected once'),
2320
                                  ],)
2321
        new_prefix = 'http://%s:%s' % (self.new_server.host,
2322
                                       self.new_server.port)
2323
        self.old_server.redirections = [
2324
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
2325
        self.old_transport = self.get_old_transport()
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2326
        self.new_server.add_user('joe', 'foo')
5247.2.26 by Vincent Ladeuil
Fix http redirection socket leaks.
2327
        cleanup_http_redirection_connections(self)
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2328
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
2329
    def create_transport_readonly_server(self):
2330
        server = self._auth_server(protocol_version=self._protocol_version)
2331
        server._url_protocol = self._url_protocol
2332
        return server
2333
2334
    def get_a(self, t):
2335
        return t.get('a')
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2336
2337
    def test_auth_on_redirected_via_do_catching_redirections(self):
2338
        self.redirections = 0
2339
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
2340
        def redirected(t, exception, redirection_notice):
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2341
            self.redirections += 1
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
2342
            redirected_t = t._redirected_to(exception.source, exception.target)
5247.2.28 by Vincent Ladeuil
Some more cleanups spotted on windows.
2343
            self.addCleanup(redirected_t.disconnect)
5247.3.29 by Vincent Ladeuil
Fix a bunch of tests that were building transport objects explicitely instead of relying on self.get_transport() leading to many wrong http implementation objects being tested.
2344
            return redirected_t
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2345
2346
        stdout = tests.StringIOWrapper()
2347
        stderr = tests.StringIOWrapper()
2348
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2349
                                            stdout=stdout, stderr=stderr)
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
2350
        self.assertEqual('redirected once',
2351
                         transport.do_catching_redirections(
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2352
                self.get_a, self.old_transport, redirected).read())
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
2353
        self.assertEqual(1, self.redirections)
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2354
        # stdin should be empty
2355
        self.assertEqual('', ui.ui_factory.stdin.readline())
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
2356
        # stdout should be empty, stderr will contains the prompts
2357
        self.assertEqual('', stdout.getvalue())
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2358
2359
    def test_auth_on_redirected_via_following_redirections(self):
2360
        self.new_server.add_user('joe', 'foo')
2361
        stdout = tests.StringIOWrapper()
2362
        stderr = tests.StringIOWrapper()
2363
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2364
                                            stdout=stdout, stderr=stderr)
2365
        t = self.old_transport
2366
        req = RedirectedRequest('GET', t.abspath('a'))
2367
        new_prefix = 'http://%s:%s' % (self.new_server.host,
2368
                                       self.new_server.port)
2369
        self.old_server.redirections = [
2370
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
5247.2.28 by Vincent Ladeuil
Some more cleanups spotted on windows.
2371
        self.assertEqual('redirected once', t._perform(req).read())
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2372
        # stdin should be empty
2373
        self.assertEqual('', ui.ui_factory.stdin.readline())
4795.4.6 by Vincent Ladeuil
Fixed as per John's review.
2374
        # stdout should be empty, stderr will contains the prompts
2375
        self.assertEqual('', stdout.getvalue())
4795.4.5 by Vincent Ladeuil
Make sure all redirection code paths can handle authentication.
2376