← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_http.py

(vila) Revise legal option names to be less drastic. (Vincent Ladeuil)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_http.py
1
 
# Copyright (C) 2005-2010 Canonical Ltd
 
1
# Copyright (C) 2005-2012, 2015 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
23
23
# TODO: Should be renamed to bzrlib.transport.http.tests?
24
24
# TODO: What about renaming to bzrlib.tests.transport.http ?
25
25
 
26
 
from cStringIO import StringIO
27
26
import httplib
28
 
import os
29
 
import select
30
27
import SimpleHTTPServer
31
28
import socket
32
29
import sys
34
31
 
35
32
import bzrlib
36
33
from bzrlib import (
37
 
    bzrdir,
38
34
    config,
 
35
    controldir,
 
36
    debug,
39
37
    errors,
40
38
    osutils,
41
39
    remote as _mod_remote,
42
40
    tests,
 
41
    trace,
43
42
    transport,
44
43
    ui,
45
 
    urlutils,
46
44
    )
47
45
from bzrlib.tests import (
48
46
    features,
49
47
    http_server,
50
48
    http_utils,
 
49
    test_server,
 
50
    )
 
51
from bzrlib.tests.scenarios import (
 
52
    load_tests_apply_scenarios,
 
53
    multiply_scenarios,
51
54
    )
52
55
from bzrlib.transport import (
53
56
    http,
63
66
    from bzrlib.transport.http._pycurl import PyCurlTransport
64
67
 
65
68
 
66
 
def load_tests(standard_tests, module, loader):
67
 
    """Multiply tests for http clients and protocol versions."""
68
 
    result = loader.suiteClass()
69
 
 
70
 
    # one for each transport implementation
71
 
    t_tests, remaining_tests = tests.split_suite_by_condition(
72
 
        standard_tests, tests.condition_isinstance((
73
 
                TestHttpTransportRegistration,
74
 
                TestHttpTransportUrls,
75
 
                Test_redirected_to,
76
 
                )))
 
69
load_tests = load_tests_apply_scenarios
 
70
 
 
71
 
 
72
def vary_by_http_client_implementation():
 
73
    """Test the two libraries we can use, pycurl and urllib."""
77
74
    transport_scenarios = [
78
75
        ('urllib', dict(_transport=_urllib.HttpTransport_urllib,
79
76
                        _server=http_server.HttpServer_urllib,
80
 
                        _qualified_prefix='http+urllib',)),
 
77
                        _url_protocol='http+urllib',)),
81
78
        ]
82
79
    if features.pycurl.available():
83
80
        transport_scenarios.append(
84
81
            ('pycurl', dict(_transport=PyCurlTransport,
85
82
                            _server=http_server.HttpServer_PyCurl,
86
 
                            _qualified_prefix='http+pycurl',)))
87
 
    tests.multiply_tests(t_tests, transport_scenarios, result)
88
 
 
89
 
    protocol_scenarios = [
90
 
            ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
91
 
            ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
92
 
            ]
93
 
 
94
 
    # some tests are parametrized by the protocol version only
95
 
    p_tests, remaining_tests = tests.split_suite_by_condition(
96
 
        remaining_tests, tests.condition_isinstance((
97
 
                TestAuthOnRedirected,
98
 
                )))
99
 
    tests.multiply_tests(p_tests, protocol_scenarios, result)
100
 
 
101
 
    # each implementation tested with each HTTP version
102
 
    tp_tests, remaining_tests = tests.split_suite_by_condition(
103
 
        remaining_tests, tests.condition_isinstance((
104
 
                SmartHTTPTunnellingTest,
105
 
                TestDoCatchRedirections,
106
 
                TestHTTPConnections,
107
 
                TestHTTPRedirections,
108
 
                TestHTTPSilentRedirections,
109
 
                TestLimitedRangeRequestServer,
110
 
                TestPost,
111
 
                TestProxyHttpServer,
112
 
                TestRanges,
113
 
                TestSpecificRequestHandler,
114
 
                )))
115
 
    tp_scenarios = tests.multiply_scenarios(transport_scenarios,
116
 
                                            protocol_scenarios)
117
 
    tests.multiply_tests(tp_tests, tp_scenarios, result)
118
 
 
119
 
    # proxy auth: each auth scheme on all http versions on all implementations.
120
 
    tppa_tests, remaining_tests = tests.split_suite_by_condition(
121
 
        remaining_tests, tests.condition_isinstance((
122
 
                TestProxyAuth,
123
 
                )))
124
 
    proxy_auth_scheme_scenarios = [
125
 
        ('basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
126
 
        ('digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
127
 
        ('basicdigest',
128
 
         dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
 
83
                            _url_protocol='http+pycurl',)))
 
84
    return transport_scenarios
 
85
 
 
86
 
 
87
def vary_by_http_protocol_version():
 
88
    """Test on http/1.0 and 1.1"""
 
89
    return [
 
90
        ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
 
91
        ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
129
92
        ]
130
 
    tppa_scenarios = tests.multiply_scenarios(tp_scenarios,
131
 
                                              proxy_auth_scheme_scenarios)
132
 
    tests.multiply_tests(tppa_tests, tppa_scenarios, result)
133
 
 
134
 
    # auth: each auth scheme on all http versions on all implementations.
135
 
    tpa_tests, remaining_tests = tests.split_suite_by_condition(
136
 
        remaining_tests, tests.condition_isinstance((
137
 
                TestAuth,
138
 
                )))
139
 
    auth_scheme_scenarios = [
 
93
 
 
94
 
 
95
def vary_by_http_auth_scheme():
 
96
    scenarios = [
140
97
        ('basic', dict(_auth_server=http_utils.HTTPBasicAuthServer)),
141
98
        ('digest', dict(_auth_server=http_utils.HTTPDigestAuthServer)),
142
99
        ('basicdigest',
143
 
         dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
144
 
        ]
145
 
    tpa_scenarios = tests.multiply_scenarios(tp_scenarios,
146
 
                                             auth_scheme_scenarios)
147
 
    tests.multiply_tests(tpa_tests, tpa_scenarios, result)
148
 
 
149
 
    # activity: on all http[s] versions on all implementations
150
 
    tpact_tests, remaining_tests = tests.split_suite_by_condition(
151
 
        remaining_tests, tests.condition_isinstance((
152
 
                TestActivity,
153
 
                )))
 
100
            dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
 
101
        ]
 
102
    # Add some attributes common to all scenarios
 
103
    for scenario_id, scenario_dict in scenarios:
 
104
        scenario_dict.update(_auth_header='Authorization',
 
105
                             _username_prompt_prefix='',
 
106
                             _password_prompt_prefix='')
 
107
    return scenarios
 
108
 
 
109
 
 
110
def vary_by_http_proxy_auth_scheme():
 
111
    scenarios = [
 
112
        ('proxy-basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
 
113
        ('proxy-digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
 
114
        ('proxy-basicdigest',
 
115
            dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
 
116
        ]
 
117
    # Add some attributes common to all scenarios
 
118
    for scenario_id, scenario_dict in scenarios:
 
119
        scenario_dict.update(_auth_header='Proxy-Authorization',
 
120
                             _username_prompt_prefix='Proxy ',
 
121
                             _password_prompt_prefix='Proxy ')
 
122
    return scenarios
 
123
 
 
124
 
 
125
def vary_by_http_activity():
154
126
    activity_scenarios = [
155
127
        ('urllib,http', dict(_activity_server=ActivityHTTPServer,
156
 
                             _transport=_urllib.HttpTransport_urllib,)),
 
128
                            _transport=_urllib.HttpTransport_urllib,)),
157
129
        ]
158
 
    if tests.HTTPSServerFeature.available():
 
130
    if features.pycurl.available():
 
131
        activity_scenarios.append(
 
132
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
 
133
                                _transport=PyCurlTransport,)),)
 
134
    if features.HTTPSServerFeature.available():
 
135
        # FIXME: Until we have a better way to handle self-signed certificates
 
136
        # (like allowing them in a test specific authentication.conf for
 
137
        # example), we need some specialized pycurl/urllib transport for tests.
 
138
        # -- vila 2012-01-20
 
139
        from bzrlib.tests import (
 
140
            ssl_certs,
 
141
            )
 
142
        class HTTPS_urllib_transport(_urllib.HttpTransport_urllib):
 
143
 
 
144
            def __init__(self, base, _from_transport=None):
 
145
                super(HTTPS_urllib_transport, self).__init__(
 
146
                    base, _from_transport=_from_transport,
 
147
                    ca_certs=ssl_certs.build_path('ca.crt'))
 
148
 
159
149
        activity_scenarios.append(
160
150
            ('urllib,https', dict(_activity_server=ActivityHTTPSServer,
161
 
                                  _transport=_urllib.HttpTransport_urllib,)),)
162
 
    if features.pycurl.available():
163
 
        activity_scenarios.append(
164
 
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
165
 
                                 _transport=PyCurlTransport,)),)
166
 
        if tests.HTTPSServerFeature.available():
167
 
            from bzrlib.tests import (
168
 
                ssl_certs,
169
 
                )
170
 
            # FIXME: Until we have a better way to handle self-signed
171
 
            # certificates (like allowing them in a test specific
172
 
            # authentication.conf for example), we need some specialized pycurl
173
 
            # transport for tests.
 
151
                                  _transport=HTTPS_urllib_transport,)),)
 
152
        if features.pycurl.available():
174
153
            class HTTPS_pycurl_transport(PyCurlTransport):
175
154
 
176
155
                def __init__(self, base, _from_transport=None):
180
159
 
181
160
            activity_scenarios.append(
182
161
                ('pycurl,https', dict(_activity_server=ActivityHTTPSServer,
183
 
                                      _transport=HTTPS_pycurl_transport,)),)
184
 
 
185
 
    tpact_scenarios = tests.multiply_scenarios(activity_scenarios,
186
 
                                               protocol_scenarios)
187
 
    tests.multiply_tests(tpact_tests, tpact_scenarios, result)
188
 
 
189
 
    # No parametrization for the remaining tests
190
 
    result.addTests(remaining_tests)
191
 
 
192
 
    return result
 
162
                                    _transport=HTTPS_pycurl_transport,)),)
 
163
    return activity_scenarios
193
164
 
194
165
 
195
166
class FakeManager(object):
228
199
        self._sock.bind(('127.0.0.1', 0))
229
200
        self.host, self.port = self._sock.getsockname()
230
201
        self._ready = threading.Event()
231
 
        self._thread = threading.Thread(target=self._accept_read_and_reply)
232
 
        self._thread.setDaemon(True)
 
202
        self._thread = test_server.TestThread(
 
203
            sync_event=self._ready, target=self._accept_read_and_reply)
233
204
        self._thread.start()
234
 
        self._ready.wait(5)
 
205
        if 'threads' in tests.selftest_debug_flags:
 
206
            sys.stderr.write('Thread started: %s\n' % (self._thread.ident,))
 
207
        self._ready.wait()
235
208
 
236
209
    def _accept_read_and_reply(self):
237
210
        self._sock.listen(1)
238
211
        self._ready.set()
239
 
        self._sock.settimeout(5)
240
 
        try:
241
 
            conn, address = self._sock.accept()
242
 
            # On win32, the accepted connection will be non-blocking to start
243
 
            # with because we're using settimeout.
244
 
            conn.setblocking(True)
 
212
        conn, address = self._sock.accept()
 
213
        if self._expect_body_tail is not None:
245
214
            while not self.received_bytes.endswith(self._expect_body_tail):
246
215
                self.received_bytes += conn.recv(4096)
247
216
            conn.sendall('HTTP/1.1 200 OK\r\n')
248
 
        except socket.timeout:
249
 
            # Make sure the client isn't stuck waiting for us to e.g. accept.
 
217
        try:
250
218
            self._sock.close()
251
219
        except socket.error:
252
220
            # The client may have already closed the socket.
254
222
 
255
223
    def stop_server(self):
256
224
        try:
257
 
            self._sock.close()
 
225
            # Issue a fake connection to wake up the server and allow it to
 
226
            # finish quickly
 
227
            fake_conn = osutils.connect_socket((self.host, self.port))
 
228
            fake_conn.close()
258
229
        except socket.error:
259
230
            # We might have already closed it.  We don't care.
260
231
            pass
261
232
        self.host = None
262
233
        self.port = None
 
234
        self._thread.join()
 
235
        if 'threads' in tests.selftest_debug_flags:
 
236
            sys.stderr.write('Thread  joined: %s\n' % (self._thread.ident,))
263
237
 
264
238
 
265
239
class TestAuthHeader(tests.TestCase):
301
275
        self.assertEqual('realm="Thou should not pass"', remainder)
302
276
 
303
277
 
 
278
class TestHTTPRangeParsing(tests.TestCase):
 
279
 
 
280
    def setUp(self):
 
281
        super(TestHTTPRangeParsing, self).setUp()
 
282
        # We focus on range  parsing here and ignore everything else
 
283
        class RequestHandler(http_server.TestingHTTPRequestHandler):
 
284
            def setup(self): pass
 
285
            def handle(self): pass
 
286
            def finish(self): pass
 
287
 
 
288
        self.req_handler = RequestHandler(None, None, None)
 
289
 
 
290
    def assertRanges(self, ranges, header, file_size):
 
291
        self.assertEquals(ranges,
 
292
                          self.req_handler._parse_ranges(header, file_size))
 
293
 
 
294
    def test_simple_range(self):
 
295
        self.assertRanges([(0,2)], 'bytes=0-2', 12)
 
296
 
 
297
    def test_tail(self):
 
298
        self.assertRanges([(8, 11)], 'bytes=-4', 12)
 
299
 
 
300
    def test_tail_bigger_than_file(self):
 
301
        self.assertRanges([(0, 11)], 'bytes=-99', 12)
 
302
 
 
303
    def test_range_without_end(self):
 
304
        self.assertRanges([(4, 11)], 'bytes=4-', 12)
 
305
 
 
306
    def test_invalid_ranges(self):
 
307
        self.assertRanges(None, 'bytes=12-22', 12)
 
308
        self.assertRanges(None, 'bytes=1-3,12-22', 12)
 
309
        self.assertRanges(None, 'bytes=-', 12)
 
310
 
 
311
 
304
312
class TestHTTPServer(tests.TestCase):
305
313
    """Test the HTTP servers implementations."""
306
314
 
309
317
 
310
318
            protocol_version = 'HTTP/0.1'
311
319
 
312
 
        server = http_server.HttpServer(BogusRequestHandler)
313
 
        try:
314
 
            self.assertRaises(httplib.UnknownProtocol, server.start_server)
315
 
        except:
316
 
            server.stop_server()
317
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
320
        self.assertRaises(httplib.UnknownProtocol,
 
321
                          http_server.HttpServer, BogusRequestHandler)
318
322
 
319
323
    def test_force_invalid_protocol(self):
320
 
        server = http_server.HttpServer(protocol_version='HTTP/0.1')
321
 
        try:
322
 
            self.assertRaises(httplib.UnknownProtocol, server.start_server)
323
 
        except:
324
 
            server.stop_server()
325
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
324
        self.assertRaises(httplib.UnknownProtocol,
 
325
                          http_server.HttpServer, protocol_version='HTTP/0.1')
326
326
 
327
327
    def test_server_start_and_stop(self):
328
328
        server = http_server.HttpServer()
 
329
        self.addCleanup(server.stop_server)
329
330
        server.start_server()
330
 
        try:
331
 
            self.assertTrue(server._http_running)
332
 
        finally:
333
 
            server.stop_server()
334
 
        self.assertFalse(server._http_running)
 
331
        self.assertTrue(server.server is not None)
 
332
        self.assertTrue(server.server.serving is not None)
 
333
        self.assertTrue(server.server.serving)
335
334
 
336
335
    def test_create_http_server_one_zero(self):
337
336
        class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
340
339
 
341
340
        server = http_server.HttpServer(RequestHandlerOneZero)
342
341
        self.start_server(server)
343
 
        self.assertIsInstance(server._httpd, http_server.TestingHTTPServer)
 
342
        self.assertIsInstance(server.server, http_server.TestingHTTPServer)
344
343
 
345
344
    def test_create_http_server_one_one(self):
346
345
        class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
349
348
 
350
349
        server = http_server.HttpServer(RequestHandlerOneOne)
351
350
        self.start_server(server)
352
 
        self.assertIsInstance(server._httpd,
 
351
        self.assertIsInstance(server.server,
353
352
                              http_server.TestingThreadingHTTPServer)
354
353
 
355
354
    def test_create_http_server_force_one_one(self):
360
359
        server = http_server.HttpServer(RequestHandlerOneZero,
361
360
                                        protocol_version='HTTP/1.1')
362
361
        self.start_server(server)
363
 
        self.assertIsInstance(server._httpd,
 
362
        self.assertIsInstance(server.server,
364
363
                              http_server.TestingThreadingHTTPServer)
365
364
 
366
365
    def test_create_http_server_force_one_zero(self):
371
370
        server = http_server.HttpServer(RequestHandlerOneOne,
372
371
                                        protocol_version='HTTP/1.0')
373
372
        self.start_server(server)
374
 
        self.assertIsInstance(server._httpd,
 
373
        self.assertIsInstance(server.server,
375
374
                              http_server.TestingHTTPServer)
376
375
 
377
376
 
385
384
    _transport = property(_get_pycurl_maybe)
386
385
 
387
386
 
388
 
class TestHttpUrls(tests.TestCase):
389
 
 
390
 
    # TODO: This should be moved to authorization tests once they
391
 
    # are written.
392
 
 
393
 
    def test_url_parsing(self):
394
 
        f = FakeManager()
395
 
        url = http.extract_auth('http://example.com', f)
396
 
        self.assertEqual('http://example.com', url)
397
 
        self.assertEqual(0, len(f.credentials))
398
 
        url = http.extract_auth(
399
 
            'http://user:pass@example.com/bzr/bzr.dev', f)
400
 
        self.assertEqual('http://example.com/bzr/bzr.dev', url)
401
 
        self.assertEqual(1, len(f.credentials))
402
 
        self.assertEqual([None, 'example.com', 'user', 'pass'],
403
 
                         f.credentials[0])
404
 
 
405
 
 
406
387
class TestHttpTransportUrls(tests.TestCase):
407
388
    """Test the http urls."""
408
389
 
 
390
    scenarios = vary_by_http_client_implementation()
 
391
 
409
392
    def test_abs_url(self):
410
393
        """Construction of absolute http URLs"""
411
 
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
 
394
        t = self._transport('http://example.com/bzr/bzr.dev/')
412
395
        eq = self.assertEqualDiff
413
 
        eq(t.abspath('.'), 'http://bazaar-vcs.org/bzr/bzr.dev')
414
 
        eq(t.abspath('foo/bar'), 'http://bazaar-vcs.org/bzr/bzr.dev/foo/bar')
415
 
        eq(t.abspath('.bzr'), 'http://bazaar-vcs.org/bzr/bzr.dev/.bzr')
 
396
        eq(t.abspath('.'), 'http://example.com/bzr/bzr.dev')
 
397
        eq(t.abspath('foo/bar'), 'http://example.com/bzr/bzr.dev/foo/bar')
 
398
        eq(t.abspath('.bzr'), 'http://example.com/bzr/bzr.dev/.bzr')
416
399
        eq(t.abspath('.bzr/1//2/./3'),
417
 
           'http://bazaar-vcs.org/bzr/bzr.dev/.bzr/1/2/3')
 
400
           'http://example.com/bzr/bzr.dev/.bzr/1/2/3')
418
401
 
419
402
    def test_invalid_http_urls(self):
420
403
        """Trap invalid construction of urls"""
421
 
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
 
404
        self._transport('http://example.com/bzr/bzr.dev/')
422
405
        self.assertRaises(errors.InvalidURL,
423
406
                          self._transport,
424
 
                          'http://http://bazaar-vcs.org/bzr/bzr.dev/')
 
407
                          'http://http://example.com/bzr/bzr.dev/')
425
408
 
426
409
    def test_http_root_urls(self):
427
410
        """Construction of URLs from server root"""
428
 
        t = self._transport('http://bzr.ozlabs.org/')
 
411
        t = self._transport('http://example.com/')
429
412
        eq = self.assertEqualDiff
430
413
        eq(t.abspath('.bzr/tree-version'),
431
 
           'http://bzr.ozlabs.org/.bzr/tree-version')
 
414
           'http://example.com/.bzr/tree-version')
432
415
 
433
416
    def test_http_impl_urls(self):
434
417
        """There are servers which ask for particular clients to connect"""
436
419
        server.start_server()
437
420
        try:
438
421
            url = server.get_url()
439
 
            self.assertTrue(url.startswith('%s://' % self._qualified_prefix))
 
422
            self.assertTrue(url.startswith('%s://' % self._url_protocol))
440
423
        finally:
441
424
            server.stop_server()
442
425
 
480
463
class TestHTTPConnections(http_utils.TestCaseWithWebserver):
481
464
    """Test the http connections."""
482
465
 
 
466
    scenarios = multiply_scenarios(
 
467
        vary_by_http_client_implementation(),
 
468
        vary_by_http_protocol_version(),
 
469
        )
 
470
 
483
471
    def setUp(self):
484
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
472
        super(TestHTTPConnections, self).setUp()
485
473
        self.build_tree(['foo/', 'foo/bar'], line_endings='binary',
486
474
                        transport=self.get_transport())
487
475
 
488
476
    def test_http_has(self):
489
477
        server = self.get_readonly_server()
490
 
        t = self._transport(server.get_url())
 
478
        t = self.get_readonly_transport()
491
479
        self.assertEqual(t.has('foo/bar'), True)
492
480
        self.assertEqual(len(server.logs), 1)
493
481
        self.assertContainsRe(server.logs[0],
495
483
 
496
484
    def test_http_has_not_found(self):
497
485
        server = self.get_readonly_server()
498
 
        t = self._transport(server.get_url())
 
486
        t = self.get_readonly_transport()
499
487
        self.assertEqual(t.has('not-found'), False)
500
488
        self.assertContainsRe(server.logs[1],
501
489
            r'"HEAD /not-found HTTP/1.." 404 - "-" "bzr/')
502
490
 
503
491
    def test_http_get(self):
504
492
        server = self.get_readonly_server()
505
 
        t = self._transport(server.get_url())
 
493
        t = self.get_readonly_transport()
506
494
        fp = t.get('foo/bar')
507
495
        self.assertEqualDiff(
508
496
            fp.read(),
530
518
class TestHttpTransportRegistration(tests.TestCase):
531
519
    """Test registrations of various http implementations"""
532
520
 
 
521
    scenarios = vary_by_http_client_implementation()
 
522
 
533
523
    def test_http_registered(self):
534
 
        t = transport.get_transport('%s://foo.com/' % self._qualified_prefix)
 
524
        t = transport.get_transport_from_url(
 
525
            '%s://foo.com/' % self._url_protocol)
535
526
        self.assertIsInstance(t, transport.Transport)
536
527
        self.assertIsInstance(t, self._transport)
537
528
 
538
529
 
539
530
class TestPost(tests.TestCase):
540
531
 
 
532
    scenarios = multiply_scenarios(
 
533
        vary_by_http_client_implementation(),
 
534
        vary_by_http_protocol_version(),
 
535
        )
 
536
 
541
537
    def test_post_body_is_received(self):
542
538
        server = RecordingServer(expect_body_tail='end-of-body',
543
 
            scheme=self._qualified_prefix)
 
539
                                 scheme=self._url_protocol)
544
540
        self.start_server(server)
545
541
        url = server.get_url()
546
 
        http_transport = self._transport(url)
 
542
        # FIXME: needs a cleanup -- vila 20100611
 
543
        http_transport = transport.get_transport_from_url(url)
547
544
        code, response = http_transport._post('abc def end-of-body')
548
545
        self.assertTrue(
549
546
            server.received_bytes.startswith('POST /.bzr/smart HTTP/1.'))
550
547
        self.assertTrue('content-length: 19\r' in server.received_bytes.lower())
 
548
        self.assertTrue('content-type: application/octet-stream\r'
 
549
                        in server.received_bytes.lower())
551
550
        # The transport should not be assuming that the server can accept
552
551
        # chunked encoding the first time it connects, because HTTP/1.1, so we
553
552
        # check for the literal string.
589
588
    Daughter classes are expected to override _req_handler_class
590
589
    """
591
590
 
 
591
    scenarios = multiply_scenarios(
 
592
        vary_by_http_client_implementation(),
 
593
        vary_by_http_protocol_version(),
 
594
        )
 
595
 
592
596
    # Provide a useful default
593
597
    _req_handler_class = http_server.TestingHTTPRequestHandler
594
598
 
595
599
    def create_transport_readonly_server(self):
596
 
        return http_server.HttpServer(self._req_handler_class,
597
 
                                      protocol_version=self._protocol_version)
 
600
        server = http_server.HttpServer(self._req_handler_class,
 
601
                                        protocol_version=self._protocol_version)
 
602
        server._url_protocol = self._url_protocol
 
603
        return server
598
604
 
599
605
    def _testing_pycurl(self):
600
606
        # TODO: This is duplicated for lots of the classes in this file
605
611
class WallRequestHandler(http_server.TestingHTTPRequestHandler):
606
612
    """Whatever request comes in, close the connection"""
607
613
 
608
 
    def handle_one_request(self):
 
614
    def _handle_one_request(self):
609
615
        """Handle a single HTTP request, by abruptly closing the connection"""
610
616
        self.close_connection = 1
611
617
 
616
622
    _req_handler_class = WallRequestHandler
617
623
 
618
624
    def test_http_has(self):
619
 
        server = self.get_readonly_server()
620
 
        t = self._transport(server.get_url())
 
625
        t = self.get_readonly_transport()
621
626
        # Unfortunately httplib (see HTTPResponse._read_status
622
627
        # for details) make no distinction between a closed
623
628
        # socket and badly formatted status line, so we can't
629
634
                          t.has, 'foo/bar')
630
635
 
631
636
    def test_http_get(self):
632
 
        server = self.get_readonly_server()
633
 
        t = self._transport(server.get_url())
 
637
        t = self.get_readonly_transport()
634
638
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
635
639
                           errors.InvalidHttpResponse),
636
640
                          t.get, 'foo/bar')
652
656
 
653
657
    _req_handler_class = BadStatusRequestHandler
654
658
 
 
659
    def setUp(self):
 
660
        super(TestBadStatusServer, self).setUp()
 
661
        # See https://bugs.launchpad.net/bzr/+bug/1451448 for details.
 
662
        # TD;LR: Running both a TCP client and server in the same process and
 
663
        # thread uncovers a race in python. The fix is to run the server in a
 
664
        # different process. Trying to fix yet another race here is not worth
 
665
        # the effort. -- vila 2015-09-06
 
666
        if 'HTTP/1.0' in self.id():
 
667
            raise tests.TestSkipped(
 
668
                'Client/Server in the same process and thread can hang')
 
669
 
655
670
    def test_http_has(self):
656
 
        server = self.get_readonly_server()
657
 
        t = self._transport(server.get_url())
658
 
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
 
671
        t = self.get_readonly_transport()
 
672
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
 
673
                           errors.InvalidHttpResponse),
 
674
                          t.has, 'foo/bar')
659
675
 
660
676
    def test_http_get(self):
661
 
        server = self.get_readonly_server()
662
 
        t = self._transport(server.get_url())
663
 
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
 
677
        t = self.get_readonly_transport()
 
678
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
 
679
                           errors.InvalidHttpResponse),
 
680
                          t.get, 'foo/bar')
664
681
 
665
682
 
666
683
class InvalidStatusRequestHandler(http_server.TestingHTTPRequestHandler):
670
687
        """Fakes handling a single HTTP request, returns a bad status"""
671
688
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
672
689
        self.wfile.write("Invalid status line\r\n")
 
690
        # If we don't close the connection pycurl will hang. Since this is a
 
691
        # stress test we don't *have* to respect the protocol, but we don't
 
692
        # have to sabotage it too much either.
 
693
        self.close_connection = True
673
694
        return False
674
695
 
675
696
 
681
702
 
682
703
    _req_handler_class = InvalidStatusRequestHandler
683
704
 
684
 
    def test_http_has(self):
685
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
686
 
            raise tests.KnownFailure(
687
 
                'pycurl hangs if the server send back garbage')
688
 
        super(TestInvalidStatusServer, self).test_http_has()
689
 
 
690
 
    def test_http_get(self):
691
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
692
 
            raise tests.KnownFailure(
693
 
                'pycurl hangs if the server send back garbage')
694
 
        super(TestInvalidStatusServer, self).test_http_get()
695
 
 
696
705
 
697
706
class BadProtocolRequestHandler(http_server.TestingHTTPRequestHandler):
698
707
    """Whatever request comes in, returns a bad protocol version"""
720
729
        super(TestBadProtocolServer, self).setUp()
721
730
 
722
731
    def test_http_has(self):
723
 
        server = self.get_readonly_server()
724
 
        t = self._transport(server.get_url())
 
732
        t = self.get_readonly_transport()
725
733
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
726
734
 
727
735
    def test_http_get(self):
728
 
        server = self.get_readonly_server()
729
 
        t = self._transport(server.get_url())
 
736
        t = self.get_readonly_transport()
730
737
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
731
738
 
732
739
 
746
753
    _req_handler_class = ForbiddenRequestHandler
747
754
 
748
755
    def test_http_has(self):
749
 
        server = self.get_readonly_server()
750
 
        t = self._transport(server.get_url())
 
756
        t = self.get_readonly_transport()
751
757
        self.assertRaises(errors.TransportError, t.has, 'foo/bar')
752
758
 
753
759
    def test_http_get(self):
754
 
        server = self.get_readonly_server()
755
 
        t = self._transport(server.get_url())
 
760
        t = self.get_readonly_transport()
756
761
        self.assertRaises(errors.TransportError, t.get, 'foo/bar')
757
762
 
758
763
 
797
802
        self.build_tree_contents([('a', '0123456789')],)
798
803
 
799
804
    def test_readv(self):
800
 
        server = self.get_readonly_server()
801
 
        t = self._transport(server.get_url())
 
805
        t = self.get_readonly_transport()
802
806
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
803
807
        self.assertEqual(l[0], (0, '0'))
804
808
        self.assertEqual(l[1], (1, '1'))
806
810
        self.assertEqual(l[3], (9, '9'))
807
811
 
808
812
    def test_readv_out_of_order(self):
809
 
        server = self.get_readonly_server()
810
 
        t = self._transport(server.get_url())
 
813
        t = self.get_readonly_transport()
811
814
        l = list(t.readv('a', ((1, 1), (9, 1), (0, 1), (3, 2))))
812
815
        self.assertEqual(l[0], (1, '1'))
813
816
        self.assertEqual(l[1], (9, '9'))
815
818
        self.assertEqual(l[3], (3, '34'))
816
819
 
817
820
    def test_readv_invalid_ranges(self):
818
 
        server = self.get_readonly_server()
819
 
        t = self._transport(server.get_url())
 
821
        t = self.get_readonly_transport()
820
822
 
821
823
        # This is intentionally reading off the end of the file
822
824
        # since we are sure that it cannot get there
830
832
 
831
833
    def test_readv_multiple_get_requests(self):
832
834
        server = self.get_readonly_server()
833
 
        t = self._transport(server.get_url())
 
835
        t = self.get_readonly_transport()
834
836
        # force transport to issue multiple requests
835
837
        t._max_readv_combine = 1
836
838
        t._max_get_ranges = 1
844
846
 
845
847
    def test_readv_get_max_size(self):
846
848
        server = self.get_readonly_server()
847
 
        t = self._transport(server.get_url())
 
849
        t = self.get_readonly_transport()
848
850
        # force transport to issue multiple requests by limiting the number of
849
851
        # bytes by request. Note that this apply to coalesced offsets only, a
850
852
        # single range will keep its size even if bigger than the limit.
859
861
 
860
862
    def test_complete_readv_leave_pipe_clean(self):
861
863
        server = self.get_readonly_server()
862
 
        t = self._transport(server.get_url())
 
864
        t = self.get_readonly_transport()
863
865
        # force transport to issue multiple requests
864
866
        t._get_max_size = 2
865
 
        l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
 
867
        list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
866
868
        # The server should have issued 3 requests
867
869
        self.assertEqual(3, server.GET_request_nb)
868
870
        self.assertEqual('0123456789', t.get_bytes('a'))
870
872
 
871
873
    def test_incomplete_readv_leave_pipe_clean(self):
872
874
        server = self.get_readonly_server()
873
 
        t = self._transport(server.get_url())
 
875
        t = self.get_readonly_transport()
874
876
        # force transport to issue multiple requests
875
877
        t._get_max_size = 2
876
878
        # Don't collapse readv results into a list so that we leave unread
945
947
    def get_multiple_ranges(self, file, file_size, ranges):
946
948
        self.send_response(206)
947
949
        self.send_header('Accept-Ranges', 'bytes')
 
950
        # XXX: this is strange; the 'random' name below seems undefined and
 
951
        # yet the tests pass -- mbp 2010-10-11 bug 658773
948
952
        boundary = "%d" % random.randint(0,0x7FFFFFFF)
949
953
        self.send_header("Content-Type",
950
954
                         "multipart/byteranges; boundary=%s" % boundary)
1012
1016
                return
1013
1017
            self.send_range_content(file, start, end - start + 1)
1014
1018
            cur += 1
1015
 
        # No final boundary
 
1019
        # Final boundary
1016
1020
        self.wfile.write(boundary_line)
1017
1021
 
1018
1022
 
1026
1030
 
1027
1031
    def test_readv_with_short_reads(self):
1028
1032
        server = self.get_readonly_server()
1029
 
        t = self._transport(server.get_url())
 
1033
        t = self.get_readonly_transport()
1030
1034
        # Force separate ranges for each offset
1031
1035
        t._bytes_to_read_before_seek = 0
1032
1036
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1047
1051
        # that mode
1048
1052
        self.assertEqual('single', t._range_hint)
1049
1053
 
 
1054
 
 
1055
class TruncatedBeforeBoundaryRequestHandler(
 
1056
    http_server.TestingHTTPRequestHandler):
 
1057
    """Truncation before a boundary, like in bug 198646"""
 
1058
 
 
1059
    _truncated_ranges = 1
 
1060
 
 
1061
    def get_multiple_ranges(self, file, file_size, ranges):
 
1062
        self.send_response(206)
 
1063
        self.send_header('Accept-Ranges', 'bytes')
 
1064
        boundary = 'tagada'
 
1065
        self.send_header('Content-Type',
 
1066
                         'multipart/byteranges; boundary=%s' % boundary)
 
1067
        boundary_line = '--%s\r\n' % boundary
 
1068
        # Calculate the Content-Length
 
1069
        content_length = 0
 
1070
        for (start, end) in ranges:
 
1071
            content_length += len(boundary_line)
 
1072
            content_length += self._header_line_length(
 
1073
                'Content-type', 'application/octet-stream')
 
1074
            content_length += self._header_line_length(
 
1075
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
 
1076
            content_length += len('\r\n') # end headers
 
1077
            content_length += end - start # + 1
 
1078
        content_length += len(boundary_line)
 
1079
        self.send_header('Content-length', content_length)
 
1080
        self.end_headers()
 
1081
 
 
1082
        # Send the multipart body
 
1083
        cur = 0
 
1084
        for (start, end) in ranges:
 
1085
            if cur + self._truncated_ranges >= len(ranges):
 
1086
                # Abruptly ends the response and close the connection
 
1087
                self.close_connection = 1
 
1088
                return
 
1089
            self.wfile.write(boundary_line)
 
1090
            self.send_header('Content-type', 'application/octet-stream')
 
1091
            self.send_header('Content-Range', 'bytes %d-%d/%d'
 
1092
                             % (start, end, file_size))
 
1093
            self.end_headers()
 
1094
            self.send_range_content(file, start, end - start + 1)
 
1095
            cur += 1
 
1096
        # Final boundary
 
1097
        self.wfile.write(boundary_line)
 
1098
 
 
1099
 
 
1100
class TestTruncatedBeforeBoundary(TestSpecificRequestHandler):
 
1101
    """Tests the case of bug 198646, disconnecting before a boundary."""
 
1102
 
 
1103
    _req_handler_class = TruncatedBeforeBoundaryRequestHandler
 
1104
 
 
1105
    def setUp(self):
 
1106
        super(TestTruncatedBeforeBoundary, self).setUp()
 
1107
        self.build_tree_contents([('a', '0123456789')],)
 
1108
 
 
1109
    def test_readv_with_short_reads(self):
 
1110
        server = self.get_readonly_server()
 
1111
        t = self.get_readonly_transport()
 
1112
        # Force separate ranges for each offset
 
1113
        t._bytes_to_read_before_seek = 0
 
1114
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
 
1115
        self.assertEqual((0, '0'), ireadv.next())
 
1116
        self.assertEqual((2, '2'), ireadv.next())
 
1117
        self.assertEqual((4, '45'), ireadv.next())
 
1118
        self.assertEqual((9, '9'), ireadv.next())
 
1119
 
 
1120
 
1050
1121
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1051
1122
    """Errors out when range specifiers exceed the limit"""
1052
1123
 
1076
1147
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1077
1148
    """Tests readv requests against a server erroring out on too much ranges."""
1078
1149
 
 
1150
    scenarios = multiply_scenarios(
 
1151
        vary_by_http_client_implementation(),
 
1152
        vary_by_http_protocol_version(),
 
1153
        )
 
1154
 
1079
1155
    # Requests with more range specifiers will error out
1080
1156
    range_limit = 3
1081
1157
 
1083
1159
        return LimitedRangeHTTPServer(range_limit=self.range_limit,
1084
1160
                                      protocol_version=self._protocol_version)
1085
1161
 
1086
 
    def get_transport(self):
1087
 
        return self._transport(self.get_readonly_server().get_url())
1088
 
 
1089
1162
    def setUp(self):
1090
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
1163
        super(TestLimitedRangeRequestServer, self).setUp()
1091
1164
        # We need to manipulate ranges that correspond to real chunks in the
1092
1165
        # response, so we build a content appropriately.
1093
1166
        filler = ''.join(['abcdefghij' for x in range(102)])
1095
1168
        self.build_tree_contents([('a', content)],)
1096
1169
 
1097
1170
    def test_few_ranges(self):
1098
 
        t = self.get_transport()
 
1171
        t = self.get_readonly_transport()
1099
1172
        l = list(t.readv('a', ((0, 4), (1024, 4), )))
1100
1173
        self.assertEqual(l[0], (0, '0000'))
1101
1174
        self.assertEqual(l[1], (1024, '0001'))
1102
1175
        self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1103
1176
 
1104
1177
    def test_more_ranges(self):
1105
 
        t = self.get_transport()
 
1178
        t = self.get_readonly_transport()
1106
1179
        l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1107
1180
        self.assertEqual(l[0], (0, '0000'))
1108
1181
        self.assertEqual(l[1], (1024, '0001'))
1119
1192
    Only the urllib implementation is tested here.
1120
1193
    """
1121
1194
 
1122
 
    def setUp(self):
1123
 
        tests.TestCase.setUp(self)
1124
 
        self._old_env = {}
1125
 
        self.addCleanup(self._restore_env)
1126
 
 
1127
 
    def _install_env(self, env):
1128
 
        for name, value in env.iteritems():
1129
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1130
 
 
1131
 
    def _restore_env(self):
1132
 
        for name, value in self._old_env.iteritems():
1133
 
            osutils.set_or_unset_env(name, value)
1134
 
 
1135
1195
    def _proxied_request(self):
1136
1196
        handler = _urllib2_wrappers.ProxyHandler()
1137
 
        request = _urllib2_wrappers.Request('GET','http://baz/buzzle')
 
1197
        request = _urllib2_wrappers.Request('GET', 'http://baz/buzzle')
1138
1198
        handler.set_proxy(request, 'http')
1139
1199
        return request
1140
1200
 
 
1201
    def assertEvaluateProxyBypass(self, expected, host, no_proxy):
 
1202
        handler = _urllib2_wrappers.ProxyHandler()
 
1203
        self.assertEquals(expected,
 
1204
                          handler.evaluate_proxy_bypass(host, no_proxy))
 
1205
 
1141
1206
    def test_empty_user(self):
1142
 
        self._install_env({'http_proxy': 'http://bar.com'})
 
1207
        self.overrideEnv('http_proxy', 'http://bar.com')
 
1208
        request = self._proxied_request()
 
1209
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
 
1210
 
 
1211
    def test_user_with_at(self):
 
1212
        self.overrideEnv('http_proxy',
 
1213
                         'http://username@domain:password@proxy_host:1234')
1143
1214
        request = self._proxied_request()
1144
1215
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
1145
1216
 
1146
1217
    def test_invalid_proxy(self):
1147
1218
        """A proxy env variable without scheme"""
1148
 
        self._install_env({'http_proxy': 'host:1234'})
 
1219
        self.overrideEnv('http_proxy', 'host:1234')
1149
1220
        self.assertRaises(errors.InvalidURL, self._proxied_request)
1150
1221
 
 
1222
    def test_evaluate_proxy_bypass_true(self):
 
1223
        """The host is not proxied"""
 
1224
        self.assertEvaluateProxyBypass(True, 'example.com', 'example.com')
 
1225
        self.assertEvaluateProxyBypass(True, 'bzr.example.com', '*example.com')
 
1226
 
 
1227
    def test_evaluate_proxy_bypass_false(self):
 
1228
        """The host is proxied"""
 
1229
        self.assertEvaluateProxyBypass(False, 'bzr.example.com', None)
 
1230
 
 
1231
    def test_evaluate_proxy_bypass_unknown(self):
 
1232
        """The host is not explicitly proxied"""
 
1233
        self.assertEvaluateProxyBypass(None, 'example.com', 'not.example.com')
 
1234
        self.assertEvaluateProxyBypass(None, 'bzr.example.com', 'example.com')
 
1235
 
 
1236
    def test_evaluate_proxy_bypass_empty_entries(self):
 
1237
        """Ignore empty entries"""
 
1238
        self.assertEvaluateProxyBypass(None, 'example.com', '')
 
1239
        self.assertEvaluateProxyBypass(None, 'example.com', ',')
 
1240
        self.assertEvaluateProxyBypass(None, 'example.com', 'foo,,bar')
 
1241
 
1151
1242
 
1152
1243
class TestProxyHttpServer(http_utils.TestCaseWithTwoWebservers):
1153
1244
    """Tests proxy server.
1158
1249
    to the file names).
1159
1250
    """
1160
1251
 
 
1252
    scenarios = multiply_scenarios(
 
1253
        vary_by_http_client_implementation(),
 
1254
        vary_by_http_protocol_version(),
 
1255
        )
 
1256
 
1161
1257
    # FIXME: We don't have an https server available, so we don't
1162
 
    # test https connections.
 
1258
    # test https connections. --vila toolongago
1163
1259
 
1164
1260
    def setUp(self):
1165
1261
        super(TestProxyHttpServer, self).setUp()
 
1262
        self.transport_secondary_server = http_utils.ProxyServer
1166
1263
        self.build_tree_contents([('foo', 'contents of foo\n'),
1167
1264
                                  ('foo-proxied', 'proxied contents of foo\n')])
1168
1265
        # Let's setup some attributes for tests
1169
 
        self.server = self.get_readonly_server()
1170
 
        self.proxy_address = '%s:%d' % (self.server.host, self.server.port)
 
1266
        server = self.get_readonly_server()
 
1267
        self.server_host_port = '%s:%d' % (server.host, server.port)
1171
1268
        if self._testing_pycurl():
1172
1269
            # Oh my ! pycurl does not check for the port as part of
1173
1270
            # no_proxy :-( So we just test the host part
1174
 
            self.no_proxy_host = self.server.host
 
1271
            self.no_proxy_host = server.host
1175
1272
        else:
1176
 
            self.no_proxy_host = self.proxy_address
 
1273
            self.no_proxy_host = self.server_host_port
1177
1274
        # The secondary server is the proxy
1178
 
        self.proxy = self.get_secondary_server()
1179
 
        self.proxy_url = self.proxy.get_url()
1180
 
        self._old_env = {}
 
1275
        self.proxy_url = self.get_secondary_url()
1181
1276
 
1182
1277
    def _testing_pycurl(self):
1183
1278
        # TODO: This is duplicated for lots of the classes in this file
1184
1279
        return (features.pycurl.available()
1185
1280
                and self._transport == PyCurlTransport)
1186
1281
 
1187
 
    def create_transport_secondary_server(self):
1188
 
        """Creates an http server that will serve files with
1189
 
        '-proxied' appended to their names.
1190
 
        """
1191
 
        return http_utils.ProxyServer(protocol_version=self._protocol_version)
1192
 
 
1193
 
    def _install_env(self, env):
1194
 
        for name, value in env.iteritems():
1195
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1196
 
 
1197
 
    def _restore_env(self):
1198
 
        for name, value in self._old_env.iteritems():
1199
 
            osutils.set_or_unset_env(name, value)
1200
 
 
1201
 
    def proxied_in_env(self, env):
1202
 
        self._install_env(env)
1203
 
        url = self.server.get_url()
1204
 
        t = self._transport(url)
1205
 
        try:
1206
 
            self.assertEqual('proxied contents of foo\n', t.get('foo').read())
1207
 
        finally:
1208
 
            self._restore_env()
1209
 
 
1210
 
    def not_proxied_in_env(self, env):
1211
 
        self._install_env(env)
1212
 
        url = self.server.get_url()
1213
 
        t = self._transport(url)
1214
 
        try:
1215
 
            self.assertEqual('contents of foo\n', t.get('foo').read())
1216
 
        finally:
1217
 
            self._restore_env()
 
1282
    def assertProxied(self):
 
1283
        t = self.get_readonly_transport()
 
1284
        self.assertEqual('proxied contents of foo\n', t.get('foo').read())
 
1285
 
 
1286
    def assertNotProxied(self):
 
1287
        t = self.get_readonly_transport()
 
1288
        self.assertEqual('contents of foo\n', t.get('foo').read())
1218
1289
 
1219
1290
    def test_http_proxy(self):
1220
 
        self.proxied_in_env({'http_proxy': self.proxy_url})
 
1291
        self.overrideEnv('http_proxy', self.proxy_url)
 
1292
        self.assertProxied()
1221
1293
 
1222
1294
    def test_HTTP_PROXY(self):
1223
1295
        if self._testing_pycurl():
1226
1298
            # about. Should we ?)
1227
1299
            raise tests.TestNotApplicable(
1228
1300
                'pycurl does not check HTTP_PROXY for security reasons')
1229
 
        self.proxied_in_env({'HTTP_PROXY': self.proxy_url})
 
1301
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
 
1302
        self.assertProxied()
1230
1303
 
1231
1304
    def test_all_proxy(self):
1232
 
        self.proxied_in_env({'all_proxy': self.proxy_url})
 
1305
        self.overrideEnv('all_proxy', self.proxy_url)
 
1306
        self.assertProxied()
1233
1307
 
1234
1308
    def test_ALL_PROXY(self):
1235
 
        self.proxied_in_env({'ALL_PROXY': self.proxy_url})
 
1309
        self.overrideEnv('ALL_PROXY', self.proxy_url)
 
1310
        self.assertProxied()
1236
1311
 
1237
1312
    def test_http_proxy_with_no_proxy(self):
1238
 
        self.not_proxied_in_env({'http_proxy': self.proxy_url,
1239
 
                                 'no_proxy': self.no_proxy_host})
 
1313
        self.overrideEnv('no_proxy', self.no_proxy_host)
 
1314
        self.overrideEnv('http_proxy', self.proxy_url)
 
1315
        self.assertNotProxied()
1240
1316
 
1241
1317
    def test_HTTP_PROXY_with_NO_PROXY(self):
1242
1318
        if self._testing_pycurl():
1243
1319
            raise tests.TestNotApplicable(
1244
1320
                'pycurl does not check HTTP_PROXY for security reasons')
1245
 
        self.not_proxied_in_env({'HTTP_PROXY': self.proxy_url,
1246
 
                                 'NO_PROXY': self.no_proxy_host})
 
1321
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
 
1322
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
 
1323
        self.assertNotProxied()
1247
1324
 
1248
1325
    def test_all_proxy_with_no_proxy(self):
1249
 
        self.not_proxied_in_env({'all_proxy': self.proxy_url,
1250
 
                                 'no_proxy': self.no_proxy_host})
 
1326
        self.overrideEnv('no_proxy', self.no_proxy_host)
 
1327
        self.overrideEnv('all_proxy', self.proxy_url)
 
1328
        self.assertNotProxied()
1251
1329
 
1252
1330
    def test_ALL_PROXY_with_NO_PROXY(self):
1253
 
        self.not_proxied_in_env({'ALL_PROXY': self.proxy_url,
1254
 
                                 'NO_PROXY': self.no_proxy_host})
 
1331
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
 
1332
        self.overrideEnv('ALL_PROXY', self.proxy_url)
 
1333
        self.assertNotProxied()
1255
1334
 
1256
1335
    def test_http_proxy_without_scheme(self):
 
1336
        self.overrideEnv('http_proxy', self.server_host_port)
1257
1337
        if self._testing_pycurl():
1258
1338
            # pycurl *ignores* invalid proxy env variables. If that ever change
1259
1339
            # in the future, this test will fail indicating that pycurl do not
1260
1340
            # ignore anymore such variables.
1261
 
            self.not_proxied_in_env({'http_proxy': self.proxy_address})
 
1341
            self.assertNotProxied()
1262
1342
        else:
1263
 
            self.assertRaises(errors.InvalidURL,
1264
 
                              self.proxied_in_env,
1265
 
                              {'http_proxy': self.proxy_address})
 
1343
            self.assertRaises(errors.InvalidURL, self.assertProxied)
1266
1344
 
1267
1345
 
1268
1346
class TestRanges(http_utils.TestCaseWithWebserver):
1269
1347
    """Test the Range header in GET methods."""
1270
1348
 
 
1349
    scenarios = multiply_scenarios(
 
1350
        vary_by_http_client_implementation(),
 
1351
        vary_by_http_protocol_version(),
 
1352
        )
 
1353
 
1271
1354
    def setUp(self):
1272
 
        http_utils.TestCaseWithWebserver.setUp(self)
 
1355
        super(TestRanges, self).setUp()
1273
1356
        self.build_tree_contents([('a', '0123456789')],)
1274
 
        server = self.get_readonly_server()
1275
 
        self.transport = self._transport(server.get_url())
1276
1357
 
1277
1358
    def create_transport_readonly_server(self):
1278
1359
        return http_server.HttpServer(protocol_version=self._protocol_version)
1279
1360
 
1280
1361
    def _file_contents(self, relpath, ranges):
 
1362
        t = self.get_readonly_transport()
1281
1363
        offsets = [ (start, end - start + 1) for start, end in ranges]
1282
 
        coalesce = self.transport._coalesce_offsets
 
1364
        coalesce = t._coalesce_offsets
1283
1365
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1284
 
        code, data = self.transport._get(relpath, coalesced)
 
1366
        code, data = t._get(relpath, coalesced)
1285
1367
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1286
1368
        for start, end in ranges:
1287
1369
            data.seek(start)
1288
1370
            yield data.read(end - start + 1)
1289
1371
 
1290
1372
    def _file_tail(self, relpath, tail_amount):
1291
 
        code, data = self.transport._get(relpath, [], tail_amount)
 
1373
        t = self.get_readonly_transport()
 
1374
        code, data = t._get(relpath, [], tail_amount)
1292
1375
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1293
1376
        data.seek(-tail_amount, 2)
1294
1377
        return data.read(tail_amount)
1313
1396
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1314
1397
    """Test redirection between http servers."""
1315
1398
 
1316
 
    def create_transport_secondary_server(self):
1317
 
        """Create the secondary server redirecting to the primary server"""
1318
 
        new = self.get_readonly_server()
1319
 
 
1320
 
        redirecting = http_utils.HTTPServerRedirecting(
1321
 
            protocol_version=self._protocol_version)
1322
 
        redirecting.redirect_to(new.host, new.port)
1323
 
        return redirecting
 
1399
    scenarios = multiply_scenarios(
 
1400
        vary_by_http_client_implementation(),
 
1401
        vary_by_http_protocol_version(),
 
1402
        )
1324
1403
 
1325
1404
    def setUp(self):
1326
1405
        super(TestHTTPRedirections, self).setUp()
1328
1407
                                  ('bundle',
1329
1408
                                  '# Bazaar revision bundle v0.9\n#\n')
1330
1409
                                  ],)
1331
 
        # The requests to the old server will be redirected to the new server
1332
 
        self.old_transport = self._transport(self.old_server.get_url())
1333
1410
 
1334
1411
    def test_redirected(self):
1335
 
        self.assertRaises(errors.RedirectRequested, self.old_transport.get, 'a')
1336
 
        t = self._transport(self.new_server.get_url())
1337
 
        self.assertEqual('0123456789', t.get('a').read())
 
1412
        self.assertRaises(errors.RedirectRequested,
 
1413
                          self.get_old_transport().get, 'a')
 
1414
        self.assertEqual('0123456789', self.get_new_transport().get('a').read())
1338
1415
 
1339
1416
 
1340
1417
class RedirectedRequest(_urllib2_wrappers.Request):
1357
1434
    test.overrideAttr(_urllib2_wrappers, 'Request', RedirectedRequest)
1358
1435
 
1359
1436
 
 
1437
def cleanup_http_redirection_connections(test):
 
1438
    # Some sockets are opened but never seen by _urllib, so we trap them at
 
1439
    # the _urllib2_wrappers level to be able to clean them up.
 
1440
    def socket_disconnect(sock):
 
1441
        try:
 
1442
            sock.shutdown(socket.SHUT_RDWR)
 
1443
            sock.close()
 
1444
        except socket.error:
 
1445
            pass
 
1446
    def connect(connection):
 
1447
        test.http_connect_orig(connection)
 
1448
        test.addCleanup(socket_disconnect, connection.sock)
 
1449
    test.http_connect_orig = test.overrideAttr(
 
1450
        _urllib2_wrappers.HTTPConnection, 'connect', connect)
 
1451
    def connect(connection):
 
1452
        test.https_connect_orig(connection)
 
1453
        test.addCleanup(socket_disconnect, connection.sock)
 
1454
    test.https_connect_orig = test.overrideAttr(
 
1455
        _urllib2_wrappers.HTTPSConnection, 'connect', connect)
 
1456
 
 
1457
 
1360
1458
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1361
1459
    """Test redirections.
1362
1460
 
1371
1469
    -- vila 20070212
1372
1470
    """
1373
1471
 
 
1472
    scenarios = multiply_scenarios(
 
1473
        vary_by_http_client_implementation(),
 
1474
        vary_by_http_protocol_version(),
 
1475
        )
 
1476
 
1374
1477
    def setUp(self):
1375
1478
        if (features.pycurl.available()
1376
1479
            and self._transport == PyCurlTransport):
1377
1480
            raise tests.TestNotApplicable(
1378
 
                "pycurl doesn't redirect silently annymore")
 
1481
                "pycurl doesn't redirect silently anymore")
1379
1482
        super(TestHTTPSilentRedirections, self).setUp()
1380
1483
        install_redirected_request(self)
 
1484
        cleanup_http_redirection_connections(self)
1381
1485
        self.build_tree_contents([('a','a'),
1382
1486
                                  ('1/',),
1383
1487
                                  ('1/a', 'redirected once'),
1391
1495
                                  ('5/a', 'redirected 5 times'),
1392
1496
                                  ],)
1393
1497
 
1394
 
        self.old_transport = self._transport(self.old_server.get_url())
1395
 
 
1396
 
    def create_transport_secondary_server(self):
1397
 
        """Create the secondary server, redirections are defined in the tests"""
1398
 
        return http_utils.HTTPServerRedirecting(
1399
 
            protocol_version=self._protocol_version)
1400
 
 
1401
1498
    def test_one_redirection(self):
1402
 
        t = self.old_transport
1403
 
 
1404
 
        req = RedirectedRequest('GET', t.abspath('a'))
 
1499
        t = self.get_old_transport()
 
1500
        req = RedirectedRequest('GET', t._remote_path('a'))
1405
1501
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1406
1502
                                       self.new_server.port)
1407
1503
        self.old_server.redirections = \
1408
1504
            [('(.*)', r'%s/1\1' % (new_prefix), 301),]
1409
 
        self.assertEqual('redirected once',t._perform(req).read())
 
1505
        self.assertEqual('redirected once', t._perform(req).read())
1410
1506
 
1411
1507
    def test_five_redirections(self):
1412
 
        t = self.old_transport
1413
 
 
1414
 
        req = RedirectedRequest('GET', t.abspath('a'))
 
1508
        t = self.get_old_transport()
 
1509
        req = RedirectedRequest('GET', t._remote_path('a'))
1415
1510
        old_prefix = 'http://%s:%s' % (self.old_server.host,
1416
1511
                                       self.old_server.port)
1417
1512
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1423
1518
            ('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1424
1519
            ('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1425
1520
            ]
1426
 
        self.assertEqual('redirected 5 times',t._perform(req).read())
 
1521
        self.assertEqual('redirected 5 times', t._perform(req).read())
1427
1522
 
1428
1523
 
1429
1524
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1430
1525
    """Test transport.do_catching_redirections."""
1431
1526
 
 
1527
    scenarios = multiply_scenarios(
 
1528
        vary_by_http_client_implementation(),
 
1529
        vary_by_http_protocol_version(),
 
1530
        )
 
1531
 
1432
1532
    def setUp(self):
1433
1533
        super(TestDoCatchRedirections, self).setUp()
1434
1534
        self.build_tree_contents([('a', '0123456789'),],)
1435
 
 
1436
 
        self.old_transport = self._transport(self.old_server.get_url())
1437
 
 
1438
 
    def get_a(self, transport):
1439
 
        return transport.get('a')
 
1535
        cleanup_http_redirection_connections(self)
 
1536
 
 
1537
        self.old_transport = self.get_old_transport()
 
1538
 
 
1539
    def get_a(self, t):
 
1540
        return t.get('a')
1440
1541
 
1441
1542
    def test_no_redirection(self):
1442
 
        t = self._transport(self.new_server.get_url())
 
1543
        t = self.get_new_transport()
1443
1544
 
1444
1545
        # We use None for redirected so that we fail if redirected
1445
1546
        self.assertEqual('0123456789',
1449
1550
    def test_one_redirection(self):
1450
1551
        self.redirections = 0
1451
1552
 
1452
 
        def redirected(transport, exception, redirection_notice):
 
1553
        def redirected(t, exception, redirection_notice):
1453
1554
            self.redirections += 1
1454
 
            dir, file = urlutils.split(exception.target)
1455
 
            return self._transport(dir)
 
1555
            redirected_t = t._redirected_to(exception.source, exception.target)
 
1556
            return redirected_t
1456
1557
 
1457
1558
        self.assertEqual('0123456789',
1458
1559
                         transport.do_catching_redirections(
1472
1573
                          self.get_a, self.old_transport, redirected)
1473
1574
 
1474
1575
 
 
1576
def _setup_authentication_config(**kwargs):
 
1577
    conf = config.AuthenticationConfig()
 
1578
    conf._get_config().update({'httptest': kwargs})
 
1579
    conf._save()
 
1580
 
 
1581
 
 
1582
class TestUrllib2AuthHandler(tests.TestCaseWithTransport):
 
1583
    """Unit tests for glue by which urllib2 asks us for authentication"""
 
1584
 
 
1585
    def test_get_user_password_without_port(self):
 
1586
        """We cope if urllib2 doesn't tell us the port.
 
1587
 
 
1588
        See https://bugs.launchpad.net/bzr/+bug/654684
 
1589
        """
 
1590
        user = 'joe'
 
1591
        password = 'foo'
 
1592
        _setup_authentication_config(scheme='http', host='localhost',
 
1593
                                     user=user, password=password)
 
1594
        handler = _urllib2_wrappers.HTTPAuthHandler()
 
1595
        got_pass = handler.get_user_password(dict(
 
1596
            user='joe',
 
1597
            protocol='http',
 
1598
            host='localhost',
 
1599
            path='/',
 
1600
            realm='Realm',
 
1601
            ))
 
1602
        self.assertEquals((user, password), got_pass)
 
1603
 
 
1604
 
1475
1605
class TestAuth(http_utils.TestCaseWithWebserver):
1476
1606
    """Test authentication scheme"""
1477
1607
 
1478
 
    _auth_header = 'Authorization'
1479
 
    _password_prompt_prefix = ''
1480
 
    _username_prompt_prefix = ''
1481
 
    # Set by load_tests
1482
 
    _auth_server = None
 
1608
    scenarios = multiply_scenarios(
 
1609
        vary_by_http_client_implementation(),
 
1610
        vary_by_http_protocol_version(),
 
1611
        vary_by_http_auth_scheme(),
 
1612
        )
1483
1613
 
1484
1614
    def setUp(self):
1485
1615
        super(TestAuth, self).setUp()
1488
1618
                                  ('b', 'contents of b\n'),])
1489
1619
 
1490
1620
    def create_transport_readonly_server(self):
1491
 
        return self._auth_server(protocol_version=self._protocol_version)
 
1621
        server = self._auth_server(protocol_version=self._protocol_version)
 
1622
        server._url_protocol = self._url_protocol
 
1623
        return server
1492
1624
 
1493
1625
    def _testing_pycurl(self):
1494
1626
        # TODO: This is duplicated for lots of the classes in this file
1507
1639
        return url
1508
1640
 
1509
1641
    def get_user_transport(self, user, password):
1510
 
        return self._transport(self.get_user_url(user, password))
 
1642
        t = transport.get_transport_from_url(
 
1643
            self.get_user_url(user, password))
 
1644
        return t
1511
1645
 
1512
1646
    def test_no_user(self):
1513
1647
        self.server.add_user('joe', 'foo')
1624
1758
        ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1625
1759
                                            stderr=tests.StringIOWrapper())
1626
1760
        # Create a minimal config file with the right password
1627
 
        conf = config.AuthenticationConfig()
1628
 
        conf._get_config().update(
1629
 
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1630
 
                          'user': user, 'password': password}})
1631
 
        conf._save()
 
1761
        _setup_authentication_config(scheme='http', port=self.server.port,
 
1762
                                     user=user, password=password)
1632
1763
        # Issue a request to the server to connect
1633
1764
        self.assertEqual('contents of a\n',t.get('a').read())
1634
1765
        # stdin should have  been left untouched
1636
1767
        # Only one 'Authentication Required' error should occur
1637
1768
        self.assertEqual(1, self.server.auth_required_errors)
1638
1769
 
1639
 
    def test_user_from_auth_conf(self):
1640
 
        if self._testing_pycurl():
1641
 
            raise tests.TestNotApplicable(
1642
 
                'pycurl does not support authentication.conf')
1643
 
        user = 'joe'
1644
 
        password = 'foo'
1645
 
        self.server.add_user(user, password)
1646
 
        # Create a minimal config file with the right password
1647
 
        conf = config.AuthenticationConfig()
1648
 
        conf._get_config().update(
1649
 
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1650
 
                          'user': user, 'password': password}})
1651
 
        conf._save()
1652
 
        t = self.get_user_transport(None, None)
1653
 
        # Issue a request to the server to connect
1654
 
        self.assertEqual('contents of a\n', t.get('a').read())
1655
 
        # Only one 'Authentication Required' error should occur
1656
 
        self.assertEqual(1, self.server.auth_required_errors)
1657
 
 
1658
1770
    def test_changing_nonce(self):
1659
1771
        if self._auth_server not in (http_utils.HTTPDigestAuthServer,
1660
1772
                                     http_utils.ProxyDigestAuthServer):
1661
1773
            raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1662
1774
        if self._testing_pycurl():
1663
 
            raise tests.KnownFailure(
 
1775
            self.knownFailure(
1664
1776
                'pycurl does not handle a nonce change')
1665
1777
        self.server.add_user('joe', 'foo')
1666
1778
        t = self.get_user_transport('joe', 'foo')
1676
1788
        # initial 'who are you' and a second 'who are you' with the new nonce)
1677
1789
        self.assertEqual(2, self.server.auth_required_errors)
1678
1790
 
 
1791
    def test_user_from_auth_conf(self):
 
1792
        if self._testing_pycurl():
 
1793
            raise tests.TestNotApplicable(
 
1794
                'pycurl does not support authentication.conf')
 
1795
        user = 'joe'
 
1796
        password = 'foo'
 
1797
        self.server.add_user(user, password)
 
1798
        _setup_authentication_config(scheme='http', port=self.server.port,
 
1799
                                     user=user, password=password)
 
1800
        t = self.get_user_transport(None, None)
 
1801
        # Issue a request to the server to connect
 
1802
        self.assertEqual('contents of a\n', t.get('a').read())
 
1803
        # Only one 'Authentication Required' error should occur
 
1804
        self.assertEqual(1, self.server.auth_required_errors)
 
1805
 
 
1806
    def test_no_credential_leaks_in_log(self):
 
1807
        self.overrideAttr(debug, 'debug_flags', set(['http']))
 
1808
        user = 'joe'
 
1809
        password = 'very-sensitive-password'
 
1810
        self.server.add_user(user, password)
 
1811
        t = self.get_user_transport(user, password)
 
1812
        # Capture the debug calls to mutter
 
1813
        self.mutters = []
 
1814
        def mutter(*args):
 
1815
            lines = args[0] % args[1:]
 
1816
            # Some calls output multiple lines, just split them now since we
 
1817
            # care about a single one later.
 
1818
            self.mutters.extend(lines.splitlines())
 
1819
        self.overrideAttr(trace, 'mutter', mutter)
 
1820
        # Issue a request to the server to connect
 
1821
        self.assertEqual(True, t.has('a'))
 
1822
        # Only one 'Authentication Required' error should occur
 
1823
        self.assertEqual(1, self.server.auth_required_errors)
 
1824
        # Since the authentification succeeded, there should be a corresponding
 
1825
        # debug line
 
1826
        sent_auth_headers = [line for line in self.mutters
 
1827
                             if line.startswith('> %s' % (self._auth_header,))]
 
1828
        self.assertLength(1, sent_auth_headers)
 
1829
        self.assertStartsWith(sent_auth_headers[0],
 
1830
                              '> %s: <masked>' % (self._auth_header,))
1679
1831
 
1680
1832
 
1681
1833
class TestProxyAuth(TestAuth):
1682
 
    """Test proxy authentication schemes."""
1683
 
 
1684
 
    _auth_header = 'Proxy-authorization'
1685
 
    _password_prompt_prefix = 'Proxy '
1686
 
    _username_prompt_prefix = 'Proxy '
 
1834
    """Test proxy authentication schemes.
 
1835
 
 
1836
    This inherits from TestAuth to tweak the setUp and filter some failing
 
1837
    tests.
 
1838
    """
 
1839
 
 
1840
    scenarios = multiply_scenarios(
 
1841
        vary_by_http_client_implementation(),
 
1842
        vary_by_http_protocol_version(),
 
1843
        vary_by_http_proxy_auth_scheme(),
 
1844
        )
1687
1845
 
1688
1846
    def setUp(self):
1689
1847
        super(TestProxyAuth, self).setUp()
1690
 
        self._old_env = {}
1691
 
        self.addCleanup(self._restore_env)
1692
1848
        # Override the contents to avoid false positives
1693
1849
        self.build_tree_contents([('a', 'not proxied contents of a\n'),
1694
1850
                                  ('b', 'not proxied contents of b\n'),
1697
1853
                                  ])
1698
1854
 
1699
1855
    def get_user_transport(self, user, password):
1700
 
        self._install_env({'all_proxy': self.get_user_url(user, password)})
1701
 
        return self._transport(self.server.get_url())
1702
 
 
1703
 
    def _install_env(self, env):
1704
 
        for name, value in env.iteritems():
1705
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1706
 
 
1707
 
    def _restore_env(self):
1708
 
        for name, value in self._old_env.iteritems():
1709
 
            osutils.set_or_unset_env(name, value)
 
1856
        self.overrideEnv('all_proxy', self.get_user_url(user, password))
 
1857
        return TestAuth.get_user_transport(self, user, password)
1710
1858
 
1711
1859
    def test_empty_pass(self):
1712
1860
        if self._testing_pycurl():
1713
1861
            import pycurl
1714
1862
            if pycurl.version_info()[1] < '7.16.0':
1715
 
                raise tests.KnownFailure(
 
1863
                self.knownFailure(
1716
1864
                    'pycurl < 7.16.0 does not handle empty proxy passwords')
1717
1865
        super(TestProxyAuth, self).test_empty_pass()
1718
1866
 
1731
1879
        self.readfile = StringIO(socket_read_content)
1732
1880
        self.writefile = StringIO()
1733
1881
        self.writefile.close = lambda: None
 
1882
        self.close = lambda: None
1734
1883
 
1735
1884
    def makefile(self, mode='r', bufsize=None):
1736
1885
        if 'r' in mode:
1741
1890
 
1742
1891
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1743
1892
 
 
1893
    scenarios = multiply_scenarios(
 
1894
        vary_by_http_client_implementation(),
 
1895
        vary_by_http_protocol_version(),
 
1896
        )
 
1897
 
1744
1898
    def setUp(self):
1745
1899
        super(SmartHTTPTunnellingTest, self).setUp()
1746
1900
        # We use the VFS layer as part of HTTP tunnelling tests.
1747
 
        self._captureVar('BZR_NO_SMART_VFS', None)
 
1901
        self.overrideEnv('BZR_NO_SMART_VFS', None)
1748
1902
        self.transport_readonly_server = http_utils.HTTPServerWithSmarts
 
1903
        self.http_server = self.get_readonly_server()
1749
1904
 
1750
1905
    def create_transport_readonly_server(self):
1751
 
        return http_utils.HTTPServerWithSmarts(
 
1906
        server = http_utils.HTTPServerWithSmarts(
1752
1907
            protocol_version=self._protocol_version)
 
1908
        server._url_protocol = self._url_protocol
 
1909
        return server
1753
1910
 
1754
 
    def test_open_bzrdir(self):
 
1911
    def test_open_controldir(self):
1755
1912
        branch = self.make_branch('relpath')
1756
 
        http_server = self.get_readonly_server()
1757
 
        url = http_server.get_url() + 'relpath'
1758
 
        bd = bzrdir.BzrDir.open(url)
 
1913
        url = self.http_server.get_url() + 'relpath'
 
1914
        bd = controldir.ControlDir.open(url)
 
1915
        self.addCleanup(bd.transport.disconnect)
1759
1916
        self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1760
1917
 
1761
1918
    def test_bulk_data(self):
1763
1920
        # The 'readv' command in the smart protocol both sends and receives
1764
1921
        # bulk data, so we use that.
1765
1922
        self.build_tree(['data-file'])
1766
 
        http_server = self.get_readonly_server()
1767
 
        http_transport = self._transport(http_server.get_url())
 
1923
        http_transport = transport.get_transport_from_url(
 
1924
            self.http_server.get_url())
1768
1925
        medium = http_transport.get_smart_medium()
1769
1926
        # Since we provide the medium, the url below will be mostly ignored
1770
1927
        # during the test, as long as the path is '/'.
1778
1935
        post_body = 'hello\n'
1779
1936
        expected_reply_body = 'ok\x012\n'
1780
1937
 
1781
 
        http_server = self.get_readonly_server()
1782
 
        http_transport = self._transport(http_server.get_url())
 
1938
        http_transport = transport.get_transport_from_url(
 
1939
            self.http_server.get_url())
1783
1940
        medium = http_transport.get_smart_medium()
1784
1941
        response = medium.send_http_smart_request(post_body)
1785
1942
        reply_body = response.read()
1786
1943
        self.assertEqual(expected_reply_body, reply_body)
1787
1944
 
1788
1945
    def test_smart_http_server_post_request_handler(self):
1789
 
        httpd = self.get_readonly_server()._get_httpd()
 
1946
        httpd = self.http_server.server
1790
1947
 
1791
1948
        socket = SampleSocket(
1792
1949
            'POST /.bzr/smart %s \r\n' % self._protocol_version
1824
1981
 
1825
1982
    def test_probe_smart_server(self):
1826
1983
        """Test error handling against server refusing smart requests."""
1827
 
        server = self.get_readonly_server()
1828
 
        t = self._transport(server.get_url())
 
1984
        t = self.get_readonly_transport()
1829
1985
        # No need to build a valid smart request here, the server will not even
1830
1986
        # try to interpret it.
1831
1987
        self.assertRaises(errors.SmartProtocolError,
1832
1988
                          t.get_smart_medium().send_http_smart_request,
1833
1989
                          'whatever')
1834
1990
 
 
1991
 
1835
1992
class Test_redirected_to(tests.TestCase):
1836
1993
 
 
1994
    scenarios = vary_by_http_client_implementation()
 
1995
 
1837
1996
    def test_redirected_to_subdir(self):
1838
1997
        t = self._transport('http://www.example.com/foo')
1839
1998
        r = t._redirected_to('http://www.example.com/foo',
1841
2000
        self.assertIsInstance(r, type(t))
1842
2001
        # Both transports share the some connection
1843
2002
        self.assertEqual(t._get_connection(), r._get_connection())
 
2003
        self.assertEquals('http://www.example.com/foo/subdir/', r.base)
1844
2004
 
1845
2005
    def test_redirected_to_self_with_slash(self):
1846
2006
        t = self._transport('http://www.example.com/foo')
1857
2017
        r = t._redirected_to('http://www.example.com/foo',
1858
2018
                             'http://foo.example.com/foo/subdir')
1859
2019
        self.assertIsInstance(r, type(t))
 
2020
        self.assertEquals('http://foo.example.com/foo/subdir/',
 
2021
            r.external_url())
1860
2022
 
1861
2023
    def test_redirected_to_same_host_sibling_protocol(self):
1862
2024
        t = self._transport('http://www.example.com/foo')
1863
2025
        r = t._redirected_to('http://www.example.com/foo',
1864
2026
                             'https://www.example.com/foo')
1865
2027
        self.assertIsInstance(r, type(t))
 
2028
        self.assertEquals('https://www.example.com/foo/',
 
2029
            r.external_url())
1866
2030
 
1867
2031
    def test_redirected_to_same_host_different_protocol(self):
1868
2032
        t = self._transport('http://www.example.com/foo')
1869
2033
        r = t._redirected_to('http://www.example.com/foo',
1870
2034
                             'ftp://www.example.com/foo')
1871
2035
        self.assertNotEquals(type(r), type(t))
 
2036
        self.assertEquals('ftp://www.example.com/foo/', r.external_url())
 
2037
 
 
2038
    def test_redirected_to_same_host_specific_implementation(self):
 
2039
        t = self._transport('http://www.example.com/foo')
 
2040
        r = t._redirected_to('http://www.example.com/foo',
 
2041
                             'https+urllib://www.example.com/foo')
 
2042
        self.assertEquals('https://www.example.com/foo/', r.external_url())
1872
2043
 
1873
2044
    def test_redirected_to_different_host_same_user(self):
1874
2045
        t = self._transport('http://joe@www.example.com/foo')
1875
2046
        r = t._redirected_to('http://www.example.com/foo',
1876
2047
                             'https://foo.example.com/foo')
1877
2048
        self.assertIsInstance(r, type(t))
1878
 
        self.assertEqual(t._user, r._user)
 
2049
        self.assertEqual(t._parsed_url.user, r._parsed_url.user)
 
2050
        self.assertEquals('https://joe@foo.example.com/foo/', r.external_url())
1879
2051
 
1880
2052
 
1881
2053
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
1890
2062
    line.
1891
2063
    """
1892
2064
 
1893
 
    def handle_one_request(self):
 
2065
    def _handle_one_request(self):
1894
2066
        tcs = self.server.test_case_server
1895
2067
        requestline = self.rfile.readline()
1896
2068
        headers = self.MessageClass(self.rfile, 0)
1934
2106
    pass
1935
2107
 
1936
2108
 
1937
 
if tests.HTTPSServerFeature.available():
 
2109
if features.HTTPSServerFeature.available():
1938
2110
    from bzrlib.tests import https_server
1939
2111
    class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
1940
2112
        pass
1948
2120
    """
1949
2121
 
1950
2122
    def setUp(self):
1951
 
        tests.TestCase.setUp(self)
1952
2123
        self.server = self._activity_server(self._protocol_version)
1953
2124
        self.server.start_server()
1954
 
        self.activities = {}
 
2125
        self.addCleanup(self.server.stop_server)
 
2126
        _activities = {} # Don't close over self and create a cycle
1955
2127
        def report_activity(t, bytes, direction):
1956
 
            count = self.activities.get(direction, 0)
 
2128
            count = _activities.get(direction, 0)
1957
2129
            count += bytes
1958
 
            self.activities[direction] = count
1959
 
 
 
2130
            _activities[direction] = count
 
2131
        self.activities = _activities
1960
2132
        # We override at class level because constructors may propagate the
1961
2133
        # bound method and render instance overriding ineffective (an
1962
2134
        # alternative would be to define a specific ui factory instead...)
1963
2135
        self.overrideAttr(self._transport, '_report_activity', report_activity)
1964
 
        self.addCleanup(self.server.stop_server)
1965
2136
 
1966
2137
    def get_transport(self):
1967
 
        return self._transport(self.server.get_url())
 
2138
        t = self._transport(self.server.get_url())
 
2139
        # FIXME: Needs cleanup -- vila 20100611
 
2140
        return t
1968
2141
 
1969
2142
    def assertActivitiesMatch(self):
1970
2143
        self.assertEqual(self.server.bytes_read,
2075
2248
'''
2076
2249
        t = self.get_transport()
2077
2250
        # We must send a single line of body bytes, see
2078
 
        # PredefinedRequestHandler.handle_one_request
 
2251
        # PredefinedRequestHandler._handle_one_request
2079
2252
        code, f = t._post('abc def end-of-body\n')
2080
2253
        self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2081
2254
        self.assertActivitiesMatch()
2083
2256
 
2084
2257
class TestActivity(tests.TestCase, TestActivityMixin):
2085
2258
 
 
2259
    scenarios = multiply_scenarios(
 
2260
        vary_by_http_activity(),
 
2261
        vary_by_http_protocol_version(),
 
2262
        )
 
2263
 
2086
2264
    def setUp(self):
 
2265
        super(TestActivity, self).setUp()
2087
2266
        TestActivityMixin.setUp(self)
2088
2267
 
2089
2268
 
2098
2277
    _protocol_version = 'HTTP/1.1'
2099
2278
 
2100
2279
    def setUp(self):
 
2280
        super(TestNoReportActivity, self).setUp()
2101
2281
        self._transport =_urllib.HttpTransport_urllib
2102
2282
        TestActivityMixin.setUp(self)
2103
2283
 
2109
2289
class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver):
2110
2290
    """Test authentication on the redirected http server."""
2111
2291
 
 
2292
    scenarios = vary_by_http_protocol_version()
 
2293
 
2112
2294
    _auth_header = 'Authorization'
2113
2295
    _password_prompt_prefix = ''
2114
2296
    _username_prompt_prefix = ''
2115
2297
    _auth_server = http_utils.HTTPBasicAuthServer
2116
2298
    _transport = _urllib.HttpTransport_urllib
2117
2299
 
2118
 
    def create_transport_readonly_server(self):
2119
 
        return self._auth_server(protocol_version=self._protocol_version)
2120
 
 
2121
 
    def create_transport_secondary_server(self):
2122
 
        """Create the secondary server redirecting to the primary server"""
2123
 
        new = self.get_readonly_server()
2124
 
 
2125
 
        redirecting = http_utils.HTTPServerRedirecting(
2126
 
            protocol_version=self._protocol_version)
2127
 
        redirecting.redirect_to(new.host, new.port)
2128
 
        return redirecting
2129
 
 
2130
2300
    def setUp(self):
2131
2301
        super(TestAuthOnRedirected, self).setUp()
2132
2302
        self.build_tree_contents([('a','a'),
2137
2307
                                       self.new_server.port)
2138
2308
        self.old_server.redirections = [
2139
2309
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2140
 
        self.old_transport = self._transport(self.old_server.get_url())
 
2310
        self.old_transport = self.get_old_transport()
2141
2311
        self.new_server.add_user('joe', 'foo')
2142
 
 
2143
 
    def get_a(self, transport):
2144
 
        return transport.get('a')
 
2312
        cleanup_http_redirection_connections(self)
 
2313
 
 
2314
    def create_transport_readonly_server(self):
 
2315
        server = self._auth_server(protocol_version=self._protocol_version)
 
2316
        server._url_protocol = self._url_protocol
 
2317
        return server
 
2318
 
 
2319
    def get_a(self, t):
 
2320
        return t.get('a')
2145
2321
 
2146
2322
    def test_auth_on_redirected_via_do_catching_redirections(self):
2147
2323
        self.redirections = 0
2148
2324
 
2149
 
        def redirected(transport, exception, redirection_notice):
 
2325
        def redirected(t, exception, redirection_notice):
2150
2326
            self.redirections += 1
2151
 
            dir, file = urlutils.split(exception.target)
2152
 
            return self._transport(dir)
 
2327
            redirected_t = t._redirected_to(exception.source, exception.target)
 
2328
            self.addCleanup(redirected_t.disconnect)
 
2329
            return redirected_t
2153
2330
 
2154
2331
        stdout = tests.StringIOWrapper()
2155
2332
        stderr = tests.StringIOWrapper()
2176
2353
                                       self.new_server.port)
2177
2354
        self.old_server.redirections = [
2178
2355
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2179
 
        self.assertEqual('redirected once',t._perform(req).read())
 
2356
        self.assertEqual('redirected once', t._perform(req).read())
2180
2357
        # stdin should be empty
2181
2358
        self.assertEqual('', ui.ui_factory.stdin.readline())
2182
2359
        # stdout should be empty, stderr will contains the prompts
2183
2360
        self.assertEqual('', stdout.getvalue())
2184
2361
 
2185