← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_http.py

  • Committer: Jelmer Vernooij
  • Date: 2012-02-20 14:15:25 UTC
  • mto: (6471.1.4 iter-child-entries)
  • mto: This revision was merged to the branch mainline in revision 6472.
  • Revision ID: jelmer@samba.org-20120220141525-9azkfei62st8yc7w
Use inventories directly in fewer places.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_http.py
1
 
# Copyright (C) 2005-2010 Canonical Ltd
 
1
# Copyright (C) 2005-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
23
23
# TODO: Should be renamed to bzrlib.transport.http.tests?
24
24
# TODO: What about renaming to bzrlib.tests.transport.http ?
25
25
 
26
 
from cStringIO import StringIO
27
26
import httplib
28
 
import os
29
 
import select
30
27
import SimpleHTTPServer
31
28
import socket
32
29
import sys
36
33
from bzrlib import (
37
34
    bzrdir,
38
35
    config,
 
36
    debug,
39
37
    errors,
40
38
    osutils,
41
39
    remote as _mod_remote,
42
40
    tests,
 
41
    trace,
43
42
    transport,
44
43
    ui,
45
 
    urlutils,
46
 
    )
47
 
from bzrlib.symbol_versioning import (
48
 
    deprecated_in,
49
44
    )
50
45
from bzrlib.tests import (
51
46
    features,
52
47
    http_server,
53
48
    http_utils,
 
49
    test_server,
 
50
    )
 
51
from bzrlib.tests.scenarios import (
 
52
    load_tests_apply_scenarios,
 
53
    multiply_scenarios,
54
54
    )
55
55
from bzrlib.transport import (
56
56
    http,
66
66
    from bzrlib.transport.http._pycurl import PyCurlTransport
67
67
 
68
68
 
69
 
def load_tests(standard_tests, module, loader):
70
 
    """Multiply tests for http clients and protocol versions."""
71
 
    result = loader.suiteClass()
72
 
 
73
 
    # one for each transport implementation
74
 
    t_tests, remaining_tests = tests.split_suite_by_condition(
75
 
        standard_tests, tests.condition_isinstance((
76
 
                TestHttpTransportRegistration,
77
 
                TestHttpTransportUrls,
78
 
                Test_redirected_to,
79
 
                )))
 
69
load_tests = load_tests_apply_scenarios
 
70
 
 
71
 
 
72
def vary_by_http_client_implementation():
 
73
    """Test the two libraries we can use, pycurl and urllib."""
80
74
    transport_scenarios = [
81
75
        ('urllib', dict(_transport=_urllib.HttpTransport_urllib,
82
76
                        _server=http_server.HttpServer_urllib,
83
 
                        _qualified_prefix='http+urllib',)),
 
77
                        _url_protocol='http+urllib',)),
84
78
        ]
85
79
    if features.pycurl.available():
86
80
        transport_scenarios.append(
87
81
            ('pycurl', dict(_transport=PyCurlTransport,
88
82
                            _server=http_server.HttpServer_PyCurl,
89
 
                            _qualified_prefix='http+pycurl',)))
90
 
    tests.multiply_tests(t_tests, transport_scenarios, result)
91
 
 
92
 
    # each implementation tested with each HTTP version
93
 
    tp_tests, remaining_tests = tests.split_suite_by_condition(
94
 
        remaining_tests, tests.condition_isinstance((
95
 
                SmartHTTPTunnellingTest,
96
 
                TestDoCatchRedirections,
97
 
                TestHTTPConnections,
98
 
                TestHTTPRedirections,
99
 
                TestHTTPSilentRedirections,
100
 
                TestLimitedRangeRequestServer,
101
 
                TestPost,
102
 
                TestProxyHttpServer,
103
 
                TestRanges,
104
 
                TestSpecificRequestHandler,
105
 
                )))
106
 
    protocol_scenarios = [
107
 
            ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
108
 
            ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
109
 
            ]
110
 
    tp_scenarios = tests.multiply_scenarios(transport_scenarios,
111
 
                                            protocol_scenarios)
112
 
    tests.multiply_tests(tp_tests, tp_scenarios, result)
113
 
 
114
 
    # proxy auth: each auth scheme on all http versions on all implementations.
115
 
    tppa_tests, remaining_tests = tests.split_suite_by_condition(
116
 
        remaining_tests, tests.condition_isinstance((
117
 
                TestProxyAuth,
118
 
                )))
119
 
    proxy_auth_scheme_scenarios = [
120
 
        ('basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
121
 
        ('digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
122
 
        ('basicdigest',
123
 
         dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
 
83
                            _url_protocol='http+pycurl',)))
 
84
    return transport_scenarios
 
85
 
 
86
 
 
87
def vary_by_http_protocol_version():
 
88
    """Test on http/1.0 and 1.1"""
 
89
    return [
 
90
        ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
 
91
        ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
124
92
        ]
125
 
    tppa_scenarios = tests.multiply_scenarios(tp_scenarios,
126
 
                                              proxy_auth_scheme_scenarios)
127
 
    tests.multiply_tests(tppa_tests, tppa_scenarios, result)
128
 
 
129
 
    # auth: each auth scheme on all http versions on all implementations.
130
 
    tpa_tests, remaining_tests = tests.split_suite_by_condition(
131
 
        remaining_tests, tests.condition_isinstance((
132
 
                TestAuth,
133
 
                )))
134
 
    auth_scheme_scenarios = [
 
93
 
 
94
 
 
95
def vary_by_http_auth_scheme():
 
96
    scenarios = [
135
97
        ('basic', dict(_auth_server=http_utils.HTTPBasicAuthServer)),
136
98
        ('digest', dict(_auth_server=http_utils.HTTPDigestAuthServer)),
137
99
        ('basicdigest',
138
 
         dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
139
 
        ]
140
 
    tpa_scenarios = tests.multiply_scenarios(tp_scenarios,
141
 
                                             auth_scheme_scenarios)
142
 
    tests.multiply_tests(tpa_tests, tpa_scenarios, result)
143
 
 
144
 
    # activity: on all http[s] versions on all implementations
145
 
    tpact_tests, remaining_tests = tests.split_suite_by_condition(
146
 
        remaining_tests, tests.condition_isinstance((
147
 
                TestActivity,
148
 
                )))
 
100
            dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
 
101
        ]
 
102
    # Add some attributes common to all scenarios
 
103
    for scenario_id, scenario_dict in scenarios:
 
104
        scenario_dict.update(_auth_header='Authorization',
 
105
                             _username_prompt_prefix='',
 
106
                             _password_prompt_prefix='')
 
107
    return scenarios
 
108
 
 
109
 
 
110
def vary_by_http_proxy_auth_scheme():
 
111
    scenarios = [
 
112
        ('proxy-basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
 
113
        ('proxy-digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
 
114
        ('proxy-basicdigest',
 
115
            dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
 
116
        ]
 
117
    # Add some attributes common to all scenarios
 
118
    for scenario_id, scenario_dict in scenarios:
 
119
        scenario_dict.update(_auth_header='Proxy-Authorization',
 
120
                             _username_prompt_prefix='Proxy ',
 
121
                             _password_prompt_prefix='Proxy ')
 
122
    return scenarios
 
123
 
 
124
 
 
125
def vary_by_http_activity():
149
126
    activity_scenarios = [
150
127
        ('urllib,http', dict(_activity_server=ActivityHTTPServer,
151
 
                             _transport=_urllib.HttpTransport_urllib,)),
 
128
                            _transport=_urllib.HttpTransport_urllib,)),
152
129
        ]
153
 
    if tests.HTTPSServerFeature.available():
 
130
    if features.pycurl.available():
 
131
        activity_scenarios.append(
 
132
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
 
133
                                _transport=PyCurlTransport,)),)
 
134
    if features.HTTPSServerFeature.available():
 
135
        # FIXME: Until we have a better way to handle self-signed certificates
 
136
        # (like allowing them in a test specific authentication.conf for
 
137
        # example), we need some specialized pycurl/urllib transport for tests.
 
138
        # -- vila 2012-01-20
 
139
        from bzrlib.tests import (
 
140
            ssl_certs,
 
141
            )
 
142
        class HTTPS_urllib_transport(_urllib.HttpTransport_urllib):
 
143
 
 
144
            def __init__(self, base, _from_transport=None):
 
145
                super(HTTPS_urllib_transport, self).__init__(
 
146
                    base, _from_transport=_from_transport,
 
147
                    ca_certs=ssl_certs.build_path('ca.crt'))
 
148
 
154
149
        activity_scenarios.append(
155
150
            ('urllib,https', dict(_activity_server=ActivityHTTPSServer,
156
 
                                  _transport=_urllib.HttpTransport_urllib,)),)
157
 
    if features.pycurl.available():
158
 
        activity_scenarios.append(
159
 
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
160
 
                                 _transport=PyCurlTransport,)),)
161
 
        if tests.HTTPSServerFeature.available():
162
 
            from bzrlib.tests import (
163
 
                ssl_certs,
164
 
                )
165
 
            # FIXME: Until we have a better way to handle self-signed
166
 
            # certificates (like allowing them in a test specific
167
 
            # authentication.conf for example), we need some specialized pycurl
168
 
            # transport for tests.
 
151
                                  _transport=HTTPS_urllib_transport,)),)
 
152
        if features.pycurl.available():
169
153
            class HTTPS_pycurl_transport(PyCurlTransport):
170
154
 
171
155
                def __init__(self, base, _from_transport=None):
175
159
 
176
160
            activity_scenarios.append(
177
161
                ('pycurl,https', dict(_activity_server=ActivityHTTPSServer,
178
 
                                      _transport=HTTPS_pycurl_transport,)),)
179
 
 
180
 
    tpact_scenarios = tests.multiply_scenarios(activity_scenarios,
181
 
                                               protocol_scenarios)
182
 
    tests.multiply_tests(tpact_tests, tpact_scenarios, result)
183
 
 
184
 
    # No parametrization for the remaining tests
185
 
    result.addTests(remaining_tests)
186
 
 
187
 
    return result
 
162
                                    _transport=HTTPS_pycurl_transport,)),)
 
163
    return activity_scenarios
188
164
 
189
165
 
190
166
class FakeManager(object):
223
199
        self._sock.bind(('127.0.0.1', 0))
224
200
        self.host, self.port = self._sock.getsockname()
225
201
        self._ready = threading.Event()
226
 
        self._thread = threading.Thread(target=self._accept_read_and_reply)
227
 
        self._thread.setDaemon(True)
 
202
        self._thread = test_server.TestThread(
 
203
            sync_event=self._ready, target=self._accept_read_and_reply)
228
204
        self._thread.start()
229
 
        self._ready.wait(5)
 
205
        if 'threads' in tests.selftest_debug_flags:
 
206
            sys.stderr.write('Thread started: %s\n' % (self._thread.ident,))
 
207
        self._ready.wait()
230
208
 
231
209
    def _accept_read_and_reply(self):
232
210
        self._sock.listen(1)
233
211
        self._ready.set()
234
 
        self._sock.settimeout(5)
235
 
        try:
236
 
            conn, address = self._sock.accept()
237
 
            # On win32, the accepted connection will be non-blocking to start
238
 
            # with because we're using settimeout.
239
 
            conn.setblocking(True)
 
212
        conn, address = self._sock.accept()
 
213
        if self._expect_body_tail is not None:
240
214
            while not self.received_bytes.endswith(self._expect_body_tail):
241
215
                self.received_bytes += conn.recv(4096)
242
216
            conn.sendall('HTTP/1.1 200 OK\r\n')
243
 
        except socket.timeout:
244
 
            # Make sure the client isn't stuck waiting for us to e.g. accept.
 
217
        try:
245
218
            self._sock.close()
246
219
        except socket.error:
247
220
            # The client may have already closed the socket.
249
222
 
250
223
    def stop_server(self):
251
224
        try:
252
 
            self._sock.close()
 
225
            # Issue a fake connection to wake up the server and allow it to
 
226
            # finish quickly
 
227
            fake_conn = osutils.connect_socket((self.host, self.port))
 
228
            fake_conn.close()
253
229
        except socket.error:
254
230
            # We might have already closed it.  We don't care.
255
231
            pass
256
232
        self.host = None
257
233
        self.port = None
 
234
        self._thread.join()
 
235
        if 'threads' in tests.selftest_debug_flags:
 
236
            sys.stderr.write('Thread  joined: %s\n' % (self._thread.ident,))
258
237
 
259
238
 
260
239
class TestAuthHeader(tests.TestCase):
296
275
        self.assertEqual('realm="Thou should not pass"', remainder)
297
276
 
298
277
 
 
278
class TestHTTPRangeParsing(tests.TestCase):
 
279
 
 
280
    def setUp(self):
 
281
        super(TestHTTPRangeParsing, self).setUp()
 
282
        # We focus on range  parsing here and ignore everything else
 
283
        class RequestHandler(http_server.TestingHTTPRequestHandler):
 
284
            def setup(self): pass
 
285
            def handle(self): pass
 
286
            def finish(self): pass
 
287
 
 
288
        self.req_handler = RequestHandler(None, None, None)
 
289
 
 
290
    def assertRanges(self, ranges, header, file_size):
 
291
        self.assertEquals(ranges,
 
292
                          self.req_handler._parse_ranges(header, file_size))
 
293
 
 
294
    def test_simple_range(self):
 
295
        self.assertRanges([(0,2)], 'bytes=0-2', 12)
 
296
 
 
297
    def test_tail(self):
 
298
        self.assertRanges([(8, 11)], 'bytes=-4', 12)
 
299
 
 
300
    def test_tail_bigger_than_file(self):
 
301
        self.assertRanges([(0, 11)], 'bytes=-99', 12)
 
302
 
 
303
    def test_range_without_end(self):
 
304
        self.assertRanges([(4, 11)], 'bytes=4-', 12)
 
305
 
 
306
    def test_invalid_ranges(self):
 
307
        self.assertRanges(None, 'bytes=12-22', 12)
 
308
        self.assertRanges(None, 'bytes=1-3,12-22', 12)
 
309
        self.assertRanges(None, 'bytes=-', 12)
 
310
 
 
311
 
299
312
class TestHTTPServer(tests.TestCase):
300
313
    """Test the HTTP servers implementations."""
301
314
 
304
317
 
305
318
            protocol_version = 'HTTP/0.1'
306
319
 
307
 
        server = http_server.HttpServer(BogusRequestHandler)
308
 
        try:
309
 
            self.assertRaises(httplib.UnknownProtocol, server.start_server)
310
 
        except:
311
 
            server.stop_server()
312
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
320
        self.assertRaises(httplib.UnknownProtocol,
 
321
                          http_server.HttpServer, BogusRequestHandler)
313
322
 
314
323
    def test_force_invalid_protocol(self):
315
 
        server = http_server.HttpServer(protocol_version='HTTP/0.1')
316
 
        try:
317
 
            self.assertRaises(httplib.UnknownProtocol, server.start_server)
318
 
        except:
319
 
            server.stop_server()
320
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
324
        self.assertRaises(httplib.UnknownProtocol,
 
325
                          http_server.HttpServer, protocol_version='HTTP/0.1')
321
326
 
322
327
    def test_server_start_and_stop(self):
323
328
        server = http_server.HttpServer()
 
329
        self.addCleanup(server.stop_server)
324
330
        server.start_server()
325
 
        try:
326
 
            self.assertTrue(server._http_running)
327
 
        finally:
328
 
            server.stop_server()
329
 
        self.assertFalse(server._http_running)
 
331
        self.assertTrue(server.server is not None)
 
332
        self.assertTrue(server.server.serving is not None)
 
333
        self.assertTrue(server.server.serving)
330
334
 
331
335
    def test_create_http_server_one_zero(self):
332
336
        class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
335
339
 
336
340
        server = http_server.HttpServer(RequestHandlerOneZero)
337
341
        self.start_server(server)
338
 
        self.assertIsInstance(server._httpd, http_server.TestingHTTPServer)
 
342
        self.assertIsInstance(server.server, http_server.TestingHTTPServer)
339
343
 
340
344
    def test_create_http_server_one_one(self):
341
345
        class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
344
348
 
345
349
        server = http_server.HttpServer(RequestHandlerOneOne)
346
350
        self.start_server(server)
347
 
        self.assertIsInstance(server._httpd,
 
351
        self.assertIsInstance(server.server,
348
352
                              http_server.TestingThreadingHTTPServer)
349
353
 
350
354
    def test_create_http_server_force_one_one(self):
355
359
        server = http_server.HttpServer(RequestHandlerOneZero,
356
360
                                        protocol_version='HTTP/1.1')
357
361
        self.start_server(server)
358
 
        self.assertIsInstance(server._httpd,
 
362
        self.assertIsInstance(server.server,
359
363
                              http_server.TestingThreadingHTTPServer)
360
364
 
361
365
    def test_create_http_server_force_one_zero(self):
366
370
        server = http_server.HttpServer(RequestHandlerOneOne,
367
371
                                        protocol_version='HTTP/1.0')
368
372
        self.start_server(server)
369
 
        self.assertIsInstance(server._httpd,
 
373
        self.assertIsInstance(server.server,
370
374
                              http_server.TestingHTTPServer)
371
375
 
372
376
 
401
405
class TestHttpTransportUrls(tests.TestCase):
402
406
    """Test the http urls."""
403
407
 
 
408
    scenarios = vary_by_http_client_implementation()
 
409
 
404
410
    def test_abs_url(self):
405
411
        """Construction of absolute http URLs"""
406
 
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
 
412
        t = self._transport('http://example.com/bzr/bzr.dev/')
407
413
        eq = self.assertEqualDiff
408
 
        eq(t.abspath('.'), 'http://bazaar-vcs.org/bzr/bzr.dev')
409
 
        eq(t.abspath('foo/bar'), 'http://bazaar-vcs.org/bzr/bzr.dev/foo/bar')
410
 
        eq(t.abspath('.bzr'), 'http://bazaar-vcs.org/bzr/bzr.dev/.bzr')
 
414
        eq(t.abspath('.'), 'http://example.com/bzr/bzr.dev')
 
415
        eq(t.abspath('foo/bar'), 'http://example.com/bzr/bzr.dev/foo/bar')
 
416
        eq(t.abspath('.bzr'), 'http://example.com/bzr/bzr.dev/.bzr')
411
417
        eq(t.abspath('.bzr/1//2/./3'),
412
 
           'http://bazaar-vcs.org/bzr/bzr.dev/.bzr/1/2/3')
 
418
           'http://example.com/bzr/bzr.dev/.bzr/1/2/3')
413
419
 
414
420
    def test_invalid_http_urls(self):
415
421
        """Trap invalid construction of urls"""
416
 
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
 
422
        self._transport('http://example.com/bzr/bzr.dev/')
417
423
        self.assertRaises(errors.InvalidURL,
418
424
                          self._transport,
419
 
                          'http://http://bazaar-vcs.org/bzr/bzr.dev/')
 
425
                          'http://http://example.com/bzr/bzr.dev/')
420
426
 
421
427
    def test_http_root_urls(self):
422
428
        """Construction of URLs from server root"""
423
 
        t = self._transport('http://bzr.ozlabs.org/')
 
429
        t = self._transport('http://example.com/')
424
430
        eq = self.assertEqualDiff
425
431
        eq(t.abspath('.bzr/tree-version'),
426
 
           'http://bzr.ozlabs.org/.bzr/tree-version')
 
432
           'http://example.com/.bzr/tree-version')
427
433
 
428
434
    def test_http_impl_urls(self):
429
435
        """There are servers which ask for particular clients to connect"""
431
437
        server.start_server()
432
438
        try:
433
439
            url = server.get_url()
434
 
            self.assertTrue(url.startswith('%s://' % self._qualified_prefix))
 
440
            self.assertTrue(url.startswith('%s://' % self._url_protocol))
435
441
        finally:
436
442
            server.stop_server()
437
443
 
475
481
class TestHTTPConnections(http_utils.TestCaseWithWebserver):
476
482
    """Test the http connections."""
477
483
 
 
484
    scenarios = multiply_scenarios(
 
485
        vary_by_http_client_implementation(),
 
486
        vary_by_http_protocol_version(),
 
487
        )
 
488
 
478
489
    def setUp(self):
479
490
        http_utils.TestCaseWithWebserver.setUp(self)
480
491
        self.build_tree(['foo/', 'foo/bar'], line_endings='binary',
482
493
 
483
494
    def test_http_has(self):
484
495
        server = self.get_readonly_server()
485
 
        t = self._transport(server.get_url())
 
496
        t = self.get_readonly_transport()
486
497
        self.assertEqual(t.has('foo/bar'), True)
487
498
        self.assertEqual(len(server.logs), 1)
488
499
        self.assertContainsRe(server.logs[0],
490
501
 
491
502
    def test_http_has_not_found(self):
492
503
        server = self.get_readonly_server()
493
 
        t = self._transport(server.get_url())
 
504
        t = self.get_readonly_transport()
494
505
        self.assertEqual(t.has('not-found'), False)
495
506
        self.assertContainsRe(server.logs[1],
496
507
            r'"HEAD /not-found HTTP/1.." 404 - "-" "bzr/')
497
508
 
498
509
    def test_http_get(self):
499
510
        server = self.get_readonly_server()
500
 
        t = self._transport(server.get_url())
 
511
        t = self.get_readonly_transport()
501
512
        fp = t.get('foo/bar')
502
513
        self.assertEqualDiff(
503
514
            fp.read(),
525
536
class TestHttpTransportRegistration(tests.TestCase):
526
537
    """Test registrations of various http implementations"""
527
538
 
 
539
    scenarios = vary_by_http_client_implementation()
 
540
 
528
541
    def test_http_registered(self):
529
 
        t = transport.get_transport('%s://foo.com/' % self._qualified_prefix)
 
542
        t = transport.get_transport_from_url(
 
543
            '%s://foo.com/' % self._url_protocol)
530
544
        self.assertIsInstance(t, transport.Transport)
531
545
        self.assertIsInstance(t, self._transport)
532
546
 
533
547
 
534
548
class TestPost(tests.TestCase):
535
549
 
 
550
    scenarios = multiply_scenarios(
 
551
        vary_by_http_client_implementation(),
 
552
        vary_by_http_protocol_version(),
 
553
        )
 
554
 
536
555
    def test_post_body_is_received(self):
537
556
        server = RecordingServer(expect_body_tail='end-of-body',
538
 
            scheme=self._qualified_prefix)
 
557
                                 scheme=self._url_protocol)
539
558
        self.start_server(server)
540
559
        url = server.get_url()
541
 
        http_transport = self._transport(url)
 
560
        # FIXME: needs a cleanup -- vila 20100611
 
561
        http_transport = transport.get_transport_from_url(url)
542
562
        code, response = http_transport._post('abc def end-of-body')
543
563
        self.assertTrue(
544
564
            server.received_bytes.startswith('POST /.bzr/smart HTTP/1.'))
545
565
        self.assertTrue('content-length: 19\r' in server.received_bytes.lower())
 
566
        self.assertTrue('content-type: application/octet-stream\r'
 
567
                        in server.received_bytes.lower())
546
568
        # The transport should not be assuming that the server can accept
547
569
        # chunked encoding the first time it connects, because HTTP/1.1, so we
548
570
        # check for the literal string.
584
606
    Daughter classes are expected to override _req_handler_class
585
607
    """
586
608
 
 
609
    scenarios = multiply_scenarios(
 
610
        vary_by_http_client_implementation(),
 
611
        vary_by_http_protocol_version(),
 
612
        )
 
613
 
587
614
    # Provide a useful default
588
615
    _req_handler_class = http_server.TestingHTTPRequestHandler
589
616
 
590
617
    def create_transport_readonly_server(self):
591
 
        return http_server.HttpServer(self._req_handler_class,
592
 
                                      protocol_version=self._protocol_version)
 
618
        server = http_server.HttpServer(self._req_handler_class,
 
619
                                        protocol_version=self._protocol_version)
 
620
        server._url_protocol = self._url_protocol
 
621
        return server
593
622
 
594
623
    def _testing_pycurl(self):
595
624
        # TODO: This is duplicated for lots of the classes in this file
600
629
class WallRequestHandler(http_server.TestingHTTPRequestHandler):
601
630
    """Whatever request comes in, close the connection"""
602
631
 
603
 
    def handle_one_request(self):
 
632
    def _handle_one_request(self):
604
633
        """Handle a single HTTP request, by abruptly closing the connection"""
605
634
        self.close_connection = 1
606
635
 
611
640
    _req_handler_class = WallRequestHandler
612
641
 
613
642
    def test_http_has(self):
614
 
        server = self.get_readonly_server()
615
 
        t = self._transport(server.get_url())
 
643
        t = self.get_readonly_transport()
616
644
        # Unfortunately httplib (see HTTPResponse._read_status
617
645
        # for details) make no distinction between a closed
618
646
        # socket and badly formatted status line, so we can't
624
652
                          t.has, 'foo/bar')
625
653
 
626
654
    def test_http_get(self):
627
 
        server = self.get_readonly_server()
628
 
        t = self._transport(server.get_url())
 
655
        t = self.get_readonly_transport()
629
656
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
630
657
                           errors.InvalidHttpResponse),
631
658
                          t.get, 'foo/bar')
648
675
    _req_handler_class = BadStatusRequestHandler
649
676
 
650
677
    def test_http_has(self):
651
 
        server = self.get_readonly_server()
652
 
        t = self._transport(server.get_url())
 
678
        t = self.get_readonly_transport()
653
679
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
654
680
 
655
681
    def test_http_get(self):
656
 
        server = self.get_readonly_server()
657
 
        t = self._transport(server.get_url())
 
682
        t = self.get_readonly_transport()
658
683
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
659
684
 
660
685
 
665
690
        """Fakes handling a single HTTP request, returns a bad status"""
666
691
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
667
692
        self.wfile.write("Invalid status line\r\n")
 
693
        # If we don't close the connection pycurl will hang. Since this is a
 
694
        # stress test we don't *have* to respect the protocol, but we don't
 
695
        # have to sabotage it too much either.
 
696
        self.close_connection = True
668
697
        return False
669
698
 
670
699
 
676
705
 
677
706
    _req_handler_class = InvalidStatusRequestHandler
678
707
 
679
 
    def test_http_has(self):
680
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
681
 
            raise tests.KnownFailure(
682
 
                'pycurl hangs if the server send back garbage')
683
 
        super(TestInvalidStatusServer, self).test_http_has()
684
 
 
685
 
    def test_http_get(self):
686
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
687
 
            raise tests.KnownFailure(
688
 
                'pycurl hangs if the server send back garbage')
689
 
        super(TestInvalidStatusServer, self).test_http_get()
690
 
 
691
708
 
692
709
class BadProtocolRequestHandler(http_server.TestingHTTPRequestHandler):
693
710
    """Whatever request comes in, returns a bad protocol version"""
715
732
        super(TestBadProtocolServer, self).setUp()
716
733
 
717
734
    def test_http_has(self):
718
 
        server = self.get_readonly_server()
719
 
        t = self._transport(server.get_url())
 
735
        t = self.get_readonly_transport()
720
736
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
721
737
 
722
738
    def test_http_get(self):
723
 
        server = self.get_readonly_server()
724
 
        t = self._transport(server.get_url())
 
739
        t = self.get_readonly_transport()
725
740
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
726
741
 
727
742
 
741
756
    _req_handler_class = ForbiddenRequestHandler
742
757
 
743
758
    def test_http_has(self):
744
 
        server = self.get_readonly_server()
745
 
        t = self._transport(server.get_url())
 
759
        t = self.get_readonly_transport()
746
760
        self.assertRaises(errors.TransportError, t.has, 'foo/bar')
747
761
 
748
762
    def test_http_get(self):
749
 
        server = self.get_readonly_server()
750
 
        t = self._transport(server.get_url())
 
763
        t = self.get_readonly_transport()
751
764
        self.assertRaises(errors.TransportError, t.get, 'foo/bar')
752
765
 
753
766
 
792
805
        self.build_tree_contents([('a', '0123456789')],)
793
806
 
794
807
    def test_readv(self):
795
 
        server = self.get_readonly_server()
796
 
        t = self._transport(server.get_url())
 
808
        t = self.get_readonly_transport()
797
809
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
798
810
        self.assertEqual(l[0], (0, '0'))
799
811
        self.assertEqual(l[1], (1, '1'))
801
813
        self.assertEqual(l[3], (9, '9'))
802
814
 
803
815
    def test_readv_out_of_order(self):
804
 
        server = self.get_readonly_server()
805
 
        t = self._transport(server.get_url())
 
816
        t = self.get_readonly_transport()
806
817
        l = list(t.readv('a', ((1, 1), (9, 1), (0, 1), (3, 2))))
807
818
        self.assertEqual(l[0], (1, '1'))
808
819
        self.assertEqual(l[1], (9, '9'))
810
821
        self.assertEqual(l[3], (3, '34'))
811
822
 
812
823
    def test_readv_invalid_ranges(self):
813
 
        server = self.get_readonly_server()
814
 
        t = self._transport(server.get_url())
 
824
        t = self.get_readonly_transport()
815
825
 
816
826
        # This is intentionally reading off the end of the file
817
827
        # since we are sure that it cannot get there
825
835
 
826
836
    def test_readv_multiple_get_requests(self):
827
837
        server = self.get_readonly_server()
828
 
        t = self._transport(server.get_url())
 
838
        t = self.get_readonly_transport()
829
839
        # force transport to issue multiple requests
830
840
        t._max_readv_combine = 1
831
841
        t._max_get_ranges = 1
839
849
 
840
850
    def test_readv_get_max_size(self):
841
851
        server = self.get_readonly_server()
842
 
        t = self._transport(server.get_url())
 
852
        t = self.get_readonly_transport()
843
853
        # force transport to issue multiple requests by limiting the number of
844
854
        # bytes by request. Note that this apply to coalesced offsets only, a
845
855
        # single range will keep its size even if bigger than the limit.
854
864
 
855
865
    def test_complete_readv_leave_pipe_clean(self):
856
866
        server = self.get_readonly_server()
857
 
        t = self._transport(server.get_url())
 
867
        t = self.get_readonly_transport()
858
868
        # force transport to issue multiple requests
859
869
        t._get_max_size = 2
860
 
        l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
 
870
        list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
861
871
        # The server should have issued 3 requests
862
872
        self.assertEqual(3, server.GET_request_nb)
863
873
        self.assertEqual('0123456789', t.get_bytes('a'))
865
875
 
866
876
    def test_incomplete_readv_leave_pipe_clean(self):
867
877
        server = self.get_readonly_server()
868
 
        t = self._transport(server.get_url())
 
878
        t = self.get_readonly_transport()
869
879
        # force transport to issue multiple requests
870
880
        t._get_max_size = 2
871
881
        # Don't collapse readv results into a list so that we leave unread
940
950
    def get_multiple_ranges(self, file, file_size, ranges):
941
951
        self.send_response(206)
942
952
        self.send_header('Accept-Ranges', 'bytes')
 
953
        # XXX: this is strange; the 'random' name below seems undefined and
 
954
        # yet the tests pass -- mbp 2010-10-11 bug 658773
943
955
        boundary = "%d" % random.randint(0,0x7FFFFFFF)
944
956
        self.send_header("Content-Type",
945
957
                         "multipart/byteranges; boundary=%s" % boundary)
1007
1019
                return
1008
1020
            self.send_range_content(file, start, end - start + 1)
1009
1021
            cur += 1
1010
 
        # No final boundary
 
1022
        # Final boundary
1011
1023
        self.wfile.write(boundary_line)
1012
1024
 
1013
1025
 
1021
1033
 
1022
1034
    def test_readv_with_short_reads(self):
1023
1035
        server = self.get_readonly_server()
1024
 
        t = self._transport(server.get_url())
 
1036
        t = self.get_readonly_transport()
1025
1037
        # Force separate ranges for each offset
1026
1038
        t._bytes_to_read_before_seek = 0
1027
1039
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1042
1054
        # that mode
1043
1055
        self.assertEqual('single', t._range_hint)
1044
1056
 
 
1057
 
 
1058
class TruncatedBeforeBoundaryRequestHandler(
 
1059
    http_server.TestingHTTPRequestHandler):
 
1060
    """Truncation before a boundary, like in bug 198646"""
 
1061
 
 
1062
    _truncated_ranges = 1
 
1063
 
 
1064
    def get_multiple_ranges(self, file, file_size, ranges):
 
1065
        self.send_response(206)
 
1066
        self.send_header('Accept-Ranges', 'bytes')
 
1067
        boundary = 'tagada'
 
1068
        self.send_header('Content-Type',
 
1069
                         'multipart/byteranges; boundary=%s' % boundary)
 
1070
        boundary_line = '--%s\r\n' % boundary
 
1071
        # Calculate the Content-Length
 
1072
        content_length = 0
 
1073
        for (start, end) in ranges:
 
1074
            content_length += len(boundary_line)
 
1075
            content_length += self._header_line_length(
 
1076
                'Content-type', 'application/octet-stream')
 
1077
            content_length += self._header_line_length(
 
1078
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
 
1079
            content_length += len('\r\n') # end headers
 
1080
            content_length += end - start # + 1
 
1081
        content_length += len(boundary_line)
 
1082
        self.send_header('Content-length', content_length)
 
1083
        self.end_headers()
 
1084
 
 
1085
        # Send the multipart body
 
1086
        cur = 0
 
1087
        for (start, end) in ranges:
 
1088
            if cur + self._truncated_ranges >= len(ranges):
 
1089
                # Abruptly ends the response and close the connection
 
1090
                self.close_connection = 1
 
1091
                return
 
1092
            self.wfile.write(boundary_line)
 
1093
            self.send_header('Content-type', 'application/octet-stream')
 
1094
            self.send_header('Content-Range', 'bytes %d-%d/%d'
 
1095
                             % (start, end, file_size))
 
1096
            self.end_headers()
 
1097
            self.send_range_content(file, start, end - start + 1)
 
1098
            cur += 1
 
1099
        # Final boundary
 
1100
        self.wfile.write(boundary_line)
 
1101
 
 
1102
 
 
1103
class TestTruncatedBeforeBoundary(TestSpecificRequestHandler):
 
1104
    """Tests the case of bug 198646, disconnecting before a boundary."""
 
1105
 
 
1106
    _req_handler_class = TruncatedBeforeBoundaryRequestHandler
 
1107
 
 
1108
    def setUp(self):
 
1109
        super(TestTruncatedBeforeBoundary, self).setUp()
 
1110
        self.build_tree_contents([('a', '0123456789')],)
 
1111
 
 
1112
    def test_readv_with_short_reads(self):
 
1113
        server = self.get_readonly_server()
 
1114
        t = self.get_readonly_transport()
 
1115
        # Force separate ranges for each offset
 
1116
        t._bytes_to_read_before_seek = 0
 
1117
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
 
1118
        self.assertEqual((0, '0'), ireadv.next())
 
1119
        self.assertEqual((2, '2'), ireadv.next())
 
1120
        self.assertEqual((4, '45'), ireadv.next())
 
1121
        self.assertEqual((9, '9'), ireadv.next())
 
1122
 
 
1123
 
1045
1124
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1046
1125
    """Errors out when range specifiers exceed the limit"""
1047
1126
 
1071
1150
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1072
1151
    """Tests readv requests against a server erroring out on too much ranges."""
1073
1152
 
 
1153
    scenarios = multiply_scenarios(
 
1154
        vary_by_http_client_implementation(),
 
1155
        vary_by_http_protocol_version(),
 
1156
        )
 
1157
 
1074
1158
    # Requests with more range specifiers will error out
1075
1159
    range_limit = 3
1076
1160
 
1078
1162
        return LimitedRangeHTTPServer(range_limit=self.range_limit,
1079
1163
                                      protocol_version=self._protocol_version)
1080
1164
 
1081
 
    def get_transport(self):
1082
 
        return self._transport(self.get_readonly_server().get_url())
1083
 
 
1084
1165
    def setUp(self):
1085
1166
        http_utils.TestCaseWithWebserver.setUp(self)
1086
1167
        # We need to manipulate ranges that correspond to real chunks in the
1090
1171
        self.build_tree_contents([('a', content)],)
1091
1172
 
1092
1173
    def test_few_ranges(self):
1093
 
        t = self.get_transport()
 
1174
        t = self.get_readonly_transport()
1094
1175
        l = list(t.readv('a', ((0, 4), (1024, 4), )))
1095
1176
        self.assertEqual(l[0], (0, '0000'))
1096
1177
        self.assertEqual(l[1], (1024, '0001'))
1097
1178
        self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1098
1179
 
1099
1180
    def test_more_ranges(self):
1100
 
        t = self.get_transport()
 
1181
        t = self.get_readonly_transport()
1101
1182
        l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1102
1183
        self.assertEqual(l[0], (0, '0000'))
1103
1184
        self.assertEqual(l[1], (1024, '0001'))
1114
1195
    Only the urllib implementation is tested here.
1115
1196
    """
1116
1197
 
1117
 
    def setUp(self):
1118
 
        tests.TestCase.setUp(self)
1119
 
        self._old_env = {}
1120
 
 
1121
 
    def tearDown(self):
1122
 
        self._restore_env()
1123
 
        tests.TestCase.tearDown(self)
1124
 
 
1125
 
    def _install_env(self, env):
1126
 
        for name, value in env.iteritems():
1127
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1128
 
 
1129
 
    def _restore_env(self):
1130
 
        for name, value in self._old_env.iteritems():
1131
 
            osutils.set_or_unset_env(name, value)
1132
 
 
1133
1198
    def _proxied_request(self):
1134
1199
        handler = _urllib2_wrappers.ProxyHandler()
1135
 
        request = _urllib2_wrappers.Request('GET','http://baz/buzzle')
 
1200
        request = _urllib2_wrappers.Request('GET', 'http://baz/buzzle')
1136
1201
        handler.set_proxy(request, 'http')
1137
1202
        return request
1138
1203
 
 
1204
    def assertEvaluateProxyBypass(self, expected, host, no_proxy):
 
1205
        handler = _urllib2_wrappers.ProxyHandler()
 
1206
        self.assertEquals(expected,
 
1207
                          handler.evaluate_proxy_bypass(host, no_proxy))
 
1208
 
1139
1209
    def test_empty_user(self):
1140
 
        self._install_env({'http_proxy': 'http://bar.com'})
 
1210
        self.overrideEnv('http_proxy', 'http://bar.com')
 
1211
        request = self._proxied_request()
 
1212
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
 
1213
 
 
1214
    def test_user_with_at(self):
 
1215
        self.overrideEnv('http_proxy',
 
1216
                         'http://username@domain:password@proxy_host:1234')
1141
1217
        request = self._proxied_request()
1142
1218
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
1143
1219
 
1144
1220
    def test_invalid_proxy(self):
1145
1221
        """A proxy env variable without scheme"""
1146
 
        self._install_env({'http_proxy': 'host:1234'})
 
1222
        self.overrideEnv('http_proxy', 'host:1234')
1147
1223
        self.assertRaises(errors.InvalidURL, self._proxied_request)
1148
1224
 
 
1225
    def test_evaluate_proxy_bypass_true(self):
 
1226
        """The host is not proxied"""
 
1227
        self.assertEvaluateProxyBypass(True, 'example.com', 'example.com')
 
1228
        self.assertEvaluateProxyBypass(True, 'bzr.example.com', '*example.com')
 
1229
 
 
1230
    def test_evaluate_proxy_bypass_false(self):
 
1231
        """The host is proxied"""
 
1232
        self.assertEvaluateProxyBypass(False, 'bzr.example.com', None)
 
1233
 
 
1234
    def test_evaluate_proxy_bypass_unknown(self):
 
1235
        """The host is not explicitly proxied"""
 
1236
        self.assertEvaluateProxyBypass(None, 'example.com', 'not.example.com')
 
1237
        self.assertEvaluateProxyBypass(None, 'bzr.example.com', 'example.com')
 
1238
 
 
1239
    def test_evaluate_proxy_bypass_empty_entries(self):
 
1240
        """Ignore empty entries"""
 
1241
        self.assertEvaluateProxyBypass(None, 'example.com', '')
 
1242
        self.assertEvaluateProxyBypass(None, 'example.com', ',')
 
1243
        self.assertEvaluateProxyBypass(None, 'example.com', 'foo,,bar')
 
1244
 
1149
1245
 
1150
1246
class TestProxyHttpServer(http_utils.TestCaseWithTwoWebservers):
1151
1247
    """Tests proxy server.
1156
1252
    to the file names).
1157
1253
    """
1158
1254
 
 
1255
    scenarios = multiply_scenarios(
 
1256
        vary_by_http_client_implementation(),
 
1257
        vary_by_http_protocol_version(),
 
1258
        )
 
1259
 
1159
1260
    # FIXME: We don't have an https server available, so we don't
1160
 
    # test https connections.
 
1261
    # test https connections. --vila toolongago
1161
1262
 
1162
1263
    def setUp(self):
1163
1264
        super(TestProxyHttpServer, self).setUp()
 
1265
        self.transport_secondary_server = http_utils.ProxyServer
1164
1266
        self.build_tree_contents([('foo', 'contents of foo\n'),
1165
1267
                                  ('foo-proxied', 'proxied contents of foo\n')])
1166
1268
        # Let's setup some attributes for tests
1167
 
        self.server = self.get_readonly_server()
1168
 
        self.proxy_address = '%s:%d' % (self.server.host, self.server.port)
 
1269
        server = self.get_readonly_server()
 
1270
        self.server_host_port = '%s:%d' % (server.host, server.port)
1169
1271
        if self._testing_pycurl():
1170
1272
            # Oh my ! pycurl does not check for the port as part of
1171
1273
            # no_proxy :-( So we just test the host part
1172
 
            self.no_proxy_host = self.server.host
 
1274
            self.no_proxy_host = server.host
1173
1275
        else:
1174
 
            self.no_proxy_host = self.proxy_address
 
1276
            self.no_proxy_host = self.server_host_port
1175
1277
        # The secondary server is the proxy
1176
 
        self.proxy = self.get_secondary_server()
1177
 
        self.proxy_url = self.proxy.get_url()
1178
 
        self._old_env = {}
 
1278
        self.proxy_url = self.get_secondary_url()
1179
1279
 
1180
1280
    def _testing_pycurl(self):
1181
1281
        # TODO: This is duplicated for lots of the classes in this file
1182
1282
        return (features.pycurl.available()
1183
1283
                and self._transport == PyCurlTransport)
1184
1284
 
1185
 
    def create_transport_secondary_server(self):
1186
 
        """Creates an http server that will serve files with
1187
 
        '-proxied' appended to their names.
1188
 
        """
1189
 
        return http_utils.ProxyServer(protocol_version=self._protocol_version)
1190
 
 
1191
 
    def _install_env(self, env):
1192
 
        for name, value in env.iteritems():
1193
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1194
 
 
1195
 
    def _restore_env(self):
1196
 
        for name, value in self._old_env.iteritems():
1197
 
            osutils.set_or_unset_env(name, value)
1198
 
 
1199
 
    def proxied_in_env(self, env):
1200
 
        self._install_env(env)
1201
 
        url = self.server.get_url()
1202
 
        t = self._transport(url)
1203
 
        try:
1204
 
            self.assertEqual('proxied contents of foo\n', t.get('foo').read())
1205
 
        finally:
1206
 
            self._restore_env()
1207
 
 
1208
 
    def not_proxied_in_env(self, env):
1209
 
        self._install_env(env)
1210
 
        url = self.server.get_url()
1211
 
        t = self._transport(url)
1212
 
        try:
1213
 
            self.assertEqual('contents of foo\n', t.get('foo').read())
1214
 
        finally:
1215
 
            self._restore_env()
 
1285
    def assertProxied(self):
 
1286
        t = self.get_readonly_transport()
 
1287
        self.assertEqual('proxied contents of foo\n', t.get('foo').read())
 
1288
 
 
1289
    def assertNotProxied(self):
 
1290
        t = self.get_readonly_transport()
 
1291
        self.assertEqual('contents of foo\n', t.get('foo').read())
1216
1292
 
1217
1293
    def test_http_proxy(self):
1218
 
        self.proxied_in_env({'http_proxy': self.proxy_url})
 
1294
        self.overrideEnv('http_proxy', self.proxy_url)
 
1295
        self.assertProxied()
1219
1296
 
1220
1297
    def test_HTTP_PROXY(self):
1221
1298
        if self._testing_pycurl():
1224
1301
            # about. Should we ?)
1225
1302
            raise tests.TestNotApplicable(
1226
1303
                'pycurl does not check HTTP_PROXY for security reasons')
1227
 
        self.proxied_in_env({'HTTP_PROXY': self.proxy_url})
 
1304
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
 
1305
        self.assertProxied()
1228
1306
 
1229
1307
    def test_all_proxy(self):
1230
 
        self.proxied_in_env({'all_proxy': self.proxy_url})
 
1308
        self.overrideEnv('all_proxy', self.proxy_url)
 
1309
        self.assertProxied()
1231
1310
 
1232
1311
    def test_ALL_PROXY(self):
1233
 
        self.proxied_in_env({'ALL_PROXY': self.proxy_url})
 
1312
        self.overrideEnv('ALL_PROXY', self.proxy_url)
 
1313
        self.assertProxied()
1234
1314
 
1235
1315
    def test_http_proxy_with_no_proxy(self):
1236
 
        self.not_proxied_in_env({'http_proxy': self.proxy_url,
1237
 
                                 'no_proxy': self.no_proxy_host})
 
1316
        self.overrideEnv('no_proxy', self.no_proxy_host)
 
1317
        self.overrideEnv('http_proxy', self.proxy_url)
 
1318
        self.assertNotProxied()
1238
1319
 
1239
1320
    def test_HTTP_PROXY_with_NO_PROXY(self):
1240
1321
        if self._testing_pycurl():
1241
1322
            raise tests.TestNotApplicable(
1242
1323
                'pycurl does not check HTTP_PROXY for security reasons')
1243
 
        self.not_proxied_in_env({'HTTP_PROXY': self.proxy_url,
1244
 
                                 'NO_PROXY': self.no_proxy_host})
 
1324
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
 
1325
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
 
1326
        self.assertNotProxied()
1245
1327
 
1246
1328
    def test_all_proxy_with_no_proxy(self):
1247
 
        self.not_proxied_in_env({'all_proxy': self.proxy_url,
1248
 
                                 'no_proxy': self.no_proxy_host})
 
1329
        self.overrideEnv('no_proxy', self.no_proxy_host)
 
1330
        self.overrideEnv('all_proxy', self.proxy_url)
 
1331
        self.assertNotProxied()
1249
1332
 
1250
1333
    def test_ALL_PROXY_with_NO_PROXY(self):
1251
 
        self.not_proxied_in_env({'ALL_PROXY': self.proxy_url,
1252
 
                                 'NO_PROXY': self.no_proxy_host})
 
1334
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
 
1335
        self.overrideEnv('ALL_PROXY', self.proxy_url)
 
1336
        self.assertNotProxied()
1253
1337
 
1254
1338
    def test_http_proxy_without_scheme(self):
 
1339
        self.overrideEnv('http_proxy', self.server_host_port)
1255
1340
        if self._testing_pycurl():
1256
1341
            # pycurl *ignores* invalid proxy env variables. If that ever change
1257
1342
            # in the future, this test will fail indicating that pycurl do not
1258
1343
            # ignore anymore such variables.
1259
 
            self.not_proxied_in_env({'http_proxy': self.proxy_address})
 
1344
            self.assertNotProxied()
1260
1345
        else:
1261
 
            self.assertRaises(errors.InvalidURL,
1262
 
                              self.proxied_in_env,
1263
 
                              {'http_proxy': self.proxy_address})
 
1346
            self.assertRaises(errors.InvalidURL, self.assertProxied)
1264
1347
 
1265
1348
 
1266
1349
class TestRanges(http_utils.TestCaseWithWebserver):
1267
1350
    """Test the Range header in GET methods."""
1268
1351
 
 
1352
    scenarios = multiply_scenarios(
 
1353
        vary_by_http_client_implementation(),
 
1354
        vary_by_http_protocol_version(),
 
1355
        )
 
1356
 
1269
1357
    def setUp(self):
1270
1358
        http_utils.TestCaseWithWebserver.setUp(self)
1271
1359
        self.build_tree_contents([('a', '0123456789')],)
1272
 
        server = self.get_readonly_server()
1273
 
        self.transport = self._transport(server.get_url())
1274
1360
 
1275
1361
    def create_transport_readonly_server(self):
1276
1362
        return http_server.HttpServer(protocol_version=self._protocol_version)
1277
1363
 
1278
1364
    def _file_contents(self, relpath, ranges):
 
1365
        t = self.get_readonly_transport()
1279
1366
        offsets = [ (start, end - start + 1) for start, end in ranges]
1280
 
        coalesce = self.transport._coalesce_offsets
 
1367
        coalesce = t._coalesce_offsets
1281
1368
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1282
 
        code, data = self.transport._get(relpath, coalesced)
 
1369
        code, data = t._get(relpath, coalesced)
1283
1370
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1284
1371
        for start, end in ranges:
1285
1372
            data.seek(start)
1286
1373
            yield data.read(end - start + 1)
1287
1374
 
1288
1375
    def _file_tail(self, relpath, tail_amount):
1289
 
        code, data = self.transport._get(relpath, [], tail_amount)
 
1376
        t = self.get_readonly_transport()
 
1377
        code, data = t._get(relpath, [], tail_amount)
1290
1378
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1291
1379
        data.seek(-tail_amount, 2)
1292
1380
        return data.read(tail_amount)
1311
1399
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1312
1400
    """Test redirection between http servers."""
1313
1401
 
1314
 
    def create_transport_secondary_server(self):
1315
 
        """Create the secondary server redirecting to the primary server"""
1316
 
        new = self.get_readonly_server()
1317
 
 
1318
 
        redirecting = http_utils.HTTPServerRedirecting(
1319
 
            protocol_version=self._protocol_version)
1320
 
        redirecting.redirect_to(new.host, new.port)
1321
 
        return redirecting
 
1402
    scenarios = multiply_scenarios(
 
1403
        vary_by_http_client_implementation(),
 
1404
        vary_by_http_protocol_version(),
 
1405
        )
1322
1406
 
1323
1407
    def setUp(self):
1324
1408
        super(TestHTTPRedirections, self).setUp()
1326
1410
                                  ('bundle',
1327
1411
                                  '# Bazaar revision bundle v0.9\n#\n')
1328
1412
                                  ],)
1329
 
        # The requests to the old server will be redirected to the new server
1330
 
        self.old_transport = self._transport(self.old_server.get_url())
1331
1413
 
1332
1414
    def test_redirected(self):
1333
 
        self.assertRaises(errors.RedirectRequested, self.old_transport.get, 'a')
1334
 
        t = self._transport(self.new_server.get_url())
1335
 
        self.assertEqual('0123456789', t.get('a').read())
1336
 
 
1337
 
    def test_read_redirected_bundle_from_url(self):
1338
 
        from bzrlib.bundle import read_bundle_from_url
1339
 
        url = self.old_transport.abspath('bundle')
1340
 
        bundle = self.applyDeprecated(deprecated_in((1, 12, 0)),
1341
 
                read_bundle_from_url, url)
1342
 
        # If read_bundle_from_url was successful we get an empty bundle
1343
 
        self.assertEqual([], bundle.revisions)
 
1415
        self.assertRaises(errors.RedirectRequested,
 
1416
                          self.get_old_transport().get, 'a')
 
1417
        self.assertEqual('0123456789', self.get_new_transport().get('a').read())
1344
1418
 
1345
1419
 
1346
1420
class RedirectedRequest(_urllib2_wrappers.Request):
1363
1437
    test.overrideAttr(_urllib2_wrappers, 'Request', RedirectedRequest)
1364
1438
 
1365
1439
 
 
1440
def cleanup_http_redirection_connections(test):
 
1441
    # Some sockets are opened but never seen by _urllib, so we trap them at
 
1442
    # the _urllib2_wrappers level to be able to clean them up.
 
1443
    def socket_disconnect(sock):
 
1444
        try:
 
1445
            sock.shutdown(socket.SHUT_RDWR)
 
1446
            sock.close()
 
1447
        except socket.error:
 
1448
            pass
 
1449
    def connect(connection):
 
1450
        test.http_connect_orig(connection)
 
1451
        test.addCleanup(socket_disconnect, connection.sock)
 
1452
    test.http_connect_orig = test.overrideAttr(
 
1453
        _urllib2_wrappers.HTTPConnection, 'connect', connect)
 
1454
    def connect(connection):
 
1455
        test.https_connect_orig(connection)
 
1456
        test.addCleanup(socket_disconnect, connection.sock)
 
1457
    test.https_connect_orig = test.overrideAttr(
 
1458
        _urllib2_wrappers.HTTPSConnection, 'connect', connect)
 
1459
 
 
1460
 
1366
1461
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1367
1462
    """Test redirections.
1368
1463
 
1377
1472
    -- vila 20070212
1378
1473
    """
1379
1474
 
 
1475
    scenarios = multiply_scenarios(
 
1476
        vary_by_http_client_implementation(),
 
1477
        vary_by_http_protocol_version(),
 
1478
        )
 
1479
 
1380
1480
    def setUp(self):
1381
1481
        if (features.pycurl.available()
1382
1482
            and self._transport == PyCurlTransport):
1383
1483
            raise tests.TestNotApplicable(
1384
 
                "pycurl doesn't redirect silently annymore")
 
1484
                "pycurl doesn't redirect silently anymore")
1385
1485
        super(TestHTTPSilentRedirections, self).setUp()
1386
1486
        install_redirected_request(self)
 
1487
        cleanup_http_redirection_connections(self)
1387
1488
        self.build_tree_contents([('a','a'),
1388
1489
                                  ('1/',),
1389
1490
                                  ('1/a', 'redirected once'),
1397
1498
                                  ('5/a', 'redirected 5 times'),
1398
1499
                                  ],)
1399
1500
 
1400
 
        self.old_transport = self._transport(self.old_server.get_url())
1401
 
 
1402
 
    def create_transport_secondary_server(self):
1403
 
        """Create the secondary server, redirections are defined in the tests"""
1404
 
        return http_utils.HTTPServerRedirecting(
1405
 
            protocol_version=self._protocol_version)
1406
 
 
1407
1501
    def test_one_redirection(self):
1408
 
        t = self.old_transport
1409
 
 
1410
 
        req = RedirectedRequest('GET', t.abspath('a'))
 
1502
        t = self.get_old_transport()
 
1503
        req = RedirectedRequest('GET', t._remote_path('a'))
1411
1504
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1412
1505
                                       self.new_server.port)
1413
1506
        self.old_server.redirections = \
1414
1507
            [('(.*)', r'%s/1\1' % (new_prefix), 301),]
1415
 
        self.assertEqual('redirected once',t._perform(req).read())
 
1508
        self.assertEqual('redirected once', t._perform(req).read())
1416
1509
 
1417
1510
    def test_five_redirections(self):
1418
 
        t = self.old_transport
1419
 
 
1420
 
        req = RedirectedRequest('GET', t.abspath('a'))
 
1511
        t = self.get_old_transport()
 
1512
        req = RedirectedRequest('GET', t._remote_path('a'))
1421
1513
        old_prefix = 'http://%s:%s' % (self.old_server.host,
1422
1514
                                       self.old_server.port)
1423
1515
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1429
1521
            ('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1430
1522
            ('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1431
1523
            ]
1432
 
        self.assertEqual('redirected 5 times',t._perform(req).read())
 
1524
        self.assertEqual('redirected 5 times', t._perform(req).read())
1433
1525
 
1434
1526
 
1435
1527
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1436
1528
    """Test transport.do_catching_redirections."""
1437
1529
 
 
1530
    scenarios = multiply_scenarios(
 
1531
        vary_by_http_client_implementation(),
 
1532
        vary_by_http_protocol_version(),
 
1533
        )
 
1534
 
1438
1535
    def setUp(self):
1439
1536
        super(TestDoCatchRedirections, self).setUp()
1440
1537
        self.build_tree_contents([('a', '0123456789'),],)
1441
 
 
1442
 
        self.old_transport = self._transport(self.old_server.get_url())
1443
 
 
1444
 
    def get_a(self, transport):
1445
 
        return transport.get('a')
 
1538
        cleanup_http_redirection_connections(self)
 
1539
 
 
1540
        self.old_transport = self.get_old_transport()
 
1541
 
 
1542
    def get_a(self, t):
 
1543
        return t.get('a')
1446
1544
 
1447
1545
    def test_no_redirection(self):
1448
 
        t = self._transport(self.new_server.get_url())
 
1546
        t = self.get_new_transport()
1449
1547
 
1450
1548
        # We use None for redirected so that we fail if redirected
1451
1549
        self.assertEqual('0123456789',
1455
1553
    def test_one_redirection(self):
1456
1554
        self.redirections = 0
1457
1555
 
1458
 
        def redirected(transport, exception, redirection_notice):
 
1556
        def redirected(t, exception, redirection_notice):
1459
1557
            self.redirections += 1
1460
 
            dir, file = urlutils.split(exception.target)
1461
 
            return self._transport(dir)
 
1558
            redirected_t = t._redirected_to(exception.source, exception.target)
 
1559
            return redirected_t
1462
1560
 
1463
1561
        self.assertEqual('0123456789',
1464
1562
                         transport.do_catching_redirections(
1478
1576
                          self.get_a, self.old_transport, redirected)
1479
1577
 
1480
1578
 
 
1579
def _setup_authentication_config(**kwargs):
 
1580
    conf = config.AuthenticationConfig()
 
1581
    conf._get_config().update({'httptest': kwargs})
 
1582
    conf._save()
 
1583
 
 
1584
 
 
1585
class TestUrllib2AuthHandler(tests.TestCaseWithTransport):
 
1586
    """Unit tests for glue by which urllib2 asks us for authentication"""
 
1587
 
 
1588
    def test_get_user_password_without_port(self):
 
1589
        """We cope if urllib2 doesn't tell us the port.
 
1590
 
 
1591
        See https://bugs.launchpad.net/bzr/+bug/654684
 
1592
        """
 
1593
        user = 'joe'
 
1594
        password = 'foo'
 
1595
        _setup_authentication_config(scheme='http', host='localhost',
 
1596
                                     user=user, password=password)
 
1597
        handler = _urllib2_wrappers.HTTPAuthHandler()
 
1598
        got_pass = handler.get_user_password(dict(
 
1599
            user='joe',
 
1600
            protocol='http',
 
1601
            host='localhost',
 
1602
            path='/',
 
1603
            realm='Realm',
 
1604
            ))
 
1605
        self.assertEquals((user, password), got_pass)
 
1606
 
 
1607
 
1481
1608
class TestAuth(http_utils.TestCaseWithWebserver):
1482
1609
    """Test authentication scheme"""
1483
1610
 
1484
 
    _auth_header = 'Authorization'
1485
 
    _password_prompt_prefix = ''
1486
 
    _username_prompt_prefix = ''
1487
 
    # Set by load_tests
1488
 
    _auth_server = None
 
1611
    scenarios = multiply_scenarios(
 
1612
        vary_by_http_client_implementation(),
 
1613
        vary_by_http_protocol_version(),
 
1614
        vary_by_http_auth_scheme(),
 
1615
        )
1489
1616
 
1490
1617
    def setUp(self):
1491
1618
        super(TestAuth, self).setUp()
1494
1621
                                  ('b', 'contents of b\n'),])
1495
1622
 
1496
1623
    def create_transport_readonly_server(self):
1497
 
        return self._auth_server(protocol_version=self._protocol_version)
 
1624
        server = self._auth_server(protocol_version=self._protocol_version)
 
1625
        server._url_protocol = self._url_protocol
 
1626
        return server
1498
1627
 
1499
1628
    def _testing_pycurl(self):
1500
1629
        # TODO: This is duplicated for lots of the classes in this file
1513
1642
        return url
1514
1643
 
1515
1644
    def get_user_transport(self, user, password):
1516
 
        return self._transport(self.get_user_url(user, password))
 
1645
        t = transport.get_transport_from_url(
 
1646
            self.get_user_url(user, password))
 
1647
        return t
1517
1648
 
1518
1649
    def test_no_user(self):
1519
1650
        self.server.add_user('joe', 'foo')
1630
1761
        ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1631
1762
                                            stderr=tests.StringIOWrapper())
1632
1763
        # Create a minimal config file with the right password
1633
 
        conf = config.AuthenticationConfig()
1634
 
        conf._get_config().update(
1635
 
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1636
 
                          'user': user, 'password': password}})
1637
 
        conf._save()
 
1764
        _setup_authentication_config(scheme='http', port=self.server.port,
 
1765
                                     user=user, password=password)
1638
1766
        # Issue a request to the server to connect
1639
1767
        self.assertEqual('contents of a\n',t.get('a').read())
1640
1768
        # stdin should have  been left untouched
1642
1770
        # Only one 'Authentication Required' error should occur
1643
1771
        self.assertEqual(1, self.server.auth_required_errors)
1644
1772
 
1645
 
    def test_user_from_auth_conf(self):
1646
 
        if self._testing_pycurl():
1647
 
            raise tests.TestNotApplicable(
1648
 
                'pycurl does not support authentication.conf')
1649
 
        user = 'joe'
1650
 
        password = 'foo'
1651
 
        self.server.add_user(user, password)
1652
 
        # Create a minimal config file with the right password
1653
 
        conf = config.AuthenticationConfig()
1654
 
        conf._get_config().update(
1655
 
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1656
 
                          'user': user, 'password': password}})
1657
 
        conf._save()
1658
 
        t = self.get_user_transport(None, None)
1659
 
        # Issue a request to the server to connect
1660
 
        self.assertEqual('contents of a\n', t.get('a').read())
1661
 
        # Only one 'Authentication Required' error should occur
1662
 
        self.assertEqual(1, self.server.auth_required_errors)
1663
 
 
1664
1773
    def test_changing_nonce(self):
1665
1774
        if self._auth_server not in (http_utils.HTTPDigestAuthServer,
1666
1775
                                     http_utils.ProxyDigestAuthServer):
1667
1776
            raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1668
1777
        if self._testing_pycurl():
1669
 
            raise tests.KnownFailure(
 
1778
            self.knownFailure(
1670
1779
                'pycurl does not handle a nonce change')
1671
1780
        self.server.add_user('joe', 'foo')
1672
1781
        t = self.get_user_transport('joe', 'foo')
1682
1791
        # initial 'who are you' and a second 'who are you' with the new nonce)
1683
1792
        self.assertEqual(2, self.server.auth_required_errors)
1684
1793
 
 
1794
    def test_user_from_auth_conf(self):
 
1795
        if self._testing_pycurl():
 
1796
            raise tests.TestNotApplicable(
 
1797
                'pycurl does not support authentication.conf')
 
1798
        user = 'joe'
 
1799
        password = 'foo'
 
1800
        self.server.add_user(user, password)
 
1801
        _setup_authentication_config(scheme='http', port=self.server.port,
 
1802
                                     user=user, password=password)
 
1803
        t = self.get_user_transport(None, None)
 
1804
        # Issue a request to the server to connect
 
1805
        self.assertEqual('contents of a\n', t.get('a').read())
 
1806
        # Only one 'Authentication Required' error should occur
 
1807
        self.assertEqual(1, self.server.auth_required_errors)
 
1808
 
 
1809
    def test_no_credential_leaks_in_log(self):
 
1810
        self.overrideAttr(debug, 'debug_flags', set(['http']))
 
1811
        user = 'joe'
 
1812
        password = 'very-sensitive-password'
 
1813
        self.server.add_user(user, password)
 
1814
        t = self.get_user_transport(user, password)
 
1815
        # Capture the debug calls to mutter
 
1816
        self.mutters = []
 
1817
        def mutter(*args):
 
1818
            lines = args[0] % args[1:]
 
1819
            # Some calls output multiple lines, just split them now since we
 
1820
            # care about a single one later.
 
1821
            self.mutters.extend(lines.splitlines())
 
1822
        self.overrideAttr(trace, 'mutter', mutter)
 
1823
        # Issue a request to the server to connect
 
1824
        self.assertEqual(True, t.has('a'))
 
1825
        # Only one 'Authentication Required' error should occur
 
1826
        self.assertEqual(1, self.server.auth_required_errors)
 
1827
        # Since the authentification succeeded, there should be a corresponding
 
1828
        # debug line
 
1829
        sent_auth_headers = [line for line in self.mutters
 
1830
                             if line.startswith('> %s' % (self._auth_header,))]
 
1831
        self.assertLength(1, sent_auth_headers)
 
1832
        self.assertStartsWith(sent_auth_headers[0],
 
1833
                              '> %s: <masked>' % (self._auth_header,))
1685
1834
 
1686
1835
 
1687
1836
class TestProxyAuth(TestAuth):
1688
 
    """Test proxy authentication schemes."""
1689
 
 
1690
 
    _auth_header = 'Proxy-authorization'
1691
 
    _password_prompt_prefix = 'Proxy '
1692
 
    _username_prompt_prefix = 'Proxy '
 
1837
    """Test proxy authentication schemes.
 
1838
 
 
1839
    This inherits from TestAuth to tweak the setUp and filter some failing
 
1840
    tests.
 
1841
    """
 
1842
 
 
1843
    scenarios = multiply_scenarios(
 
1844
        vary_by_http_client_implementation(),
 
1845
        vary_by_http_protocol_version(),
 
1846
        vary_by_http_proxy_auth_scheme(),
 
1847
        )
1693
1848
 
1694
1849
    def setUp(self):
1695
1850
        super(TestProxyAuth, self).setUp()
1696
 
        self._old_env = {}
1697
 
        self.addCleanup(self._restore_env)
1698
1851
        # Override the contents to avoid false positives
1699
1852
        self.build_tree_contents([('a', 'not proxied contents of a\n'),
1700
1853
                                  ('b', 'not proxied contents of b\n'),
1703
1856
                                  ])
1704
1857
 
1705
1858
    def get_user_transport(self, user, password):
1706
 
        self._install_env({'all_proxy': self.get_user_url(user, password)})
1707
 
        return self._transport(self.server.get_url())
1708
 
 
1709
 
    def _install_env(self, env):
1710
 
        for name, value in env.iteritems():
1711
 
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1712
 
 
1713
 
    def _restore_env(self):
1714
 
        for name, value in self._old_env.iteritems():
1715
 
            osutils.set_or_unset_env(name, value)
 
1859
        self.overrideEnv('all_proxy', self.get_user_url(user, password))
 
1860
        return TestAuth.get_user_transport(self, user, password)
1716
1861
 
1717
1862
    def test_empty_pass(self):
1718
1863
        if self._testing_pycurl():
1719
1864
            import pycurl
1720
1865
            if pycurl.version_info()[1] < '7.16.0':
1721
 
                raise tests.KnownFailure(
 
1866
                self.knownFailure(
1722
1867
                    'pycurl < 7.16.0 does not handle empty proxy passwords')
1723
1868
        super(TestProxyAuth, self).test_empty_pass()
1724
1869
 
1737
1882
        self.readfile = StringIO(socket_read_content)
1738
1883
        self.writefile = StringIO()
1739
1884
        self.writefile.close = lambda: None
 
1885
        self.close = lambda: None
1740
1886
 
1741
1887
    def makefile(self, mode='r', bufsize=None):
1742
1888
        if 'r' in mode:
1747
1893
 
1748
1894
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1749
1895
 
 
1896
    scenarios = multiply_scenarios(
 
1897
        vary_by_http_client_implementation(),
 
1898
        vary_by_http_protocol_version(),
 
1899
        )
 
1900
 
1750
1901
    def setUp(self):
1751
1902
        super(SmartHTTPTunnellingTest, self).setUp()
1752
1903
        # We use the VFS layer as part of HTTP tunnelling tests.
1753
 
        self._captureVar('BZR_NO_SMART_VFS', None)
 
1904
        self.overrideEnv('BZR_NO_SMART_VFS', None)
1754
1905
        self.transport_readonly_server = http_utils.HTTPServerWithSmarts
 
1906
        self.http_server = self.get_readonly_server()
1755
1907
 
1756
1908
    def create_transport_readonly_server(self):
1757
 
        return http_utils.HTTPServerWithSmarts(
 
1909
        server = http_utils.HTTPServerWithSmarts(
1758
1910
            protocol_version=self._protocol_version)
 
1911
        server._url_protocol = self._url_protocol
 
1912
        return server
1759
1913
 
1760
1914
    def test_open_bzrdir(self):
1761
1915
        branch = self.make_branch('relpath')
1762
 
        http_server = self.get_readonly_server()
1763
 
        url = http_server.get_url() + 'relpath'
 
1916
        url = self.http_server.get_url() + 'relpath'
1764
1917
        bd = bzrdir.BzrDir.open(url)
 
1918
        self.addCleanup(bd.transport.disconnect)
1765
1919
        self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1766
1920
 
1767
1921
    def test_bulk_data(self):
1769
1923
        # The 'readv' command in the smart protocol both sends and receives
1770
1924
        # bulk data, so we use that.
1771
1925
        self.build_tree(['data-file'])
1772
 
        http_server = self.get_readonly_server()
1773
 
        http_transport = self._transport(http_server.get_url())
 
1926
        http_transport = transport.get_transport_from_url(
 
1927
            self.http_server.get_url())
1774
1928
        medium = http_transport.get_smart_medium()
1775
1929
        # Since we provide the medium, the url below will be mostly ignored
1776
1930
        # during the test, as long as the path is '/'.
1784
1938
        post_body = 'hello\n'
1785
1939
        expected_reply_body = 'ok\x012\n'
1786
1940
 
1787
 
        http_server = self.get_readonly_server()
1788
 
        http_transport = self._transport(http_server.get_url())
 
1941
        http_transport = transport.get_transport_from_url(
 
1942
            self.http_server.get_url())
1789
1943
        medium = http_transport.get_smart_medium()
1790
1944
        response = medium.send_http_smart_request(post_body)
1791
1945
        reply_body = response.read()
1792
1946
        self.assertEqual(expected_reply_body, reply_body)
1793
1947
 
1794
1948
    def test_smart_http_server_post_request_handler(self):
1795
 
        httpd = self.get_readonly_server()._get_httpd()
 
1949
        httpd = self.http_server.server
1796
1950
 
1797
1951
        socket = SampleSocket(
1798
1952
            'POST /.bzr/smart %s \r\n' % self._protocol_version
1830
1984
 
1831
1985
    def test_probe_smart_server(self):
1832
1986
        """Test error handling against server refusing smart requests."""
1833
 
        server = self.get_readonly_server()
1834
 
        t = self._transport(server.get_url())
 
1987
        t = self.get_readonly_transport()
1835
1988
        # No need to build a valid smart request here, the server will not even
1836
1989
        # try to interpret it.
1837
1990
        self.assertRaises(errors.SmartProtocolError,
1838
1991
                          t.get_smart_medium().send_http_smart_request,
1839
1992
                          'whatever')
1840
1993
 
 
1994
 
1841
1995
class Test_redirected_to(tests.TestCase):
1842
1996
 
 
1997
    scenarios = vary_by_http_client_implementation()
 
1998
 
1843
1999
    def test_redirected_to_subdir(self):
1844
2000
        t = self._transport('http://www.example.com/foo')
1845
2001
        r = t._redirected_to('http://www.example.com/foo',
1847
2003
        self.assertIsInstance(r, type(t))
1848
2004
        # Both transports share the some connection
1849
2005
        self.assertEqual(t._get_connection(), r._get_connection())
 
2006
        self.assertEquals('http://www.example.com/foo/subdir/', r.base)
1850
2007
 
1851
2008
    def test_redirected_to_self_with_slash(self):
1852
2009
        t = self._transport('http://www.example.com/foo')
1863
2020
        r = t._redirected_to('http://www.example.com/foo',
1864
2021
                             'http://foo.example.com/foo/subdir')
1865
2022
        self.assertIsInstance(r, type(t))
 
2023
        self.assertEquals('http://foo.example.com/foo/subdir/',
 
2024
            r.external_url())
1866
2025
 
1867
2026
    def test_redirected_to_same_host_sibling_protocol(self):
1868
2027
        t = self._transport('http://www.example.com/foo')
1869
2028
        r = t._redirected_to('http://www.example.com/foo',
1870
2029
                             'https://www.example.com/foo')
1871
2030
        self.assertIsInstance(r, type(t))
 
2031
        self.assertEquals('https://www.example.com/foo/',
 
2032
            r.external_url())
1872
2033
 
1873
2034
    def test_redirected_to_same_host_different_protocol(self):
1874
2035
        t = self._transport('http://www.example.com/foo')
1875
2036
        r = t._redirected_to('http://www.example.com/foo',
1876
2037
                             'ftp://www.example.com/foo')
1877
2038
        self.assertNotEquals(type(r), type(t))
 
2039
        self.assertEquals('ftp://www.example.com/foo/', r.external_url())
 
2040
 
 
2041
    def test_redirected_to_same_host_specific_implementation(self):
 
2042
        t = self._transport('http://www.example.com/foo')
 
2043
        r = t._redirected_to('http://www.example.com/foo',
 
2044
                             'https+urllib://www.example.com/foo')
 
2045
        self.assertEquals('https://www.example.com/foo/', r.external_url())
1878
2046
 
1879
2047
    def test_redirected_to_different_host_same_user(self):
1880
2048
        t = self._transport('http://joe@www.example.com/foo')
1881
2049
        r = t._redirected_to('http://www.example.com/foo',
1882
2050
                             'https://foo.example.com/foo')
1883
2051
        self.assertIsInstance(r, type(t))
1884
 
        self.assertEqual(t._user, r._user)
 
2052
        self.assertEqual(t._parsed_url.user, r._parsed_url.user)
 
2053
        self.assertEquals('https://joe@foo.example.com/foo/', r.external_url())
1885
2054
 
1886
2055
 
1887
2056
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
1896
2065
    line.
1897
2066
    """
1898
2067
 
1899
 
    def handle_one_request(self):
 
2068
    def _handle_one_request(self):
1900
2069
        tcs = self.server.test_case_server
1901
2070
        requestline = self.rfile.readline()
1902
2071
        headers = self.MessageClass(self.rfile, 0)
1940
2109
    pass
1941
2110
 
1942
2111
 
1943
 
if tests.HTTPSServerFeature.available():
 
2112
if features.HTTPSServerFeature.available():
1944
2113
    from bzrlib.tests import https_server
1945
2114
    class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
1946
2115
        pass
1957
2126
        tests.TestCase.setUp(self)
1958
2127
        self.server = self._activity_server(self._protocol_version)
1959
2128
        self.server.start_server()
1960
 
        self.activities = {}
 
2129
        self.addCleanup(self.server.stop_server)
 
2130
        _activities = {} # Don't close over self and create a cycle
1961
2131
        def report_activity(t, bytes, direction):
1962
 
            count = self.activities.get(direction, 0)
 
2132
            count = _activities.get(direction, 0)
1963
2133
            count += bytes
1964
 
            self.activities[direction] = count
1965
 
 
 
2134
            _activities[direction] = count
 
2135
        self.activities = _activities
1966
2136
        # We override at class level because constructors may propagate the
1967
2137
        # bound method and render instance overriding ineffective (an
1968
2138
        # alternative would be to define a specific ui factory instead...)
1969
 
        self.orig_report_activity = self._transport._report_activity
1970
 
        self._transport._report_activity = report_activity
1971
 
 
1972
 
    def tearDown(self):
1973
 
        self._transport._report_activity = self.orig_report_activity
1974
 
        self.server.stop_server()
1975
 
        tests.TestCase.tearDown(self)
 
2139
        self.overrideAttr(self._transport, '_report_activity', report_activity)
1976
2140
 
1977
2141
    def get_transport(self):
1978
 
        return self._transport(self.server.get_url())
 
2142
        t = self._transport(self.server.get_url())
 
2143
        # FIXME: Needs cleanup -- vila 20100611
 
2144
        return t
1979
2145
 
1980
2146
    def assertActivitiesMatch(self):
1981
2147
        self.assertEqual(self.server.bytes_read,
2086
2252
'''
2087
2253
        t = self.get_transport()
2088
2254
        # We must send a single line of body bytes, see
2089
 
        # PredefinedRequestHandler.handle_one_request
 
2255
        # PredefinedRequestHandler._handle_one_request
2090
2256
        code, f = t._post('abc def end-of-body\n')
2091
2257
        self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2092
2258
        self.assertActivitiesMatch()
2094
2260
 
2095
2261
class TestActivity(tests.TestCase, TestActivityMixin):
2096
2262
 
 
2263
    scenarios = multiply_scenarios(
 
2264
        vary_by_http_activity(),
 
2265
        vary_by_http_protocol_version(),
 
2266
        )
 
2267
 
2097
2268
    def setUp(self):
2098
 
        tests.TestCase.setUp(self)
2099
 
        self.server = self._activity_server(self._protocol_version)
2100
 
        self.server.start_server()
2101
 
        self.activities = {}
2102
 
        def report_activity(t, bytes, direction):
2103
 
            count = self.activities.get(direction, 0)
2104
 
            count += bytes
2105
 
            self.activities[direction] = count
2106
 
 
2107
 
        # We override at class level because constructors may propagate the
2108
 
        # bound method and render instance overriding ineffective (an
2109
 
        # alternative would be to define a specific ui factory instead...)
2110
 
        self.orig_report_activity = self._transport._report_activity
2111
 
        self._transport._report_activity = report_activity
2112
 
 
2113
 
    def tearDown(self):
2114
 
        self._transport._report_activity = self.orig_report_activity
2115
 
        self.server.stop_server()
2116
 
        tests.TestCase.tearDown(self)
 
2269
        TestActivityMixin.setUp(self)
2117
2270
 
2118
2271
 
2119
2272
class TestNoReportActivity(tests.TestCase, TestActivityMixin):
2120
2273
 
 
2274
    # Unlike TestActivity, we are really testing ReportingFileSocket and
 
2275
    # ReportingSocket, so we don't need all the parametrization. Since
 
2276
    # ReportingFileSocket and ReportingSocket are wrappers, it's easier to
 
2277
    # test them through their use by the transport than directly (that's a
 
2278
    # bit less clean but far more simpler and effective).
 
2279
    _activity_server = ActivityHTTPServer
 
2280
    _protocol_version = 'HTTP/1.1'
 
2281
 
2121
2282
    def setUp(self):
2122
 
        tests.TestCase.setUp(self)
2123
 
        # Unlike TestActivity, we are really testing ReportingFileSocket and
2124
 
        # ReportingSocket, so we don't need all the parametrization. Since
2125
 
        # ReportingFileSocket and ReportingSocket are wrappers, it's easier to
2126
 
        # test them through their use by the transport than directly (that's a
2127
 
        # bit less clean but far more simpler and effective).
2128
 
        self.server = ActivityHTTPServer('HTTP/1.1')
2129
 
        self._transport=_urllib.HttpTransport_urllib
2130
 
 
2131
 
        self.server.start_server()
2132
 
 
2133
 
        # We override at class level because constructors may propagate the
2134
 
        # bound method and render instance overriding ineffective (an
2135
 
        # alternative would be to define a specific ui factory instead...)
2136
 
        self.orig_report_activity = self._transport._report_activity
2137
 
        self._transport._report_activity = None
2138
 
 
2139
 
    def tearDown(self):
2140
 
        self._transport._report_activity = self.orig_report_activity
2141
 
        self.server.stop_server()
2142
 
        tests.TestCase.tearDown(self)
 
2283
        self._transport =_urllib.HttpTransport_urllib
 
2284
        TestActivityMixin.setUp(self)
2143
2285
 
2144
2286
    def assertActivitiesMatch(self):
2145
2287
        # Nothing to check here
2149
2291
class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver):
2150
2292
    """Test authentication on the redirected http server."""
2151
2293
 
 
2294
    scenarios = vary_by_http_protocol_version()
 
2295
 
2152
2296
    _auth_header = 'Authorization'
2153
2297
    _password_prompt_prefix = ''
2154
2298
    _username_prompt_prefix = ''
2155
2299
    _auth_server = http_utils.HTTPBasicAuthServer
2156
2300
    _transport = _urllib.HttpTransport_urllib
2157
2301
 
2158
 
    def create_transport_readonly_server(self):
2159
 
        return self._auth_server()
2160
 
 
2161
 
    def create_transport_secondary_server(self):
2162
 
        """Create the secondary server redirecting to the primary server"""
2163
 
        new = self.get_readonly_server()
2164
 
 
2165
 
        redirecting = http_utils.HTTPServerRedirecting()
2166
 
        redirecting.redirect_to(new.host, new.port)
2167
 
        return redirecting
2168
 
 
2169
2302
    def setUp(self):
2170
2303
        super(TestAuthOnRedirected, self).setUp()
2171
2304
        self.build_tree_contents([('a','a'),
2176
2309
                                       self.new_server.port)
2177
2310
        self.old_server.redirections = [
2178
2311
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2179
 
        self.old_transport = self._transport(self.old_server.get_url())
 
2312
        self.old_transport = self.get_old_transport()
2180
2313
        self.new_server.add_user('joe', 'foo')
2181
 
 
2182
 
    def get_a(self, transport):
2183
 
        return transport.get('a')
 
2314
        cleanup_http_redirection_connections(self)
 
2315
 
 
2316
    def create_transport_readonly_server(self):
 
2317
        server = self._auth_server(protocol_version=self._protocol_version)
 
2318
        server._url_protocol = self._url_protocol
 
2319
        return server
 
2320
 
 
2321
    def get_a(self, t):
 
2322
        return t.get('a')
2184
2323
 
2185
2324
    def test_auth_on_redirected_via_do_catching_redirections(self):
2186
2325
        self.redirections = 0
2187
2326
 
2188
 
        def redirected(transport, exception, redirection_notice):
 
2327
        def redirected(t, exception, redirection_notice):
2189
2328
            self.redirections += 1
2190
 
            dir, file = urlutils.split(exception.target)
2191
 
            return self._transport(dir)
 
2329
            redirected_t = t._redirected_to(exception.source, exception.target)
 
2330
            self.addCleanup(redirected_t.disconnect)
 
2331
            return redirected_t
2192
2332
 
2193
2333
        stdout = tests.StringIOWrapper()
2194
2334
        stderr = tests.StringIOWrapper()
2215
2355
                                       self.new_server.port)
2216
2356
        self.old_server.redirections = [
2217
2357
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2218
 
        self.assertEqual('redirected once',t._perform(req).read())
 
2358
        self.assertEqual('redirected once', t._perform(req).read())
2219
2359
        # stdin should be empty
2220
2360
        self.assertEqual('', ui.ui_factory.stdin.readline())
2221
2361
        # stdout should be empty, stderr will contains the prompts
2222
2362
        self.assertEqual('', stdout.getvalue())
2223
2363
 
2224