~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_http.py

  • Committer: Jelmer Vernooij
  • Date: 2009-06-09 00:59:51 UTC
  • mto: (4443.1.1 bzr.dev)
  • mto: This revision was merged to the branch mainline in revision 4444.
  • Revision ID: jelmer@samba.org-20090609005951-apv900cdk35o2ygh
Move squashing of XML-invalid characters to XMLSerializer.

Show diffs side-by-side

added added

removed removed

Lines of Context:
1
 
# Copyright (C) 2005-2011 Canonical Ltd
 
1
# Copyright (C) 2005, 2006, 2007, 2008, 2009 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
23
23
# TODO: Should be renamed to bzrlib.transport.http.tests?
24
24
# TODO: What about renaming to bzrlib.tests.transport.http ?
25
25
 
 
26
from cStringIO import StringIO
26
27
import httplib
 
28
import os
 
29
import select
27
30
import SimpleHTTPServer
28
31
import socket
29
32
import sys
32
35
import bzrlib
33
36
from bzrlib import (
34
37
    bzrdir,
35
 
    cethread,
36
38
    config,
37
 
    debug,
38
39
    errors,
39
40
    osutils,
40
41
    remote as _mod_remote,
41
42
    tests,
42
 
    trace,
43
43
    transport,
44
44
    ui,
 
45
    urlutils,
 
46
    )
 
47
from bzrlib.symbol_versioning import (
 
48
    deprecated_in,
45
49
    )
46
50
from bzrlib.tests import (
47
 
    features,
48
51
    http_server,
49
52
    http_utils,
50
 
    test_server,
51
 
    )
52
 
from bzrlib.tests.scenarios import (
53
 
    load_tests_apply_scenarios,
54
 
    multiply_scenarios,
55
53
    )
56
54
from bzrlib.transport import (
57
55
    http,
63
61
    )
64
62
 
65
63
 
66
 
if features.pycurl.available():
 
64
try:
67
65
    from bzrlib.transport.http._pycurl import PyCurlTransport
68
 
 
69
 
 
70
 
load_tests = load_tests_apply_scenarios
71
 
 
72
 
 
73
 
def vary_by_http_client_implementation():
74
 
    """Test the two libraries we can use, pycurl and urllib."""
 
66
    pycurl_present = True
 
67
except errors.DependencyNotPresent:
 
68
    pycurl_present = False
 
69
 
 
70
 
 
71
def load_tests(standard_tests, module, loader):
 
72
    """Multiply tests for http clients and protocol versions."""
 
73
    result = loader.suiteClass()
 
74
 
 
75
    # one for each transport implementation
 
76
    t_tests, remaining_tests = tests.split_suite_by_condition(
 
77
        standard_tests, tests.condition_isinstance((
 
78
                TestHttpTransportRegistration,
 
79
                TestHttpTransportUrls,
 
80
                Test_redirected_to,
 
81
                )))
75
82
    transport_scenarios = [
76
83
        ('urllib', dict(_transport=_urllib.HttpTransport_urllib,
77
84
                        _server=http_server.HttpServer_urllib,
78
 
                        _url_protocol='http+urllib',)),
 
85
                        _qualified_prefix='http+urllib',)),
79
86
        ]
80
 
    if features.pycurl.available():
 
87
    if pycurl_present:
81
88
        transport_scenarios.append(
82
89
            ('pycurl', dict(_transport=PyCurlTransport,
83
90
                            _server=http_server.HttpServer_PyCurl,
84
 
                            _url_protocol='http+pycurl',)))
85
 
    return transport_scenarios
86
 
 
87
 
 
88
 
def vary_by_http_protocol_version():
89
 
    """Test on http/1.0 and 1.1"""
90
 
    return [
91
 
        ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
92
 
        ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
 
91
                            _qualified_prefix='http+pycurl',)))
 
92
    tests.multiply_tests(t_tests, transport_scenarios, result)
 
93
 
 
94
    # each implementation tested with each HTTP version
 
95
    tp_tests, remaining_tests = tests.split_suite_by_condition(
 
96
        remaining_tests, tests.condition_isinstance((
 
97
                SmartHTTPTunnellingTest,
 
98
                TestDoCatchRedirections,
 
99
                TestHTTPConnections,
 
100
                TestHTTPRedirections,
 
101
                TestHTTPSilentRedirections,
 
102
                TestLimitedRangeRequestServer,
 
103
                TestPost,
 
104
                TestProxyHttpServer,
 
105
                TestRanges,
 
106
                TestSpecificRequestHandler,
 
107
                )))
 
108
    protocol_scenarios = [
 
109
            ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
 
110
            ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
 
111
            ]
 
112
    tp_scenarios = tests.multiply_scenarios(transport_scenarios,
 
113
                                            protocol_scenarios)
 
114
    tests.multiply_tests(tp_tests, tp_scenarios, result)
 
115
 
 
116
    # proxy auth: each auth scheme on all http versions on all implementations.
 
117
    tppa_tests, remaining_tests = tests.split_suite_by_condition(
 
118
        remaining_tests, tests.condition_isinstance((
 
119
                TestProxyAuth,
 
120
                )))
 
121
    proxy_auth_scheme_scenarios = [
 
122
        ('basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
 
123
        ('digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
 
124
        ('basicdigest',
 
125
         dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
93
126
        ]
94
 
 
95
 
 
96
 
def vary_by_http_auth_scheme():
97
 
    scenarios = [
 
127
    tppa_scenarios = tests.multiply_scenarios(tp_scenarios,
 
128
                                              proxy_auth_scheme_scenarios)
 
129
    tests.multiply_tests(tppa_tests, tppa_scenarios, result)
 
130
 
 
131
    # auth: each auth scheme on all http versions on all implementations.
 
132
    tpa_tests, remaining_tests = tests.split_suite_by_condition(
 
133
        remaining_tests, tests.condition_isinstance((
 
134
                TestAuth,
 
135
                )))
 
136
    auth_scheme_scenarios = [
98
137
        ('basic', dict(_auth_server=http_utils.HTTPBasicAuthServer)),
99
138
        ('digest', dict(_auth_server=http_utils.HTTPDigestAuthServer)),
100
139
        ('basicdigest',
101
 
            dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
102
 
        ]
103
 
    # Add some attributes common to all scenarios
104
 
    for scenario_id, scenario_dict in scenarios:
105
 
        scenario_dict.update(_auth_header='Authorization',
106
 
                             _username_prompt_prefix='',
107
 
                             _password_prompt_prefix='')
108
 
    return scenarios
109
 
 
110
 
 
111
 
def vary_by_http_proxy_auth_scheme():
112
 
    scenarios = [
113
 
        ('proxy-basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
114
 
        ('proxy-digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
115
 
        ('proxy-basicdigest',
116
 
            dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
117
 
        ]
118
 
    # Add some attributes common to all scenarios
119
 
    for scenario_id, scenario_dict in scenarios:
120
 
        scenario_dict.update(_auth_header='Proxy-Authorization',
121
 
                             _username_prompt_prefix='Proxy ',
122
 
                             _password_prompt_prefix='Proxy ')
123
 
    return scenarios
124
 
 
125
 
 
126
 
def vary_by_http_activity():
 
140
         dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
 
141
        ]
 
142
    tpa_scenarios = tests.multiply_scenarios(tp_scenarios,
 
143
                                             auth_scheme_scenarios)
 
144
    tests.multiply_tests(tpa_tests, tpa_scenarios, result)
 
145
 
 
146
    # activity: on all http[s] versions on all implementations
 
147
    tpact_tests, remaining_tests = tests.split_suite_by_condition(
 
148
        remaining_tests, tests.condition_isinstance((
 
149
                TestActivity,
 
150
                )))
127
151
    activity_scenarios = [
128
152
        ('urllib,http', dict(_activity_server=ActivityHTTPServer,
129
 
                            _transport=_urllib.HttpTransport_urllib,)),
 
153
                             _transport=_urllib.HttpTransport_urllib,)),
130
154
        ]
131
 
    if features.HTTPSServerFeature.available():
 
155
    if tests.HTTPSServerFeature.available():
132
156
        activity_scenarios.append(
133
157
            ('urllib,https', dict(_activity_server=ActivityHTTPSServer,
134
 
                                _transport=_urllib.HttpTransport_urllib,)),)
135
 
    if features.pycurl.available():
 
158
                                  _transport=_urllib.HttpTransport_urllib,)),)
 
159
    if pycurl_present:
136
160
        activity_scenarios.append(
137
161
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
138
 
                                _transport=PyCurlTransport,)),)
139
 
        if features.HTTPSServerFeature.available():
 
162
                                 _transport=PyCurlTransport,)),)
 
163
        if tests.HTTPSServerFeature.available():
140
164
            from bzrlib.tests import (
141
165
                ssl_certs,
142
166
                )
153
177
 
154
178
            activity_scenarios.append(
155
179
                ('pycurl,https', dict(_activity_server=ActivityHTTPSServer,
156
 
                                    _transport=HTTPS_pycurl_transport,)),)
157
 
    return activity_scenarios
 
180
                                      _transport=HTTPS_pycurl_transport,)),)
 
181
 
 
182
    tpact_scenarios = tests.multiply_scenarios(activity_scenarios,
 
183
                                               protocol_scenarios)
 
184
    tests.multiply_tests(tpact_tests, tpact_scenarios, result)
 
185
 
 
186
    # No parametrization for the remaining tests
 
187
    result.addTests(remaining_tests)
 
188
 
 
189
    return result
158
190
 
159
191
 
160
192
class FakeManager(object):
172
204
    It records the bytes sent to it, and replies with a 200.
173
205
    """
174
206
 
175
 
    def __init__(self, expect_body_tail=None, scheme=''):
 
207
    def __init__(self, expect_body_tail=None):
176
208
        """Constructor.
177
209
 
178
210
        :type expect_body_tail: str
183
215
        self.host = None
184
216
        self.port = None
185
217
        self.received_bytes = ''
186
 
        self.scheme = scheme
187
 
 
188
 
    def get_url(self):
189
 
        return '%s://%s:%s/' % (self.scheme, self.host, self.port)
190
 
 
191
 
    def start_server(self):
 
218
 
 
219
    def setUp(self):
192
220
        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
193
221
        self._sock.bind(('127.0.0.1', 0))
194
222
        self.host, self.port = self._sock.getsockname()
195
223
        self._ready = threading.Event()
196
 
        self._thread = test_server.TestThread(
197
 
            sync_event=self._ready, target=self._accept_read_and_reply)
 
224
        self._thread = threading.Thread(target=self._accept_read_and_reply)
 
225
        self._thread.setDaemon(True)
198
226
        self._thread.start()
199
 
        if 'threads' in tests.selftest_debug_flags:
200
 
            sys.stderr.write('Thread started: %s\n' % (self._thread.ident,))
201
 
        self._ready.wait()
 
227
        self._ready.wait(5)
202
228
 
203
229
    def _accept_read_and_reply(self):
204
230
        self._sock.listen(1)
205
231
        self._ready.set()
206
 
        conn, address = self._sock.accept()
207
 
        if self._expect_body_tail is not None:
 
232
        self._sock.settimeout(5)
 
233
        try:
 
234
            conn, address = self._sock.accept()
 
235
            # On win32, the accepted connection will be non-blocking to start
 
236
            # with because we're using settimeout.
 
237
            conn.setblocking(True)
208
238
            while not self.received_bytes.endswith(self._expect_body_tail):
209
239
                self.received_bytes += conn.recv(4096)
210
240
            conn.sendall('HTTP/1.1 200 OK\r\n')
211
 
        try:
 
241
        except socket.timeout:
 
242
            # Make sure the client isn't stuck waiting for us to e.g. accept.
212
243
            self._sock.close()
213
244
        except socket.error:
214
245
            # The client may have already closed the socket.
215
246
            pass
216
247
 
217
 
    def stop_server(self):
 
248
    def tearDown(self):
218
249
        try:
219
 
            # Issue a fake connection to wake up the server and allow it to
220
 
            # finish quickly
221
 
            fake_conn = osutils.connect_socket((self.host, self.port))
222
 
            fake_conn.close()
 
250
            self._sock.close()
223
251
        except socket.error:
224
252
            # We might have already closed it.  We don't care.
225
253
            pass
226
254
        self.host = None
227
255
        self.port = None
228
 
        self._thread.join()
229
 
        if 'threads' in tests.selftest_debug_flags:
230
 
            sys.stderr.write('Thread  joined: %s\n' % (self._thread.ident,))
231
256
 
232
257
 
233
258
class TestAuthHeader(tests.TestCase):
240
265
 
241
266
    def test_empty_header(self):
242
267
        scheme, remainder = self.parse_header('')
243
 
        self.assertEqual('', scheme)
 
268
        self.assertEquals('', scheme)
244
269
        self.assertIs(None, remainder)
245
270
 
246
271
    def test_negotiate_header(self):
247
272
        scheme, remainder = self.parse_header('Negotiate')
248
 
        self.assertEqual('negotiate', scheme)
 
273
        self.assertEquals('negotiate', scheme)
249
274
        self.assertIs(None, remainder)
250
275
 
251
276
    def test_basic_header(self):
252
277
        scheme, remainder = self.parse_header(
253
278
            'Basic realm="Thou should not pass"')
254
 
        self.assertEqual('basic', scheme)
255
 
        self.assertEqual('realm="Thou should not pass"', remainder)
 
279
        self.assertEquals('basic', scheme)
 
280
        self.assertEquals('realm="Thou should not pass"', remainder)
256
281
 
257
282
    def test_basic_extract_realm(self):
258
283
        scheme, remainder = self.parse_header(
260
285
            _urllib2_wrappers.BasicAuthHandler)
261
286
        match, realm = self.auth_handler.extract_realm(remainder)
262
287
        self.assertTrue(match is not None)
263
 
        self.assertEqual('Thou should not pass', realm)
 
288
        self.assertEquals('Thou should not pass', realm)
264
289
 
265
290
    def test_digest_header(self):
266
291
        scheme, remainder = self.parse_header(
267
292
            'Digest realm="Thou should not pass"')
268
 
        self.assertEqual('digest', scheme)
269
 
        self.assertEqual('realm="Thou should not pass"', remainder)
270
 
 
271
 
 
272
 
class TestHTTPRangeParsing(tests.TestCase):
273
 
 
274
 
    def setUp(self):
275
 
        super(TestHTTPRangeParsing, self).setUp()
276
 
        # We focus on range  parsing here and ignore everything else
277
 
        class RequestHandler(http_server.TestingHTTPRequestHandler):
278
 
            def setup(self): pass
279
 
            def handle(self): pass
280
 
            def finish(self): pass
281
 
 
282
 
        self.req_handler = RequestHandler(None, None, None)
283
 
 
284
 
    def assertRanges(self, ranges, header, file_size):
285
 
        self.assertEquals(ranges,
286
 
                          self.req_handler._parse_ranges(header, file_size))
287
 
 
288
 
    def test_simple_range(self):
289
 
        self.assertRanges([(0,2)], 'bytes=0-2', 12)
290
 
 
291
 
    def test_tail(self):
292
 
        self.assertRanges([(8, 11)], 'bytes=-4', 12)
293
 
 
294
 
    def test_tail_bigger_than_file(self):
295
 
        self.assertRanges([(0, 11)], 'bytes=-99', 12)
296
 
 
297
 
    def test_range_without_end(self):
298
 
        self.assertRanges([(4, 11)], 'bytes=4-', 12)
299
 
 
300
 
    def test_invalid_ranges(self):
301
 
        self.assertRanges(None, 'bytes=12-22', 12)
302
 
        self.assertRanges(None, 'bytes=1-3,12-22', 12)
303
 
        self.assertRanges(None, 'bytes=-', 12)
 
293
        self.assertEquals('digest', scheme)
 
294
        self.assertEquals('realm="Thou should not pass"', remainder)
304
295
 
305
296
 
306
297
class TestHTTPServer(tests.TestCase):
311
302
 
312
303
            protocol_version = 'HTTP/0.1'
313
304
 
314
 
        self.assertRaises(httplib.UnknownProtocol,
315
 
                          http_server.HttpServer, BogusRequestHandler)
 
305
        server = http_server.HttpServer(BogusRequestHandler)
 
306
        try:
 
307
            self.assertRaises(httplib.UnknownProtocol,server.setUp)
 
308
        except:
 
309
            server.tearDown()
 
310
            self.fail('HTTP Server creation did not raise UnknownProtocol')
316
311
 
317
312
    def test_force_invalid_protocol(self):
318
 
        self.assertRaises(httplib.UnknownProtocol,
319
 
                          http_server.HttpServer, protocol_version='HTTP/0.1')
 
313
        server = http_server.HttpServer(protocol_version='HTTP/0.1')
 
314
        try:
 
315
            self.assertRaises(httplib.UnknownProtocol,server.setUp)
 
316
        except:
 
317
            server.tearDown()
 
318
            self.fail('HTTP Server creation did not raise UnknownProtocol')
320
319
 
321
320
    def test_server_start_and_stop(self):
322
321
        server = http_server.HttpServer()
323
 
        self.addCleanup(server.stop_server)
324
 
        server.start_server()
325
 
        self.assertTrue(server.server is not None)
326
 
        self.assertTrue(server.server.serving is not None)
327
 
        self.assertTrue(server.server.serving)
 
322
        server.setUp()
 
323
        self.assertTrue(server._http_running)
 
324
        server.tearDown()
 
325
        self.assertFalse(server._http_running)
328
326
 
329
327
    def test_create_http_server_one_zero(self):
330
328
        class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
332
330
            protocol_version = 'HTTP/1.0'
333
331
 
334
332
        server = http_server.HttpServer(RequestHandlerOneZero)
335
 
        self.start_server(server)
336
 
        self.assertIsInstance(server.server, http_server.TestingHTTPServer)
 
333
        server.setUp()
 
334
        self.addCleanup(server.tearDown)
 
335
        self.assertIsInstance(server._httpd, http_server.TestingHTTPServer)
337
336
 
338
337
    def test_create_http_server_one_one(self):
339
338
        class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
341
340
            protocol_version = 'HTTP/1.1'
342
341
 
343
342
        server = http_server.HttpServer(RequestHandlerOneOne)
344
 
        self.start_server(server)
345
 
        self.assertIsInstance(server.server,
 
343
        server.setUp()
 
344
        self.addCleanup(server.tearDown)
 
345
        self.assertIsInstance(server._httpd,
346
346
                              http_server.TestingThreadingHTTPServer)
347
347
 
348
348
    def test_create_http_server_force_one_one(self):
352
352
 
353
353
        server = http_server.HttpServer(RequestHandlerOneZero,
354
354
                                        protocol_version='HTTP/1.1')
355
 
        self.start_server(server)
356
 
        self.assertIsInstance(server.server,
 
355
        server.setUp()
 
356
        self.addCleanup(server.tearDown)
 
357
        self.assertIsInstance(server._httpd,
357
358
                              http_server.TestingThreadingHTTPServer)
358
359
 
359
360
    def test_create_http_server_force_one_zero(self):
363
364
 
364
365
        server = http_server.HttpServer(RequestHandlerOneOne,
365
366
                                        protocol_version='HTTP/1.0')
366
 
        self.start_server(server)
367
 
        self.assertIsInstance(server.server,
 
367
        server.setUp()
 
368
        self.addCleanup(server.tearDown)
 
369
        self.assertIsInstance(server._httpd,
368
370
                              http_server.TestingHTTPServer)
369
371
 
370
372
 
372
374
    """Test case to inherit from if pycurl is present"""
373
375
 
374
376
    def _get_pycurl_maybe(self):
375
 
        self.requireFeature(features.pycurl)
376
 
        return PyCurlTransport
 
377
        try:
 
378
            from bzrlib.transport.http._pycurl import PyCurlTransport
 
379
            return PyCurlTransport
 
380
        except errors.DependencyNotPresent:
 
381
            raise tests.TestSkipped('pycurl not present')
377
382
 
378
383
    _transport = property(_get_pycurl_maybe)
379
384
 
386
391
    def test_url_parsing(self):
387
392
        f = FakeManager()
388
393
        url = http.extract_auth('http://example.com', f)
389
 
        self.assertEqual('http://example.com', url)
390
 
        self.assertEqual(0, len(f.credentials))
 
394
        self.assertEquals('http://example.com', url)
 
395
        self.assertEquals(0, len(f.credentials))
391
396
        url = http.extract_auth(
392
 
            'http://user:pass@example.com/bzr/bzr.dev', f)
393
 
        self.assertEqual('http://example.com/bzr/bzr.dev', url)
394
 
        self.assertEqual(1, len(f.credentials))
395
 
        self.assertEqual([None, 'example.com', 'user', 'pass'],
396
 
                         f.credentials[0])
 
397
            'http://user:pass@www.bazaar-vcs.org/bzr/bzr.dev', f)
 
398
        self.assertEquals('http://www.bazaar-vcs.org/bzr/bzr.dev', url)
 
399
        self.assertEquals(1, len(f.credentials))
 
400
        self.assertEquals([None, 'www.bazaar-vcs.org', 'user', 'pass'],
 
401
                          f.credentials[0])
397
402
 
398
403
 
399
404
class TestHttpTransportUrls(tests.TestCase):
400
405
    """Test the http urls."""
401
406
 
402
 
    scenarios = vary_by_http_client_implementation()
403
 
 
404
407
    def test_abs_url(self):
405
408
        """Construction of absolute http URLs"""
406
 
        t = self._transport('http://example.com/bzr/bzr.dev/')
 
409
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
407
410
        eq = self.assertEqualDiff
408
 
        eq(t.abspath('.'), 'http://example.com/bzr/bzr.dev')
409
 
        eq(t.abspath('foo/bar'), 'http://example.com/bzr/bzr.dev/foo/bar')
410
 
        eq(t.abspath('.bzr'), 'http://example.com/bzr/bzr.dev/.bzr')
 
411
        eq(t.abspath('.'), 'http://bazaar-vcs.org/bzr/bzr.dev')
 
412
        eq(t.abspath('foo/bar'), 'http://bazaar-vcs.org/bzr/bzr.dev/foo/bar')
 
413
        eq(t.abspath('.bzr'), 'http://bazaar-vcs.org/bzr/bzr.dev/.bzr')
411
414
        eq(t.abspath('.bzr/1//2/./3'),
412
 
           'http://example.com/bzr/bzr.dev/.bzr/1/2/3')
 
415
           'http://bazaar-vcs.org/bzr/bzr.dev/.bzr/1/2/3')
413
416
 
414
417
    def test_invalid_http_urls(self):
415
418
        """Trap invalid construction of urls"""
416
 
        self._transport('http://example.com/bzr/bzr.dev/')
 
419
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
417
420
        self.assertRaises(errors.InvalidURL,
418
421
                          self._transport,
419
 
                          'http://http://example.com/bzr/bzr.dev/')
 
422
                          'http://http://bazaar-vcs.org/bzr/bzr.dev/')
420
423
 
421
424
    def test_http_root_urls(self):
422
425
        """Construction of URLs from server root"""
423
 
        t = self._transport('http://example.com/')
 
426
        t = self._transport('http://bzr.ozlabs.org/')
424
427
        eq = self.assertEqualDiff
425
428
        eq(t.abspath('.bzr/tree-version'),
426
 
           'http://example.com/.bzr/tree-version')
 
429
           'http://bzr.ozlabs.org/.bzr/tree-version')
427
430
 
428
431
    def test_http_impl_urls(self):
429
432
        """There are servers which ask for particular clients to connect"""
430
433
        server = self._server()
431
 
        server.start_server()
432
434
        try:
 
435
            server.setUp()
433
436
            url = server.get_url()
434
 
            self.assertTrue(url.startswith('%s://' % self._url_protocol))
 
437
            self.assertTrue(url.startswith('%s://' % self._qualified_prefix))
435
438
        finally:
436
 
            server.stop_server()
 
439
            server.tearDown()
437
440
 
438
441
 
439
442
class TestHttps_pycurl(TestWithTransport_pycurl, tests.TestCase):
448
451
        https by supplying a fake version_info that do not
449
452
        support it.
450
453
        """
451
 
        self.requireFeature(features.pycurl)
452
 
        # Import the module locally now that we now it's available.
453
 
        pycurl = features.pycurl.module
 
454
        try:
 
455
            import pycurl
 
456
        except ImportError:
 
457
            raise tests.TestSkipped('pycurl not present')
454
458
 
455
 
        self.overrideAttr(pycurl, 'version_info',
456
 
                          # Fake the pycurl version_info This was taken from
457
 
                          # a windows pycurl without SSL (thanks to bialix)
458
 
                          lambda : (2,
459
 
                                    '7.13.2',
460
 
                                    462082,
461
 
                                    'i386-pc-win32',
462
 
                                    2576,
463
 
                                    None,
464
 
                                    0,
465
 
                                    None,
466
 
                                    ('ftp', 'gopher', 'telnet',
467
 
                                     'dict', 'ldap', 'http', 'file'),
468
 
                                    None,
469
 
                                    0,
470
 
                                    None))
471
 
        self.assertRaises(errors.DependencyNotPresent, self._transport,
472
 
                          'https://launchpad.net')
 
459
        version_info_orig = pycurl.version_info
 
460
        try:
 
461
            # Now that we have pycurl imported, we can fake its version_info
 
462
            # This was taken from a windows pycurl without SSL
 
463
            # (thanks to bialix)
 
464
            pycurl.version_info = lambda : (2,
 
465
                                            '7.13.2',
 
466
                                            462082,
 
467
                                            'i386-pc-win32',
 
468
                                            2576,
 
469
                                            None,
 
470
                                            0,
 
471
                                            None,
 
472
                                            ('ftp', 'gopher', 'telnet',
 
473
                                             'dict', 'ldap', 'http', 'file'),
 
474
                                            None,
 
475
                                            0,
 
476
                                            None)
 
477
            self.assertRaises(errors.DependencyNotPresent, self._transport,
 
478
                              'https://launchpad.net')
 
479
        finally:
 
480
            # Restore the right function
 
481
            pycurl.version_info = version_info_orig
473
482
 
474
483
 
475
484
class TestHTTPConnections(http_utils.TestCaseWithWebserver):
476
485
    """Test the http connections."""
477
486
 
478
 
    scenarios = multiply_scenarios(
479
 
        vary_by_http_client_implementation(),
480
 
        vary_by_http_protocol_version(),
481
 
        )
482
 
 
483
487
    def setUp(self):
484
488
        http_utils.TestCaseWithWebserver.setUp(self)
485
489
        self.build_tree(['foo/', 'foo/bar'], line_endings='binary',
487
491
 
488
492
    def test_http_has(self):
489
493
        server = self.get_readonly_server()
490
 
        t = self.get_readonly_transport()
 
494
        t = self._transport(server.get_url())
491
495
        self.assertEqual(t.has('foo/bar'), True)
492
496
        self.assertEqual(len(server.logs), 1)
493
497
        self.assertContainsRe(server.logs[0],
495
499
 
496
500
    def test_http_has_not_found(self):
497
501
        server = self.get_readonly_server()
498
 
        t = self.get_readonly_transport()
 
502
        t = self._transport(server.get_url())
499
503
        self.assertEqual(t.has('not-found'), False)
500
504
        self.assertContainsRe(server.logs[1],
501
505
            r'"HEAD /not-found HTTP/1.." 404 - "-" "bzr/')
502
506
 
503
507
    def test_http_get(self):
504
508
        server = self.get_readonly_server()
505
 
        t = self.get_readonly_transport()
 
509
        t = self._transport(server.get_url())
506
510
        fp = t.get('foo/bar')
507
511
        self.assertEqualDiff(
508
512
            fp.read(),
530
534
class TestHttpTransportRegistration(tests.TestCase):
531
535
    """Test registrations of various http implementations"""
532
536
 
533
 
    scenarios = vary_by_http_client_implementation()
534
 
 
535
537
    def test_http_registered(self):
536
 
        t = transport.get_transport('%s://foo.com/' % self._url_protocol)
 
538
        t = transport.get_transport('%s://foo.com/' % self._qualified_prefix)
537
539
        self.assertIsInstance(t, transport.Transport)
538
540
        self.assertIsInstance(t, self._transport)
539
541
 
540
542
 
541
543
class TestPost(tests.TestCase):
542
544
 
543
 
    scenarios = multiply_scenarios(
544
 
        vary_by_http_client_implementation(),
545
 
        vary_by_http_protocol_version(),
546
 
        )
547
 
 
548
545
    def test_post_body_is_received(self):
549
 
        server = RecordingServer(expect_body_tail='end-of-body',
550
 
                                 scheme=self._url_protocol)
551
 
        self.start_server(server)
552
 
        url = server.get_url()
553
 
        # FIXME: needs a cleanup -- vila 20100611
554
 
        http_transport = transport.get_transport(url)
 
546
        server = RecordingServer(expect_body_tail='end-of-body')
 
547
        server.setUp()
 
548
        self.addCleanup(server.tearDown)
 
549
        scheme = self._qualified_prefix
 
550
        url = '%s://%s:%s/' % (scheme, server.host, server.port)
 
551
        http_transport = self._transport(url)
555
552
        code, response = http_transport._post('abc def end-of-body')
556
553
        self.assertTrue(
557
554
            server.received_bytes.startswith('POST /.bzr/smart HTTP/1.'))
558
555
        self.assertTrue('content-length: 19\r' in server.received_bytes.lower())
559
 
        self.assertTrue('content-type: application/octet-stream\r'
560
 
                        in server.received_bytes.lower())
561
556
        # The transport should not be assuming that the server can accept
562
557
        # chunked encoding the first time it connects, because HTTP/1.1, so we
563
558
        # check for the literal string.
599
594
    Daughter classes are expected to override _req_handler_class
600
595
    """
601
596
 
602
 
    scenarios = multiply_scenarios(
603
 
        vary_by_http_client_implementation(),
604
 
        vary_by_http_protocol_version(),
605
 
        )
606
 
 
607
597
    # Provide a useful default
608
598
    _req_handler_class = http_server.TestingHTTPRequestHandler
609
599
 
610
600
    def create_transport_readonly_server(self):
611
 
        server = http_server.HttpServer(self._req_handler_class,
612
 
                                        protocol_version=self._protocol_version)
613
 
        server._url_protocol = self._url_protocol
614
 
        return server
 
601
        return http_server.HttpServer(self._req_handler_class,
 
602
                                      protocol_version=self._protocol_version)
615
603
 
616
604
    def _testing_pycurl(self):
617
 
        # TODO: This is duplicated for lots of the classes in this file
618
 
        return (features.pycurl.available()
619
 
                and self._transport == PyCurlTransport)
 
605
        return pycurl_present and self._transport == PyCurlTransport
620
606
 
621
607
 
622
608
class WallRequestHandler(http_server.TestingHTTPRequestHandler):
623
609
    """Whatever request comes in, close the connection"""
624
610
 
625
 
    def _handle_one_request(self):
 
611
    def handle_one_request(self):
626
612
        """Handle a single HTTP request, by abruptly closing the connection"""
627
613
        self.close_connection = 1
628
614
 
633
619
    _req_handler_class = WallRequestHandler
634
620
 
635
621
    def test_http_has(self):
636
 
        t = self.get_readonly_transport()
 
622
        server = self.get_readonly_server()
 
623
        t = self._transport(server.get_url())
637
624
        # Unfortunately httplib (see HTTPResponse._read_status
638
625
        # for details) make no distinction between a closed
639
626
        # socket and badly formatted status line, so we can't
640
627
        # just test for ConnectionError, we have to test
641
 
        # InvalidHttpResponse too. And pycurl may raise ConnectionReset
642
 
        # instead of ConnectionError too.
643
 
        self.assertRaises(( errors.ConnectionError, errors.ConnectionReset,
644
 
                            errors.InvalidHttpResponse),
 
628
        # InvalidHttpResponse too.
 
629
        self.assertRaises((errors.ConnectionError, errors.InvalidHttpResponse),
645
630
                          t.has, 'foo/bar')
646
631
 
647
632
    def test_http_get(self):
648
 
        t = self.get_readonly_transport()
649
 
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
650
 
                           errors.InvalidHttpResponse),
 
633
        server = self.get_readonly_server()
 
634
        t = self._transport(server.get_url())
 
635
        self.assertRaises((errors.ConnectionError, errors.InvalidHttpResponse),
651
636
                          t.get, 'foo/bar')
652
637
 
653
638
 
668
653
    _req_handler_class = BadStatusRequestHandler
669
654
 
670
655
    def test_http_has(self):
671
 
        t = self.get_readonly_transport()
 
656
        server = self.get_readonly_server()
 
657
        t = self._transport(server.get_url())
672
658
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
673
659
 
674
660
    def test_http_get(self):
675
 
        t = self.get_readonly_transport()
 
661
        server = self.get_readonly_server()
 
662
        t = self._transport(server.get_url())
676
663
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
677
664
 
678
665
 
683
670
        """Fakes handling a single HTTP request, returns a bad status"""
684
671
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
685
672
        self.wfile.write("Invalid status line\r\n")
686
 
        # If we don't close the connection pycurl will hang. Since this is a
687
 
        # stress test we don't *have* to respect the protocol, but we don't
688
 
        # have to sabotage it too much either.
689
 
        self.close_connection = True
690
673
        return False
691
674
 
692
675
 
698
681
 
699
682
    _req_handler_class = InvalidStatusRequestHandler
700
683
 
 
684
    def test_http_has(self):
 
685
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
 
686
            raise tests.KnownFailure(
 
687
                'pycurl hangs if the server send back garbage')
 
688
        super(TestInvalidStatusServer, self).test_http_has()
 
689
 
 
690
    def test_http_get(self):
 
691
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
 
692
            raise tests.KnownFailure(
 
693
                'pycurl hangs if the server send back garbage')
 
694
        super(TestInvalidStatusServer, self).test_http_get()
 
695
 
701
696
 
702
697
class BadProtocolRequestHandler(http_server.TestingHTTPRequestHandler):
703
698
    """Whatever request comes in, returns a bad protocol version"""
719
714
    _req_handler_class = BadProtocolRequestHandler
720
715
 
721
716
    def setUp(self):
722
 
        if self._testing_pycurl():
 
717
        if pycurl_present and self._transport == PyCurlTransport:
723
718
            raise tests.TestNotApplicable(
724
719
                "pycurl doesn't check the protocol version")
725
720
        super(TestBadProtocolServer, self).setUp()
726
721
 
727
722
    def test_http_has(self):
728
 
        t = self.get_readonly_transport()
 
723
        server = self.get_readonly_server()
 
724
        t = self._transport(server.get_url())
729
725
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
730
726
 
731
727
    def test_http_get(self):
732
 
        t = self.get_readonly_transport()
 
728
        server = self.get_readonly_server()
 
729
        t = self._transport(server.get_url())
733
730
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
734
731
 
735
732
 
749
746
    _req_handler_class = ForbiddenRequestHandler
750
747
 
751
748
    def test_http_has(self):
752
 
        t = self.get_readonly_transport()
 
749
        server = self.get_readonly_server()
 
750
        t = self._transport(server.get_url())
753
751
        self.assertRaises(errors.TransportError, t.has, 'foo/bar')
754
752
 
755
753
    def test_http_get(self):
756
 
        t = self.get_readonly_transport()
 
754
        server = self.get_readonly_server()
 
755
        t = self._transport(server.get_url())
757
756
        self.assertRaises(errors.TransportError, t.get, 'foo/bar')
758
757
 
759
758
 
765
764
        self.assertEqual(None, server.host)
766
765
        self.assertEqual(None, server.port)
767
766
 
768
 
    def test_setUp_and_stop(self):
 
767
    def test_setUp_and_tearDown(self):
769
768
        server = RecordingServer(expect_body_tail=None)
770
 
        server.start_server()
 
769
        server.setUp()
771
770
        try:
772
771
            self.assertNotEqual(None, server.host)
773
772
            self.assertNotEqual(None, server.port)
774
773
        finally:
775
 
            server.stop_server()
 
774
            server.tearDown()
776
775
        self.assertEqual(None, server.host)
777
776
        self.assertEqual(None, server.port)
778
777
 
779
778
    def test_send_receive_bytes(self):
780
 
        server = RecordingServer(expect_body_tail='c', scheme='http')
781
 
        self.start_server(server)
 
779
        server = RecordingServer(expect_body_tail='c')
 
780
        server.setUp()
 
781
        self.addCleanup(server.tearDown)
782
782
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
783
783
        sock.connect((server.host, server.port))
784
784
        sock.sendall('abc')
798
798
        self.build_tree_contents([('a', '0123456789')],)
799
799
 
800
800
    def test_readv(self):
801
 
        t = self.get_readonly_transport()
 
801
        server = self.get_readonly_server()
 
802
        t = self._transport(server.get_url())
802
803
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
803
804
        self.assertEqual(l[0], (0, '0'))
804
805
        self.assertEqual(l[1], (1, '1'))
806
807
        self.assertEqual(l[3], (9, '9'))
807
808
 
808
809
    def test_readv_out_of_order(self):
809
 
        t = self.get_readonly_transport()
 
810
        server = self.get_readonly_server()
 
811
        t = self._transport(server.get_url())
810
812
        l = list(t.readv('a', ((1, 1), (9, 1), (0, 1), (3, 2))))
811
813
        self.assertEqual(l[0], (1, '1'))
812
814
        self.assertEqual(l[1], (9, '9'))
814
816
        self.assertEqual(l[3], (3, '34'))
815
817
 
816
818
    def test_readv_invalid_ranges(self):
817
 
        t = self.get_readonly_transport()
 
819
        server = self.get_readonly_server()
 
820
        t = self._transport(server.get_url())
818
821
 
819
822
        # This is intentionally reading off the end of the file
820
823
        # since we are sure that it cannot get there
828
831
 
829
832
    def test_readv_multiple_get_requests(self):
830
833
        server = self.get_readonly_server()
831
 
        t = self.get_readonly_transport()
 
834
        t = self._transport(server.get_url())
832
835
        # force transport to issue multiple requests
833
836
        t._max_readv_combine = 1
834
837
        t._max_get_ranges = 1
842
845
 
843
846
    def test_readv_get_max_size(self):
844
847
        server = self.get_readonly_server()
845
 
        t = self.get_readonly_transport()
 
848
        t = self._transport(server.get_url())
846
849
        # force transport to issue multiple requests by limiting the number of
847
850
        # bytes by request. Note that this apply to coalesced offsets only, a
848
851
        # single range will keep its size even if bigger than the limit.
857
860
 
858
861
    def test_complete_readv_leave_pipe_clean(self):
859
862
        server = self.get_readonly_server()
860
 
        t = self.get_readonly_transport()
 
863
        t = self._transport(server.get_url())
861
864
        # force transport to issue multiple requests
862
865
        t._get_max_size = 2
863
 
        list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
 
866
        l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
864
867
        # The server should have issued 3 requests
865
868
        self.assertEqual(3, server.GET_request_nb)
866
869
        self.assertEqual('0123456789', t.get_bytes('a'))
868
871
 
869
872
    def test_incomplete_readv_leave_pipe_clean(self):
870
873
        server = self.get_readonly_server()
871
 
        t = self.get_readonly_transport()
 
874
        t = self._transport(server.get_url())
872
875
        # force transport to issue multiple requests
873
876
        t._get_max_size = 2
874
877
        # Don't collapse readv results into a list so that we leave unread
943
946
    def get_multiple_ranges(self, file, file_size, ranges):
944
947
        self.send_response(206)
945
948
        self.send_header('Accept-Ranges', 'bytes')
946
 
        # XXX: this is strange; the 'random' name below seems undefined and
947
 
        # yet the tests pass -- mbp 2010-10-11 bug 658773
948
949
        boundary = "%d" % random.randint(0,0x7FFFFFFF)
949
950
        self.send_header("Content-Type",
950
951
                         "multipart/byteranges; boundary=%s" % boundary)
1012
1013
                return
1013
1014
            self.send_range_content(file, start, end - start + 1)
1014
1015
            cur += 1
1015
 
        # Final boundary
 
1016
        # No final boundary
1016
1017
        self.wfile.write(boundary_line)
1017
1018
 
1018
1019
 
1026
1027
 
1027
1028
    def test_readv_with_short_reads(self):
1028
1029
        server = self.get_readonly_server()
1029
 
        t = self.get_readonly_transport()
 
1030
        t = self._transport(server.get_url())
1030
1031
        # Force separate ranges for each offset
1031
1032
        t._bytes_to_read_before_seek = 0
1032
1033
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1047
1048
        # that mode
1048
1049
        self.assertEqual('single', t._range_hint)
1049
1050
 
1050
 
 
1051
 
class TruncatedBeforeBoundaryRequestHandler(
1052
 
    http_server.TestingHTTPRequestHandler):
1053
 
    """Truncation before a boundary, like in bug 198646"""
1054
 
 
1055
 
    _truncated_ranges = 1
1056
 
 
1057
 
    def get_multiple_ranges(self, file, file_size, ranges):
1058
 
        self.send_response(206)
1059
 
        self.send_header('Accept-Ranges', 'bytes')
1060
 
        boundary = 'tagada'
1061
 
        self.send_header('Content-Type',
1062
 
                         'multipart/byteranges; boundary=%s' % boundary)
1063
 
        boundary_line = '--%s\r\n' % boundary
1064
 
        # Calculate the Content-Length
1065
 
        content_length = 0
1066
 
        for (start, end) in ranges:
1067
 
            content_length += len(boundary_line)
1068
 
            content_length += self._header_line_length(
1069
 
                'Content-type', 'application/octet-stream')
1070
 
            content_length += self._header_line_length(
1071
 
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
1072
 
            content_length += len('\r\n') # end headers
1073
 
            content_length += end - start # + 1
1074
 
        content_length += len(boundary_line)
1075
 
        self.send_header('Content-length', content_length)
1076
 
        self.end_headers()
1077
 
 
1078
 
        # Send the multipart body
1079
 
        cur = 0
1080
 
        for (start, end) in ranges:
1081
 
            if cur + self._truncated_ranges >= len(ranges):
1082
 
                # Abruptly ends the response and close the connection
1083
 
                self.close_connection = 1
1084
 
                return
1085
 
            self.wfile.write(boundary_line)
1086
 
            self.send_header('Content-type', 'application/octet-stream')
1087
 
            self.send_header('Content-Range', 'bytes %d-%d/%d'
1088
 
                             % (start, end, file_size))
1089
 
            self.end_headers()
1090
 
            self.send_range_content(file, start, end - start + 1)
1091
 
            cur += 1
1092
 
        # Final boundary
1093
 
        self.wfile.write(boundary_line)
1094
 
 
1095
 
 
1096
 
class TestTruncatedBeforeBoundary(TestSpecificRequestHandler):
1097
 
    """Tests the case of bug 198646, disconnecting before a boundary."""
1098
 
 
1099
 
    _req_handler_class = TruncatedBeforeBoundaryRequestHandler
1100
 
 
1101
 
    def setUp(self):
1102
 
        super(TestTruncatedBeforeBoundary, self).setUp()
1103
 
        self.build_tree_contents([('a', '0123456789')],)
1104
 
 
1105
 
    def test_readv_with_short_reads(self):
1106
 
        server = self.get_readonly_server()
1107
 
        t = self.get_readonly_transport()
1108
 
        # Force separate ranges for each offset
1109
 
        t._bytes_to_read_before_seek = 0
1110
 
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1111
 
        self.assertEqual((0, '0'), ireadv.next())
1112
 
        self.assertEqual((2, '2'), ireadv.next())
1113
 
        self.assertEqual((4, '45'), ireadv.next())
1114
 
        self.assertEqual((9, '9'), ireadv.next())
1115
 
 
1116
 
 
1117
1051
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1118
1052
    """Errors out when range specifiers exceed the limit"""
1119
1053
 
1143
1077
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1144
1078
    """Tests readv requests against a server erroring out on too much ranges."""
1145
1079
 
1146
 
    scenarios = multiply_scenarios(
1147
 
        vary_by_http_client_implementation(),
1148
 
        vary_by_http_protocol_version(),
1149
 
        )
1150
 
 
1151
1080
    # Requests with more range specifiers will error out
1152
1081
    range_limit = 3
1153
1082
 
1155
1084
        return LimitedRangeHTTPServer(range_limit=self.range_limit,
1156
1085
                                      protocol_version=self._protocol_version)
1157
1086
 
 
1087
    def get_transport(self):
 
1088
        return self._transport(self.get_readonly_server().get_url())
 
1089
 
1158
1090
    def setUp(self):
1159
1091
        http_utils.TestCaseWithWebserver.setUp(self)
1160
1092
        # We need to manipulate ranges that correspond to real chunks in the
1164
1096
        self.build_tree_contents([('a', content)],)
1165
1097
 
1166
1098
    def test_few_ranges(self):
1167
 
        t = self.get_readonly_transport()
 
1099
        t = self.get_transport()
1168
1100
        l = list(t.readv('a', ((0, 4), (1024, 4), )))
1169
1101
        self.assertEqual(l[0], (0, '0000'))
1170
1102
        self.assertEqual(l[1], (1024, '0001'))
1171
1103
        self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1172
1104
 
1173
1105
    def test_more_ranges(self):
1174
 
        t = self.get_readonly_transport()
 
1106
        t = self.get_transport()
1175
1107
        l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1176
1108
        self.assertEqual(l[0], (0, '0000'))
1177
1109
        self.assertEqual(l[1], (1024, '0001'))
1188
1120
    Only the urllib implementation is tested here.
1189
1121
    """
1190
1122
 
 
1123
    def setUp(self):
 
1124
        tests.TestCase.setUp(self)
 
1125
        self._old_env = {}
 
1126
 
 
1127
    def tearDown(self):
 
1128
        self._restore_env()
 
1129
        tests.TestCase.tearDown(self)
 
1130
 
 
1131
    def _install_env(self, env):
 
1132
        for name, value in env.iteritems():
 
1133
            self._old_env[name] = osutils.set_or_unset_env(name, value)
 
1134
 
 
1135
    def _restore_env(self):
 
1136
        for name, value in self._old_env.iteritems():
 
1137
            osutils.set_or_unset_env(name, value)
 
1138
 
1191
1139
    def _proxied_request(self):
1192
1140
        handler = _urllib2_wrappers.ProxyHandler()
1193
 
        request = _urllib2_wrappers.Request('GET', 'http://baz/buzzle')
 
1141
        request = _urllib2_wrappers.Request('GET','http://baz/buzzle')
1194
1142
        handler.set_proxy(request, 'http')
1195
1143
        return request
1196
1144
 
1197
 
    def assertEvaluateProxyBypass(self, expected, host, no_proxy):
1198
 
        handler = _urllib2_wrappers.ProxyHandler()
1199
 
        self.assertEquals(expected,
1200
 
                          handler.evaluate_proxy_bypass(host, no_proxy))
1201
 
 
1202
1145
    def test_empty_user(self):
1203
 
        self.overrideEnv('http_proxy', 'http://bar.com')
1204
 
        request = self._proxied_request()
1205
 
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
1206
 
 
1207
 
    def test_user_with_at(self):
1208
 
        self.overrideEnv('http_proxy',
1209
 
                         'http://username@domain:password@proxy_host:1234')
 
1146
        self._install_env({'http_proxy': 'http://bar.com'})
1210
1147
        request = self._proxied_request()
1211
1148
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
1212
1149
 
1213
1150
    def test_invalid_proxy(self):
1214
1151
        """A proxy env variable without scheme"""
1215
 
        self.overrideEnv('http_proxy', 'host:1234')
 
1152
        self._install_env({'http_proxy': 'host:1234'})
1216
1153
        self.assertRaises(errors.InvalidURL, self._proxied_request)
1217
1154
 
1218
 
    def test_evaluate_proxy_bypass_true(self):
1219
 
        """The host is not proxied"""
1220
 
        self.assertEvaluateProxyBypass(True, 'example.com', 'example.com')
1221
 
        self.assertEvaluateProxyBypass(True, 'bzr.example.com', '*example.com')
1222
 
 
1223
 
    def test_evaluate_proxy_bypass_false(self):
1224
 
        """The host is proxied"""
1225
 
        self.assertEvaluateProxyBypass(False, 'bzr.example.com', None)
1226
 
 
1227
 
    def test_evaluate_proxy_bypass_unknown(self):
1228
 
        """The host is not explicitly proxied"""
1229
 
        self.assertEvaluateProxyBypass(None, 'example.com', 'not.example.com')
1230
 
        self.assertEvaluateProxyBypass(None, 'bzr.example.com', 'example.com')
1231
 
 
1232
 
    def test_evaluate_proxy_bypass_empty_entries(self):
1233
 
        """Ignore empty entries"""
1234
 
        self.assertEvaluateProxyBypass(None, 'example.com', '')
1235
 
        self.assertEvaluateProxyBypass(None, 'example.com', ',')
1236
 
        self.assertEvaluateProxyBypass(None, 'example.com', 'foo,,bar')
1237
 
 
1238
1155
 
1239
1156
class TestProxyHttpServer(http_utils.TestCaseWithTwoWebservers):
1240
1157
    """Tests proxy server.
1245
1162
    to the file names).
1246
1163
    """
1247
1164
 
1248
 
    scenarios = multiply_scenarios(
1249
 
        vary_by_http_client_implementation(),
1250
 
        vary_by_http_protocol_version(),
1251
 
        )
1252
 
 
1253
1165
    # FIXME: We don't have an https server available, so we don't
1254
 
    # test https connections. --vila toolongago
 
1166
    # test https connections.
1255
1167
 
1256
1168
    def setUp(self):
1257
1169
        super(TestProxyHttpServer, self).setUp()
1258
 
        self.transport_secondary_server = http_utils.ProxyServer
1259
1170
        self.build_tree_contents([('foo', 'contents of foo\n'),
1260
1171
                                  ('foo-proxied', 'proxied contents of foo\n')])
1261
1172
        # Let's setup some attributes for tests
1262
 
        server = self.get_readonly_server()
1263
 
        self.server_host_port = '%s:%d' % (server.host, server.port)
 
1173
        self.server = self.get_readonly_server()
 
1174
        self.proxy_address = '%s:%d' % (self.server.host, self.server.port)
1264
1175
        if self._testing_pycurl():
1265
1176
            # Oh my ! pycurl does not check for the port as part of
1266
1177
            # no_proxy :-( So we just test the host part
1267
 
            self.no_proxy_host = server.host
 
1178
            self.no_proxy_host = 'localhost'
1268
1179
        else:
1269
 
            self.no_proxy_host = self.server_host_port
 
1180
            self.no_proxy_host = self.proxy_address
1270
1181
        # The secondary server is the proxy
1271
 
        self.proxy_url = self.get_secondary_url()
 
1182
        self.proxy = self.get_secondary_server()
 
1183
        self.proxy_url = self.proxy.get_url()
 
1184
        self._old_env = {}
1272
1185
 
1273
1186
    def _testing_pycurl(self):
1274
 
        # TODO: This is duplicated for lots of the classes in this file
1275
 
        return (features.pycurl.available()
1276
 
                and self._transport == PyCurlTransport)
1277
 
 
1278
 
    def assertProxied(self):
1279
 
        t = self.get_readonly_transport()
1280
 
        self.assertEqual('proxied contents of foo\n', t.get('foo').read())
1281
 
 
1282
 
    def assertNotProxied(self):
1283
 
        t = self.get_readonly_transport()
1284
 
        self.assertEqual('contents of foo\n', t.get('foo').read())
 
1187
        return pycurl_present and self._transport == PyCurlTransport
 
1188
 
 
1189
    def create_transport_secondary_server(self):
 
1190
        """Creates an http server that will serve files with
 
1191
        '-proxied' appended to their names.
 
1192
        """
 
1193
        return http_utils.ProxyServer(protocol_version=self._protocol_version)
 
1194
 
 
1195
    def _install_env(self, env):
 
1196
        for name, value in env.iteritems():
 
1197
            self._old_env[name] = osutils.set_or_unset_env(name, value)
 
1198
 
 
1199
    def _restore_env(self):
 
1200
        for name, value in self._old_env.iteritems():
 
1201
            osutils.set_or_unset_env(name, value)
 
1202
 
 
1203
    def proxied_in_env(self, env):
 
1204
        self._install_env(env)
 
1205
        url = self.server.get_url()
 
1206
        t = self._transport(url)
 
1207
        try:
 
1208
            self.assertEqual('proxied contents of foo\n', t.get('foo').read())
 
1209
        finally:
 
1210
            self._restore_env()
 
1211
 
 
1212
    def not_proxied_in_env(self, env):
 
1213
        self._install_env(env)
 
1214
        url = self.server.get_url()
 
1215
        t = self._transport(url)
 
1216
        try:
 
1217
            self.assertEqual('contents of foo\n', t.get('foo').read())
 
1218
        finally:
 
1219
            self._restore_env()
1285
1220
 
1286
1221
    def test_http_proxy(self):
1287
 
        self.overrideEnv('http_proxy', self.proxy_url)
1288
 
        self.assertProxied()
 
1222
        self.proxied_in_env({'http_proxy': self.proxy_url})
1289
1223
 
1290
1224
    def test_HTTP_PROXY(self):
1291
1225
        if self._testing_pycurl():
1294
1228
            # about. Should we ?)
1295
1229
            raise tests.TestNotApplicable(
1296
1230
                'pycurl does not check HTTP_PROXY for security reasons')
1297
 
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
1298
 
        self.assertProxied()
 
1231
        self.proxied_in_env({'HTTP_PROXY': self.proxy_url})
1299
1232
 
1300
1233
    def test_all_proxy(self):
1301
 
        self.overrideEnv('all_proxy', self.proxy_url)
1302
 
        self.assertProxied()
 
1234
        self.proxied_in_env({'all_proxy': self.proxy_url})
1303
1235
 
1304
1236
    def test_ALL_PROXY(self):
1305
 
        self.overrideEnv('ALL_PROXY', self.proxy_url)
1306
 
        self.assertProxied()
 
1237
        self.proxied_in_env({'ALL_PROXY': self.proxy_url})
1307
1238
 
1308
1239
    def test_http_proxy_with_no_proxy(self):
1309
 
        self.overrideEnv('no_proxy', self.no_proxy_host)
1310
 
        self.overrideEnv('http_proxy', self.proxy_url)
1311
 
        self.assertNotProxied()
 
1240
        self.not_proxied_in_env({'http_proxy': self.proxy_url,
 
1241
                                 'no_proxy': self.no_proxy_host})
1312
1242
 
1313
1243
    def test_HTTP_PROXY_with_NO_PROXY(self):
1314
1244
        if self._testing_pycurl():
1315
1245
            raise tests.TestNotApplicable(
1316
1246
                'pycurl does not check HTTP_PROXY for security reasons')
1317
 
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
1318
 
        self.overrideEnv('HTTP_PROXY', self.proxy_url)
1319
 
        self.assertNotProxied()
 
1247
        self.not_proxied_in_env({'HTTP_PROXY': self.proxy_url,
 
1248
                                 'NO_PROXY': self.no_proxy_host})
1320
1249
 
1321
1250
    def test_all_proxy_with_no_proxy(self):
1322
 
        self.overrideEnv('no_proxy', self.no_proxy_host)
1323
 
        self.overrideEnv('all_proxy', self.proxy_url)
1324
 
        self.assertNotProxied()
 
1251
        self.not_proxied_in_env({'all_proxy': self.proxy_url,
 
1252
                                 'no_proxy': self.no_proxy_host})
1325
1253
 
1326
1254
    def test_ALL_PROXY_with_NO_PROXY(self):
1327
 
        self.overrideEnv('NO_PROXY', self.no_proxy_host)
1328
 
        self.overrideEnv('ALL_PROXY', self.proxy_url)
1329
 
        self.assertNotProxied()
 
1255
        self.not_proxied_in_env({'ALL_PROXY': self.proxy_url,
 
1256
                                 'NO_PROXY': self.no_proxy_host})
1330
1257
 
1331
1258
    def test_http_proxy_without_scheme(self):
1332
 
        self.overrideEnv('http_proxy', self.server_host_port)
1333
1259
        if self._testing_pycurl():
1334
1260
            # pycurl *ignores* invalid proxy env variables. If that ever change
1335
1261
            # in the future, this test will fail indicating that pycurl do not
1336
1262
            # ignore anymore such variables.
1337
 
            self.assertNotProxied()
 
1263
            self.not_proxied_in_env({'http_proxy': self.proxy_address})
1338
1264
        else:
1339
 
            self.assertRaises(errors.InvalidURL, self.assertProxied)
 
1265
            self.assertRaises(errors.InvalidURL,
 
1266
                              self.proxied_in_env,
 
1267
                              {'http_proxy': self.proxy_address})
1340
1268
 
1341
1269
 
1342
1270
class TestRanges(http_utils.TestCaseWithWebserver):
1343
1271
    """Test the Range header in GET methods."""
1344
1272
 
1345
 
    scenarios = multiply_scenarios(
1346
 
        vary_by_http_client_implementation(),
1347
 
        vary_by_http_protocol_version(),
1348
 
        )
1349
 
 
1350
1273
    def setUp(self):
1351
1274
        http_utils.TestCaseWithWebserver.setUp(self)
1352
1275
        self.build_tree_contents([('a', '0123456789')],)
 
1276
        server = self.get_readonly_server()
 
1277
        self.transport = self._transport(server.get_url())
1353
1278
 
1354
1279
    def create_transport_readonly_server(self):
1355
1280
        return http_server.HttpServer(protocol_version=self._protocol_version)
1356
1281
 
1357
1282
    def _file_contents(self, relpath, ranges):
1358
 
        t = self.get_readonly_transport()
1359
1283
        offsets = [ (start, end - start + 1) for start, end in ranges]
1360
 
        coalesce = t._coalesce_offsets
 
1284
        coalesce = self.transport._coalesce_offsets
1361
1285
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1362
 
        code, data = t._get(relpath, coalesced)
 
1286
        code, data = self.transport._get(relpath, coalesced)
1363
1287
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1364
1288
        for start, end in ranges:
1365
1289
            data.seek(start)
1366
1290
            yield data.read(end - start + 1)
1367
1291
 
1368
1292
    def _file_tail(self, relpath, tail_amount):
1369
 
        t = self.get_readonly_transport()
1370
 
        code, data = t._get(relpath, [], tail_amount)
 
1293
        code, data = self.transport._get(relpath, [], tail_amount)
1371
1294
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1372
1295
        data.seek(-tail_amount, 2)
1373
1296
        return data.read(tail_amount)
1392
1315
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1393
1316
    """Test redirection between http servers."""
1394
1317
 
1395
 
    scenarios = multiply_scenarios(
1396
 
        vary_by_http_client_implementation(),
1397
 
        vary_by_http_protocol_version(),
1398
 
        )
 
1318
    def create_transport_secondary_server(self):
 
1319
        """Create the secondary server redirecting to the primary server"""
 
1320
        new = self.get_readonly_server()
 
1321
 
 
1322
        redirecting = http_utils.HTTPServerRedirecting(
 
1323
            protocol_version=self._protocol_version)
 
1324
        redirecting.redirect_to(new.host, new.port)
 
1325
        return redirecting
1399
1326
 
1400
1327
    def setUp(self):
1401
1328
        super(TestHTTPRedirections, self).setUp()
1403
1330
                                  ('bundle',
1404
1331
                                  '# Bazaar revision bundle v0.9\n#\n')
1405
1332
                                  ],)
 
1333
        # The requests to the old server will be redirected to the new server
 
1334
        self.old_transport = self._transport(self.old_server.get_url())
1406
1335
 
1407
1336
    def test_redirected(self):
1408
 
        self.assertRaises(errors.RedirectRequested,
1409
 
                          self.get_old_transport().get, 'a')
1410
 
        self.assertEqual('0123456789', self.get_new_transport().get('a').read())
 
1337
        self.assertRaises(errors.RedirectRequested, self.old_transport.get, 'a')
 
1338
        t = self._transport(self.new_server.get_url())
 
1339
        self.assertEqual('0123456789', t.get('a').read())
 
1340
 
 
1341
    def test_read_redirected_bundle_from_url(self):
 
1342
        from bzrlib.bundle import read_bundle_from_url
 
1343
        url = self.old_transport.abspath('bundle')
 
1344
        bundle = self.applyDeprecated(deprecated_in((1, 12, 0)),
 
1345
                read_bundle_from_url, url)
 
1346
        # If read_bundle_from_url was successful we get an empty bundle
 
1347
        self.assertEqual([], bundle.revisions)
1411
1348
 
1412
1349
 
1413
1350
class RedirectedRequest(_urllib2_wrappers.Request):
1426
1363
        self.follow_redirections = True
1427
1364
 
1428
1365
 
1429
 
def install_redirected_request(test):
1430
 
    test.overrideAttr(_urllib2_wrappers, 'Request', RedirectedRequest)
1431
 
 
1432
 
 
1433
 
def cleanup_http_redirection_connections(test):
1434
 
    # Some sockets are opened but never seen by _urllib, so we trap them at
1435
 
    # the _urllib2_wrappers level to be able to clean them up.
1436
 
    def socket_disconnect(sock):
1437
 
        try:
1438
 
            sock.shutdown(socket.SHUT_RDWR)
1439
 
            sock.close()
1440
 
        except socket.error:
1441
 
            pass
1442
 
    def connect(connection):
1443
 
        test.http_connect_orig(connection)
1444
 
        test.addCleanup(socket_disconnect, connection.sock)
1445
 
    test.http_connect_orig = test.overrideAttr(
1446
 
        _urllib2_wrappers.HTTPConnection, 'connect', connect)
1447
 
    def connect(connection):
1448
 
        test.https_connect_orig(connection)
1449
 
        test.addCleanup(socket_disconnect, connection.sock)
1450
 
    test.https_connect_orig = test.overrideAttr(
1451
 
        _urllib2_wrappers.HTTPSConnection, 'connect', connect)
1452
 
 
1453
 
 
1454
1366
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1455
1367
    """Test redirections.
1456
1368
 
1465
1377
    -- vila 20070212
1466
1378
    """
1467
1379
 
1468
 
    scenarios = multiply_scenarios(
1469
 
        vary_by_http_client_implementation(),
1470
 
        vary_by_http_protocol_version(),
1471
 
        )
1472
 
 
1473
1380
    def setUp(self):
1474
 
        if (features.pycurl.available()
1475
 
            and self._transport == PyCurlTransport):
 
1381
        if pycurl_present and self._transport == PyCurlTransport:
1476
1382
            raise tests.TestNotApplicable(
1477
 
                "pycurl doesn't redirect silently anymore")
 
1383
                "pycurl doesn't redirect silently annymore")
1478
1384
        super(TestHTTPSilentRedirections, self).setUp()
1479
 
        install_redirected_request(self)
1480
 
        cleanup_http_redirection_connections(self)
 
1385
        self.setup_redirected_request()
 
1386
        self.addCleanup(self.cleanup_redirected_request)
1481
1387
        self.build_tree_contents([('a','a'),
1482
1388
                                  ('1/',),
1483
1389
                                  ('1/a', 'redirected once'),
1491
1397
                                  ('5/a', 'redirected 5 times'),
1492
1398
                                  ],)
1493
1399
 
 
1400
        self.old_transport = self._transport(self.old_server.get_url())
 
1401
 
 
1402
    def setup_redirected_request(self):
 
1403
        self.original_class = _urllib2_wrappers.Request
 
1404
        _urllib2_wrappers.Request = RedirectedRequest
 
1405
 
 
1406
    def cleanup_redirected_request(self):
 
1407
        _urllib2_wrappers.Request = self.original_class
 
1408
 
 
1409
    def create_transport_secondary_server(self):
 
1410
        """Create the secondary server, redirections are defined in the tests"""
 
1411
        return http_utils.HTTPServerRedirecting(
 
1412
            protocol_version=self._protocol_version)
 
1413
 
1494
1414
    def test_one_redirection(self):
1495
 
        t = self.get_old_transport()
1496
 
        req = RedirectedRequest('GET', t._remote_path('a'))
 
1415
        t = self.old_transport
 
1416
 
 
1417
        req = RedirectedRequest('GET', t.abspath('a'))
 
1418
        req.follow_redirections = True
1497
1419
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1498
1420
                                       self.new_server.port)
1499
1421
        self.old_server.redirections = \
1500
1422
            [('(.*)', r'%s/1\1' % (new_prefix), 301),]
1501
 
        self.assertEqual('redirected once', t._perform(req).read())
 
1423
        self.assertEquals('redirected once',t._perform(req).read())
1502
1424
 
1503
1425
    def test_five_redirections(self):
1504
 
        t = self.get_old_transport()
1505
 
        req = RedirectedRequest('GET', t._remote_path('a'))
 
1426
        t = self.old_transport
 
1427
 
 
1428
        req = RedirectedRequest('GET', t.abspath('a'))
 
1429
        req.follow_redirections = True
1506
1430
        old_prefix = 'http://%s:%s' % (self.old_server.host,
1507
1431
                                       self.old_server.port)
1508
1432
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1514
1438
            ('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1515
1439
            ('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1516
1440
            ]
1517
 
        self.assertEqual('redirected 5 times', t._perform(req).read())
 
1441
        self.assertEquals('redirected 5 times',t._perform(req).read())
1518
1442
 
1519
1443
 
1520
1444
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1521
1445
    """Test transport.do_catching_redirections."""
1522
1446
 
1523
 
    scenarios = multiply_scenarios(
1524
 
        vary_by_http_client_implementation(),
1525
 
        vary_by_http_protocol_version(),
1526
 
        )
1527
 
 
1528
1447
    def setUp(self):
1529
1448
        super(TestDoCatchRedirections, self).setUp()
1530
1449
        self.build_tree_contents([('a', '0123456789'),],)
1531
 
        cleanup_http_redirection_connections(self)
1532
 
 
1533
 
        self.old_transport = self.get_old_transport()
1534
 
 
1535
 
    def get_a(self, t):
1536
 
        return t.get('a')
 
1450
 
 
1451
        self.old_transport = self._transport(self.old_server.get_url())
 
1452
 
 
1453
    def get_a(self, transport):
 
1454
        return transport.get('a')
1537
1455
 
1538
1456
    def test_no_redirection(self):
1539
 
        t = self.get_new_transport()
 
1457
        t = self._transport(self.new_server.get_url())
1540
1458
 
1541
1459
        # We use None for redirected so that we fail if redirected
1542
 
        self.assertEqual('0123456789',
1543
 
                         transport.do_catching_redirections(
 
1460
        self.assertEquals('0123456789',
 
1461
                          transport.do_catching_redirections(
1544
1462
                self.get_a, t, None).read())
1545
1463
 
1546
1464
    def test_one_redirection(self):
1547
1465
        self.redirections = 0
1548
1466
 
1549
 
        def redirected(t, exception, redirection_notice):
 
1467
        def redirected(transport, exception, redirection_notice):
1550
1468
            self.redirections += 1
1551
 
            redirected_t = t._redirected_to(exception.source, exception.target)
1552
 
            return redirected_t
 
1469
            dir, file = urlutils.split(exception.target)
 
1470
            return self._transport(dir)
1553
1471
 
1554
 
        self.assertEqual('0123456789',
1555
 
                         transport.do_catching_redirections(
 
1472
        self.assertEquals('0123456789',
 
1473
                          transport.do_catching_redirections(
1556
1474
                self.get_a, self.old_transport, redirected).read())
1557
 
        self.assertEqual(1, self.redirections)
 
1475
        self.assertEquals(1, self.redirections)
1558
1476
 
1559
1477
    def test_redirection_loop(self):
1560
1478
 
1569
1487
                          self.get_a, self.old_transport, redirected)
1570
1488
 
1571
1489
 
1572
 
def _setup_authentication_config(**kwargs):
1573
 
    conf = config.AuthenticationConfig()
1574
 
    conf._get_config().update({'httptest': kwargs})
1575
 
    conf._save()
1576
 
 
1577
 
 
1578
 
class TestUrllib2AuthHandler(tests.TestCaseWithTransport):
1579
 
    """Unit tests for glue by which urllib2 asks us for authentication"""
1580
 
 
1581
 
    def test_get_user_password_without_port(self):
1582
 
        """We cope if urllib2 doesn't tell us the port.
1583
 
 
1584
 
        See https://bugs.launchpad.net/bzr/+bug/654684
1585
 
        """
1586
 
        user = 'joe'
1587
 
        password = 'foo'
1588
 
        _setup_authentication_config(scheme='http', host='localhost',
1589
 
                                     user=user, password=password)
1590
 
        handler = _urllib2_wrappers.HTTPAuthHandler()
1591
 
        got_pass = handler.get_user_password(dict(
1592
 
            user='joe',
1593
 
            protocol='http',
1594
 
            host='localhost',
1595
 
            path='/',
1596
 
            realm='Realm',
1597
 
            ))
1598
 
        self.assertEquals((user, password), got_pass)
1599
 
 
1600
 
 
1601
1490
class TestAuth(http_utils.TestCaseWithWebserver):
1602
1491
    """Test authentication scheme"""
1603
1492
 
1604
 
    scenarios = multiply_scenarios(
1605
 
        vary_by_http_client_implementation(),
1606
 
        vary_by_http_protocol_version(),
1607
 
        vary_by_http_auth_scheme(),
1608
 
        )
 
1493
    _auth_header = 'Authorization'
 
1494
    _password_prompt_prefix = ''
 
1495
    _username_prompt_prefix = ''
 
1496
    # Set by load_tests
 
1497
    _auth_server = None
1609
1498
 
1610
1499
    def setUp(self):
1611
1500
        super(TestAuth, self).setUp()
1614
1503
                                  ('b', 'contents of b\n'),])
1615
1504
 
1616
1505
    def create_transport_readonly_server(self):
1617
 
        server = self._auth_server(protocol_version=self._protocol_version)
1618
 
        server._url_protocol = self._url_protocol
1619
 
        return server
 
1506
        return self._auth_server(protocol_version=self._protocol_version)
1620
1507
 
1621
1508
    def _testing_pycurl(self):
1622
 
        # TODO: This is duplicated for lots of the classes in this file
1623
 
        return (features.pycurl.available()
1624
 
                and self._transport == PyCurlTransport)
 
1509
        return pycurl_present and self._transport == PyCurlTransport
1625
1510
 
1626
1511
    def get_user_url(self, user, password):
1627
1512
        """Build an url embedding user and password"""
1635
1520
        return url
1636
1521
 
1637
1522
    def get_user_transport(self, user, password):
1638
 
        t = transport.get_transport(self.get_user_url(user, password))
1639
 
        return t
 
1523
        return self._transport(self.get_user_url(user, password))
1640
1524
 
1641
1525
    def test_no_user(self):
1642
1526
        self.server.add_user('joe', 'foo')
1693
1577
        self.assertEqual('', ui.ui_factory.stdin.readline())
1694
1578
        stderr.seek(0)
1695
1579
        expected_prompt = self._expected_username_prompt(t._unqualified_scheme)
1696
 
        self.assertEqual(expected_prompt, stderr.read(len(expected_prompt)))
1697
 
        self.assertEqual('', stdout.getvalue())
 
1580
        self.assertEquals(expected_prompt, stderr.read(len(expected_prompt)))
 
1581
        self.assertEquals('', stdout.getvalue())
1698
1582
        self._check_password_prompt(t._unqualified_scheme, 'joe',
1699
1583
                                    stderr.readline())
1700
1584
 
1715
1599
        self.assertEqual('', ui.ui_factory.stdin.readline())
1716
1600
        self._check_password_prompt(t._unqualified_scheme, 'joe',
1717
1601
                                    stderr.getvalue())
1718
 
        self.assertEqual('', stdout.getvalue())
 
1602
        self.assertEquals('', stdout.getvalue())
1719
1603
        # And we shouldn't prompt again for a different request
1720
1604
        # against the same transport.
1721
1605
        self.assertEqual('contents of b\n',t.get('b').read())
1731
1615
                              % (scheme.upper(),
1732
1616
                                 user, self.server.host, self.server.port,
1733
1617
                                 self.server.auth_realm)))
1734
 
        self.assertEqual(expected_prompt, actual_prompt)
 
1618
        self.assertEquals(expected_prompt, actual_prompt)
1735
1619
 
1736
1620
    def _expected_username_prompt(self, scheme):
1737
1621
        return (self._username_prompt_prefix
1751
1635
        self.server.add_user(user, password)
1752
1636
        t = self.get_user_transport(user, None)
1753
1637
        ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1754
 
                                            stderr=tests.StringIOWrapper())
 
1638
                                            stdout=tests.StringIOWrapper())
1755
1639
        # Create a minimal config file with the right password
1756
 
        _setup_authentication_config(scheme='http', port=self.server.port,
1757
 
                                     user=user, password=password)
 
1640
        conf = config.AuthenticationConfig()
 
1641
        conf._get_config().update(
 
1642
            {'httptest': {'scheme': 'http', 'port': self.server.port,
 
1643
                          'user': user, 'password': password}})
 
1644
        conf._save()
1758
1645
        # Issue a request to the server to connect
1759
1646
        self.assertEqual('contents of a\n',t.get('a').read())
1760
1647
        # stdin should have  been left untouched
1762
1649
        # Only one 'Authentication Required' error should occur
1763
1650
        self.assertEqual(1, self.server.auth_required_errors)
1764
1651
 
 
1652
    def test_user_from_auth_conf(self):
 
1653
        if self._testing_pycurl():
 
1654
            raise tests.TestNotApplicable(
 
1655
                'pycurl does not support authentication.conf')
 
1656
        user = 'joe'
 
1657
        password = 'foo'
 
1658
        self.server.add_user(user, password)
 
1659
        # Create a minimal config file with the right password
 
1660
        conf = config.AuthenticationConfig()
 
1661
        conf._get_config().update(
 
1662
            {'httptest': {'scheme': 'http', 'port': self.server.port,
 
1663
                          'user': user, 'password': password}})
 
1664
        conf._save()
 
1665
        t = self.get_user_transport(None, None)
 
1666
        # Issue a request to the server to connect
 
1667
        self.assertEqual('contents of a\n', t.get('a').read())
 
1668
        # Only one 'Authentication Required' error should occur
 
1669
        self.assertEqual(1, self.server.auth_required_errors)
 
1670
 
1765
1671
    def test_changing_nonce(self):
1766
1672
        if self._auth_server not in (http_utils.HTTPDigestAuthServer,
1767
1673
                                     http_utils.ProxyDigestAuthServer):
1768
1674
            raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1769
1675
        if self._testing_pycurl():
1770
 
            self.knownFailure(
 
1676
            raise tests.KnownFailure(
1771
1677
                'pycurl does not handle a nonce change')
1772
1678
        self.server.add_user('joe', 'foo')
1773
1679
        t = self.get_user_transport('joe', 'foo')
1783
1689
        # initial 'who are you' and a second 'who are you' with the new nonce)
1784
1690
        self.assertEqual(2, self.server.auth_required_errors)
1785
1691
 
1786
 
    def test_user_from_auth_conf(self):
1787
 
        if self._testing_pycurl():
1788
 
            raise tests.TestNotApplicable(
1789
 
                'pycurl does not support authentication.conf')
1790
 
        user = 'joe'
1791
 
        password = 'foo'
1792
 
        self.server.add_user(user, password)
1793
 
        _setup_authentication_config(scheme='http', port=self.server.port,
1794
 
                                     user=user, password=password)
1795
 
        t = self.get_user_transport(None, None)
1796
 
        # Issue a request to the server to connect
1797
 
        self.assertEqual('contents of a\n', t.get('a').read())
1798
 
        # Only one 'Authentication Required' error should occur
1799
 
        self.assertEqual(1, self.server.auth_required_errors)
1800
 
 
1801
 
    def test_no_credential_leaks_in_log(self):
1802
 
        self.overrideAttr(debug, 'debug_flags', set(['http']))
1803
 
        user = 'joe'
1804
 
        password = 'very-sensitive-password'
1805
 
        self.server.add_user(user, password)
1806
 
        t = self.get_user_transport(user, password)
1807
 
        # Capture the debug calls to mutter
1808
 
        self.mutters = []
1809
 
        def mutter(*args):
1810
 
            lines = args[0] % args[1:]
1811
 
            # Some calls output multiple lines, just split them now since we
1812
 
            # care about a single one later.
1813
 
            self.mutters.extend(lines.splitlines())
1814
 
        self.overrideAttr(trace, 'mutter', mutter)
1815
 
        # Issue a request to the server to connect
1816
 
        self.assertEqual(True, t.has('a'))
1817
 
        # Only one 'Authentication Required' error should occur
1818
 
        self.assertEqual(1, self.server.auth_required_errors)
1819
 
        # Since the authentification succeeded, there should be a corresponding
1820
 
        # debug line
1821
 
        sent_auth_headers = [line for line in self.mutters
1822
 
                             if line.startswith('> %s' % (self._auth_header,))]
1823
 
        self.assertLength(1, sent_auth_headers)
1824
 
        self.assertStartsWith(sent_auth_headers[0],
1825
 
                              '> %s: <masked>' % (self._auth_header,))
1826
1692
 
1827
1693
 
1828
1694
class TestProxyAuth(TestAuth):
1829
 
    """Test proxy authentication schemes.
1830
 
 
1831
 
    This inherits from TestAuth to tweak the setUp and filter some failing
1832
 
    tests.
1833
 
    """
1834
 
 
1835
 
    scenarios = multiply_scenarios(
1836
 
        vary_by_http_client_implementation(),
1837
 
        vary_by_http_protocol_version(),
1838
 
        vary_by_http_proxy_auth_scheme(),
1839
 
        )
 
1695
    """Test proxy authentication schemes."""
 
1696
 
 
1697
    _auth_header = 'Proxy-authorization'
 
1698
    _password_prompt_prefix = 'Proxy '
 
1699
    _username_prompt_prefix = 'Proxy '
1840
1700
 
1841
1701
    def setUp(self):
1842
1702
        super(TestProxyAuth, self).setUp()
 
1703
        self._old_env = {}
 
1704
        self.addCleanup(self._restore_env)
1843
1705
        # Override the contents to avoid false positives
1844
1706
        self.build_tree_contents([('a', 'not proxied contents of a\n'),
1845
1707
                                  ('b', 'not proxied contents of b\n'),
1848
1710
                                  ])
1849
1711
 
1850
1712
    def get_user_transport(self, user, password):
1851
 
        self.overrideEnv('all_proxy', self.get_user_url(user, password))
1852
 
        return TestAuth.get_user_transport(self, user, password)
 
1713
        self._install_env({'all_proxy': self.get_user_url(user, password)})
 
1714
        return self._transport(self.server.get_url())
 
1715
 
 
1716
    def _install_env(self, env):
 
1717
        for name, value in env.iteritems():
 
1718
            self._old_env[name] = osutils.set_or_unset_env(name, value)
 
1719
 
 
1720
    def _restore_env(self):
 
1721
        for name, value in self._old_env.iteritems():
 
1722
            osutils.set_or_unset_env(name, value)
1853
1723
 
1854
1724
    def test_empty_pass(self):
1855
1725
        if self._testing_pycurl():
1856
1726
            import pycurl
1857
1727
            if pycurl.version_info()[1] < '7.16.0':
1858
 
                self.knownFailure(
 
1728
                raise tests.KnownFailure(
1859
1729
                    'pycurl < 7.16.0 does not handle empty proxy passwords')
1860
1730
        super(TestProxyAuth, self).test_empty_pass()
1861
1731
 
1874
1744
        self.readfile = StringIO(socket_read_content)
1875
1745
        self.writefile = StringIO()
1876
1746
        self.writefile.close = lambda: None
1877
 
        self.close = lambda: None
1878
1747
 
1879
1748
    def makefile(self, mode='r', bufsize=None):
1880
1749
        if 'r' in mode:
1885
1754
 
1886
1755
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1887
1756
 
1888
 
    scenarios = multiply_scenarios(
1889
 
        vary_by_http_client_implementation(),
1890
 
        vary_by_http_protocol_version(),
1891
 
        )
1892
 
 
1893
1757
    def setUp(self):
1894
1758
        super(SmartHTTPTunnellingTest, self).setUp()
1895
1759
        # We use the VFS layer as part of HTTP tunnelling tests.
1896
 
        self.overrideEnv('BZR_NO_SMART_VFS', None)
 
1760
        self._captureVar('BZR_NO_SMART_VFS', None)
1897
1761
        self.transport_readonly_server = http_utils.HTTPServerWithSmarts
1898
 
        self.http_server = self.get_readonly_server()
1899
1762
 
1900
1763
    def create_transport_readonly_server(self):
1901
 
        server = http_utils.HTTPServerWithSmarts(
 
1764
        return http_utils.HTTPServerWithSmarts(
1902
1765
            protocol_version=self._protocol_version)
1903
 
        server._url_protocol = self._url_protocol
1904
 
        return server
1905
1766
 
1906
1767
    def test_open_bzrdir(self):
1907
1768
        branch = self.make_branch('relpath')
1908
 
        url = self.http_server.get_url() + 'relpath'
 
1769
        http_server = self.get_readonly_server()
 
1770
        url = http_server.get_url() + 'relpath'
1909
1771
        bd = bzrdir.BzrDir.open(url)
1910
 
        self.addCleanup(bd.transport.disconnect)
1911
1772
        self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1912
1773
 
1913
1774
    def test_bulk_data(self):
1915
1776
        # The 'readv' command in the smart protocol both sends and receives
1916
1777
        # bulk data, so we use that.
1917
1778
        self.build_tree(['data-file'])
1918
 
        http_transport = transport.get_transport(self.http_server.get_url())
 
1779
        http_server = self.get_readonly_server()
 
1780
        http_transport = self._transport(http_server.get_url())
1919
1781
        medium = http_transport.get_smart_medium()
1920
1782
        # Since we provide the medium, the url below will be mostly ignored
1921
1783
        # during the test, as long as the path is '/'.
1929
1791
        post_body = 'hello\n'
1930
1792
        expected_reply_body = 'ok\x012\n'
1931
1793
 
1932
 
        http_transport = transport.get_transport(self.http_server.get_url())
 
1794
        http_server = self.get_readonly_server()
 
1795
        http_transport = self._transport(http_server.get_url())
1933
1796
        medium = http_transport.get_smart_medium()
1934
1797
        response = medium.send_http_smart_request(post_body)
1935
1798
        reply_body = response.read()
1936
1799
        self.assertEqual(expected_reply_body, reply_body)
1937
1800
 
1938
1801
    def test_smart_http_server_post_request_handler(self):
1939
 
        httpd = self.http_server.server
 
1802
        httpd = self.get_readonly_server()._get_httpd()
1940
1803
 
1941
1804
        socket = SampleSocket(
1942
1805
            'POST /.bzr/smart %s \r\n' % self._protocol_version
1974
1837
 
1975
1838
    def test_probe_smart_server(self):
1976
1839
        """Test error handling against server refusing smart requests."""
1977
 
        t = self.get_readonly_transport()
 
1840
        server = self.get_readonly_server()
 
1841
        t = self._transport(server.get_url())
1978
1842
        # No need to build a valid smart request here, the server will not even
1979
1843
        # try to interpret it.
1980
1844
        self.assertRaises(errors.SmartProtocolError,
1981
1845
                          t.get_smart_medium().send_http_smart_request,
1982
1846
                          'whatever')
1983
1847
 
1984
 
 
1985
1848
class Test_redirected_to(tests.TestCase):
1986
1849
 
1987
 
    scenarios = vary_by_http_client_implementation()
1988
 
 
1989
1850
    def test_redirected_to_subdir(self):
1990
1851
        t = self._transport('http://www.example.com/foo')
1991
1852
        r = t._redirected_to('http://www.example.com/foo',
1992
1853
                             'http://www.example.com/foo/subdir')
1993
1854
        self.assertIsInstance(r, type(t))
1994
1855
        # Both transports share the some connection
1995
 
        self.assertEqual(t._get_connection(), r._get_connection())
 
1856
        self.assertEquals(t._get_connection(), r._get_connection())
1996
1857
 
1997
1858
    def test_redirected_to_self_with_slash(self):
1998
1859
        t = self._transport('http://www.example.com/foo')
2002
1863
        # Both transports share the some connection (one can argue that we
2003
1864
        # should return the exact same transport here, but that seems
2004
1865
        # overkill).
2005
 
        self.assertEqual(t._get_connection(), r._get_connection())
 
1866
        self.assertEquals(t._get_connection(), r._get_connection())
2006
1867
 
2007
1868
    def test_redirected_to_host(self):
2008
1869
        t = self._transport('http://www.example.com/foo')
2027
1888
        r = t._redirected_to('http://www.example.com/foo',
2028
1889
                             'https://foo.example.com/foo')
2029
1890
        self.assertIsInstance(r, type(t))
2030
 
        self.assertEqual(t._parsed_url.user, r._parsed_url.user)
 
1891
        self.assertEquals(t._user, r._user)
2031
1892
 
2032
1893
 
2033
1894
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
2042
1903
    line.
2043
1904
    """
2044
1905
 
2045
 
    def _handle_one_request(self):
 
1906
    def handle_one_request(self):
2046
1907
        tcs = self.server.test_case_server
2047
1908
        requestline = self.rfile.readline()
2048
1909
        headers = self.MessageClass(self.rfile, 0)
2086
1947
    pass
2087
1948
 
2088
1949
 
2089
 
if features.HTTPSServerFeature.available():
 
1950
if tests.HTTPSServerFeature.available():
2090
1951
    from bzrlib.tests import https_server
2091
1952
    class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
2092
1953
        pass
2093
1954
 
2094
1955
 
2095
 
class TestActivityMixin(object):
 
1956
class TestActivity(tests.TestCase):
2096
1957
    """Test socket activity reporting.
2097
1958
 
2098
1959
    We use a special purpose server to control the bytes sent and received and
2102
1963
    def setUp(self):
2103
1964
        tests.TestCase.setUp(self)
2104
1965
        self.server = self._activity_server(self._protocol_version)
2105
 
        self.server.start_server()
2106
 
        _activities = {} # Don't close over self and create a cycle
 
1966
        self.server.setUp()
 
1967
        self.activities = {}
2107
1968
        def report_activity(t, bytes, direction):
2108
 
            count = _activities.get(direction, 0)
 
1969
            count = self.activities.get(direction, 0)
2109
1970
            count += bytes
2110
 
            _activities[direction] = count
2111
 
        self.activities = _activities
 
1971
            self.activities[direction] = count
2112
1972
 
2113
1973
        # We override at class level because constructors may propagate the
2114
1974
        # bound method and render instance overriding ineffective (an
2115
1975
        # alternative would be to define a specific ui factory instead...)
2116
 
        self.overrideAttr(self._transport, '_report_activity', report_activity)
2117
 
        self.addCleanup(self.server.stop_server)
 
1976
        self.orig_report_activity = self._transport._report_activity
 
1977
        self._transport._report_activity = report_activity
 
1978
 
 
1979
    def tearDown(self):
 
1980
        self._transport._report_activity = self.orig_report_activity
 
1981
        self.server.tearDown()
 
1982
        tests.TestCase.tearDown(self)
2118
1983
 
2119
1984
    def get_transport(self):
2120
 
        t = self._transport(self.server.get_url())
2121
 
        # FIXME: Needs cleanup -- vila 20100611
2122
 
        return t
 
1985
        return self._transport(self.server.get_url())
2123
1986
 
2124
1987
    def assertActivitiesMatch(self):
2125
1988
        self.assertEqual(self.server.bytes_read,
2230
2093
'''
2231
2094
        t = self.get_transport()
2232
2095
        # We must send a single line of body bytes, see
2233
 
        # PredefinedRequestHandler._handle_one_request
 
2096
        # PredefinedRequestHandler.handle_one_request
2234
2097
        code, f = t._post('abc def end-of-body\n')
2235
2098
        self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2236
2099
        self.assertActivitiesMatch()
2237
 
 
2238
 
 
2239
 
class TestActivity(tests.TestCase, TestActivityMixin):
2240
 
 
2241
 
    scenarios = multiply_scenarios(
2242
 
        vary_by_http_activity(),
2243
 
        vary_by_http_protocol_version(),
2244
 
        )
2245
 
 
2246
 
    def setUp(self):
2247
 
        TestActivityMixin.setUp(self)
2248
 
 
2249
 
 
2250
 
class TestNoReportActivity(tests.TestCase, TestActivityMixin):
2251
 
 
2252
 
    # Unlike TestActivity, we are really testing ReportingFileSocket and
2253
 
    # ReportingSocket, so we don't need all the parametrization. Since
2254
 
    # ReportingFileSocket and ReportingSocket are wrappers, it's easier to
2255
 
    # test them through their use by the transport than directly (that's a
2256
 
    # bit less clean but far more simpler and effective).
2257
 
    _activity_server = ActivityHTTPServer
2258
 
    _protocol_version = 'HTTP/1.1'
2259
 
 
2260
 
    def setUp(self):
2261
 
        self._transport =_urllib.HttpTransport_urllib
2262
 
        TestActivityMixin.setUp(self)
2263
 
 
2264
 
    def assertActivitiesMatch(self):
2265
 
        # Nothing to check here
2266
 
        pass
2267
 
 
2268
 
 
2269
 
class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver):
2270
 
    """Test authentication on the redirected http server."""
2271
 
 
2272
 
    scenarios = vary_by_http_protocol_version()
2273
 
 
2274
 
    _auth_header = 'Authorization'
2275
 
    _password_prompt_prefix = ''
2276
 
    _username_prompt_prefix = ''
2277
 
    _auth_server = http_utils.HTTPBasicAuthServer
2278
 
    _transport = _urllib.HttpTransport_urllib
2279
 
 
2280
 
    def setUp(self):
2281
 
        super(TestAuthOnRedirected, self).setUp()
2282
 
        self.build_tree_contents([('a','a'),
2283
 
                                  ('1/',),
2284
 
                                  ('1/a', 'redirected once'),
2285
 
                                  ],)
2286
 
        new_prefix = 'http://%s:%s' % (self.new_server.host,
2287
 
                                       self.new_server.port)
2288
 
        self.old_server.redirections = [
2289
 
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2290
 
        self.old_transport = self.get_old_transport()
2291
 
        self.new_server.add_user('joe', 'foo')
2292
 
        cleanup_http_redirection_connections(self)
2293
 
 
2294
 
    def create_transport_readonly_server(self):
2295
 
        server = self._auth_server(protocol_version=self._protocol_version)
2296
 
        server._url_protocol = self._url_protocol
2297
 
        return server
2298
 
 
2299
 
    def get_a(self, t):
2300
 
        return t.get('a')
2301
 
 
2302
 
    def test_auth_on_redirected_via_do_catching_redirections(self):
2303
 
        self.redirections = 0
2304
 
 
2305
 
        def redirected(t, exception, redirection_notice):
2306
 
            self.redirections += 1
2307
 
            redirected_t = t._redirected_to(exception.source, exception.target)
2308
 
            self.addCleanup(redirected_t.disconnect)
2309
 
            return redirected_t
2310
 
 
2311
 
        stdout = tests.StringIOWrapper()
2312
 
        stderr = tests.StringIOWrapper()
2313
 
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2314
 
                                            stdout=stdout, stderr=stderr)
2315
 
        self.assertEqual('redirected once',
2316
 
                         transport.do_catching_redirections(
2317
 
                self.get_a, self.old_transport, redirected).read())
2318
 
        self.assertEqual(1, self.redirections)
2319
 
        # stdin should be empty
2320
 
        self.assertEqual('', ui.ui_factory.stdin.readline())
2321
 
        # stdout should be empty, stderr will contains the prompts
2322
 
        self.assertEqual('', stdout.getvalue())
2323
 
 
2324
 
    def test_auth_on_redirected_via_following_redirections(self):
2325
 
        self.new_server.add_user('joe', 'foo')
2326
 
        stdout = tests.StringIOWrapper()
2327
 
        stderr = tests.StringIOWrapper()
2328
 
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
2329
 
                                            stdout=stdout, stderr=stderr)
2330
 
        t = self.old_transport
2331
 
        req = RedirectedRequest('GET', t.abspath('a'))
2332
 
        new_prefix = 'http://%s:%s' % (self.new_server.host,
2333
 
                                       self.new_server.port)
2334
 
        self.old_server.redirections = [
2335
 
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
2336
 
        self.assertEqual('redirected once', t._perform(req).read())
2337
 
        # stdin should be empty
2338
 
        self.assertEqual('', ui.ui_factory.stdin.readline())
2339
 
        # stdout should be empty, stderr will contains the prompts
2340
 
        self.assertEqual('', stdout.getvalue())
2341