← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_http.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2010-09-01 08:02:42 UTC
  • mfrom: (5390.3.3 faster-revert-593560)
  • Revision ID: pqm@pqm.ubuntu.com-20100901080242-esg62ody4frwmy66
(spiv) Avoid repeatedly calling self.target.all_file_ids() in
 InterTree.iter_changes. (Andrew Bennetts)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_http.py
1
 
# Copyright (C) 2005, 2006, 2007, 2008, 2009 Canonical Ltd
 
1
# Copyright (C) 2005-2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
44
44
    ui,
45
45
    urlutils,
46
46
    )
47
 
from bzrlib.symbol_versioning import (
48
 
    deprecated_in,
49
 
    )
50
47
from bzrlib.tests import (
 
48
    features,
51
49
    http_server,
52
50
    http_utils,
 
51
    test_server,
53
52
    )
54
53
from bzrlib.transport import (
55
54
    http,
61
60
    )
62
61
 
63
62
 
64
 
try:
 
63
if features.pycurl.available():
65
64
    from bzrlib.transport.http._pycurl import PyCurlTransport
66
 
    pycurl_present = True
67
 
except errors.DependencyNotPresent:
68
 
    pycurl_present = False
69
65
 
70
66
 
71
67
def load_tests(standard_tests, module, loader):
82
78
    transport_scenarios = [
83
79
        ('urllib', dict(_transport=_urllib.HttpTransport_urllib,
84
80
                        _server=http_server.HttpServer_urllib,
85
 
                        _qualified_prefix='http+urllib',)),
 
81
                        _url_protocol='http+urllib',)),
86
82
        ]
87
 
    if pycurl_present:
 
83
    if features.pycurl.available():
88
84
        transport_scenarios.append(
89
85
            ('pycurl', dict(_transport=PyCurlTransport,
90
86
                            _server=http_server.HttpServer_PyCurl,
91
 
                            _qualified_prefix='http+pycurl',)))
 
87
                            _url_protocol='http+pycurl',)))
92
88
    tests.multiply_tests(t_tests, transport_scenarios, result)
93
89
 
 
90
    protocol_scenarios = [
 
91
            ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
 
92
            ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
 
93
            ]
 
94
 
 
95
    # some tests are parametrized by the protocol version only
 
96
    p_tests, remaining_tests = tests.split_suite_by_condition(
 
97
        remaining_tests, tests.condition_isinstance((
 
98
                TestAuthOnRedirected,
 
99
                )))
 
100
    tests.multiply_tests(p_tests, protocol_scenarios, result)
 
101
 
94
102
    # each implementation tested with each HTTP version
95
103
    tp_tests, remaining_tests = tests.split_suite_by_condition(
96
104
        remaining_tests, tests.condition_isinstance((
105
113
                TestRanges,
106
114
                TestSpecificRequestHandler,
107
115
                )))
108
 
    protocol_scenarios = [
109
 
            ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
110
 
            ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
111
 
            ]
112
116
    tp_scenarios = tests.multiply_scenarios(transport_scenarios,
113
117
                                            protocol_scenarios)
114
118
    tests.multiply_tests(tp_tests, tp_scenarios, result)
115
119
 
 
120
    # proxy auth: each auth scheme on all http versions on all implementations.
 
121
    tppa_tests, remaining_tests = tests.split_suite_by_condition(
 
122
        remaining_tests, tests.condition_isinstance((
 
123
                TestProxyAuth,
 
124
                )))
 
125
    proxy_auth_scheme_scenarios = [
 
126
        ('basic', dict(_auth_server=http_utils.ProxyBasicAuthServer)),
 
127
        ('digest', dict(_auth_server=http_utils.ProxyDigestAuthServer)),
 
128
        ('basicdigest',
 
129
         dict(_auth_server=http_utils.ProxyBasicAndDigestAuthServer)),
 
130
        ]
 
131
    tppa_scenarios = tests.multiply_scenarios(tp_scenarios,
 
132
                                              proxy_auth_scheme_scenarios)
 
133
    tests.multiply_tests(tppa_tests, tppa_scenarios, result)
 
134
 
116
135
    # auth: each auth scheme on all http versions on all implementations.
117
136
    tpa_tests, remaining_tests = tests.split_suite_by_condition(
118
137
        remaining_tests, tests.condition_isinstance((
119
138
                TestAuth,
120
139
                )))
121
140
    auth_scheme_scenarios = [
122
 
        ('basic', dict(_auth_scheme='basic')),
123
 
        ('digest', dict(_auth_scheme='digest')),
 
141
        ('basic', dict(_auth_server=http_utils.HTTPBasicAuthServer)),
 
142
        ('digest', dict(_auth_server=http_utils.HTTPDigestAuthServer)),
 
143
        ('basicdigest',
 
144
         dict(_auth_server=http_utils.HTTPBasicAndDigestAuthServer)),
124
145
        ]
125
146
    tpa_scenarios = tests.multiply_scenarios(tp_scenarios,
126
 
        auth_scheme_scenarios)
 
147
                                             auth_scheme_scenarios)
127
148
    tests.multiply_tests(tpa_tests, tpa_scenarios, result)
128
149
 
129
 
    # activity: activity on all http versions on all implementations
 
150
    # activity: on all http[s] versions on all implementations
130
151
    tpact_tests, remaining_tests = tests.split_suite_by_condition(
131
152
        remaining_tests, tests.condition_isinstance((
132
153
                TestActivity,
133
154
                )))
134
155
    activity_scenarios = [
135
 
        ('http', dict(_activity_server=ActivityHTTPServer)),
 
156
        ('urllib,http', dict(_activity_server=ActivityHTTPServer,
 
157
                             _transport=_urllib.HttpTransport_urllib,)),
136
158
        ]
137
159
    if tests.HTTPSServerFeature.available():
138
160
        activity_scenarios.append(
139
 
            ('https', dict(_activity_server=ActivityHTTPSServer)))
140
 
    tpact_scenarios = tests.multiply_scenarios(tp_scenarios,
141
 
        activity_scenarios)
 
161
            ('urllib,https', dict(_activity_server=ActivityHTTPSServer,
 
162
                                  _transport=_urllib.HttpTransport_urllib,)),)
 
163
    if features.pycurl.available():
 
164
        activity_scenarios.append(
 
165
            ('pycurl,http', dict(_activity_server=ActivityHTTPServer,
 
166
                                 _transport=PyCurlTransport,)),)
 
167
        if tests.HTTPSServerFeature.available():
 
168
            from bzrlib.tests import (
 
169
                ssl_certs,
 
170
                )
 
171
            # FIXME: Until we have a better way to handle self-signed
 
172
            # certificates (like allowing them in a test specific
 
173
            # authentication.conf for example), we need some specialized pycurl
 
174
            # transport for tests.
 
175
            class HTTPS_pycurl_transport(PyCurlTransport):
 
176
 
 
177
                def __init__(self, base, _from_transport=None):
 
178
                    super(HTTPS_pycurl_transport, self).__init__(
 
179
                        base, _from_transport)
 
180
                    self.cabundle = str(ssl_certs.build_path('ca.crt'))
 
181
 
 
182
            activity_scenarios.append(
 
183
                ('pycurl,https', dict(_activity_server=ActivityHTTPSServer,
 
184
                                      _transport=HTTPS_pycurl_transport,)),)
 
185
 
 
186
    tpact_scenarios = tests.multiply_scenarios(activity_scenarios,
 
187
                                               protocol_scenarios)
142
188
    tests.multiply_tests(tpact_tests, tpact_scenarios, result)
143
189
 
144
190
    # No parametrization for the remaining tests
162
208
    It records the bytes sent to it, and replies with a 200.
163
209
    """
164
210
 
165
 
    def __init__(self, expect_body_tail=None):
 
211
    def __init__(self, expect_body_tail=None, scheme=''):
166
212
        """Constructor.
167
213
 
168
214
        :type expect_body_tail: str
173
219
        self.host = None
174
220
        self.port = None
175
221
        self.received_bytes = ''
176
 
 
177
 
    def setUp(self):
 
222
        self.scheme = scheme
 
223
 
 
224
    def get_url(self):
 
225
        return '%s://%s:%s/' % (self.scheme, self.host, self.port)
 
226
 
 
227
    def start_server(self):
178
228
        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
179
229
        self._sock.bind(('127.0.0.1', 0))
180
230
        self.host, self.port = self._sock.getsockname()
181
231
        self._ready = threading.Event()
182
 
        self._thread = threading.Thread(target=self._accept_read_and_reply)
183
 
        self._thread.setDaemon(True)
 
232
        self._thread = test_server.ThreadWithException(
 
233
            event=self._ready, target=self._accept_read_and_reply)
184
234
        self._thread.start()
185
 
        self._ready.wait(5)
 
235
        if 'threads' in tests.selftest_debug_flags:
 
236
            sys.stderr.write('Thread started: %s\n' % (self._thread.ident,))
 
237
        self._ready.wait()
186
238
 
187
239
    def _accept_read_and_reply(self):
188
240
        self._sock.listen(1)
189
241
        self._ready.set()
190
 
        self._sock.settimeout(5)
191
 
        try:
192
 
            conn, address = self._sock.accept()
193
 
            # On win32, the accepted connection will be non-blocking to start
194
 
            # with because we're using settimeout.
195
 
            conn.setblocking(True)
 
242
        conn, address = self._sock.accept()
 
243
        if self._expect_body_tail is not None:
196
244
            while not self.received_bytes.endswith(self._expect_body_tail):
197
245
                self.received_bytes += conn.recv(4096)
198
246
            conn.sendall('HTTP/1.1 200 OK\r\n')
199
 
        except socket.timeout:
200
 
            # Make sure the client isn't stuck waiting for us to e.g. accept.
 
247
        try:
201
248
            self._sock.close()
202
249
        except socket.error:
203
250
            # The client may have already closed the socket.
204
251
            pass
205
252
 
206
 
    def tearDown(self):
 
253
    def stop_server(self):
207
254
        try:
208
 
            self._sock.close()
 
255
            # Issue a fake connection to wake up the server and allow it to
 
256
            # finish quickly
 
257
            fake_conn = osutils.connect_socket((self.host, self.port))
 
258
            fake_conn.close()
209
259
        except socket.error:
210
260
            # We might have already closed it.  We don't care.
211
261
            pass
212
262
        self.host = None
213
263
        self.port = None
 
264
        self._thread.join()
 
265
        if 'threads' in tests.selftest_debug_flags:
 
266
            sys.stderr.write('Thread  joined: %s\n' % (self._thread.ident,))
214
267
 
215
268
 
216
269
class TestAuthHeader(tests.TestCase):
217
270
 
218
 
    def parse_header(self, header):
219
 
        ah =  _urllib2_wrappers.AbstractAuthHandler()
220
 
        return ah._parse_auth_header(header)
 
271
    def parse_header(self, header, auth_handler_class=None):
 
272
        if auth_handler_class is None:
 
273
            auth_handler_class = _urllib2_wrappers.AbstractAuthHandler
 
274
        self.auth_handler =  auth_handler_class()
 
275
        return self.auth_handler._parse_auth_header(header)
221
276
 
222
277
    def test_empty_header(self):
223
278
        scheme, remainder = self.parse_header('')
224
 
        self.assertEquals('', scheme)
 
279
        self.assertEqual('', scheme)
225
280
        self.assertIs(None, remainder)
226
281
 
227
282
    def test_negotiate_header(self):
228
283
        scheme, remainder = self.parse_header('Negotiate')
229
 
        self.assertEquals('negotiate', scheme)
 
284
        self.assertEqual('negotiate', scheme)
230
285
        self.assertIs(None, remainder)
231
286
 
232
287
    def test_basic_header(self):
233
288
        scheme, remainder = self.parse_header(
234
289
            'Basic realm="Thou should not pass"')
235
 
        self.assertEquals('basic', scheme)
236
 
        self.assertEquals('realm="Thou should not pass"', remainder)
 
290
        self.assertEqual('basic', scheme)
 
291
        self.assertEqual('realm="Thou should not pass"', remainder)
 
292
 
 
293
    def test_basic_extract_realm(self):
 
294
        scheme, remainder = self.parse_header(
 
295
            'Basic realm="Thou should not pass"',
 
296
            _urllib2_wrappers.BasicAuthHandler)
 
297
        match, realm = self.auth_handler.extract_realm(remainder)
 
298
        self.assertTrue(match is not None)
 
299
        self.assertEqual('Thou should not pass', realm)
237
300
 
238
301
    def test_digest_header(self):
239
302
        scheme, remainder = self.parse_header(
240
303
            'Digest realm="Thou should not pass"')
241
 
        self.assertEquals('digest', scheme)
242
 
        self.assertEquals('realm="Thou should not pass"', remainder)
 
304
        self.assertEqual('digest', scheme)
 
305
        self.assertEqual('realm="Thou should not pass"', remainder)
243
306
 
244
307
 
245
308
class TestHTTPServer(tests.TestCase):
250
313
 
251
314
            protocol_version = 'HTTP/0.1'
252
315
 
253
 
        server = http_server.HttpServer(BogusRequestHandler)
254
 
        try:
255
 
            self.assertRaises(httplib.UnknownProtocol,server.setUp)
256
 
        except:
257
 
            server.tearDown()
258
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
316
        self.assertRaises(httplib.UnknownProtocol,
 
317
                          http_server.HttpServer, BogusRequestHandler)
259
318
 
260
319
    def test_force_invalid_protocol(self):
261
 
        server = http_server.HttpServer(protocol_version='HTTP/0.1')
262
 
        try:
263
 
            self.assertRaises(httplib.UnknownProtocol,server.setUp)
264
 
        except:
265
 
            server.tearDown()
266
 
            self.fail('HTTP Server creation did not raise UnknownProtocol')
 
320
        self.assertRaises(httplib.UnknownProtocol,
 
321
                          http_server.HttpServer, protocol_version='HTTP/0.1')
267
322
 
268
323
    def test_server_start_and_stop(self):
269
324
        server = http_server.HttpServer()
270
 
        server.setUp()
271
 
        self.assertTrue(server._http_running)
272
 
        server.tearDown()
273
 
        self.assertFalse(server._http_running)
 
325
        self.addCleanup(server.stop_server)
 
326
        server.start_server()
 
327
        self.assertTrue(server.server is not None)
 
328
        self.assertTrue(server.server.serving is not None)
 
329
        self.assertTrue(server.server.serving)
274
330
 
275
331
    def test_create_http_server_one_zero(self):
276
332
        class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
278
334
            protocol_version = 'HTTP/1.0'
279
335
 
280
336
        server = http_server.HttpServer(RequestHandlerOneZero)
281
 
        server.setUp()
282
 
        self.addCleanup(server.tearDown)
283
 
        self.assertIsInstance(server._httpd, http_server.TestingHTTPServer)
 
337
        self.start_server(server)
 
338
        self.assertIsInstance(server.server, http_server.TestingHTTPServer)
284
339
 
285
340
    def test_create_http_server_one_one(self):
286
341
        class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
288
343
            protocol_version = 'HTTP/1.1'
289
344
 
290
345
        server = http_server.HttpServer(RequestHandlerOneOne)
291
 
        server.setUp()
292
 
        self.addCleanup(server.tearDown)
293
 
        self.assertIsInstance(server._httpd,
 
346
        self.start_server(server)
 
347
        self.assertIsInstance(server.server,
294
348
                              http_server.TestingThreadingHTTPServer)
295
349
 
296
350
    def test_create_http_server_force_one_one(self):
300
354
 
301
355
        server = http_server.HttpServer(RequestHandlerOneZero,
302
356
                                        protocol_version='HTTP/1.1')
303
 
        server.setUp()
304
 
        self.addCleanup(server.tearDown)
305
 
        self.assertIsInstance(server._httpd,
 
357
        self.start_server(server)
 
358
        self.assertIsInstance(server.server,
306
359
                              http_server.TestingThreadingHTTPServer)
307
360
 
308
361
    def test_create_http_server_force_one_zero(self):
312
365
 
313
366
        server = http_server.HttpServer(RequestHandlerOneOne,
314
367
                                        protocol_version='HTTP/1.0')
315
 
        server.setUp()
316
 
        self.addCleanup(server.tearDown)
317
 
        self.assertIsInstance(server._httpd,
 
368
        self.start_server(server)
 
369
        self.assertIsInstance(server.server,
318
370
                              http_server.TestingHTTPServer)
319
371
 
320
372
 
322
374
    """Test case to inherit from if pycurl is present"""
323
375
 
324
376
    def _get_pycurl_maybe(self):
325
 
        try:
326
 
            from bzrlib.transport.http._pycurl import PyCurlTransport
327
 
            return PyCurlTransport
328
 
        except errors.DependencyNotPresent:
329
 
            raise tests.TestSkipped('pycurl not present')
 
377
        self.requireFeature(features.pycurl)
 
378
        return PyCurlTransport
330
379
 
331
380
    _transport = property(_get_pycurl_maybe)
332
381
 
339
388
    def test_url_parsing(self):
340
389
        f = FakeManager()
341
390
        url = http.extract_auth('http://example.com', f)
342
 
        self.assertEquals('http://example.com', url)
343
 
        self.assertEquals(0, len(f.credentials))
 
391
        self.assertEqual('http://example.com', url)
 
392
        self.assertEqual(0, len(f.credentials))
344
393
        url = http.extract_auth(
345
 
            'http://user:pass@www.bazaar-vcs.org/bzr/bzr.dev', f)
346
 
        self.assertEquals('http://www.bazaar-vcs.org/bzr/bzr.dev', url)
347
 
        self.assertEquals(1, len(f.credentials))
348
 
        self.assertEquals([None, 'www.bazaar-vcs.org', 'user', 'pass'],
349
 
                          f.credentials[0])
 
394
            'http://user:pass@example.com/bzr/bzr.dev', f)
 
395
        self.assertEqual('http://example.com/bzr/bzr.dev', url)
 
396
        self.assertEqual(1, len(f.credentials))
 
397
        self.assertEqual([None, 'example.com', 'user', 'pass'],
 
398
                         f.credentials[0])
350
399
 
351
400
 
352
401
class TestHttpTransportUrls(tests.TestCase):
379
428
    def test_http_impl_urls(self):
380
429
        """There are servers which ask for particular clients to connect"""
381
430
        server = self._server()
 
431
        server.start_server()
382
432
        try:
383
 
            server.setUp()
384
433
            url = server.get_url()
385
 
            self.assertTrue(url.startswith('%s://' % self._qualified_prefix))
 
434
            self.assertTrue(url.startswith('%s://' % self._url_protocol))
386
435
        finally:
387
 
            server.tearDown()
 
436
            server.stop_server()
388
437
 
389
438
 
390
439
class TestHttps_pycurl(TestWithTransport_pycurl, tests.TestCase):
399
448
        https by supplying a fake version_info that do not
400
449
        support it.
401
450
        """
402
 
        try:
403
 
            import pycurl
404
 
        except ImportError:
405
 
            raise tests.TestSkipped('pycurl not present')
 
451
        self.requireFeature(features.pycurl)
 
452
        # Import the module locally now that we now it's available.
 
453
        pycurl = features.pycurl.module
406
454
 
407
 
        version_info_orig = pycurl.version_info
408
 
        try:
409
 
            # Now that we have pycurl imported, we can fake its version_info
410
 
            # This was taken from a windows pycurl without SSL
411
 
            # (thanks to bialix)
412
 
            pycurl.version_info = lambda : (2,
413
 
                                            '7.13.2',
414
 
                                            462082,
415
 
                                            'i386-pc-win32',
416
 
                                            2576,
417
 
                                            None,
418
 
                                            0,
419
 
                                            None,
420
 
                                            ('ftp', 'gopher', 'telnet',
421
 
                                             'dict', 'ldap', 'http', 'file'),
422
 
                                            None,
423
 
                                            0,
424
 
                                            None)
425
 
            self.assertRaises(errors.DependencyNotPresent, self._transport,
426
 
                              'https://launchpad.net')
427
 
        finally:
428
 
            # Restore the right function
429
 
            pycurl.version_info = version_info_orig
 
455
        self.overrideAttr(pycurl, 'version_info',
 
456
                          # Fake the pycurl version_info This was taken from
 
457
                          # a windows pycurl without SSL (thanks to bialix)
 
458
                          lambda : (2,
 
459
                                    '7.13.2',
 
460
                                    462082,
 
461
                                    'i386-pc-win32',
 
462
                                    2576,
 
463
                                    None,
 
464
                                    0,
 
465
                                    None,
 
466
                                    ('ftp', 'gopher', 'telnet',
 
467
                                     'dict', 'ldap', 'http', 'file'),
 
468
                                    None,
 
469
                                    0,
 
470
                                    None))
 
471
        self.assertRaises(errors.DependencyNotPresent, self._transport,
 
472
                          'https://launchpad.net')
430
473
 
431
474
 
432
475
class TestHTTPConnections(http_utils.TestCaseWithWebserver):
439
482
 
440
483
    def test_http_has(self):
441
484
        server = self.get_readonly_server()
442
 
        t = self._transport(server.get_url())
 
485
        t = self.get_readonly_transport()
443
486
        self.assertEqual(t.has('foo/bar'), True)
444
487
        self.assertEqual(len(server.logs), 1)
445
488
        self.assertContainsRe(server.logs[0],
447
490
 
448
491
    def test_http_has_not_found(self):
449
492
        server = self.get_readonly_server()
450
 
        t = self._transport(server.get_url())
 
493
        t = self.get_readonly_transport()
451
494
        self.assertEqual(t.has('not-found'), False)
452
495
        self.assertContainsRe(server.logs[1],
453
496
            r'"HEAD /not-found HTTP/1.." 404 - "-" "bzr/')
454
497
 
455
498
    def test_http_get(self):
456
499
        server = self.get_readonly_server()
457
 
        t = self._transport(server.get_url())
 
500
        t = self.get_readonly_transport()
458
501
        fp = t.get('foo/bar')
459
502
        self.assertEqualDiff(
460
503
            fp.read(),
483
526
    """Test registrations of various http implementations"""
484
527
 
485
528
    def test_http_registered(self):
486
 
        t = transport.get_transport('%s://foo.com/' % self._qualified_prefix)
 
529
        t = transport.get_transport('%s://foo.com/' % self._url_protocol)
487
530
        self.assertIsInstance(t, transport.Transport)
488
531
        self.assertIsInstance(t, self._transport)
489
532
 
491
534
class TestPost(tests.TestCase):
492
535
 
493
536
    def test_post_body_is_received(self):
494
 
        server = RecordingServer(expect_body_tail='end-of-body')
495
 
        server.setUp()
496
 
        self.addCleanup(server.tearDown)
497
 
        scheme = self._qualified_prefix
498
 
        url = '%s://%s:%s/' % (scheme, server.host, server.port)
499
 
        http_transport = self._transport(url)
 
537
        server = RecordingServer(expect_body_tail='end-of-body',
 
538
                                 scheme=self._url_protocol)
 
539
        self.start_server(server)
 
540
        url = server.get_url()
 
541
        # FIXME: needs a cleanup -- vila 20100611
 
542
        http_transport = transport.get_transport(url)
500
543
        code, response = http_transport._post('abc def end-of-body')
501
544
        self.assertTrue(
502
545
            server.received_bytes.startswith('POST /.bzr/smart HTTP/1.'))
546
589
    _req_handler_class = http_server.TestingHTTPRequestHandler
547
590
 
548
591
    def create_transport_readonly_server(self):
549
 
        return http_server.HttpServer(self._req_handler_class,
550
 
                                      protocol_version=self._protocol_version)
 
592
        server = http_server.HttpServer(self._req_handler_class,
 
593
                                        protocol_version=self._protocol_version)
 
594
        server._url_protocol = self._url_protocol
 
595
        return server
551
596
 
552
597
    def _testing_pycurl(self):
553
 
        return pycurl_present and self._transport == PyCurlTransport
 
598
        # TODO: This is duplicated for lots of the classes in this file
 
599
        return (features.pycurl.available()
 
600
                and self._transport == PyCurlTransport)
554
601
 
555
602
 
556
603
class WallRequestHandler(http_server.TestingHTTPRequestHandler):
557
604
    """Whatever request comes in, close the connection"""
558
605
 
559
 
    def handle_one_request(self):
 
606
    def _handle_one_request(self):
560
607
        """Handle a single HTTP request, by abruptly closing the connection"""
561
608
        self.close_connection = 1
562
609
 
567
614
    _req_handler_class = WallRequestHandler
568
615
 
569
616
    def test_http_has(self):
570
 
        server = self.get_readonly_server()
571
 
        t = self._transport(server.get_url())
 
617
        t = self.get_readonly_transport()
572
618
        # Unfortunately httplib (see HTTPResponse._read_status
573
619
        # for details) make no distinction between a closed
574
620
        # socket and badly formatted status line, so we can't
575
621
        # just test for ConnectionError, we have to test
576
 
        # InvalidHttpResponse too.
577
 
        self.assertRaises((errors.ConnectionError, errors.InvalidHttpResponse),
 
622
        # InvalidHttpResponse too. And pycurl may raise ConnectionReset
 
623
        # instead of ConnectionError too.
 
624
        self.assertRaises(( errors.ConnectionError, errors.ConnectionReset,
 
625
                            errors.InvalidHttpResponse),
578
626
                          t.has, 'foo/bar')
579
627
 
580
628
    def test_http_get(self):
581
 
        server = self.get_readonly_server()
582
 
        t = self._transport(server.get_url())
583
 
        self.assertRaises((errors.ConnectionError, errors.InvalidHttpResponse),
 
629
        t = self.get_readonly_transport()
 
630
        self.assertRaises((errors.ConnectionError, errors.ConnectionReset,
 
631
                           errors.InvalidHttpResponse),
584
632
                          t.get, 'foo/bar')
585
633
 
586
634
 
601
649
    _req_handler_class = BadStatusRequestHandler
602
650
 
603
651
    def test_http_has(self):
604
 
        server = self.get_readonly_server()
605
 
        t = self._transport(server.get_url())
 
652
        t = self.get_readonly_transport()
606
653
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
607
654
 
608
655
    def test_http_get(self):
609
 
        server = self.get_readonly_server()
610
 
        t = self._transport(server.get_url())
 
656
        t = self.get_readonly_transport()
611
657
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
612
658
 
613
659
 
618
664
        """Fakes handling a single HTTP request, returns a bad status"""
619
665
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
620
666
        self.wfile.write("Invalid status line\r\n")
 
667
        # If we don't close the connection pycurl will hang. Since this is a
 
668
        # stress test we don't *have* to respect the protocol, but we don't
 
669
        # have to sabotage it too much either.
 
670
        self.close_connection = True
621
671
        return False
622
672
 
623
673
 
629
679
 
630
680
    _req_handler_class = InvalidStatusRequestHandler
631
681
 
632
 
    def test_http_has(self):
633
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
634
 
            raise tests.KnownFailure(
635
 
                'pycurl hangs if the server send back garbage')
636
 
        super(TestInvalidStatusServer, self).test_http_has()
637
 
 
638
 
    def test_http_get(self):
639
 
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
640
 
            raise tests.KnownFailure(
641
 
                'pycurl hangs if the server send back garbage')
642
 
        super(TestInvalidStatusServer, self).test_http_get()
643
 
 
644
682
 
645
683
class BadProtocolRequestHandler(http_server.TestingHTTPRequestHandler):
646
684
    """Whatever request comes in, returns a bad protocol version"""
662
700
    _req_handler_class = BadProtocolRequestHandler
663
701
 
664
702
    def setUp(self):
665
 
        if pycurl_present and self._transport == PyCurlTransport:
 
703
        if self._testing_pycurl():
666
704
            raise tests.TestNotApplicable(
667
705
                "pycurl doesn't check the protocol version")
668
706
        super(TestBadProtocolServer, self).setUp()
669
707
 
670
708
    def test_http_has(self):
671
 
        server = self.get_readonly_server()
672
 
        t = self._transport(server.get_url())
 
709
        t = self.get_readonly_transport()
673
710
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
674
711
 
675
712
    def test_http_get(self):
676
 
        server = self.get_readonly_server()
677
 
        t = self._transport(server.get_url())
 
713
        t = self.get_readonly_transport()
678
714
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
679
715
 
680
716
 
694
730
    _req_handler_class = ForbiddenRequestHandler
695
731
 
696
732
    def test_http_has(self):
697
 
        server = self.get_readonly_server()
698
 
        t = self._transport(server.get_url())
 
733
        t = self.get_readonly_transport()
699
734
        self.assertRaises(errors.TransportError, t.has, 'foo/bar')
700
735
 
701
736
    def test_http_get(self):
702
 
        server = self.get_readonly_server()
703
 
        t = self._transport(server.get_url())
 
737
        t = self.get_readonly_transport()
704
738
        self.assertRaises(errors.TransportError, t.get, 'foo/bar')
705
739
 
706
740
 
712
746
        self.assertEqual(None, server.host)
713
747
        self.assertEqual(None, server.port)
714
748
 
715
 
    def test_setUp_and_tearDown(self):
 
749
    def test_setUp_and_stop(self):
716
750
        server = RecordingServer(expect_body_tail=None)
717
 
        server.setUp()
 
751
        server.start_server()
718
752
        try:
719
753
            self.assertNotEqual(None, server.host)
720
754
            self.assertNotEqual(None, server.port)
721
755
        finally:
722
 
            server.tearDown()
 
756
            server.stop_server()
723
757
        self.assertEqual(None, server.host)
724
758
        self.assertEqual(None, server.port)
725
759
 
726
760
    def test_send_receive_bytes(self):
727
 
        server = RecordingServer(expect_body_tail='c')
728
 
        server.setUp()
729
 
        self.addCleanup(server.tearDown)
 
761
        server = RecordingServer(expect_body_tail='c', scheme='http')
 
762
        self.start_server(server)
730
763
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
731
764
        sock.connect((server.host, server.port))
732
765
        sock.sendall('abc')
746
779
        self.build_tree_contents([('a', '0123456789')],)
747
780
 
748
781
    def test_readv(self):
749
 
        server = self.get_readonly_server()
750
 
        t = self._transport(server.get_url())
 
782
        t = self.get_readonly_transport()
751
783
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
752
784
        self.assertEqual(l[0], (0, '0'))
753
785
        self.assertEqual(l[1], (1, '1'))
755
787
        self.assertEqual(l[3], (9, '9'))
756
788
 
757
789
    def test_readv_out_of_order(self):
758
 
        server = self.get_readonly_server()
759
 
        t = self._transport(server.get_url())
 
790
        t = self.get_readonly_transport()
760
791
        l = list(t.readv('a', ((1, 1), (9, 1), (0, 1), (3, 2))))
761
792
        self.assertEqual(l[0], (1, '1'))
762
793
        self.assertEqual(l[1], (9, '9'))
764
795
        self.assertEqual(l[3], (3, '34'))
765
796
 
766
797
    def test_readv_invalid_ranges(self):
767
 
        server = self.get_readonly_server()
768
 
        t = self._transport(server.get_url())
 
798
        t = self.get_readonly_transport()
769
799
 
770
800
        # This is intentionally reading off the end of the file
771
801
        # since we are sure that it cannot get there
779
809
 
780
810
    def test_readv_multiple_get_requests(self):
781
811
        server = self.get_readonly_server()
782
 
        t = self._transport(server.get_url())
 
812
        t = self.get_readonly_transport()
783
813
        # force transport to issue multiple requests
784
814
        t._max_readv_combine = 1
785
815
        t._max_get_ranges = 1
793
823
 
794
824
    def test_readv_get_max_size(self):
795
825
        server = self.get_readonly_server()
796
 
        t = self._transport(server.get_url())
 
826
        t = self.get_readonly_transport()
797
827
        # force transport to issue multiple requests by limiting the number of
798
828
        # bytes by request. Note that this apply to coalesced offsets only, a
799
829
        # single range will keep its size even if bigger than the limit.
808
838
 
809
839
    def test_complete_readv_leave_pipe_clean(self):
810
840
        server = self.get_readonly_server()
811
 
        t = self._transport(server.get_url())
 
841
        t = self.get_readonly_transport()
812
842
        # force transport to issue multiple requests
813
843
        t._get_max_size = 2
814
844
        l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
819
849
 
820
850
    def test_incomplete_readv_leave_pipe_clean(self):
821
851
        server = self.get_readonly_server()
822
 
        t = self._transport(server.get_url())
 
852
        t = self.get_readonly_transport()
823
853
        # force transport to issue multiple requests
824
854
        t._get_max_size = 2
825
855
        # Don't collapse readv results into a list so that we leave unread
975
1005
 
976
1006
    def test_readv_with_short_reads(self):
977
1007
        server = self.get_readonly_server()
978
 
        t = self._transport(server.get_url())
 
1008
        t = self.get_readonly_transport()
979
1009
        # Force separate ranges for each offset
980
1010
        t._bytes_to_read_before_seek = 0
981
1011
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
1032
1062
        return LimitedRangeHTTPServer(range_limit=self.range_limit,
1033
1063
                                      protocol_version=self._protocol_version)
1034
1064
 
1035
 
    def get_transport(self):
1036
 
        return self._transport(self.get_readonly_server().get_url())
1037
 
 
1038
1065
    def setUp(self):
1039
1066
        http_utils.TestCaseWithWebserver.setUp(self)
1040
1067
        # We need to manipulate ranges that correspond to real chunks in the
1044
1071
        self.build_tree_contents([('a', content)],)
1045
1072
 
1046
1073
    def test_few_ranges(self):
1047
 
        t = self.get_transport()
 
1074
        t = self.get_readonly_transport()
1048
1075
        l = list(t.readv('a', ((0, 4), (1024, 4), )))
1049
1076
        self.assertEqual(l[0], (0, '0000'))
1050
1077
        self.assertEqual(l[1], (1024, '0001'))
1051
1078
        self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1052
1079
 
1053
1080
    def test_more_ranges(self):
1054
 
        t = self.get_transport()
 
1081
        t = self.get_readonly_transport()
1055
1082
        l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1056
1083
        self.assertEqual(l[0], (0, '0000'))
1057
1084
        self.assertEqual(l[1], (1024, '0001'))
1071
1098
    def setUp(self):
1072
1099
        tests.TestCase.setUp(self)
1073
1100
        self._old_env = {}
1074
 
 
1075
 
    def tearDown(self):
1076
 
        self._restore_env()
1077
 
        tests.TestCase.tearDown(self)
 
1101
        self.addCleanup(self._restore_env)
1078
1102
 
1079
1103
    def _install_env(self, env):
1080
1104
        for name, value in env.iteritems():
1111
1135
    """
1112
1136
 
1113
1137
    # FIXME: We don't have an https server available, so we don't
1114
 
    # test https connections.
 
1138
    # test https connections. --vila toolongago
1115
1139
 
1116
1140
    def setUp(self):
1117
1141
        super(TestProxyHttpServer, self).setUp()
 
1142
        self.transport_secondary_server = http_utils.ProxyServer
1118
1143
        self.build_tree_contents([('foo', 'contents of foo\n'),
1119
1144
                                  ('foo-proxied', 'proxied contents of foo\n')])
1120
1145
        # Let's setup some attributes for tests
1121
 
        self.server = self.get_readonly_server()
1122
 
        self.proxy_address = '%s:%d' % (self.server.host, self.server.port)
 
1146
        server = self.get_readonly_server()
 
1147
        self.server_host_port = '%s:%d' % (server.host, server.port)
1123
1148
        if self._testing_pycurl():
1124
1149
            # Oh my ! pycurl does not check for the port as part of
1125
1150
            # no_proxy :-( So we just test the host part
1126
 
            self.no_proxy_host = 'localhost'
 
1151
            self.no_proxy_host = server.host
1127
1152
        else:
1128
 
            self.no_proxy_host = self.proxy_address
 
1153
            self.no_proxy_host = self.server_host_port
1129
1154
        # The secondary server is the proxy
1130
 
        self.proxy = self.get_secondary_server()
1131
 
        self.proxy_url = self.proxy.get_url()
 
1155
        self.proxy_url = self.get_secondary_url()
1132
1156
        self._old_env = {}
1133
1157
 
1134
1158
    def _testing_pycurl(self):
1135
 
        return pycurl_present and self._transport == PyCurlTransport
1136
 
 
1137
 
    def create_transport_secondary_server(self):
1138
 
        """Creates an http server that will serve files with
1139
 
        '-proxied' appended to their names.
1140
 
        """
1141
 
        return http_utils.ProxyServer(protocol_version=self._protocol_version)
 
1159
        # TODO: This is duplicated for lots of the classes in this file
 
1160
        return (features.pycurl.available()
 
1161
                and self._transport == PyCurlTransport)
1142
1162
 
1143
1163
    def _install_env(self, env):
1144
1164
        for name, value in env.iteritems():
1150
1170
 
1151
1171
    def proxied_in_env(self, env):
1152
1172
        self._install_env(env)
1153
 
        url = self.server.get_url()
1154
 
        t = self._transport(url)
 
1173
        t = self.get_readonly_transport()
1155
1174
        try:
1156
1175
            self.assertEqual('proxied contents of foo\n', t.get('foo').read())
1157
1176
        finally:
1159
1178
 
1160
1179
    def not_proxied_in_env(self, env):
1161
1180
        self._install_env(env)
1162
 
        url = self.server.get_url()
1163
 
        t = self._transport(url)
 
1181
        t = self.get_readonly_transport()
1164
1182
        try:
1165
1183
            self.assertEqual('contents of foo\n', t.get('foo').read())
1166
1184
        finally:
1208
1226
            # pycurl *ignores* invalid proxy env variables. If that ever change
1209
1227
            # in the future, this test will fail indicating that pycurl do not
1210
1228
            # ignore anymore such variables.
1211
 
            self.not_proxied_in_env({'http_proxy': self.proxy_address})
 
1229
            self.not_proxied_in_env({'http_proxy': self.server_host_port})
1212
1230
        else:
1213
1231
            self.assertRaises(errors.InvalidURL,
1214
1232
                              self.proxied_in_env,
1215
 
                              {'http_proxy': self.proxy_address})
 
1233
                              {'http_proxy': self.server_host_port})
1216
1234
 
1217
1235
 
1218
1236
class TestRanges(http_utils.TestCaseWithWebserver):
1221
1239
    def setUp(self):
1222
1240
        http_utils.TestCaseWithWebserver.setUp(self)
1223
1241
        self.build_tree_contents([('a', '0123456789')],)
1224
 
        server = self.get_readonly_server()
1225
 
        self.transport = self._transport(server.get_url())
1226
1242
 
1227
1243
    def create_transport_readonly_server(self):
1228
1244
        return http_server.HttpServer(protocol_version=self._protocol_version)
1229
1245
 
1230
1246
    def _file_contents(self, relpath, ranges):
 
1247
        t = self.get_readonly_transport()
1231
1248
        offsets = [ (start, end - start + 1) for start, end in ranges]
1232
 
        coalesce = self.transport._coalesce_offsets
 
1249
        coalesce = t._coalesce_offsets
1233
1250
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1234
 
        code, data = self.transport._get(relpath, coalesced)
 
1251
        code, data = t._get(relpath, coalesced)
1235
1252
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1236
1253
        for start, end in ranges:
1237
1254
            data.seek(start)
1238
1255
            yield data.read(end - start + 1)
1239
1256
 
1240
1257
    def _file_tail(self, relpath, tail_amount):
1241
 
        code, data = self.transport._get(relpath, [], tail_amount)
 
1258
        t = self.get_readonly_transport()
 
1259
        code, data = t._get(relpath, [], tail_amount)
1242
1260
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1243
1261
        data.seek(-tail_amount, 2)
1244
1262
        return data.read(tail_amount)
1263
1281
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1264
1282
    """Test redirection between http servers."""
1265
1283
 
1266
 
    def create_transport_secondary_server(self):
1267
 
        """Create the secondary server redirecting to the primary server"""
1268
 
        new = self.get_readonly_server()
1269
 
 
1270
 
        redirecting = http_utils.HTTPServerRedirecting(
1271
 
            protocol_version=self._protocol_version)
1272
 
        redirecting.redirect_to(new.host, new.port)
1273
 
        return redirecting
1274
 
 
1275
1284
    def setUp(self):
1276
1285
        super(TestHTTPRedirections, self).setUp()
1277
1286
        self.build_tree_contents([('a', '0123456789'),
1278
1287
                                  ('bundle',
1279
1288
                                  '# Bazaar revision bundle v0.9\n#\n')
1280
1289
                                  ],)
1281
 
        # The requests to the old server will be redirected to the new server
1282
 
        self.old_transport = self._transport(self.old_server.get_url())
1283
1290
 
1284
1291
    def test_redirected(self):
1285
 
        self.assertRaises(errors.RedirectRequested, self.old_transport.get, 'a')
1286
 
        t = self._transport(self.new_server.get_url())
1287
 
        self.assertEqual('0123456789', t.get('a').read())
1288
 
 
1289
 
    def test_read_redirected_bundle_from_url(self):
1290
 
        from bzrlib.bundle import read_bundle_from_url
1291
 
        url = self.old_transport.abspath('bundle')
1292
 
        bundle = self.applyDeprecated(deprecated_in((1, 12, 0)),
1293
 
                read_bundle_from_url, url)
1294
 
        # If read_bundle_from_url was successful we get an empty bundle
1295
 
        self.assertEqual([], bundle.revisions)
 
1292
        self.assertRaises(errors.RedirectRequested,
 
1293
                          self.get_old_transport().get, 'a')
 
1294
        self.assertEqual('0123456789', self.get_new_transport().get('a').read())
1296
1295
 
1297
1296
 
1298
1297
class RedirectedRequest(_urllib2_wrappers.Request):
1307
1306
        # Since the tests using this class will replace
1308
1307
        # _urllib2_wrappers.Request, we can't just call the base class __init__
1309
1308
        # or we'll loop.
1310
 
        RedirectedRequest.init_orig(self, method, url, args, kwargs)
 
1309
        RedirectedRequest.init_orig(self, method, url, *args, **kwargs)
1311
1310
        self.follow_redirections = True
1312
1311
 
1313
1312
 
 
1313
def install_redirected_request(test):
 
1314
    test.overrideAttr(_urllib2_wrappers, 'Request', RedirectedRequest)
 
1315
 
 
1316
 
 
1317
def cleanup_http_redirection_connections(test):
 
1318
    # Some sockets are opened but never seen by _urllib, so we trap them at
 
1319
    # the _urllib2_wrappers level to be able to clean them up.
 
1320
    def socket_disconnect(sock):
 
1321
        try:
 
1322
            sock.shutdown(socket.SHUT_RDWR)
 
1323
            sock.close()
 
1324
        except socket.error:
 
1325
            pass
 
1326
    def connect(connection):
 
1327
        test.http_connect_orig(connection)
 
1328
        test.addCleanup(socket_disconnect, connection.sock)
 
1329
    test.http_connect_orig = test.overrideAttr(
 
1330
        _urllib2_wrappers.HTTPConnection, 'connect', connect)
 
1331
    def connect(connection):
 
1332
        test.https_connect_orig(connection)
 
1333
        test.addCleanup(socket_disconnect, connection.sock)
 
1334
    test.https_connect_orig = test.overrideAttr(
 
1335
        _urllib2_wrappers.HTTPSConnection, 'connect', connect)
 
1336
 
 
1337
 
1314
1338
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1315
1339
    """Test redirections.
1316
1340
 
1326
1350
    """
1327
1351
 
1328
1352
    def setUp(self):
1329
 
        if pycurl_present and self._transport == PyCurlTransport:
 
1353
        if (features.pycurl.available()
 
1354
            and self._transport == PyCurlTransport):
1330
1355
            raise tests.TestNotApplicable(
1331
 
                "pycurl doesn't redirect silently annymore")
 
1356
                "pycurl doesn't redirect silently anymore")
1332
1357
        super(TestHTTPSilentRedirections, self).setUp()
1333
 
        self.setup_redirected_request()
1334
 
        self.addCleanup(self.cleanup_redirected_request)
 
1358
        install_redirected_request(self)
 
1359
        cleanup_http_redirection_connections(self)
1335
1360
        self.build_tree_contents([('a','a'),
1336
1361
                                  ('1/',),
1337
1362
                                  ('1/a', 'redirected once'),
1345
1370
                                  ('5/a', 'redirected 5 times'),
1346
1371
                                  ],)
1347
1372
 
1348
 
        self.old_transport = self._transport(self.old_server.get_url())
1349
 
 
1350
 
    def setup_redirected_request(self):
1351
 
        self.original_class = _urllib2_wrappers.Request
1352
 
        _urllib2_wrappers.Request = RedirectedRequest
1353
 
 
1354
 
    def cleanup_redirected_request(self):
1355
 
        _urllib2_wrappers.Request = self.original_class
1356
 
 
1357
 
    def create_transport_secondary_server(self):
1358
 
        """Create the secondary server, redirections are defined in the tests"""
1359
 
        return http_utils.HTTPServerRedirecting(
1360
 
            protocol_version=self._protocol_version)
1361
 
 
1362
1373
    def test_one_redirection(self):
1363
 
        t = self.old_transport
1364
 
 
1365
 
        req = RedirectedRequest('GET', t.abspath('a'))
1366
 
        req.follow_redirections = True
 
1374
        t = self.get_old_transport()
 
1375
        req = RedirectedRequest('GET', t._remote_path('a'))
1367
1376
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1368
1377
                                       self.new_server.port)
1369
1378
        self.old_server.redirections = \
1370
1379
            [('(.*)', r'%s/1\1' % (new_prefix), 301),]
1371
 
        self.assertEquals('redirected once',t._perform(req).read())
 
1380
        self.assertEqual('redirected once', t._perform(req).read())
1372
1381
 
1373
1382
    def test_five_redirections(self):
1374
 
        t = self.old_transport
1375
 
 
1376
 
        req = RedirectedRequest('GET', t.abspath('a'))
1377
 
        req.follow_redirections = True
 
1383
        t = self.get_old_transport()
 
1384
        req = RedirectedRequest('GET', t._remote_path('a'))
1378
1385
        old_prefix = 'http://%s:%s' % (self.old_server.host,
1379
1386
                                       self.old_server.port)
1380
1387
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1386
1393
            ('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1387
1394
            ('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1388
1395
            ]
1389
 
        self.assertEquals('redirected 5 times',t._perform(req).read())
 
1396
        self.assertEqual('redirected 5 times', t._perform(req).read())
1390
1397
 
1391
1398
 
1392
1399
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1395
1402
    def setUp(self):
1396
1403
        super(TestDoCatchRedirections, self).setUp()
1397
1404
        self.build_tree_contents([('a', '0123456789'),],)
1398
 
 
1399
 
        self.old_transport = self._transport(self.old_server.get_url())
1400
 
 
1401
 
    def get_a(self, transport):
1402
 
        return transport.get('a')
 
1405
        cleanup_http_redirection_connections(self)
 
1406
 
 
1407
        self.old_transport = self.get_old_transport()
 
1408
 
 
1409
    def get_a(self, t):
 
1410
        return t.get('a')
1403
1411
 
1404
1412
    def test_no_redirection(self):
1405
 
        t = self._transport(self.new_server.get_url())
 
1413
        t = self.get_new_transport()
1406
1414
 
1407
1415
        # We use None for redirected so that we fail if redirected
1408
 
        self.assertEquals('0123456789',
1409
 
                          transport.do_catching_redirections(
 
1416
        self.assertEqual('0123456789',
 
1417
                         transport.do_catching_redirections(
1410
1418
                self.get_a, t, None).read())
1411
1419
 
1412
1420
    def test_one_redirection(self):
1413
1421
        self.redirections = 0
1414
1422
 
1415
 
        def redirected(transport, exception, redirection_notice):
 
1423
        def redirected(t, exception, redirection_notice):
1416
1424
            self.redirections += 1
1417
 
            dir, file = urlutils.split(exception.target)
1418
 
            return self._transport(dir)
 
1425
            redirected_t = t._redirected_to(exception.source, exception.target)
 
1426
            return redirected_t
1419
1427
 
1420
 
        self.assertEquals('0123456789',
1421
 
                          transport.do_catching_redirections(
 
1428
        self.assertEqual('0123456789',
 
1429
                         transport.do_catching_redirections(
1422
1430
                self.get_a, self.old_transport, redirected).read())
1423
 
        self.assertEquals(1, self.redirections)
 
1431
        self.assertEqual(1, self.redirections)
1424
1432
 
1425
1433
    def test_redirection_loop(self):
1426
1434
 
1440
1448
 
1441
1449
    _auth_header = 'Authorization'
1442
1450
    _password_prompt_prefix = ''
 
1451
    _username_prompt_prefix = ''
 
1452
    # Set by load_tests
 
1453
    _auth_server = None
1443
1454
 
1444
1455
    def setUp(self):
1445
1456
        super(TestAuth, self).setUp()
1448
1459
                                  ('b', 'contents of b\n'),])
1449
1460
 
1450
1461
    def create_transport_readonly_server(self):
1451
 
        if self._auth_scheme == 'basic':
1452
 
            server = http_utils.HTTPBasicAuthServer(
1453
 
                protocol_version=self._protocol_version)
1454
 
        else:
1455
 
            if self._auth_scheme != 'digest':
1456
 
                raise AssertionError('Unknown auth scheme: %r'
1457
 
                                     % self._auth_scheme)
1458
 
            server = http_utils.HTTPDigestAuthServer(
1459
 
                protocol_version=self._protocol_version)
 
1462
        server = self._auth_server(protocol_version=self._protocol_version)
 
1463
        server._url_protocol = self._url_protocol
1460
1464
        return server
1461
1465
 
1462
1466
    def _testing_pycurl(self):
1463
 
        return pycurl_present and self._transport == PyCurlTransport
 
1467
        # TODO: This is duplicated for lots of the classes in this file
 
1468
        return (features.pycurl.available()
 
1469
                and self._transport == PyCurlTransport)
1464
1470
 
1465
1471
    def get_user_url(self, user, password):
1466
1472
        """Build an url embedding user and password"""
1474
1480
        return url
1475
1481
 
1476
1482
    def get_user_transport(self, user, password):
1477
 
        return self._transport(self.get_user_url(user, password))
 
1483
        t = transport.get_transport(self.get_user_url(user, password))
 
1484
        return t
1478
1485
 
1479
1486
    def test_no_user(self):
1480
1487
        self.server.add_user('joe', 'foo')
1514
1521
        # initial 'who are you' and 'this is not you, who are you')
1515
1522
        self.assertEqual(2, self.server.auth_required_errors)
1516
1523
 
 
1524
    def test_prompt_for_username(self):
 
1525
        if self._testing_pycurl():
 
1526
            raise tests.TestNotApplicable(
 
1527
                'pycurl cannot prompt, it handles auth by embedding'
 
1528
                ' user:pass in urls only')
 
1529
 
 
1530
        self.server.add_user('joe', 'foo')
 
1531
        t = self.get_user_transport(None, None)
 
1532
        stdout = tests.StringIOWrapper()
 
1533
        stderr = tests.StringIOWrapper()
 
1534
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
 
1535
                                            stdout=stdout, stderr=stderr)
 
1536
        self.assertEqual('contents of a\n',t.get('a').read())
 
1537
        # stdin should be empty
 
1538
        self.assertEqual('', ui.ui_factory.stdin.readline())
 
1539
        stderr.seek(0)
 
1540
        expected_prompt = self._expected_username_prompt(t._unqualified_scheme)
 
1541
        self.assertEqual(expected_prompt, stderr.read(len(expected_prompt)))
 
1542
        self.assertEqual('', stdout.getvalue())
 
1543
        self._check_password_prompt(t._unqualified_scheme, 'joe',
 
1544
                                    stderr.readline())
 
1545
 
1517
1546
    def test_prompt_for_password(self):
1518
1547
        if self._testing_pycurl():
1519
1548
            raise tests.TestNotApplicable(
1523
1552
        self.server.add_user('joe', 'foo')
1524
1553
        t = self.get_user_transport('joe', None)
1525
1554
        stdout = tests.StringIOWrapper()
1526
 
        ui.ui_factory = tests.TestUIFactory(stdin='foo\n', stdout=stdout)
1527
 
        self.assertEqual('contents of a\n',t.get('a').read())
 
1555
        stderr = tests.StringIOWrapper()
 
1556
        ui.ui_factory = tests.TestUIFactory(stdin='foo\n',
 
1557
                                            stdout=stdout, stderr=stderr)
 
1558
        self.assertEqual('contents of a\n', t.get('a').read())
1528
1559
        # stdin should be empty
1529
1560
        self.assertEqual('', ui.ui_factory.stdin.readline())
1530
1561
        self._check_password_prompt(t._unqualified_scheme, 'joe',
1531
 
                                    stdout.getvalue())
 
1562
                                    stderr.getvalue())
 
1563
        self.assertEqual('', stdout.getvalue())
1532
1564
        # And we shouldn't prompt again for a different request
1533
1565
        # against the same transport.
1534
1566
        self.assertEqual('contents of b\n',t.get('b').read())
1544
1576
                              % (scheme.upper(),
1545
1577
                                 user, self.server.host, self.server.port,
1546
1578
                                 self.server.auth_realm)))
1547
 
        self.assertEquals(expected_prompt, actual_prompt)
 
1579
        self.assertEqual(expected_prompt, actual_prompt)
 
1580
 
 
1581
    def _expected_username_prompt(self, scheme):
 
1582
        return (self._username_prompt_prefix
 
1583
                + "%s %s:%d, Realm: '%s' username: " % (scheme.upper(),
 
1584
                                 self.server.host, self.server.port,
 
1585
                                 self.server.auth_realm))
1548
1586
 
1549
1587
    def test_no_prompt_for_password_when_using_auth_config(self):
1550
1588
        if self._testing_pycurl():
1558
1596
        self.server.add_user(user, password)
1559
1597
        t = self.get_user_transport(user, None)
1560
1598
        ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1561
 
                                            stdout=tests.StringIOWrapper())
 
1599
                                            stderr=tests.StringIOWrapper())
1562
1600
        # Create a minimal config file with the right password
1563
1601
        conf = config.AuthenticationConfig()
1564
1602
        conf._get_config().update(
1592
1630
        self.assertEqual(1, self.server.auth_required_errors)
1593
1631
 
1594
1632
    def test_changing_nonce(self):
1595
 
        if self._auth_scheme != 'digest':
1596
 
            raise tests.TestNotApplicable('HTTP auth digest only test')
 
1633
        if self._auth_server not in (http_utils.HTTPDigestAuthServer,
 
1634
                                     http_utils.ProxyDigestAuthServer):
 
1635
            raise tests.TestNotApplicable('HTTP/proxy auth digest only test')
1597
1636
        if self._testing_pycurl():
1598
1637
            raise tests.KnownFailure(
1599
1638
                'pycurl does not handle a nonce change')
1617
1656
    """Test proxy authentication schemes."""
1618
1657
 
1619
1658
    _auth_header = 'Proxy-authorization'
1620
 
    _password_prompt_prefix='Proxy '
 
1659
    _password_prompt_prefix = 'Proxy '
 
1660
    _username_prompt_prefix = 'Proxy '
1621
1661
 
1622
1662
    def setUp(self):
1623
1663
        super(TestProxyAuth, self).setUp()
1630
1670
                                  ('b-proxied', 'contents of b\n'),
1631
1671
                                  ])
1632
1672
 
1633
 
    def create_transport_readonly_server(self):
1634
 
        if self._auth_scheme == 'basic':
1635
 
            server = http_utils.ProxyBasicAuthServer(
1636
 
                protocol_version=self._protocol_version)
1637
 
        else:
1638
 
            if self._auth_scheme != 'digest':
1639
 
                raise AssertionError('Unknown auth scheme: %r'
1640
 
                                     % self._auth_scheme)
1641
 
            server = http_utils.ProxyDigestAuthServer(
1642
 
                protocol_version=self._protocol_version)
1643
 
        return server
1644
 
 
1645
1673
    def get_user_transport(self, user, password):
1646
1674
        self._install_env({'all_proxy': self.get_user_url(user, password)})
1647
 
        return self._transport(self.server.get_url())
 
1675
        return TestAuth.get_user_transport(self, user, password)
1648
1676
 
1649
1677
    def _install_env(self, env):
1650
1678
        for name, value in env.iteritems():
1677
1705
        self.readfile = StringIO(socket_read_content)
1678
1706
        self.writefile = StringIO()
1679
1707
        self.writefile.close = lambda: None
 
1708
        self.close = lambda: None
1680
1709
 
1681
1710
    def makefile(self, mode='r', bufsize=None):
1682
1711
        if 'r' in mode:
1692
1721
        # We use the VFS layer as part of HTTP tunnelling tests.
1693
1722
        self._captureVar('BZR_NO_SMART_VFS', None)
1694
1723
        self.transport_readonly_server = http_utils.HTTPServerWithSmarts
 
1724
        self.http_server = self.get_readonly_server()
1695
1725
 
1696
1726
    def create_transport_readonly_server(self):
1697
 
        return http_utils.HTTPServerWithSmarts(
 
1727
        server = http_utils.HTTPServerWithSmarts(
1698
1728
            protocol_version=self._protocol_version)
 
1729
        server._url_protocol = self._url_protocol
 
1730
        return server
1699
1731
 
1700
1732
    def test_open_bzrdir(self):
1701
1733
        branch = self.make_branch('relpath')
1702
 
        http_server = self.get_readonly_server()
1703
 
        url = http_server.get_url() + 'relpath'
 
1734
        url = self.http_server.get_url() + 'relpath'
1704
1735
        bd = bzrdir.BzrDir.open(url)
 
1736
        self.addCleanup(bd.transport.disconnect)
1705
1737
        self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1706
1738
 
1707
1739
    def test_bulk_data(self):
1709
1741
        # The 'readv' command in the smart protocol both sends and receives
1710
1742
        # bulk data, so we use that.
1711
1743
        self.build_tree(['data-file'])
1712
 
        http_server = self.get_readonly_server()
1713
 
        http_transport = self._transport(http_server.get_url())
 
1744
        http_transport = transport.get_transport(self.http_server.get_url())
1714
1745
        medium = http_transport.get_smart_medium()
1715
1746
        # Since we provide the medium, the url below will be mostly ignored
1716
1747
        # during the test, as long as the path is '/'.
1724
1755
        post_body = 'hello\n'
1725
1756
        expected_reply_body = 'ok\x012\n'
1726
1757
 
1727
 
        http_server = self.get_readonly_server()
1728
 
        http_transport = self._transport(http_server.get_url())
 
1758
        http_transport = transport.get_transport(self.http_server.get_url())
1729
1759
        medium = http_transport.get_smart_medium()
1730
1760
        response = medium.send_http_smart_request(post_body)
1731
1761
        reply_body = response.read()
1732
1762
        self.assertEqual(expected_reply_body, reply_body)
1733
1763
 
1734
1764
    def test_smart_http_server_post_request_handler(self):
1735
 
        httpd = self.get_readonly_server()._get_httpd()
 
1765
        httpd = self.http_server.server
1736
1766
 
1737
1767
        socket = SampleSocket(
1738
1768
            'POST /.bzr/smart %s \r\n' % self._protocol_version
1770
1800
 
1771
1801
    def test_probe_smart_server(self):
1772
1802
        """Test error handling against server refusing smart requests."""
1773
 
        server = self.get_readonly_server()
1774
 
        t = self._transport(server.get_url())
 
1803
        t = self.get_readonly_transport()
1775
1804
        # No need to build a valid smart request here, the server will not even
1776
1805
        # try to interpret it.
1777
1806
        self.assertRaises(errors.SmartProtocolError,
1778
1807
                          t.get_smart_medium().send_http_smart_request,
1779
1808
                          'whatever')
1780
1809
 
 
1810
 
1781
1811
class Test_redirected_to(tests.TestCase):
1782
1812
 
1783
1813
    def test_redirected_to_subdir(self):
1786
1816
                             'http://www.example.com/foo/subdir')
1787
1817
        self.assertIsInstance(r, type(t))
1788
1818
        # Both transports share the some connection
1789
 
        self.assertEquals(t._get_connection(), r._get_connection())
 
1819
        self.assertEqual(t._get_connection(), r._get_connection())
1790
1820
 
1791
1821
    def test_redirected_to_self_with_slash(self):
1792
1822
        t = self._transport('http://www.example.com/foo')
1796
1826
        # Both transports share the some connection (one can argue that we
1797
1827
        # should return the exact same transport here, but that seems
1798
1828
        # overkill).
1799
 
        self.assertEquals(t._get_connection(), r._get_connection())
 
1829
        self.assertEqual(t._get_connection(), r._get_connection())
1800
1830
 
1801
1831
    def test_redirected_to_host(self):
1802
1832
        t = self._transport('http://www.example.com/foo')
1821
1851
        r = t._redirected_to('http://www.example.com/foo',
1822
1852
                             'https://foo.example.com/foo')
1823
1853
        self.assertIsInstance(r, type(t))
1824
 
        self.assertEquals(t._user, r._user)
 
1854
        self.assertEqual(t._user, r._user)
1825
1855
 
1826
1856
 
1827
1857
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
1836
1866
    line.
1837
1867
    """
1838
1868
 
1839
 
    def handle_one_request(self):
 
1869
    def _handle_one_request(self):
1840
1870
        tcs = self.server.test_case_server
1841
1871
        requestline = self.rfile.readline()
1842
1872
        headers = self.MessageClass(self.rfile, 0)
1886
1916
        pass
1887
1917
 
1888
1918
 
1889
 
class TestActivity(tests.TestCase):
 
1919
class TestActivityMixin(object):
1890
1920
    """Test socket activity reporting.
1891
1921
 
1892
1922
    We use a special purpose server to control the bytes sent and received and
1896
1926
    def setUp(self):
1897
1927
        tests.TestCase.setUp(self)
1898
1928
        self.server = self._activity_server(self._protocol_version)
1899
 
        self.server.setUp()
 
1929
        self.server.start_server()
1900
1930
        self.activities = {}
1901
1931
        def report_activity(t, bytes, direction):
1902
1932
            count = self.activities.get(direction, 0)
1905
1935
 
1906
1936
        # We override at class level because constructors may propagate the
1907
1937
        # bound method and render instance overriding ineffective (an
1908
 
        # alternative would be be to define a specific ui factory instead...)
1909
 
        self.orig_report_activity = self._transport._report_activity
1910
 
        self._transport._report_activity = report_activity
1911
 
 
1912
 
    def tearDown(self):
1913
 
        self._transport._report_activity = self.orig_report_activity
1914
 
        self.server.tearDown()
1915
 
        tests.TestCase.tearDown(self)
 
1938
        # alternative would be to define a specific ui factory instead...)
 
1939
        self.overrideAttr(self._transport, '_report_activity', report_activity)
 
1940
        self.addCleanup(self.server.stop_server)
1916
1941
 
1917
1942
    def get_transport(self):
1918
 
        return self._transport(self.server.get_url())
 
1943
        t = self._transport(self.server.get_url())
 
1944
        # FIXME: Needs cleanup -- vila 20100611
 
1945
        return t
1919
1946
 
1920
1947
    def assertActivitiesMatch(self):
1921
1948
        self.assertEqual(self.server.bytes_read,
2026
2053
'''
2027
2054
        t = self.get_transport()
2028
2055
        # We must send a single line of body bytes, see
2029
 
        # PredefinedRequestHandler.handle_one_request
 
2056
        # PredefinedRequestHandler._handle_one_request
2030
2057
        code, f = t._post('abc def end-of-body\n')
2031
2058
        self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2032
2059
        self.assertActivitiesMatch()
 
2060
 
 
2061
 
 
2062
class TestActivity(tests.TestCase, TestActivityMixin):
 
2063
 
 
2064
    def setUp(self):
 
2065
        TestActivityMixin.setUp(self)
 
2066
 
 
2067
 
 
2068
class TestNoReportActivity(tests.TestCase, TestActivityMixin):
 
2069
 
 
2070
    # Unlike TestActivity, we are really testing ReportingFileSocket and
 
2071
    # ReportingSocket, so we don't need all the parametrization. Since
 
2072
    # ReportingFileSocket and ReportingSocket are wrappers, it's easier to
 
2073
    # test them through their use by the transport than directly (that's a
 
2074
    # bit less clean but far more simpler and effective).
 
2075
    _activity_server = ActivityHTTPServer
 
2076
    _protocol_version = 'HTTP/1.1'
 
2077
 
 
2078
    def setUp(self):
 
2079
        self._transport =_urllib.HttpTransport_urllib
 
2080
        TestActivityMixin.setUp(self)
 
2081
 
 
2082
    def assertActivitiesMatch(self):
 
2083
        # Nothing to check here
 
2084
        pass
 
2085
 
 
2086
 
 
2087
class TestAuthOnRedirected(http_utils.TestCaseWithRedirectedWebserver):
 
2088
    """Test authentication on the redirected http server."""
 
2089
 
 
2090
    _auth_header = 'Authorization'
 
2091
    _password_prompt_prefix = ''
 
2092
    _username_prompt_prefix = ''
 
2093
    _auth_server = http_utils.HTTPBasicAuthServer
 
2094
    _transport = _urllib.HttpTransport_urllib
 
2095
 
 
2096
    def setUp(self):
 
2097
        super(TestAuthOnRedirected, self).setUp()
 
2098
        self.build_tree_contents([('a','a'),
 
2099
                                  ('1/',),
 
2100
                                  ('1/a', 'redirected once'),
 
2101
                                  ],)
 
2102
        new_prefix = 'http://%s:%s' % (self.new_server.host,
 
2103
                                       self.new_server.port)
 
2104
        self.old_server.redirections = [
 
2105
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
 
2106
        self.old_transport = self.get_old_transport()
 
2107
        self.new_server.add_user('joe', 'foo')
 
2108
        cleanup_http_redirection_connections(self)
 
2109
 
 
2110
    def create_transport_readonly_server(self):
 
2111
        server = self._auth_server(protocol_version=self._protocol_version)
 
2112
        server._url_protocol = self._url_protocol
 
2113
        return server
 
2114
 
 
2115
    def get_a(self, t):
 
2116
        return t.get('a')
 
2117
 
 
2118
    def test_auth_on_redirected_via_do_catching_redirections(self):
 
2119
        self.redirections = 0
 
2120
 
 
2121
        def redirected(t, exception, redirection_notice):
 
2122
            self.redirections += 1
 
2123
            redirected_t = t._redirected_to(exception.source, exception.target)
 
2124
            self.addCleanup(redirected_t.disconnect)
 
2125
            return redirected_t
 
2126
 
 
2127
        stdout = tests.StringIOWrapper()
 
2128
        stderr = tests.StringIOWrapper()
 
2129
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
 
2130
                                            stdout=stdout, stderr=stderr)
 
2131
        self.assertEqual('redirected once',
 
2132
                         transport.do_catching_redirections(
 
2133
                self.get_a, self.old_transport, redirected).read())
 
2134
        self.assertEqual(1, self.redirections)
 
2135
        # stdin should be empty
 
2136
        self.assertEqual('', ui.ui_factory.stdin.readline())
 
2137
        # stdout should be empty, stderr will contains the prompts
 
2138
        self.assertEqual('', stdout.getvalue())
 
2139
 
 
2140
    def test_auth_on_redirected_via_following_redirections(self):
 
2141
        self.new_server.add_user('joe', 'foo')
 
2142
        stdout = tests.StringIOWrapper()
 
2143
        stderr = tests.StringIOWrapper()
 
2144
        ui.ui_factory = tests.TestUIFactory(stdin='joe\nfoo\n',
 
2145
                                            stdout=stdout, stderr=stderr)
 
2146
        t = self.old_transport
 
2147
        req = RedirectedRequest('GET', t.abspath('a'))
 
2148
        new_prefix = 'http://%s:%s' % (self.new_server.host,
 
2149
                                       self.new_server.port)
 
2150
        self.old_server.redirections = [
 
2151
            ('(.*)', r'%s/1\1' % (new_prefix), 301),]
 
2152
        self.assertEqual('redirected once', t._perform(req).read())
 
2153
        # stdin should be empty
 
2154
        self.assertEqual('', ui.ui_factory.stdin.readline())
 
2155
        # stdout should be empty, stderr will contains the prompts
 
2156
        self.assertEqual('', stdout.getvalue())
 
2157
 
 
2158