~bzr-pqm/bzr/bzr.dev

3995.2.2 by Martin Pool
Cope with read_bundle_from_url deprecation in test_http
1
# Copyright (C) 2005, 2006, 2007, 2008, 2009 Canonical Ltd
1540.3.24 by Martin Pool
Add new protocol 'http+pycurl' that always uses PyCurl.
2
#
1540.3.15 by Martin Pool
[merge] large merge to sync with bzr.dev
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1540.3.24 by Martin Pool
Add new protocol 'http+pycurl' that always uses PyCurl.
7
#
1540.3.15 by Martin Pool
[merge] large merge to sync with bzr.dev
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1540.3.24 by Martin Pool
Add new protocol 'http+pycurl' that always uses PyCurl.
12
#
1540.3.15 by Martin Pool
[merge] large merge to sync with bzr.dev
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
1185.16.68 by Martin Pool
- http url fixes suggested by Robey Pointer, and tests
16
3111.1.30 by Vincent Ladeuil
Update NEWS. Some cosmetic changes.
17
"""Tests for HTTP implementations.
3111.1.10 by Vincent Ladeuil
Finish http parameterization, 24 auth tests failing for pycurl (not
18
3111.1.30 by Vincent Ladeuil
Update NEWS. Some cosmetic changes.
19
This module defines a load_tests() method that parametrize tests classes for
20
transport implementation, http protocol versions and authentication schemes.
3111.1.10 by Vincent Ladeuil
Finish http parameterization, 24 auth tests failing for pycurl (not
21
"""
1540.3.3 by Martin Pool
Review updates of pycurl transport
22
1540.3.22 by Martin Pool
[patch] Add TestCase.assertIsInstance
23
# TODO: Should be renamed to bzrlib.transport.http.tests?
2167.3.5 by v.ladeuil+lp at free
Tests for proxies, covering #74759.
24
# TODO: What about renaming to bzrlib.tests.transport.http ?
1540.3.22 by Martin Pool
[patch] Add TestCase.assertIsInstance
25
2363.4.12 by Vincent Ladeuil
Take jam's review comments into account. Fix typos, give better
26
from cStringIO import StringIO
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
27
import httplib
2167.3.5 by v.ladeuil+lp at free
Tests for proxies, covering #74759.
28
import os
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
29
import select
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
30
import SimpleHTTPServer
2000.2.2 by John Arbash Meinel
Update the urllib.has test.
31
import socket
2420.1.20 by Vincent Ladeuil
Fix test failure on pqm.
32
import sys
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
33
import threading
2000.2.2 by John Arbash Meinel
Update the urllib.has test.
34
1553.1.2 by James Henstridge
Add a test to make sure the user-agent header is being sent correctly.
35
import bzrlib
2164.2.22 by Vincent Ladeuil
Take Aaron's review comments into account.
36
from bzrlib import (
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
37
    bzrdir,
2900.2.6 by Vincent Ladeuil
Make http aware of authentication config.
38
    config,
2164.2.22 by Vincent Ladeuil
Take Aaron's review comments into account.
39
    errors,
40
    osutils,
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
41
    remote as _mod_remote,
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
42
    tests,
3111.1.10 by Vincent Ladeuil
Finish http parameterization, 24 auth tests failing for pycurl (not
43
    transport,
2363.4.12 by Vincent Ladeuil
Take jam's review comments into account. Fix typos, give better
44
    ui,
2164.2.22 by Vincent Ladeuil
Take Aaron's review comments into account.
45
    urlutils,
46
    )
3995.2.2 by Martin Pool
Cope with read_bundle_from_url deprecation in test_http
47
from bzrlib.symbol_versioning import (
48
    deprecated_in,
49
    )
3102.1.1 by Vincent Ladeuil
Rename bzrlib/test/HTTPTestUtils.py to bzrlib/tests/http_utils.py and fix
50
from bzrlib.tests import (
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
51
    http_server,
3111.1.7 by Vincent Ladeuil
Further refactoring.
52
    http_utils,
3102.1.1 by Vincent Ladeuil
Rename bzrlib/test/HTTPTestUtils.py to bzrlib/tests/http_utils.py and fix
53
    )
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
54
from bzrlib.transport import (
55
    http,
56
    remote,
57
    )
2004.3.3 by vila
Better (but still incomplete) design for bogus servers.
58
from bzrlib.transport.http import (
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
59
    _urllib,
2164.2.29 by Vincent Ladeuil
Test the http redirection at the request level even if it's not
60
    _urllib2_wrappers,
2004.3.3 by vila
Better (but still incomplete) design for bogus servers.
61
    )
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
62
63
64
try:
65
    from bzrlib.transport.http._pycurl import PyCurlTransport
66
    pycurl_present = True
67
except errors.DependencyNotPresent:
68
    pycurl_present = False
69
70
71
def load_tests(standard_tests, module, loader):
72
    """Multiply tests for http clients and protocol versions."""
3945.1.7 by Vincent Ladeuil
Test against https.
73
    result = loader.suiteClass()
74
4084.5.1 by Robert Collins
Bulk update all test adaptation into a single approach, using multiply_tests rather than test adapters.
75
    # one for each transport implementation
3945.1.7 by Vincent Ladeuil
Test against https.
76
    t_tests, remaining_tests = tests.split_suite_by_condition(
4084.5.1 by Robert Collins
Bulk update all test adaptation into a single approach, using multiply_tests rather than test adapters.
77
        standard_tests, tests.condition_isinstance((
3945.1.7 by Vincent Ladeuil
Test against https.
78
                TestHttpTransportRegistration,
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
79
                TestHttpTransportUrls,
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
80
                Test_redirected_to,
3945.1.7 by Vincent Ladeuil
Test against https.
81
                )))
82
    transport_scenarios = [
83
        ('urllib', dict(_transport=_urllib.HttpTransport_urllib,
84
                        _server=http_server.HttpServer_urllib,
85
                        _qualified_prefix='http+urllib',)),
86
        ]
87
    if pycurl_present:
88
        transport_scenarios.append(
89
            ('pycurl', dict(_transport=PyCurlTransport,
90
                            _server=http_server.HttpServer_PyCurl,
91
                            _qualified_prefix='http+pycurl',)))
4084.5.1 by Robert Collins
Bulk update all test adaptation into a single approach, using multiply_tests rather than test adapters.
92
    tests.multiply_tests(t_tests, transport_scenarios, result)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
93
4084.5.1 by Robert Collins
Bulk update all test adaptation into a single approach, using multiply_tests rather than test adapters.
94
    # each implementation tested with each HTTP version
3945.1.7 by Vincent Ladeuil
Test against https.
95
    tp_tests, remaining_tests = tests.split_suite_by_condition(
96
        remaining_tests, tests.condition_isinstance((
97
                SmartHTTPTunnellingTest,
98
                TestDoCatchRedirections,
99
                TestHTTPConnections,
100
                TestHTTPRedirections,
101
                TestHTTPSilentRedirections,
102
                TestLimitedRangeRequestServer,
103
                TestPost,
104
                TestProxyHttpServer,
105
                TestRanges,
106
                TestSpecificRequestHandler,
107
                )))
108
    protocol_scenarios = [
109
            ('HTTP/1.0',  dict(_protocol_version='HTTP/1.0')),
110
            ('HTTP/1.1',  dict(_protocol_version='HTTP/1.1')),
111
            ]
4084.5.1 by Robert Collins
Bulk update all test adaptation into a single approach, using multiply_tests rather than test adapters.
112
    tp_scenarios = tests.multiply_scenarios(transport_scenarios,
3945.1.7 by Vincent Ladeuil
Test against https.
113
                                            protocol_scenarios)
4084.5.1 by Robert Collins
Bulk update all test adaptation into a single approach, using multiply_tests rather than test adapters.
114
    tests.multiply_tests(tp_tests, tp_scenarios, result)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
115
4084.5.1 by Robert Collins
Bulk update all test adaptation into a single approach, using multiply_tests rather than test adapters.
116
    # auth: each auth scheme on all http versions on all implementations.
3945.1.7 by Vincent Ladeuil
Test against https.
117
    tpa_tests, remaining_tests = tests.split_suite_by_condition(
118
        remaining_tests, tests.condition_isinstance((
119
                TestAuth,
120
                )))
121
    auth_scheme_scenarios = [
122
        ('basic', dict(_auth_scheme='basic')),
123
        ('digest', dict(_auth_scheme='digest')),
124
        ]
4084.5.1 by Robert Collins
Bulk update all test adaptation into a single approach, using multiply_tests rather than test adapters.
125
    tpa_scenarios = tests.multiply_scenarios(tp_scenarios,
126
        auth_scheme_scenarios)
127
    tests.multiply_tests(tpa_tests, tpa_scenarios, result)
3945.1.7 by Vincent Ladeuil
Test against https.
128
4084.5.1 by Robert Collins
Bulk update all test adaptation into a single approach, using multiply_tests rather than test adapters.
129
    # activity: activity on all http versions on all implementations
3945.1.7 by Vincent Ladeuil
Test against https.
130
    tpact_tests, remaining_tests = tests.split_suite_by_condition(
131
        remaining_tests, tests.condition_isinstance((
132
                TestActivity,
133
                )))
134
    activity_scenarios = [
135
        ('http', dict(_activity_server=ActivityHTTPServer)),
136
        ]
137
    if tests.HTTPSServerFeature.available():
138
        activity_scenarios.append(
4084.5.1 by Robert Collins
Bulk update all test adaptation into a single approach, using multiply_tests rather than test adapters.
139
            ('https', dict(_activity_server=ActivityHTTPSServer)))
140
    tpact_scenarios = tests.multiply_scenarios(tp_scenarios,
141
        activity_scenarios)
142
    tests.multiply_tests(tpact_tests, tpact_scenarios, result)
3945.1.7 by Vincent Ladeuil
Test against https.
143
144
    # No parametrization for the remaining tests
145
    result.addTests(remaining_tests)
146
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
147
    return result
1185.40.20 by Robey Pointer
allow user:pass@ info in http urls to be used for auth; this should be easily expandable later to use auth config files
148
1786.1.8 by John Arbash Meinel
[merge] Johan Rydberg test updates
149
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
150
class FakeManager(object):
1786.1.8 by John Arbash Meinel
[merge] Johan Rydberg test updates
151
1185.40.20 by Robey Pointer
allow user:pass@ info in http urls to be used for auth; this should be easily expandable later to use auth config files
152
    def __init__(self):
153
        self.credentials = []
2004.3.1 by vila
Test ConnectionError exceptions.
154
1185.40.20 by Robey Pointer
allow user:pass@ info in http urls to be used for auth; this should be easily expandable later to use auth config files
155
    def add_password(self, realm, host, username, password):
156
        self.credentials.append([realm, host, username, password])
157
1553.1.2 by James Henstridge
Add a test to make sure the user-agent header is being sent correctly.
158
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
159
class RecordingServer(object):
160
    """A fake HTTP server.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
161
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
162
    It records the bytes sent to it, and replies with a 200.
163
    """
164
165
    def __init__(self, expect_body_tail=None):
166
        """Constructor.
167
168
        :type expect_body_tail: str
169
        :param expect_body_tail: a reply won't be sent until this string is
170
            received.
171
        """
172
        self._expect_body_tail = expect_body_tail
173
        self.host = None
174
        self.port = None
175
        self.received_bytes = ''
176
177
    def setUp(self):
178
        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
179
        self._sock.bind(('127.0.0.1', 0))
180
        self.host, self.port = self._sock.getsockname()
181
        self._ready = threading.Event()
182
        self._thread = threading.Thread(target=self._accept_read_and_reply)
183
        self._thread.setDaemon(True)
184
        self._thread.start()
185
        self._ready.wait(5)
186
187
    def _accept_read_and_reply(self):
188
        self._sock.listen(1)
189
        self._ready.set()
190
        self._sock.settimeout(5)
191
        try:
192
            conn, address = self._sock.accept()
193
            # On win32, the accepted connection will be non-blocking to start
194
            # with because we're using settimeout.
195
            conn.setblocking(True)
196
            while not self.received_bytes.endswith(self._expect_body_tail):
197
                self.received_bytes += conn.recv(4096)
198
            conn.sendall('HTTP/1.1 200 OK\r\n')
199
        except socket.timeout:
200
            # Make sure the client isn't stuck waiting for us to e.g. accept.
201
            self._sock.close()
202
        except socket.error:
203
            # The client may have already closed the socket.
204
            pass
205
206
    def tearDown(self):
207
        try:
208
            self._sock.close()
209
        except socket.error:
210
            # We might have already closed it.  We don't care.
211
            pass
212
        self.host = None
213
        self.port = None
214
215
4050.2.2 by Vincent Ladeuil
Ensures all auth handlers correctly parse all auth headers.
216
class TestAuthHeader(tests.TestCase):
217
218
    def parse_header(self, header):
219
        ah =  _urllib2_wrappers.AbstractAuthHandler()
220
        return ah._parse_auth_header(header)
221
222
    def test_empty_header(self):
223
        scheme, remainder = self.parse_header('')
224
        self.assertEquals('', scheme)
225
        self.assertIs(None, remainder)
226
227
    def test_negotiate_header(self):
228
        scheme, remainder = self.parse_header('Negotiate')
229
        self.assertEquals('negotiate', scheme)
230
        self.assertIs(None, remainder)
231
232
    def test_basic_header(self):
233
        scheme, remainder = self.parse_header(
234
            'Basic realm="Thou should not pass"')
235
        self.assertEquals('basic', scheme)
236
        self.assertEquals('realm="Thou should not pass"', remainder)
237
238
    def test_digest_header(self):
239
        scheme, remainder = self.parse_header(
240
            'Digest realm="Thou should not pass"')
241
        self.assertEquals('digest', scheme)
242
        self.assertEquals('realm="Thou should not pass"', remainder)
243
244
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
245
class TestHTTPServer(tests.TestCase):
246
    """Test the HTTP servers implementations."""
247
248
    def test_invalid_protocol(self):
249
        class BogusRequestHandler(http_server.TestingHTTPRequestHandler):
250
251
            protocol_version = 'HTTP/0.1'
252
253
        server = http_server.HttpServer(BogusRequestHandler)
254
        try:
255
            self.assertRaises(httplib.UnknownProtocol,server.setUp)
256
        except:
257
            server.tearDown()
258
            self.fail('HTTP Server creation did not raise UnknownProtocol')
259
3111.1.17 by Vincent Ladeuil
Add tests for the protocol version parameter.
260
    def test_force_invalid_protocol(self):
261
        server = http_server.HttpServer(protocol_version='HTTP/0.1')
262
        try:
263
            self.assertRaises(httplib.UnknownProtocol,server.setUp)
264
        except:
265
            server.tearDown()
266
            self.fail('HTTP Server creation did not raise UnknownProtocol')
267
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
268
    def test_server_start_and_stop(self):
269
        server = http_server.HttpServer()
270
        server.setUp()
271
        self.assertTrue(server._http_running)
272
        server.tearDown()
273
        self.assertFalse(server._http_running)
274
275
    def test_create_http_server_one_zero(self):
276
        class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
277
278
            protocol_version = 'HTTP/1.0'
279
280
        server = http_server.HttpServer(RequestHandlerOneZero)
281
        server.setUp()
3111.1.17 by Vincent Ladeuil
Add tests for the protocol version parameter.
282
        self.addCleanup(server.tearDown)
283
        self.assertIsInstance(server._httpd, http_server.TestingHTTPServer)
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
284
285
    def test_create_http_server_one_one(self):
286
        class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
287
288
            protocol_version = 'HTTP/1.1'
289
290
        server = http_server.HttpServer(RequestHandlerOneOne)
291
        server.setUp()
3111.1.17 by Vincent Ladeuil
Add tests for the protocol version parameter.
292
        self.addCleanup(server.tearDown)
293
        self.assertIsInstance(server._httpd,
294
                              http_server.TestingThreadingHTTPServer)
295
296
    def test_create_http_server_force_one_one(self):
297
        class RequestHandlerOneZero(http_server.TestingHTTPRequestHandler):
298
299
            protocol_version = 'HTTP/1.0'
300
301
        server = http_server.HttpServer(RequestHandlerOneZero,
302
                                        protocol_version='HTTP/1.1')
303
        server.setUp()
304
        self.addCleanup(server.tearDown)
305
        self.assertIsInstance(server._httpd,
306
                              http_server.TestingThreadingHTTPServer)
307
308
    def test_create_http_server_force_one_zero(self):
309
        class RequestHandlerOneOne(http_server.TestingHTTPRequestHandler):
310
311
            protocol_version = 'HTTP/1.1'
312
313
        server = http_server.HttpServer(RequestHandlerOneOne,
314
                                        protocol_version='HTTP/1.0')
315
        server.setUp()
316
        self.addCleanup(server.tearDown)
317
        self.assertIsInstance(server._httpd,
318
                              http_server.TestingHTTPServer)
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
319
320
2294.3.1 by Vincent Ladeuil
Fix #85305 by issuing an exception instead of a traceback.
321
class TestWithTransport_pycurl(object):
322
    """Test case to inherit from if pycurl is present"""
323
324
    def _get_pycurl_maybe(self):
325
        try:
326
            from bzrlib.transport.http._pycurl import PyCurlTransport
327
            return PyCurlTransport
328
        except errors.DependencyNotPresent:
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
329
            raise tests.TestSkipped('pycurl not present')
2294.3.1 by Vincent Ladeuil
Fix #85305 by issuing an exception instead of a traceback.
330
331
    _transport = property(_get_pycurl_maybe)
332
333
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
334
class TestHttpUrls(tests.TestCase):
1786.1.8 by John Arbash Meinel
[merge] Johan Rydberg test updates
335
2294.3.1 by Vincent Ladeuil
Fix #85305 by issuing an exception instead of a traceback.
336
    # TODO: This should be moved to authorization tests once they
337
    # are written.
2004.1.40 by v.ladeuil+lp at free
Fix the race condition again and correct some small typos to be in
338
1185.40.20 by Robey Pointer
allow user:pass@ info in http urls to be used for auth; this should be easily expandable later to use auth config files
339
    def test_url_parsing(self):
340
        f = FakeManager()
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
341
        url = http.extract_auth('http://example.com', f)
1185.40.20 by Robey Pointer
allow user:pass@ info in http urls to be used for auth; this should be easily expandable later to use auth config files
342
        self.assertEquals('http://example.com', url)
343
        self.assertEquals(0, len(f.credentials))
3111.1.30 by Vincent Ladeuil
Update NEWS. Some cosmetic changes.
344
        url = http.extract_auth(
345
            'http://user:pass@www.bazaar-vcs.org/bzr/bzr.dev', f)
1185.50.94 by John Arbash Meinel
Updated web page url to http://bazaar-vcs.org
346
        self.assertEquals('http://www.bazaar-vcs.org/bzr/bzr.dev', url)
1185.40.20 by Robey Pointer
allow user:pass@ info in http urls to be used for auth; this should be easily expandable later to use auth config files
347
        self.assertEquals(1, len(f.credentials))
2004.3.1 by vila
Test ConnectionError exceptions.
348
        self.assertEquals([None, 'www.bazaar-vcs.org', 'user', 'pass'],
349
                          f.credentials[0])
350
2294.3.1 by Vincent Ladeuil
Fix #85305 by issuing an exception instead of a traceback.
351
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
352
class TestHttpTransportUrls(tests.TestCase):
353
    """Test the http urls."""
354
355
    def test_abs_url(self):
356
        """Construction of absolute http URLs"""
357
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
358
        eq = self.assertEqualDiff
359
        eq(t.abspath('.'), 'http://bazaar-vcs.org/bzr/bzr.dev')
360
        eq(t.abspath('foo/bar'), 'http://bazaar-vcs.org/bzr/bzr.dev/foo/bar')
361
        eq(t.abspath('.bzr'), 'http://bazaar-vcs.org/bzr/bzr.dev/.bzr')
362
        eq(t.abspath('.bzr/1//2/./3'),
363
           'http://bazaar-vcs.org/bzr/bzr.dev/.bzr/1/2/3')
364
365
    def test_invalid_http_urls(self):
366
        """Trap invalid construction of urls"""
367
        t = self._transport('http://bazaar-vcs.org/bzr/bzr.dev/')
368
        self.assertRaises(errors.InvalidURL,
369
                          self._transport,
370
                          'http://http://bazaar-vcs.org/bzr/bzr.dev/')
371
372
    def test_http_root_urls(self):
373
        """Construction of URLs from server root"""
374
        t = self._transport('http://bzr.ozlabs.org/')
375
        eq = self.assertEqualDiff
376
        eq(t.abspath('.bzr/tree-version'),
377
           'http://bzr.ozlabs.org/.bzr/tree-version')
378
379
    def test_http_impl_urls(self):
380
        """There are servers which ask for particular clients to connect"""
381
        server = self._server()
382
        try:
383
            server.setUp()
384
            url = server.get_url()
385
            self.assertTrue(url.startswith('%s://' % self._qualified_prefix))
386
        finally:
387
            server.tearDown()
388
389
3111.1.9 by Vincent Ladeuil
Most refactoring regarding parameterization for urllib/pycurl and custom
390
class TestHttps_pycurl(TestWithTransport_pycurl, tests.TestCase):
2294.3.1 by Vincent Ladeuil
Fix #85305 by issuing an exception instead of a traceback.
391
392
    # TODO: This should really be moved into another pycurl
393
    # specific test. When https tests will be implemented, take
394
    # this one into account.
395
    def test_pycurl_without_https_support(self):
396
        """Test that pycurl without SSL do not fail with a traceback.
397
398
        For the purpose of the test, we force pycurl to ignore
399
        https by supplying a fake version_info that do not
400
        support it.
401
        """
402
        try:
403
            import pycurl
404
        except ImportError:
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
405
            raise tests.TestSkipped('pycurl not present')
3111.1.14 by Vincent Ladeuil
Fix test leakage.
406
407
        version_info_orig = pycurl.version_info
408
        try:
409
            # Now that we have pycurl imported, we can fake its version_info
410
            # This was taken from a windows pycurl without SSL
411
            # (thanks to bialix)
412
            pycurl.version_info = lambda : (2,
413
                                            '7.13.2',
414
                                            462082,
415
                                            'i386-pc-win32',
416
                                            2576,
417
                                            None,
418
                                            0,
419
                                            None,
420
                                            ('ftp', 'gopher', 'telnet',
421
                                             'dict', 'ldap', 'http', 'file'),
422
                                            None,
423
                                            0,
424
                                            None)
425
            self.assertRaises(errors.DependencyNotPresent, self._transport,
426
                              'https://launchpad.net')
427
        finally:
428
            # Restore the right function
429
            pycurl.version_info = version_info_orig
2294.3.1 by Vincent Ladeuil
Fix #85305 by issuing an exception instead of a traceback.
430
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
431
432
class TestHTTPConnections(http_utils.TestCaseWithWebserver):
433
    """Test the http connections."""
434
435
    def setUp(self):
436
        http_utils.TestCaseWithWebserver.setUp(self)
437
        self.build_tree(['foo/', 'foo/bar'], line_endings='binary',
438
                        transport=self.get_transport())
439
440
    def test_http_has(self):
441
        server = self.get_readonly_server()
442
        t = self._transport(server.get_url())
443
        self.assertEqual(t.has('foo/bar'), True)
444
        self.assertEqual(len(server.logs), 1)
445
        self.assertContainsRe(server.logs[0],
446
            r'"HEAD /foo/bar HTTP/1.." (200|302) - "-" "bzr/')
447
448
    def test_http_has_not_found(self):
449
        server = self.get_readonly_server()
450
        t = self._transport(server.get_url())
451
        self.assertEqual(t.has('not-found'), False)
452
        self.assertContainsRe(server.logs[1],
453
            r'"HEAD /not-found HTTP/1.." 404 - "-" "bzr/')
454
455
    def test_http_get(self):
456
        server = self.get_readonly_server()
457
        t = self._transport(server.get_url())
458
        fp = t.get('foo/bar')
459
        self.assertEqualDiff(
460
            fp.read(),
461
            'contents of foo/bar\n')
462
        self.assertEqual(len(server.logs), 1)
463
        self.assertTrue(server.logs[0].find(
464
            '"GET /foo/bar HTTP/1.1" 200 - "-" "bzr/%s'
465
            % bzrlib.__version__) > -1)
466
467
    def test_has_on_bogus_host(self):
468
        # Get a free address and don't 'accept' on it, so that we
469
        # can be sure there is no http handler there, but set a
470
        # reasonable timeout to not slow down tests too much.
471
        default_timeout = socket.getdefaulttimeout()
472
        try:
473
            socket.setdefaulttimeout(2)
474
            s = socket.socket()
475
            s.bind(('localhost', 0))
476
            t = self._transport('http://%s:%s/' % s.getsockname())
477
            self.assertRaises(errors.ConnectionError, t.has, 'foo/bar')
478
        finally:
479
            socket.setdefaulttimeout(default_timeout)
480
481
482
class TestHttpTransportRegistration(tests.TestCase):
483
    """Test registrations of various http implementations"""
484
485
    def test_http_registered(self):
486
        t = transport.get_transport('%s://foo.com/' % self._qualified_prefix)
487
        self.assertIsInstance(t, transport.Transport)
488
        self.assertIsInstance(t, self._transport)
489
490
491
class TestPost(tests.TestCase):
492
493
    def test_post_body_is_received(self):
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
494
        server = RecordingServer(expect_body_tail='end-of-body')
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
495
        server.setUp()
496
        self.addCleanup(server.tearDown)
497
        scheme = self._qualified_prefix
498
        url = '%s://%s:%s/' % (scheme, server.host, server.port)
499
        http_transport = self._transport(url)
500
        code, response = http_transport._post('abc def end-of-body')
501
        self.assertTrue(
502
            server.received_bytes.startswith('POST /.bzr/smart HTTP/1.'))
503
        self.assertTrue('content-length: 19\r' in server.received_bytes.lower())
504
        # The transport should not be assuming that the server can accept
505
        # chunked encoding the first time it connects, because HTTP/1.1, so we
506
        # check for the literal string.
507
        self.assertTrue(
508
            server.received_bytes.endswith('\r\n\r\nabc def end-of-body'))
509
510
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
511
class TestRangeHeader(tests.TestCase):
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
512
    """Test range_header method"""
513
514
    def check_header(self, value, ranges=[], tail=0):
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
515
        offsets = [ (start, end - start + 1) for start, end in ranges]
3111.1.10 by Vincent Ladeuil
Finish http parameterization, 24 auth tests failing for pycurl (not
516
        coalesce = transport.Transport._coalesce_offsets
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
517
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
518
        range_header = http.HttpTransportBase._range_header
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
519
        self.assertEqual(value, range_header(coalesced, tail))
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
520
521
    def test_range_header_single(self):
2520.2.1 by Vincent Ladeuil
First step to fix #115209 use _coalesce_offsets like other transports.
522
        self.check_header('0-9', ranges=[(0,9)])
523
        self.check_header('100-109', ranges=[(100,109)])
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
524
525
    def test_range_header_tail(self):
1786.1.36 by John Arbash Meinel
pycurl expects us to just set the range of bytes, not including bytes=
526
        self.check_header('-10', tail=10)
527
        self.check_header('-50', tail=50)
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
528
529
    def test_range_header_multi(self):
1786.1.36 by John Arbash Meinel
pycurl expects us to just set the range of bytes, not including bytes=
530
        self.check_header('0-9,100-200,300-5000',
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
531
                          ranges=[(0,9), (100, 200), (300,5000)])
532
533
    def test_range_header_mixed(self):
1786.1.36 by John Arbash Meinel
pycurl expects us to just set the range of bytes, not including bytes=
534
        self.check_header('0-9,300-5000,-50',
1786.1.28 by John Arbash Meinel
Update and add tests for the HttpTransportBase.range_header
535
                          ranges=[(0,9), (300,5000)],
536
                          tail=50)
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
537
2004.1.15 by v.ladeuil+lp at free
Better design for bogus servers. Both urllib and pycurl pass tests.
538
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
539
class TestSpecificRequestHandler(http_utils.TestCaseWithWebserver):
540
    """Tests a specific request handler.
541
3111.1.31 by Vincent Ladeuil
Review feeback.
542
    Daughter classes are expected to override _req_handler_class
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
543
    """
544
545
    # Provide a useful default
546
    _req_handler_class = http_server.TestingHTTPRequestHandler
547
548
    def create_transport_readonly_server(self):
549
        return http_server.HttpServer(self._req_handler_class,
550
                                      protocol_version=self._protocol_version)
551
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
552
    def _testing_pycurl(self):
553
        return pycurl_present and self._transport == PyCurlTransport
554
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
555
556
class WallRequestHandler(http_server.TestingHTTPRequestHandler):
557
    """Whatever request comes in, close the connection"""
558
559
    def handle_one_request(self):
560
        """Handle a single HTTP request, by abruptly closing the connection"""
561
        self.close_connection = 1
562
563
564
class TestWallServer(TestSpecificRequestHandler):
565
    """Tests exceptions during the connection phase"""
566
567
    _req_handler_class = WallRequestHandler
568
569
    def test_http_has(self):
570
        server = self.get_readonly_server()
571
        t = self._transport(server.get_url())
572
        # Unfortunately httplib (see HTTPResponse._read_status
573
        # for details) make no distinction between a closed
574
        # socket and badly formatted status line, so we can't
575
        # just test for ConnectionError, we have to test
576
        # InvalidHttpResponse too.
577
        self.assertRaises((errors.ConnectionError, errors.InvalidHttpResponse),
578
                          t.has, 'foo/bar')
579
580
    def test_http_get(self):
581
        server = self.get_readonly_server()
582
        t = self._transport(server.get_url())
583
        self.assertRaises((errors.ConnectionError, errors.InvalidHttpResponse),
584
                          t.get, 'foo/bar')
585
586
587
class BadStatusRequestHandler(http_server.TestingHTTPRequestHandler):
588
    """Whatever request comes in, returns a bad status"""
589
590
    def parse_request(self):
591
        """Fakes handling a single HTTP request, returns a bad status"""
592
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
593
        self.send_response(0, "Bad status")
594
        self.close_connection = 1
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
595
        return False
596
597
598
class TestBadStatusServer(TestSpecificRequestHandler):
599
    """Tests bad status from server."""
600
601
    _req_handler_class = BadStatusRequestHandler
602
603
    def test_http_has(self):
604
        server = self.get_readonly_server()
605
        t = self._transport(server.get_url())
606
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
607
608
    def test_http_get(self):
609
        server = self.get_readonly_server()
610
        t = self._transport(server.get_url())
611
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
612
613
614
class InvalidStatusRequestHandler(http_server.TestingHTTPRequestHandler):
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
615
    """Whatever request comes in, returns an invalid status"""
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
616
617
    def parse_request(self):
618
        """Fakes handling a single HTTP request, returns a bad status"""
619
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
620
        self.wfile.write("Invalid status line\r\n")
621
        return False
622
623
624
class TestInvalidStatusServer(TestBadStatusServer):
625
    """Tests invalid status from server.
626
627
    Both implementations raises the same error as for a bad status.
628
    """
629
630
    _req_handler_class = InvalidStatusRequestHandler
631
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
632
    def test_http_has(self):
633
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
634
            raise tests.KnownFailure(
635
                'pycurl hangs if the server send back garbage')
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
636
        super(TestInvalidStatusServer, self).test_http_has()
637
638
    def test_http_get(self):
639
        if self._testing_pycurl() and self._protocol_version == 'HTTP/1.1':
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
640
            raise tests.KnownFailure(
641
                'pycurl hangs if the server send back garbage')
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
642
        super(TestInvalidStatusServer, self).test_http_get()
643
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
644
645
class BadProtocolRequestHandler(http_server.TestingHTTPRequestHandler):
646
    """Whatever request comes in, returns a bad protocol version"""
647
648
    def parse_request(self):
649
        """Fakes handling a single HTTP request, returns a bad status"""
650
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
651
        # Returns an invalid protocol version, but curl just
652
        # ignores it and those cannot be tested.
653
        self.wfile.write("%s %d %s\r\n" % ('HTTP/0.0',
654
                                           404,
655
                                           'Look at my protocol version'))
656
        return False
657
658
659
class TestBadProtocolServer(TestSpecificRequestHandler):
660
    """Tests bad protocol from server."""
661
662
    _req_handler_class = BadProtocolRequestHandler
663
664
    def setUp(self):
665
        if pycurl_present and self._transport == PyCurlTransport:
666
            raise tests.TestNotApplicable(
667
                "pycurl doesn't check the protocol version")
668
        super(TestBadProtocolServer, self).setUp()
669
670
    def test_http_has(self):
671
        server = self.get_readonly_server()
672
        t = self._transport(server.get_url())
673
        self.assertRaises(errors.InvalidHttpResponse, t.has, 'foo/bar')
674
675
    def test_http_get(self):
676
        server = self.get_readonly_server()
677
        t = self._transport(server.get_url())
678
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'foo/bar')
679
680
681
class ForbiddenRequestHandler(http_server.TestingHTTPRequestHandler):
682
    """Whatever request comes in, returns a 403 code"""
683
684
    def parse_request(self):
685
        """Handle a single HTTP request, by replying we cannot handle it"""
686
        ignored = http_server.TestingHTTPRequestHandler.parse_request(self)
687
        self.send_error(403)
688
        return False
689
690
691
class TestForbiddenServer(TestSpecificRequestHandler):
692
    """Tests forbidden server"""
693
694
    _req_handler_class = ForbiddenRequestHandler
695
696
    def test_http_has(self):
697
        server = self.get_readonly_server()
698
        t = self._transport(server.get_url())
699
        self.assertRaises(errors.TransportError, t.has, 'foo/bar')
700
701
    def test_http_get(self):
702
        server = self.get_readonly_server()
703
        t = self._transport(server.get_url())
704
        self.assertRaises(errors.TransportError, t.get, 'foo/bar')
705
706
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
707
class TestRecordingServer(tests.TestCase):
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
708
709
    def test_create(self):
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
710
        server = RecordingServer(expect_body_tail=None)
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
711
        self.assertEqual('', server.received_bytes)
712
        self.assertEqual(None, server.host)
713
        self.assertEqual(None, server.port)
714
715
    def test_setUp_and_tearDown(self):
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
716
        server = RecordingServer(expect_body_tail=None)
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
717
        server.setUp()
718
        try:
719
            self.assertNotEqual(None, server.host)
720
            self.assertNotEqual(None, server.port)
721
        finally:
722
            server.tearDown()
723
        self.assertEqual(None, server.host)
724
        self.assertEqual(None, server.port)
725
726
    def test_send_receive_bytes(self):
3111.1.29 by Vincent Ladeuil
Cancel RecordingServer move, that was useless.
727
        server = RecordingServer(expect_body_tail='c')
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
728
        server.setUp()
729
        self.addCleanup(server.tearDown)
730
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
731
        sock.connect((server.host, server.port))
732
        sock.sendall('abc')
733
        self.assertEqual('HTTP/1.1 200 OK\r\n',
2091.1.1 by Martin Pool
Avoid MSG_WAITALL as it doesn't work on Windows
734
                         osutils.recv_all(sock, 4096))
2018.2.9 by Andrew Bennetts
(Andrew Bennetts, Robert Collins) Add test_http.RecordingServer, and use it to
735
        self.assertEqual('abc', server.received_bytes)
2004.1.29 by v.ladeuil+lp at free
New tests for http range requests handling.
736
737
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
738
class TestRangeRequestServer(TestSpecificRequestHandler):
739
    """Tests readv requests against server.
740
741
    We test against default "normal" server.
742
    """
743
744
    def setUp(self):
745
        super(TestRangeRequestServer, self).setUp()
746
        self.build_tree_contents([('a', '0123456789')],)
747
748
    def test_readv(self):
749
        server = self.get_readonly_server()
750
        t = self._transport(server.get_url())
751
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
752
        self.assertEqual(l[0], (0, '0'))
753
        self.assertEqual(l[1], (1, '1'))
754
        self.assertEqual(l[2], (3, '34'))
755
        self.assertEqual(l[3], (9, '9'))
756
757
    def test_readv_out_of_order(self):
758
        server = self.get_readonly_server()
759
        t = self._transport(server.get_url())
760
        l = list(t.readv('a', ((1, 1), (9, 1), (0, 1), (3, 2))))
761
        self.assertEqual(l[0], (1, '1'))
762
        self.assertEqual(l[1], (9, '9'))
763
        self.assertEqual(l[2], (0, '0'))
764
        self.assertEqual(l[3], (3, '34'))
765
766
    def test_readv_invalid_ranges(self):
767
        server = self.get_readonly_server()
768
        t = self._transport(server.get_url())
769
770
        # This is intentionally reading off the end of the file
771
        # since we are sure that it cannot get there
772
        self.assertListRaises((errors.InvalidRange, errors.ShortReadvError,),
773
                              t.readv, 'a', [(1,1), (8,10)])
774
775
        # This is trying to seek past the end of the file, it should
776
        # also raise a special error
777
        self.assertListRaises((errors.InvalidRange, errors.ShortReadvError,),
778
                              t.readv, 'a', [(12,2)])
779
780
    def test_readv_multiple_get_requests(self):
781
        server = self.get_readonly_server()
782
        t = self._transport(server.get_url())
783
        # force transport to issue multiple requests
784
        t._max_readv_combine = 1
785
        t._max_get_ranges = 1
786
        l = list(t.readv('a', ((0, 1), (1, 1), (3, 2), (9, 1))))
787
        self.assertEqual(l[0], (0, '0'))
788
        self.assertEqual(l[1], (1, '1'))
789
        self.assertEqual(l[2], (3, '34'))
790
        self.assertEqual(l[3], (9, '9'))
791
        # The server should have issued 4 requests
792
        self.assertEqual(4, server.GET_request_nb)
793
794
    def test_readv_get_max_size(self):
795
        server = self.get_readonly_server()
796
        t = self._transport(server.get_url())
797
        # force transport to issue multiple requests by limiting the number of
798
        # bytes by request. Note that this apply to coalesced offsets only, a
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
799
        # single range will keep its size even if bigger than the limit.
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
800
        t._get_max_size = 2
801
        l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
802
        self.assertEqual(l[0], (0, '0'))
803
        self.assertEqual(l[1], (1, '1'))
804
        self.assertEqual(l[2], (2, '2345'))
805
        self.assertEqual(l[3], (6, '6789'))
806
        # The server should have issued 3 requests
807
        self.assertEqual(3, server.GET_request_nb)
808
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
809
    def test_complete_readv_leave_pipe_clean(self):
810
        server = self.get_readonly_server()
811
        t = self._transport(server.get_url())
812
        # force transport to issue multiple requests
813
        t._get_max_size = 2
814
        l = list(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
815
        # The server should have issued 3 requests
816
        self.assertEqual(3, server.GET_request_nb)
817
        self.assertEqual('0123456789', t.get_bytes('a'))
818
        self.assertEqual(4, server.GET_request_nb)
819
820
    def test_incomplete_readv_leave_pipe_clean(self):
821
        server = self.get_readonly_server()
822
        t = self._transport(server.get_url())
823
        # force transport to issue multiple requests
824
        t._get_max_size = 2
825
        # Don't collapse readv results into a list so that we leave unread
826
        # bytes on the socket
827
        ireadv = iter(t.readv('a', ((0, 1), (1, 1), (2, 4), (6, 4))))
828
        self.assertEqual((0, '0'), ireadv.next())
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
829
        # The server should have issued one request so far
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
830
        self.assertEqual(1, server.GET_request_nb)
831
        self.assertEqual('0123456789', t.get_bytes('a'))
832
        # get_bytes issued an additional request, the readv pending ones are
833
        # lost
834
        self.assertEqual(2, server.GET_request_nb)
835
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
836
837
class SingleRangeRequestHandler(http_server.TestingHTTPRequestHandler):
838
    """Always reply to range request as if they were single.
839
840
    Don't be explicit about it, just to annoy the clients.
841
    """
842
843
    def get_multiple_ranges(self, file, file_size, ranges):
844
        """Answer as if it was a single range request and ignores the rest"""
845
        (start, end) = ranges[0]
846
        return self.get_single_range(file, file_size, start, end)
847
848
849
class TestSingleRangeRequestServer(TestRangeRequestServer):
850
    """Test readv against a server which accept only single range requests"""
851
852
    _req_handler_class = SingleRangeRequestHandler
853
854
855
class SingleOnlyRangeRequestHandler(http_server.TestingHTTPRequestHandler):
856
    """Only reply to simple range requests, errors out on multiple"""
857
858
    def get_multiple_ranges(self, file, file_size, ranges):
859
        """Refuses the multiple ranges request"""
860
        if len(ranges) > 1:
861
            file.close()
862
            self.send_error(416, "Requested range not satisfiable")
863
            return
864
        (start, end) = ranges[0]
865
        return self.get_single_range(file, file_size, start, end)
866
867
868
class TestSingleOnlyRangeRequestServer(TestRangeRequestServer):
869
    """Test readv against a server which only accept single range requests"""
870
871
    _req_handler_class = SingleOnlyRangeRequestHandler
872
873
874
class NoRangeRequestHandler(http_server.TestingHTTPRequestHandler):
875
    """Ignore range requests without notice"""
876
877
    def do_GET(self):
878
        # Update the statistics
879
        self.server.test_case_server.GET_request_nb += 1
880
        # Just bypass the range handling done by TestingHTTPRequestHandler
881
        return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
882
883
884
class TestNoRangeRequestServer(TestRangeRequestServer):
885
    """Test readv against a server which do not accept range requests"""
886
887
    _req_handler_class = NoRangeRequestHandler
888
889
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
890
class MultipleRangeWithoutContentLengthRequestHandler(
891
    http_server.TestingHTTPRequestHandler):
892
    """Reply to multiple range requests without content length header."""
893
894
    def get_multiple_ranges(self, file, file_size, ranges):
895
        self.send_response(206)
896
        self.send_header('Accept-Ranges', 'bytes')
897
        boundary = "%d" % random.randint(0,0x7FFFFFFF)
898
        self.send_header("Content-Type",
899
                         "multipart/byteranges; boundary=%s" % boundary)
900
        self.end_headers()
901
        for (start, end) in ranges:
902
            self.wfile.write("--%s\r\n" % boundary)
903
            self.send_header("Content-type", 'application/octet-stream')
904
            self.send_header("Content-Range", "bytes %d-%d/%d" % (start,
905
                                                                  end,
906
                                                                  file_size))
907
            self.end_headers()
908
            self.send_range_content(file, start, end - start + 1)
909
        # Final boundary
910
        self.wfile.write("--%s\r\n" % boundary)
911
912
913
class TestMultipleRangeWithoutContentLengthServer(TestRangeRequestServer):
914
915
    _req_handler_class = MultipleRangeWithoutContentLengthRequestHandler
916
3146.3.2 by Vincent Ladeuil
Fix #179368 by keeping the current range hint on ShortReadvErrors.
917
918
class TruncatedMultipleRangeRequestHandler(
919
    http_server.TestingHTTPRequestHandler):
920
    """Reply to multiple range requests truncating the last ones.
921
922
    This server generates responses whose Content-Length describes all the
923
    ranges, but fail to include the last ones leading to client short reads.
924
    This has been observed randomly with lighttpd (bug #179368).
925
    """
926
927
    _truncated_ranges = 2
928
929
    def get_multiple_ranges(self, file, file_size, ranges):
930
        self.send_response(206)
931
        self.send_header('Accept-Ranges', 'bytes')
932
        boundary = 'tagada'
933
        self.send_header('Content-Type',
934
                         'multipart/byteranges; boundary=%s' % boundary)
935
        boundary_line = '--%s\r\n' % boundary
936
        # Calculate the Content-Length
937
        content_length = 0
938
        for (start, end) in ranges:
939
            content_length += len(boundary_line)
940
            content_length += self._header_line_length(
941
                'Content-type', 'application/octet-stream')
942
            content_length += self._header_line_length(
943
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
944
            content_length += len('\r\n') # end headers
945
            content_length += end - start # + 1
946
        content_length += len(boundary_line)
947
        self.send_header('Content-length', content_length)
948
        self.end_headers()
949
950
        # Send the multipart body
951
        cur = 0
952
        for (start, end) in ranges:
953
            self.wfile.write(boundary_line)
954
            self.send_header('Content-type', 'application/octet-stream')
955
            self.send_header('Content-Range', 'bytes %d-%d/%d'
956
                             % (start, end, file_size))
957
            self.end_headers()
958
            if cur + self._truncated_ranges >= len(ranges):
959
                # Abruptly ends the response and close the connection
960
                self.close_connection = 1
961
                return
962
            self.send_range_content(file, start, end - start + 1)
963
            cur += 1
964
        # No final boundary
965
        self.wfile.write(boundary_line)
966
967
968
class TestTruncatedMultipleRangeServer(TestSpecificRequestHandler):
969
970
    _req_handler_class = TruncatedMultipleRangeRequestHandler
971
972
    def setUp(self):
973
        super(TestTruncatedMultipleRangeServer, self).setUp()
974
        self.build_tree_contents([('a', '0123456789')],)
975
976
    def test_readv_with_short_reads(self):
977
        server = self.get_readonly_server()
978
        t = self._transport(server.get_url())
979
        # Force separate ranges for each offset
980
        t._bytes_to_read_before_seek = 0
981
        ireadv = iter(t.readv('a', ((0, 1), (2, 1), (4, 2), (9, 1))))
982
        self.assertEqual((0, '0'), ireadv.next())
983
        self.assertEqual((2, '2'), ireadv.next())
984
        if not self._testing_pycurl():
985
            # Only one request have been issued so far (except for pycurl that
986
            # try to read the whole response at once)
987
            self.assertEqual(1, server.GET_request_nb)
988
        self.assertEqual((4, '45'), ireadv.next())
989
        self.assertEqual((9, '9'), ireadv.next())
990
        # Both implementations issue 3 requests but:
991
        # - urllib does two multiple (4 ranges, then 2 ranges) then a single
992
        #   range,
993
        # - pycurl does two multiple (4 ranges, 4 ranges) then a single range
994
        self.assertEqual(3, server.GET_request_nb)
995
        # Finally the client have tried a single range request and stays in
996
        # that mode
997
        self.assertEqual('single', t._range_hint)
998
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
999
class LimitedRangeRequestHandler(http_server.TestingHTTPRequestHandler):
1000
    """Errors out when range specifiers exceed the limit"""
1001
1002
    def get_multiple_ranges(self, file, file_size, ranges):
1003
        """Refuses the multiple ranges request"""
1004
        tcs = self.server.test_case_server
1005
        if tcs.range_limit is not None and len(ranges) > tcs.range_limit:
1006
            file.close()
1007
            # Emulate apache behavior
1008
            self.send_error(400, "Bad Request")
1009
            return
1010
        return http_server.TestingHTTPRequestHandler.get_multiple_ranges(
1011
            self, file, file_size, ranges)
1012
1013
1014
class LimitedRangeHTTPServer(http_server.HttpServer):
1015
    """An HttpServer erroring out on requests with too much range specifiers"""
1016
1017
    def __init__(self, request_handler=LimitedRangeRequestHandler,
1018
                 protocol_version=None,
1019
                 range_limit=None):
1020
        http_server.HttpServer.__init__(self, request_handler,
1021
                                        protocol_version=protocol_version)
1022
        self.range_limit = range_limit
1023
1024
1025
class TestLimitedRangeRequestServer(http_utils.TestCaseWithWebserver):
1026
    """Tests readv requests against a server erroring out on too much ranges."""
1027
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
1028
    # Requests with more range specifiers will error out
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1029
    range_limit = 3
1030
1031
    def create_transport_readonly_server(self):
1032
        return LimitedRangeHTTPServer(range_limit=self.range_limit,
1033
                                      protocol_version=self._protocol_version)
1034
1035
    def get_transport(self):
1036
        return self._transport(self.get_readonly_server().get_url())
1037
1038
    def setUp(self):
1039
        http_utils.TestCaseWithWebserver.setUp(self)
1040
        # We need to manipulate ranges that correspond to real chunks in the
1041
        # response, so we build a content appropriately.
1042
        filler = ''.join(['abcdefghij' for x in range(102)])
1043
        content = ''.join(['%04d' % v + filler for v in range(16)])
1044
        self.build_tree_contents([('a', content)],)
1045
1046
    def test_few_ranges(self):
1047
        t = self.get_transport()
1048
        l = list(t.readv('a', ((0, 4), (1024, 4), )))
1049
        self.assertEqual(l[0], (0, '0000'))
1050
        self.assertEqual(l[1], (1024, '0001'))
1051
        self.assertEqual(1, self.get_readonly_server().GET_request_nb)
1052
1053
    def test_more_ranges(self):
1054
        t = self.get_transport()
1055
        l = list(t.readv('a', ((0, 4), (1024, 4), (4096, 4), (8192, 4))))
1056
        self.assertEqual(l[0], (0, '0000'))
1057
        self.assertEqual(l[1], (1024, '0001'))
1058
        self.assertEqual(l[2], (4096, '0004'))
1059
        self.assertEqual(l[3], (8192, '0008'))
1060
        # The server will refuse to serve the first request (too much ranges),
3199.1.2 by Vincent Ladeuil
Fix two more leaked log files.
1061
        # a second request will succeed.
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1062
        self.assertEqual(2, self.get_readonly_server().GET_request_nb)
1063
1064
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
1065
class TestHttpProxyWhiteBox(tests.TestCase):
2298.7.1 by Vincent Ladeuil
Fix bug #87765: proxy env variables without scheme should cause
1066
    """Whitebox test proxy http authorization.
1067
2420.1.3 by Vincent Ladeuil
Implement http proxy basic authentication.
1068
    Only the urllib implementation is tested here.
2298.7.1 by Vincent Ladeuil
Fix bug #87765: proxy env variables without scheme should cause
1069
    """
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1070
1071
    def setUp(self):
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
1072
        tests.TestCase.setUp(self)
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1073
        self._old_env = {}
1074
1075
    def tearDown(self):
1076
        self._restore_env()
3199.1.2 by Vincent Ladeuil
Fix two more leaked log files.
1077
        tests.TestCase.tearDown(self)
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1078
1079
    def _install_env(self, env):
1080
        for name, value in env.iteritems():
2420.1.2 by Vincent Ladeuil
Define tests for http proxy basic authentication. They fail.
1081
            self._old_env[name] = osutils.set_or_unset_env(name, value)
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1082
1083
    def _restore_env(self):
1084
        for name, value in self._old_env.iteritems():
1085
            osutils.set_or_unset_env(name, value)
1086
1087
    def _proxied_request(self):
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
1088
        handler = _urllib2_wrappers.ProxyHandler()
1089
        request = _urllib2_wrappers.Request('GET','http://baz/buzzle')
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1090
        handler.set_proxy(request, 'http')
1091
        return request
1092
1093
    def test_empty_user(self):
1094
        self._install_env({'http_proxy': 'http://bar.com'})
1095
        request = self._proxied_request()
1096
        self.assertFalse(request.headers.has_key('Proxy-authorization'))
1097
2298.7.1 by Vincent Ladeuil
Fix bug #87765: proxy env variables without scheme should cause
1098
    def test_invalid_proxy(self):
1099
        """A proxy env variable without scheme"""
1100
        self._install_env({'http_proxy': 'host:1234'})
1101
        self.assertRaises(errors.InvalidURL, self._proxied_request)
2273.2.2 by v.ladeuil+lp at free
Really fix bug #83954, with tests.
1102
1103
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1104
class TestProxyHttpServer(http_utils.TestCaseWithTwoWebservers):
1105
    """Tests proxy server.
1106
1107
    Be aware that we do not setup a real proxy here. Instead, we
1108
    check that the *connection* goes through the proxy by serving
1109
    different content (the faked proxy server append '-proxied'
1110
    to the file names).
1111
    """
1112
1113
    # FIXME: We don't have an https server available, so we don't
1114
    # test https connections.
1115
1116
    def setUp(self):
1117
        super(TestProxyHttpServer, self).setUp()
1118
        self.build_tree_contents([('foo', 'contents of foo\n'),
1119
                                  ('foo-proxied', 'proxied contents of foo\n')])
1120
        # Let's setup some attributes for tests
1121
        self.server = self.get_readonly_server()
1122
        self.proxy_address = '%s:%d' % (self.server.host, self.server.port)
1123
        if self._testing_pycurl():
1124
            # Oh my ! pycurl does not check for the port as part of
1125
            # no_proxy :-( So we just test the host part
1126
            self.no_proxy_host = 'localhost'
1127
        else:
1128
            self.no_proxy_host = self.proxy_address
1129
        # The secondary server is the proxy
1130
        self.proxy = self.get_secondary_server()
1131
        self.proxy_url = self.proxy.get_url()
1132
        self._old_env = {}
1133
1134
    def _testing_pycurl(self):
1135
        return pycurl_present and self._transport == PyCurlTransport
1136
1137
    def create_transport_secondary_server(self):
1138
        """Creates an http server that will serve files with
1139
        '-proxied' appended to their names.
1140
        """
1141
        return http_utils.ProxyServer(protocol_version=self._protocol_version)
1142
1143
    def _install_env(self, env):
1144
        for name, value in env.iteritems():
1145
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1146
1147
    def _restore_env(self):
1148
        for name, value in self._old_env.iteritems():
1149
            osutils.set_or_unset_env(name, value)
1150
1151
    def proxied_in_env(self, env):
1152
        self._install_env(env)
1153
        url = self.server.get_url()
1154
        t = self._transport(url)
1155
        try:
3734.2.8 by Vincent Ladeuil
Catch spurious exceptions (python-2.6) when SocketServer is shut down.
1156
            self.assertEqual('proxied contents of foo\n', t.get('foo').read())
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1157
        finally:
1158
            self._restore_env()
1159
1160
    def not_proxied_in_env(self, env):
1161
        self._install_env(env)
1162
        url = self.server.get_url()
1163
        t = self._transport(url)
1164
        try:
3734.2.8 by Vincent Ladeuil
Catch spurious exceptions (python-2.6) when SocketServer is shut down.
1165
            self.assertEqual('contents of foo\n', t.get('foo').read())
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1166
        finally:
1167
            self._restore_env()
1168
1169
    def test_http_proxy(self):
1170
        self.proxied_in_env({'http_proxy': self.proxy_url})
1171
1172
    def test_HTTP_PROXY(self):
1173
        if self._testing_pycurl():
1174
            # pycurl does not check HTTP_PROXY for security reasons
1175
            # (for use in a CGI context that we do not care
1176
            # about. Should we ?)
1177
            raise tests.TestNotApplicable(
1178
                'pycurl does not check HTTP_PROXY for security reasons')
1179
        self.proxied_in_env({'HTTP_PROXY': self.proxy_url})
1180
1181
    def test_all_proxy(self):
1182
        self.proxied_in_env({'all_proxy': self.proxy_url})
1183
1184
    def test_ALL_PROXY(self):
1185
        self.proxied_in_env({'ALL_PROXY': self.proxy_url})
1186
1187
    def test_http_proxy_with_no_proxy(self):
1188
        self.not_proxied_in_env({'http_proxy': self.proxy_url,
1189
                                 'no_proxy': self.no_proxy_host})
1190
1191
    def test_HTTP_PROXY_with_NO_PROXY(self):
1192
        if self._testing_pycurl():
1193
            raise tests.TestNotApplicable(
1194
                'pycurl does not check HTTP_PROXY for security reasons')
1195
        self.not_proxied_in_env({'HTTP_PROXY': self.proxy_url,
1196
                                 'NO_PROXY': self.no_proxy_host})
1197
1198
    def test_all_proxy_with_no_proxy(self):
1199
        self.not_proxied_in_env({'all_proxy': self.proxy_url,
1200
                                 'no_proxy': self.no_proxy_host})
1201
1202
    def test_ALL_PROXY_with_NO_PROXY(self):
1203
        self.not_proxied_in_env({'ALL_PROXY': self.proxy_url,
1204
                                 'NO_PROXY': self.no_proxy_host})
1205
1206
    def test_http_proxy_without_scheme(self):
1207
        if self._testing_pycurl():
1208
            # pycurl *ignores* invalid proxy env variables. If that ever change
1209
            # in the future, this test will fail indicating that pycurl do not
1210
            # ignore anymore such variables.
1211
            self.not_proxied_in_env({'http_proxy': self.proxy_address})
1212
        else:
1213
            self.assertRaises(errors.InvalidURL,
1214
                              self.proxied_in_env,
1215
                              {'http_proxy': self.proxy_address})
1216
1217
1218
class TestRanges(http_utils.TestCaseWithWebserver):
1219
    """Test the Range header in GET methods."""
1220
1221
    def setUp(self):
1222
        http_utils.TestCaseWithWebserver.setUp(self)
1223
        self.build_tree_contents([('a', '0123456789')],)
1224
        server = self.get_readonly_server()
1225
        self.transport = self._transport(server.get_url())
1226
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
1227
    def create_transport_readonly_server(self):
1228
        return http_server.HttpServer(protocol_version=self._protocol_version)
1229
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1230
    def _file_contents(self, relpath, ranges):
1231
        offsets = [ (start, end - start + 1) for start, end in ranges]
1232
        coalesce = self.transport._coalesce_offsets
1233
        coalesced = list(coalesce(offsets, limit=0, fudge_factor=0))
1234
        code, data = self.transport._get(relpath, coalesced)
1235
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1236
        for start, end in ranges:
1237
            data.seek(start)
1238
            yield data.read(end - start + 1)
1239
1240
    def _file_tail(self, relpath, tail_amount):
1241
        code, data = self.transport._get(relpath, [], tail_amount)
1242
        self.assertTrue(code in (200, 206),'_get returns: %d' % code)
1243
        data.seek(-tail_amount, 2)
1244
        return data.read(tail_amount)
1245
1246
    def test_range_header(self):
1247
        # Valid ranges
1248
        map(self.assertEqual,['0', '234'],
1249
            list(self._file_contents('a', [(0,0), (2,4)])),)
1250
1251
    def test_range_header_tail(self):
1252
        self.assertEqual('789', self._file_tail('a', 3))
1253
1254
    def test_syntactically_invalid_range_header(self):
1255
        self.assertListRaises(errors.InvalidHttpRange,
1256
                          self._file_contents, 'a', [(4, 3)])
1257
1258
    def test_semantically_invalid_range_header(self):
1259
        self.assertListRaises(errors.InvalidHttpRange,
1260
                          self._file_contents, 'a', [(42, 128)])
1261
1262
1263
class TestHTTPRedirections(http_utils.TestCaseWithRedirectedWebserver):
1264
    """Test redirection between http servers."""
1265
1266
    def create_transport_secondary_server(self):
1267
        """Create the secondary server redirecting to the primary server"""
1268
        new = self.get_readonly_server()
1269
1270
        redirecting = http_utils.HTTPServerRedirecting(
1271
            protocol_version=self._protocol_version)
1272
        redirecting.redirect_to(new.host, new.port)
1273
        return redirecting
1274
1275
    def setUp(self):
1276
        super(TestHTTPRedirections, self).setUp()
1277
        self.build_tree_contents([('a', '0123456789'),
1278
                                  ('bundle',
1279
                                  '# Bazaar revision bundle v0.9\n#\n')
1280
                                  ],)
3878.4.1 by Vincent Ladeuil
Fix bug #245964 by preserving decorators during redirections (when
1281
        # The requests to the old server will be redirected to the new server
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1282
        self.old_transport = self._transport(self.old_server.get_url())
1283
1284
    def test_redirected(self):
1285
        self.assertRaises(errors.RedirectRequested, self.old_transport.get, 'a')
1286
        t = self._transport(self.new_server.get_url())
1287
        self.assertEqual('0123456789', t.get('a').read())
1288
1289
    def test_read_redirected_bundle_from_url(self):
1290
        from bzrlib.bundle import read_bundle_from_url
1291
        url = self.old_transport.abspath('bundle')
3995.2.2 by Martin Pool
Cope with read_bundle_from_url deprecation in test_http
1292
        bundle = self.applyDeprecated(deprecated_in((1, 12, 0)),
1293
                read_bundle_from_url, url)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1294
        # If read_bundle_from_url was successful we get an empty bundle
1295
        self.assertEqual([], bundle.revisions)
1296
1297
1298
class RedirectedRequest(_urllib2_wrappers.Request):
1299
    """Request following redirections. """
1300
1301
    init_orig = _urllib2_wrappers.Request.__init__
1302
1303
    def __init__(self, method, url, *args, **kwargs):
1304
        """Constructor.
1305
1306
        """
1307
        # Since the tests using this class will replace
1308
        # _urllib2_wrappers.Request, we can't just call the base class __init__
1309
        # or we'll loop.
1310
        RedirectedRequest.init_orig(self, method, url, args, kwargs)
1311
        self.follow_redirections = True
1312
1313
1314
class TestHTTPSilentRedirections(http_utils.TestCaseWithRedirectedWebserver):
1315
    """Test redirections.
1316
1317
    http implementations do not redirect silently anymore (they
1318
    do not redirect at all in fact). The mechanism is still in
1319
    place at the _urllib2_wrappers.Request level and these tests
1320
    exercise it.
1321
1322
    For the pycurl implementation
1323
    the redirection have been deleted as we may deprecate pycurl
1324
    and I have no place to keep a working implementation.
1325
    -- vila 20070212
1326
    """
1327
1328
    def setUp(self):
1329
        if pycurl_present and self._transport == PyCurlTransport:
1330
            raise tests.TestNotApplicable(
1331
                "pycurl doesn't redirect silently annymore")
1332
        super(TestHTTPSilentRedirections, self).setUp()
1333
        self.setup_redirected_request()
1334
        self.addCleanup(self.cleanup_redirected_request)
1335
        self.build_tree_contents([('a','a'),
1336
                                  ('1/',),
1337
                                  ('1/a', 'redirected once'),
1338
                                  ('2/',),
1339
                                  ('2/a', 'redirected twice'),
1340
                                  ('3/',),
1341
                                  ('3/a', 'redirected thrice'),
1342
                                  ('4/',),
1343
                                  ('4/a', 'redirected 4 times'),
1344
                                  ('5/',),
1345
                                  ('5/a', 'redirected 5 times'),
1346
                                  ],)
1347
1348
        self.old_transport = self._transport(self.old_server.get_url())
1349
1350
    def setup_redirected_request(self):
1351
        self.original_class = _urllib2_wrappers.Request
1352
        _urllib2_wrappers.Request = RedirectedRequest
1353
1354
    def cleanup_redirected_request(self):
1355
        _urllib2_wrappers.Request = self.original_class
1356
1357
    def create_transport_secondary_server(self):
1358
        """Create the secondary server, redirections are defined in the tests"""
1359
        return http_utils.HTTPServerRedirecting(
1360
            protocol_version=self._protocol_version)
1361
1362
    def test_one_redirection(self):
1363
        t = self.old_transport
1364
1365
        req = RedirectedRequest('GET', t.abspath('a'))
1366
        req.follow_redirections = True
1367
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1368
                                       self.new_server.port)
1369
        self.old_server.redirections = \
1370
            [('(.*)', r'%s/1\1' % (new_prefix), 301),]
1371
        self.assertEquals('redirected once',t._perform(req).read())
1372
1373
    def test_five_redirections(self):
1374
        t = self.old_transport
1375
1376
        req = RedirectedRequest('GET', t.abspath('a'))
1377
        req.follow_redirections = True
1378
        old_prefix = 'http://%s:%s' % (self.old_server.host,
1379
                                       self.old_server.port)
1380
        new_prefix = 'http://%s:%s' % (self.new_server.host,
1381
                                       self.new_server.port)
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
1382
        self.old_server.redirections = [
1383
            ('/1(.*)', r'%s/2\1' % (old_prefix), 302),
1384
            ('/2(.*)', r'%s/3\1' % (old_prefix), 303),
1385
            ('/3(.*)', r'%s/4\1' % (old_prefix), 307),
1386
            ('/4(.*)', r'%s/5\1' % (new_prefix), 301),
1387
            ('(/[^/]+)', r'%s/1\1' % (old_prefix), 301),
1388
            ]
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1389
        self.assertEquals('redirected 5 times',t._perform(req).read())
1390
1391
1392
class TestDoCatchRedirections(http_utils.TestCaseWithRedirectedWebserver):
1393
    """Test transport.do_catching_redirections."""
1394
1395
    def setUp(self):
1396
        super(TestDoCatchRedirections, self).setUp()
1397
        self.build_tree_contents([('a', '0123456789'),],)
1398
1399
        self.old_transport = self._transport(self.old_server.get_url())
1400
1401
    def get_a(self, transport):
1402
        return transport.get('a')
1403
1404
    def test_no_redirection(self):
1405
        t = self._transport(self.new_server.get_url())
1406
1407
        # We use None for redirected so that we fail if redirected
1408
        self.assertEquals('0123456789',
1409
                          transport.do_catching_redirections(
1410
                self.get_a, t, None).read())
1411
1412
    def test_one_redirection(self):
1413
        self.redirections = 0
1414
1415
        def redirected(transport, exception, redirection_notice):
1416
            self.redirections += 1
1417
            dir, file = urlutils.split(exception.target)
1418
            return self._transport(dir)
1419
1420
        self.assertEquals('0123456789',
1421
                          transport.do_catching_redirections(
1422
                self.get_a, self.old_transport, redirected).read())
1423
        self.assertEquals(1, self.redirections)
1424
1425
    def test_redirection_loop(self):
1426
1427
        def redirected(transport, exception, redirection_notice):
1428
            # By using the redirected url as a base dir for the
1429
            # *old* transport, we create a loop: a => a/a =>
1430
            # a/a/a
1431
            return self.old_transport.clone(exception.target)
1432
1433
        self.assertRaises(errors.TooManyRedirections,
1434
                          transport.do_catching_redirections,
1435
                          self.get_a, self.old_transport, redirected)
1436
1437
1438
class TestAuth(http_utils.TestCaseWithWebserver):
1439
    """Test authentication scheme"""
1440
1441
    _auth_header = 'Authorization'
1442
    _password_prompt_prefix = ''
1443
1444
    def setUp(self):
1445
        super(TestAuth, self).setUp()
1446
        self.server = self.get_readonly_server()
1447
        self.build_tree_contents([('a', 'contents of a\n'),
1448
                                  ('b', 'contents of b\n'),])
1449
1450
    def create_transport_readonly_server(self):
1451
        if self._auth_scheme == 'basic':
1452
            server = http_utils.HTTPBasicAuthServer(
1453
                protocol_version=self._protocol_version)
1454
        else:
1455
            if self._auth_scheme != 'digest':
1456
                raise AssertionError('Unknown auth scheme: %r'
1457
                                     % self._auth_scheme)
1458
            server = http_utils.HTTPDigestAuthServer(
1459
                protocol_version=self._protocol_version)
1460
        return server
1461
1462
    def _testing_pycurl(self):
1463
        return pycurl_present and self._transport == PyCurlTransport
1464
3910.2.4 by Vincent Ladeuil
Fixed as per John's review.
1465
    def get_user_url(self, user, password):
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1466
        """Build an url embedding user and password"""
1467
        url = '%s://' % self.server._url_protocol
1468
        if user is not None:
1469
            url += user
1470
            if password is not None:
1471
                url += ':' + password
1472
            url += '@'
1473
        url += '%s:%s/' % (self.server.host, self.server.port)
1474
        return url
1475
3910.2.4 by Vincent Ladeuil
Fixed as per John's review.
1476
    def get_user_transport(self, user, password):
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1477
        return self._transport(self.get_user_url(user, password))
1478
1479
    def test_no_user(self):
1480
        self.server.add_user('joe', 'foo')
3910.2.4 by Vincent Ladeuil
Fixed as per John's review.
1481
        t = self.get_user_transport(None, None)
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1482
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1483
        # Only one 'Authentication Required' error should occur
1484
        self.assertEqual(1, self.server.auth_required_errors)
1485
1486
    def test_empty_pass(self):
1487
        self.server.add_user('joe', '')
1488
        t = self.get_user_transport('joe', '')
1489
        self.assertEqual('contents of a\n', t.get('a').read())
1490
        # Only one 'Authentication Required' error should occur
1491
        self.assertEqual(1, self.server.auth_required_errors)
1492
1493
    def test_user_pass(self):
1494
        self.server.add_user('joe', 'foo')
1495
        t = self.get_user_transport('joe', 'foo')
1496
        self.assertEqual('contents of a\n', t.get('a').read())
1497
        # Only one 'Authentication Required' error should occur
1498
        self.assertEqual(1, self.server.auth_required_errors)
1499
1500
    def test_unknown_user(self):
1501
        self.server.add_user('joe', 'foo')
1502
        t = self.get_user_transport('bill', 'foo')
1503
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1504
        # Two 'Authentication Required' errors should occur (the
1505
        # initial 'who are you' and 'I don't know you, who are
1506
        # you').
1507
        self.assertEqual(2, self.server.auth_required_errors)
1508
1509
    def test_wrong_pass(self):
1510
        self.server.add_user('joe', 'foo')
1511
        t = self.get_user_transport('joe', 'bar')
1512
        self.assertRaises(errors.InvalidHttpResponse, t.get, 'a')
1513
        # Two 'Authentication Required' errors should occur (the
1514
        # initial 'who are you' and 'this is not you, who are you')
1515
        self.assertEqual(2, self.server.auth_required_errors)
1516
1517
    def test_prompt_for_password(self):
1518
        if self._testing_pycurl():
1519
            raise tests.TestNotApplicable(
1520
                'pycurl cannot prompt, it handles auth by embedding'
1521
                ' user:pass in urls only')
1522
1523
        self.server.add_user('joe', 'foo')
1524
        t = self.get_user_transport('joe', None)
1525
        stdout = tests.StringIOWrapper()
1526
        ui.ui_factory = tests.TestUIFactory(stdin='foo\n', stdout=stdout)
1527
        self.assertEqual('contents of a\n',t.get('a').read())
1528
        # stdin should be empty
1529
        self.assertEqual('', ui.ui_factory.stdin.readline())
1530
        self._check_password_prompt(t._unqualified_scheme, 'joe',
1531
                                    stdout.getvalue())
1532
        # And we shouldn't prompt again for a different request
1533
        # against the same transport.
1534
        self.assertEqual('contents of b\n',t.get('b').read())
1535
        t2 = t.clone()
1536
        # And neither against a clone
1537
        self.assertEqual('contents of b\n',t2.get('b').read())
1538
        # Only one 'Authentication Required' error should occur
1539
        self.assertEqual(1, self.server.auth_required_errors)
1540
1541
    def _check_password_prompt(self, scheme, user, actual_prompt):
1542
        expected_prompt = (self._password_prompt_prefix
1543
                           + ("%s %s@%s:%d, Realm: '%s' password: "
1544
                              % (scheme.upper(),
1545
                                 user, self.server.host, self.server.port,
1546
                                 self.server.auth_realm)))
1547
        self.assertEquals(expected_prompt, actual_prompt)
1548
1549
    def test_no_prompt_for_password_when_using_auth_config(self):
1550
        if self._testing_pycurl():
1551
            raise tests.TestNotApplicable(
1552
                'pycurl does not support authentication.conf'
1553
                ' since it cannot prompt')
1554
1555
        user =' joe'
1556
        password = 'foo'
1557
        stdin_content = 'bar\n'  # Not the right password
1558
        self.server.add_user(user, password)
1559
        t = self.get_user_transport(user, None)
1560
        ui.ui_factory = tests.TestUIFactory(stdin=stdin_content,
1561
                                            stdout=tests.StringIOWrapper())
1562
        # Create a minimal config file with the right password
1563
        conf = config.AuthenticationConfig()
1564
        conf._get_config().update(
1565
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1566
                          'user': user, 'password': password}})
1567
        conf._save()
1568
        # Issue a request to the server to connect
1569
        self.assertEqual('contents of a\n',t.get('a').read())
1570
        # stdin should have  been left untouched
1571
        self.assertEqual(stdin_content, ui.ui_factory.stdin.readline())
1572
        # Only one 'Authentication Required' error should occur
1573
        self.assertEqual(1, self.server.auth_required_errors)
1574
3910.2.2 by Vincent Ladeuil
Fix bug #300347 by allowing querying authentication.conf if no
1575
    def test_user_from_auth_conf(self):
1576
        if self._testing_pycurl():
1577
            raise tests.TestNotApplicable(
1578
                'pycurl does not support authentication.conf')
3910.2.3 by Ben Jansen
Made tweaks requested by John Arbash Meinel.
1579
        user = 'joe'
3910.2.2 by Vincent Ladeuil
Fix bug #300347 by allowing querying authentication.conf if no
1580
        password = 'foo'
1581
        self.server.add_user(user, password)
1582
        # Create a minimal config file with the right password
1583
        conf = config.AuthenticationConfig()
1584
        conf._get_config().update(
1585
            {'httptest': {'scheme': 'http', 'port': self.server.port,
1586
                          'user': user, 'password': password}})
1587
        conf._save()
3910.2.4 by Vincent Ladeuil
Fixed as per John's review.
1588
        t = self.get_user_transport(None, None)
3910.2.2 by Vincent Ladeuil
Fix bug #300347 by allowing querying authentication.conf if no
1589
        # Issue a request to the server to connect
3910.2.3 by Ben Jansen
Made tweaks requested by John Arbash Meinel.
1590
        self.assertEqual('contents of a\n', t.get('a').read())
3910.2.2 by Vincent Ladeuil
Fix bug #300347 by allowing querying authentication.conf if no
1591
        # Only one 'Authentication Required' error should occur
1592
        self.assertEqual(1, self.server.auth_required_errors)
1593
3111.1.26 by Vincent Ladeuil
Re-add a test lost in refactoring.
1594
    def test_changing_nonce(self):
1595
        if self._auth_scheme != 'digest':
1596
            raise tests.TestNotApplicable('HTTP auth digest only test')
1597
        if self._testing_pycurl():
1598
            raise tests.KnownFailure(
1599
                'pycurl does not handle a nonce change')
1600
        self.server.add_user('joe', 'foo')
1601
        t = self.get_user_transport('joe', 'foo')
1602
        self.assertEqual('contents of a\n', t.get('a').read())
1603
        self.assertEqual('contents of b\n', t.get('b').read())
1604
        # Only one 'Authentication Required' error should have
1605
        # occured so far
1606
        self.assertEqual(1, self.server.auth_required_errors)
1607
        # The server invalidates the current nonce
1608
        self.server.auth_nonce = self.server.auth_nonce + '. No, now!'
1609
        self.assertEqual('contents of a\n', t.get('a').read())
1610
        # Two 'Authentication Required' errors should occur (the
1611
        # initial 'who are you' and a second 'who are you' with the new nonce)
1612
        self.assertEqual(2, self.server.auth_required_errors)
1613
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1614
1615
1616
class TestProxyAuth(TestAuth):
1617
    """Test proxy authentication schemes."""
1618
1619
    _auth_header = 'Proxy-authorization'
1620
    _password_prompt_prefix='Proxy '
1621
1622
    def setUp(self):
1623
        super(TestProxyAuth, self).setUp()
1624
        self._old_env = {}
1625
        self.addCleanup(self._restore_env)
1626
        # Override the contents to avoid false positives
1627
        self.build_tree_contents([('a', 'not proxied contents of a\n'),
1628
                                  ('b', 'not proxied contents of b\n'),
1629
                                  ('a-proxied', 'contents of a\n'),
1630
                                  ('b-proxied', 'contents of b\n'),
1631
                                  ])
1632
1633
    def create_transport_readonly_server(self):
1634
        if self._auth_scheme == 'basic':
1635
            server = http_utils.ProxyBasicAuthServer(
1636
                protocol_version=self._protocol_version)
1637
        else:
1638
            if self._auth_scheme != 'digest':
1639
                raise AssertionError('Unknown auth scheme: %r'
1640
                                     % self._auth_scheme)
1641
            server = http_utils.ProxyDigestAuthServer(
1642
                protocol_version=self._protocol_version)
1643
        return server
1644
3910.2.4 by Vincent Ladeuil
Fixed as per John's review.
1645
    def get_user_transport(self, user, password):
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
1646
        self._install_env({'all_proxy': self.get_user_url(user, password)})
1647
        return self._transport(self.server.get_url())
1648
1649
    def _install_env(self, env):
1650
        for name, value in env.iteritems():
1651
            self._old_env[name] = osutils.set_or_unset_env(name, value)
1652
1653
    def _restore_env(self):
1654
        for name, value in self._old_env.iteritems():
1655
            osutils.set_or_unset_env(name, value)
1656
1657
    def test_empty_pass(self):
1658
        if self._testing_pycurl():
1659
            import pycurl
1660
            if pycurl.version_info()[1] < '7.16.0':
1661
                raise tests.KnownFailure(
1662
                    'pycurl < 7.16.0 does not handle empty proxy passwords')
1663
        super(TestProxyAuth, self).test_empty_pass()
1664
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1665
1666
class SampleSocket(object):
1667
    """A socket-like object for use in testing the HTTP request handler."""
1668
1669
    def __init__(self, socket_read_content):
1670
        """Constructs a sample socket.
1671
1672
        :param socket_read_content: a byte sequence
1673
        """
1674
        # Use plain python StringIO so we can monkey-patch the close method to
1675
        # not discard the contents.
1676
        from StringIO import StringIO
1677
        self.readfile = StringIO(socket_read_content)
1678
        self.writefile = StringIO()
1679
        self.writefile.close = lambda: None
1680
1681
    def makefile(self, mode='r', bufsize=None):
1682
        if 'r' in mode:
1683
            return self.readfile
1684
        else:
1685
            return self.writefile
1686
1687
1688
class SmartHTTPTunnellingTest(tests.TestCaseWithTransport):
1689
1690
    def setUp(self):
1691
        super(SmartHTTPTunnellingTest, self).setUp()
1692
        # We use the VFS layer as part of HTTP tunnelling tests.
1693
        self._captureVar('BZR_NO_SMART_VFS', None)
1694
        self.transport_readonly_server = http_utils.HTTPServerWithSmarts
1695
1696
    def create_transport_readonly_server(self):
1697
        return http_utils.HTTPServerWithSmarts(
1698
            protocol_version=self._protocol_version)
1699
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
1700
    def test_open_bzrdir(self):
1701
        branch = self.make_branch('relpath')
1702
        http_server = self.get_readonly_server()
1703
        url = http_server.get_url() + 'relpath'
1704
        bd = bzrdir.BzrDir.open(url)
1705
        self.assertIsInstance(bd, _mod_remote.RemoteBzrDir)
1706
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1707
    def test_bulk_data(self):
1708
        # We should be able to send and receive bulk data in a single message.
1709
        # The 'readv' command in the smart protocol both sends and receives
1710
        # bulk data, so we use that.
1711
        self.build_tree(['data-file'])
1712
        http_server = self.get_readonly_server()
1713
        http_transport = self._transport(http_server.get_url())
1714
        medium = http_transport.get_smart_medium()
1715
        # Since we provide the medium, the url below will be mostly ignored
1716
        # during the test, as long as the path is '/'.
1717
        remote_transport = remote.RemoteTransport('bzr://fake_host/',
1718
                                                  medium=medium)
1719
        self.assertEqual(
1720
            [(0, "c")], list(remote_transport.readv("data-file", [(0,1)])))
1721
1722
    def test_http_send_smart_request(self):
1723
1724
        post_body = 'hello\n'
3245.4.59 by Andrew Bennetts
Various tweaks in response to Martin's review.
1725
        expected_reply_body = 'ok\x012\n'
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1726
1727
        http_server = self.get_readonly_server()
1728
        http_transport = self._transport(http_server.get_url())
1729
        medium = http_transport.get_smart_medium()
1730
        response = medium.send_http_smart_request(post_body)
1731
        reply_body = response.read()
1732
        self.assertEqual(expected_reply_body, reply_body)
1733
1734
    def test_smart_http_server_post_request_handler(self):
1735
        httpd = self.get_readonly_server()._get_httpd()
1736
1737
        socket = SampleSocket(
1738
            'POST /.bzr/smart %s \r\n' % self._protocol_version
1739
            # HTTP/1.1 posts must have a Content-Length (but it doesn't hurt
1740
            # for 1.0)
1741
            + 'Content-Length: 6\r\n'
1742
            '\r\n'
1743
            'hello\n')
1744
        # Beware: the ('localhost', 80) below is the
1745
        # client_address parameter, but we don't have one because
1746
        # we have defined a socket which is not bound to an
1747
        # address. The test framework never uses this client
1748
        # address, so far...
1749
        request_handler = http_utils.SmartRequestHandler(socket,
1750
                                                         ('localhost', 80),
1751
                                                         httpd)
1752
        response = socket.writefile.getvalue()
1753
        self.assertStartsWith(response, '%s 200 ' % self._protocol_version)
1754
        # This includes the end of the HTTP headers, and all the body.
3245.4.59 by Andrew Bennetts
Various tweaks in response to Martin's review.
1755
        expected_end_of_response = '\r\n\r\nok\x012\n'
3111.1.25 by Vincent Ladeuil
Fix the smart server failing test and use it against protocol combinations.
1756
        self.assertEndsWith(response, expected_end_of_response)
1757
1758
3430.3.4 by Vincent Ladeuil
Of course we can write tests !
1759
class ForbiddenRequestHandler(http_server.TestingHTTPRequestHandler):
1760
    """No smart server here request handler."""
1761
1762
    def do_POST(self):
1763
        self.send_error(403, "Forbidden")
1764
1765
1766
class SmartClientAgainstNotSmartServer(TestSpecificRequestHandler):
1767
    """Test smart client behaviour against an http server without smarts."""
1768
1769
    _req_handler_class = ForbiddenRequestHandler
1770
1771
    def test_probe_smart_server(self):
1772
        """Test error handling against server refusing smart requests."""
1773
        server = self.get_readonly_server()
1774
        t = self._transport(server.get_url())
1775
        # No need to build a valid smart request here, the server will not even
1776
        # try to interpret it.
1777
        self.assertRaises(errors.SmartProtocolError,
3734.3.1 by Vincent Ladeuil
Fix SmartHTTPMedium refactoring related test.
1778
                          t.get_smart_medium().send_http_smart_request,
1779
                          'whatever')
3430.3.4 by Vincent Ladeuil
Of course we can write tests !
1780
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
1781
class Test_redirected_to(tests.TestCase):
1782
1783
    def test_redirected_to_subdir(self):
1784
        t = self._transport('http://www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
1785
        r = t._redirected_to('http://www.example.com/foo',
1786
                             'http://www.example.com/foo/subdir')
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
1787
        self.assertIsInstance(r, type(t))
1788
        # Both transports share the some connection
1789
        self.assertEquals(t._get_connection(), r._get_connection())
1790
3878.4.3 by Vincent Ladeuil
Fix bug #303959 by returning a transport based on the same url
1791
    def test_redirected_to_self_with_slash(self):
1792
        t = self._transport('http://www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
1793
        r = t._redirected_to('http://www.example.com/foo',
1794
                             'http://www.example.com/foo/')
3878.4.3 by Vincent Ladeuil
Fix bug #303959 by returning a transport based on the same url
1795
        self.assertIsInstance(r, type(t))
1796
        # Both transports share the some connection (one can argue that we
1797
        # should return the exact same transport here, but that seems
1798
        # overkill).
1799
        self.assertEquals(t._get_connection(), r._get_connection())
1800
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
1801
    def test_redirected_to_host(self):
1802
        t = self._transport('http://www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
1803
        r = t._redirected_to('http://www.example.com/foo',
1804
                             'http://foo.example.com/foo/subdir')
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
1805
        self.assertIsInstance(r, type(t))
1806
1807
    def test_redirected_to_same_host_sibling_protocol(self):
1808
        t = self._transport('http://www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
1809
        r = t._redirected_to('http://www.example.com/foo',
1810
                             'https://www.example.com/foo')
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
1811
        self.assertIsInstance(r, type(t))
1812
1813
    def test_redirected_to_same_host_different_protocol(self):
1814
        t = self._transport('http://www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
1815
        r = t._redirected_to('http://www.example.com/foo',
1816
                             'ftp://www.example.com/foo')
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
1817
        self.assertNotEquals(type(r), type(t))
1818
1819
    def test_redirected_to_different_host_same_user(self):
1820
        t = self._transport('http://joe@www.example.com/foo')
3878.4.5 by Vincent Ladeuil
Don't use the exception as a parameter for _redirected_to.
1821
        r = t._redirected_to('http://www.example.com/foo',
1822
                             'https://foo.example.com/foo')
3878.4.2 by Vincent Ladeuil
Fix bug #265070 by providing a finer sieve for accepted redirections.
1823
        self.assertIsInstance(r, type(t))
1824
        self.assertEquals(t._user, r._user)
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
1825
1826
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
1827
class PredefinedRequestHandler(http_server.TestingHTTPRequestHandler):
1828
    """Request handler for a unique and pre-defined request.
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
1829
1830
    The only thing we care about here is how many bytes travel on the wire. But
1831
    since we want to measure it for a real http client, we have to send it
1832
    correct responses.
1833
1834
    We expect to receive a *single* request nothing more (and we won't even
1835
    check what request it is, we just measure the bytes read until an empty
1836
    line.
1837
    """
1838
1839
    def handle_one_request(self):
1840
        tcs = self.server.test_case_server
1841
        requestline = self.rfile.readline()
1842
        headers = self.MessageClass(self.rfile, 0)
1843
        # We just read: the request, the headers, an empty line indicating the
1844
        # end of the headers.
1845
        bytes_read = len(requestline)
1846
        for line in headers.headers:
1847
            bytes_read += len(line)
1848
        bytes_read += len('\r\n')
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
1849
        if requestline.startswith('POST'):
1850
            # The body should be a single line (or we don't know where it ends
1851
            # and we don't want to issue a blocking read)
1852
            body = self.rfile.readline()
1853
            bytes_read += len(body)
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
1854
        tcs.bytes_read = bytes_read
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
1855
1856
        # We set the bytes written *before* issuing the write, the client is
1857
        # supposed to consume every produced byte *before* checking that value.
3945.1.7 by Vincent Ladeuil
Test against https.
1858
1859
        # Doing the oppposite may lead to test failure: we may be interrupted
1860
        # after the write but before updating the value. The client can then
1861
        # continue and read the value *before* we can update it. And yes,
1862
        # this has been observed -- vila 20090129
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
1863
        tcs.bytes_written = len(tcs.canned_response)
1864
        self.wfile.write(tcs.canned_response)
1865
1866
1867
class ActivityServerMixin(object):
1868
1869
    def __init__(self, protocol_version):
1870
        super(ActivityServerMixin, self).__init__(
1871
            request_handler=PredefinedRequestHandler,
1872
            protocol_version=protocol_version)
1873
        # Bytes read and written by the server
1874
        self.bytes_read = 0
1875
        self.bytes_written = 0
1876
        self.canned_response = None
1877
1878
1879
class ActivityHTTPServer(ActivityServerMixin, http_server.HttpServer):
1880
    pass
1881
1882
1883
if tests.HTTPSServerFeature.available():
1884
    from bzrlib.tests import https_server
1885
    class ActivityHTTPSServer(ActivityServerMixin, https_server.HTTPSServer):
1886
        pass
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
1887
1888
1889
class TestActivity(tests.TestCase):
1890
    """Test socket activity reporting.
1891
1892
    We use a special purpose server to control the bytes sent and received and
1893
    be able to predict the activity on the client socket.
1894
    """
1895
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
1896
    def setUp(self):
1897
        tests.TestCase.setUp(self)
1898
        self.server = self._activity_server(self._protocol_version)
1899
        self.server.setUp()
1900
        self.activities = {}
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
1901
        def report_activity(t, bytes, direction):
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
1902
            count = self.activities.get(direction, 0)
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
1903
            count += bytes
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
1904
            self.activities[direction] = count
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
1905
1906
        # We override at class level because constructors may propagate the
1907
        # bound method and render instance overriding ineffective (an
1908
        # alternative would be be to define a specific ui factory instead...)
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
1909
        self.orig_report_activity = self._transport._report_activity
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
1910
        self._transport._report_activity = report_activity
1911
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
1912
    def tearDown(self):
1913
        self._transport._report_activity = self.orig_report_activity
1914
        self.server.tearDown()
1915
        tests.TestCase.tearDown(self)
1916
1917
    def get_transport(self):
1918
        return self._transport(self.server.get_url())
1919
1920
    def assertActivitiesMatch(self):
1921
        self.assertEqual(self.server.bytes_read,
1922
                         self.activities.get('write', 0), 'written bytes')
1923
        self.assertEqual(self.server.bytes_written,
1924
                         self.activities.get('read', 0), 'read bytes')
1925
1926
    def test_get(self):
1927
        self.server.canned_response = '''HTTP/1.1 200 OK\r
1928
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
1929
Server: Apache/2.0.54 (Fedora)\r
1930
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
1931
ETag: "56691-23-38e9ae00"\r
1932
Accept-Ranges: bytes\r
1933
Content-Length: 35\r
1934
Connection: close\r
1935
Content-Type: text/plain; charset=UTF-8\r
1936
\r
1937
Bazaar-NG meta directory, format 1
1938
'''
1939
        t = self.get_transport()
3945.1.5 by Vincent Ladeuil
Start implementing http activity reporting at socket level.
1940
        self.assertEqual('Bazaar-NG meta directory, format 1\n',
1941
                         t.get('foo/bar').read())
3945.1.8 by Vincent Ladeuil
Add more tests, fix pycurl double handling, revert previous tracking.
1942
        self.assertActivitiesMatch()
1943
1944
    def test_has(self):
1945
        self.server.canned_response = '''HTTP/1.1 200 OK\r
1946
Server: SimpleHTTP/0.6 Python/2.5.2\r
1947
Date: Thu, 29 Jan 2009 20:21:47 GMT\r
1948
Content-type: application/octet-stream\r
1949
Content-Length: 20\r
1950
Last-Modified: Thu, 29 Jan 2009 20:21:47 GMT\r
1951
\r
1952
'''
1953
        t = self.get_transport()
1954
        self.assertTrue(t.has('foo/bar'))
1955
        self.assertActivitiesMatch()
1956
1957
    def test_readv(self):
1958
        self.server.canned_response = '''HTTP/1.1 206 Partial Content\r
1959
Date: Tue, 11 Jul 2006 04:49:48 GMT\r
1960
Server: Apache/2.0.54 (Fedora)\r
1961
Last-Modified: Thu, 06 Jul 2006 20:22:05 GMT\r
1962
ETag: "238a3c-16ec2-805c5540"\r
1963
Accept-Ranges: bytes\r
1964
Content-Length: 1534\r
1965
Connection: close\r
1966
Content-Type: multipart/byteranges; boundary=418470f848b63279b\r
1967
\r
1968
\r
1969
--418470f848b63279b\r
1970
Content-type: text/plain; charset=UTF-8\r
1971
Content-range: bytes 0-254/93890\r
1972
\r
1973
mbp@sourcefrog.net-20050309040815-13242001617e4a06
1974
mbp@sourcefrog.net-20050309040929-eee0eb3e6d1e7627
1975
mbp@sourcefrog.net-20050309040957-6cad07f466bb0bb8
1976
mbp@sourcefrog.net-20050309041501-c840e09071de3b67
1977
mbp@sourcefrog.net-20050309044615-c24a3250be83220a
1978
\r
1979
--418470f848b63279b\r
1980
Content-type: text/plain; charset=UTF-8\r
1981
Content-range: bytes 1000-2049/93890\r
1982
\r
1983
40-fd4ec249b6b139ab
1984
mbp@sourcefrog.net-20050311063625-07858525021f270b
1985
mbp@sourcefrog.net-20050311231934-aa3776aff5200bb9
1986
mbp@sourcefrog.net-20050311231953-73aeb3a131c3699a
1987
mbp@sourcefrog.net-20050311232353-f5e33da490872c6a
1988
mbp@sourcefrog.net-20050312071639-0a8f59a34a024ff0
1989
mbp@sourcefrog.net-20050312073432-b2c16a55e0d6e9fb
1990
mbp@sourcefrog.net-20050312073831-a47c3335ece1920f
1991
mbp@sourcefrog.net-20050312085412-13373aa129ccbad3
1992
mbp@sourcefrog.net-20050313052251-2bf004cb96b39933
1993
mbp@sourcefrog.net-20050313052856-3edd84094687cb11
1994
mbp@sourcefrog.net-20050313053233-e30a4f28aef48f9d
1995
mbp@sourcefrog.net-20050313053853-7c64085594ff3072
1996
mbp@sourcefrog.net-20050313054757-a86c3f5871069e22
1997
mbp@sourcefrog.net-20050313061422-418f1f73b94879b9
1998
mbp@sourcefrog.net-20050313120651-497bd231b19df600
1999
mbp@sourcefrog.net-20050314024931-eae0170ef25a5d1a
2000
mbp@sourcefrog.net-20050314025438-d52099f915fe65fc
2001
mbp@sourcefrog.net-20050314025539-637a636692c055cf
2002
mbp@sourcefrog.net-20050314025737-55eb441f430ab4ba
2003
mbp@sourcefrog.net-20050314025901-d74aa93bb7ee8f62
2004
mbp@source\r
2005
--418470f848b63279b--\r
2006
'''
2007
        t = self.get_transport()
2008
        # Remember that the request is ignored and that the ranges below
2009
        # doesn't have to match the canned response.
2010
        l = list(t.readv('/foo/bar', ((0, 255), (1000, 1050))))
2011
        self.assertEqual(2, len(l))
2012
        self.assertActivitiesMatch()
2013
2014
    def test_post(self):
2015
        self.server.canned_response = '''HTTP/1.1 200 OK\r
2016
Date: Tue, 11 Jul 2006 04:32:56 GMT\r
2017
Server: Apache/2.0.54 (Fedora)\r
2018
Last-Modified: Sun, 23 Apr 2006 19:35:20 GMT\r
2019
ETag: "56691-23-38e9ae00"\r
2020
Accept-Ranges: bytes\r
2021
Content-Length: 35\r
2022
Connection: close\r
2023
Content-Type: text/plain; charset=UTF-8\r
2024
\r
2025
lalala whatever as long as itsssss
2026
'''
2027
        t = self.get_transport()
2028
        # We must send a single line of body bytes, see
2029
        # PredefinedRequestHandler.handle_one_request
2030
        code, f = t._post('abc def end-of-body\n')
2031
        self.assertEqual('lalala whatever as long as itsssss\n', f.read())
2032
        self.assertActivitiesMatch()