~bzr-pqm/bzr/bzr.dev

2004.1.40 by v.ladeuil+lp at free
Fix the race condition again and correct some small typos to be in
1
# Copyright (C) 2006 Canonical Ltd
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
import BaseHTTPServer
18
import errno
19
import os
20
from SimpleHTTPServer import SimpleHTTPRequestHandler
21
import socket
2146.1.1 by Alexander Belchenko
fixes for test suite: forgotten imports in HttpServer.py
22
import posixpath
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
23
import random
24
import re
25
import sys
26
import threading
27
import time
2146.1.1 by Alexander Belchenko
fixes for test suite: forgotten imports in HttpServer.py
28
import urllib
29
import urlparse
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
30
31
from bzrlib.transport import Server
2381.1.2 by Robert Collins
Fixup the test changes made for hpss to be clean and self contained.
32
from bzrlib.transport.local import LocalURLServer
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
33
34
35
class WebserverNotAvailable(Exception):
36
    pass
37
38
39
class BadWebserverPath(ValueError):
40
    def __str__(self):
41
        return 'path %s is not in %s' % self.args
42
43
44
class TestingHTTPRequestHandler(SimpleHTTPRequestHandler):
2420.1.10 by Vincent Ladeuil
Doc fixes.
45
    """Handles one request.
46
47
    A TestingHTTPRequestHandler is instantiated for every request
2420.1.12 by Vincent Ladeuil
Cometic changes.
48
    received by the associated server.
2420.1.10 by Vincent Ladeuil
Doc fixes.
49
    """
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
50
51
    def log_message(self, format, *args):
2164.2.28 by Vincent Ladeuil
TestingHTTPServer.test_case_server renamed from test_case to avoid confusions.
52
        tcs = self.server.test_case_server
53
        tcs.log('webserver - %s - - [%s] %s "%s" "%s"',
54
                self.address_string(),
55
                self.log_date_time_string(),
56
                format % args,
57
                self.headers.get('referer', '-'),
58
                self.headers.get('user-agent', '-'))
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
59
60
    def handle_one_request(self):
61
        """Handle a single HTTP request.
62
2831.6.1 by Vincent Ladeuil
Remove some more noise from test suite.
63
        We catch all socket errors occurring when the client close the
64
        connection early to avoid polluting the test results.
65
        """
66
        try:
2953.2.1 by Vincent Ladeuil
Fix #158972 by not using timeout for HttpServer.
67
            SimpleHTTPRequestHandler.handle_one_request(self)
2831.6.1 by Vincent Ladeuil
Remove some more noise from test suite.
68
        except socket.error, e:
69
            if (len(e.args) > 0
70
                and e.args[0] in (errno.EPIPE, errno.ECONNRESET,
71
                                  errno.ECONNABORTED,)):
72
                self.close_connection = 1
73
                pass
74
            else:
75
                raise
76
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
77
    _range_regexp = re.compile(r'^(?P<start>\d+)-(?P<end>\d+)$')
78
    _tail_regexp = re.compile(r'^-(?P<tail>\d+)$')
79
80
    def parse_ranges(self, ranges_header):
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
81
        """Parse the range header value and returns ranges and tail.
82
83
        RFC2616 14.35 says that syntactically invalid range
84
        specifiers MUST be ignored. In that case, we return 0 for
85
        tail and [] for ranges.
86
        """
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
87
        tail = 0
88
        ranges = []
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
89
        if not ranges_header.startswith('bytes='):
90
            # Syntactically invalid header
91
            return 0, []
92
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
93
        ranges_header = ranges_header[len('bytes='):]
94
        for range_str in ranges_header.split(','):
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
95
            # FIXME: RFC2616 says end is optional and default to file_size
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
96
            range_match = self._range_regexp.match(range_str)
97
            if range_match is not None:
2182.2.2 by v.ladeuil+lp at free
Thanks again to Aaron, the http server RFC2616 compliance
98
                start = int(range_match.group('start'))
99
                end = int(range_match.group('end'))
100
                if start > end:
101
                    # Syntactically invalid range
102
                    return 0, []
103
                ranges.append((start, end))
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
104
            else:
105
                tail_match = self._tail_regexp.match(range_str)
106
                if tail_match is not None:
107
                    tail = int(tail_match.group('tail'))
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
108
                else:
109
                    # Syntactically invalid range
110
                    return 0, []
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
111
        return tail, ranges
112
113
    def send_range_content(self, file, start, length):
114
        file.seek(start)
115
        self.wfile.write(file.read(length))
116
117
    def get_single_range(self, file, file_size, start, end):
118
        self.send_response(206)
119
        length = end - start + 1
120
        self.send_header('Accept-Ranges', 'bytes')
121
        self.send_header("Content-Length", "%d" % length)
122
123
        self.send_header("Content-Type", 'application/octet-stream')
124
        self.send_header("Content-Range", "bytes %d-%d/%d" % (start,
125
                                                              end,
126
                                                              file_size))
127
        self.end_headers()
128
        self.send_range_content(file, start, length)
129
130
    def get_multiple_ranges(self, file, file_size, ranges):
131
        self.send_response(206)
132
        self.send_header('Accept-Ranges', 'bytes')
133
        boundary = "%d" % random.randint(0,0x7FFFFFFF)
134
        self.send_header("Content-Type",
135
                         "multipart/byteranges; boundary=%s" % boundary)
136
        self.end_headers()
137
        for (start, end) in ranges:
138
            self.wfile.write("--%s\r\n" % boundary)
139
            self.send_header("Content-type", 'application/octet-stream')
140
            self.send_header("Content-Range", "bytes %d-%d/%d" % (start,
141
                                                                  end,
142
                                                                  file_size))
143
            self.end_headers()
144
            self.send_range_content(file, start, end - start + 1)
145
            self.wfile.write("--%s\r\n" % boundary)
146
147
    def do_GET(self):
148
        """Serve a GET request.
149
150
        Handles the Range header.
151
        """
152
153
        path = self.translate_path(self.path)
154
        ranges_header_value = self.headers.get('Range')
155
        if ranges_header_value is None or os.path.isdir(path):
156
            # Let the mother class handle most cases
157
            return SimpleHTTPRequestHandler.do_GET(self)
158
159
        try:
160
            # Always read in binary mode. Opening files in text
161
            # mode may cause newline translations, making the
162
            # actual size of the content transmitted *less* than
163
            # the content-length!
164
            file = open(path, 'rb')
165
        except IOError:
166
            self.send_error(404, "File not found")
2000.3.9 by v.ladeuil+lp at free
The tests that would have help avoid bug #73948 and all that mess :)
167
            return
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
168
169
        file_size = os.fstat(file.fileno())[6]
170
        tail, ranges = self.parse_ranges(ranges_header_value)
171
        # Normalize tail into ranges
172
        if tail != 0:
173
            ranges.append((file_size - tail, file_size))
174
2182.2.2 by v.ladeuil+lp at free
Thanks again to Aaron, the http server RFC2616 compliance
175
        self._satisfiable_ranges = True
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
176
        if len(ranges) == 0:
2182.2.2 by v.ladeuil+lp at free
Thanks again to Aaron, the http server RFC2616 compliance
177
            self._satisfiable_ranges = False
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
178
        else:
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
179
            def check_range(range_specifier):
180
                start, end = range_specifier
2182.2.2 by v.ladeuil+lp at free
Thanks again to Aaron, the http server RFC2616 compliance
181
                # RFC2616 14.35, ranges are invalid if start >= file_size
182
                if start >= file_size:
183
                    self._satisfiable_ranges = False # Side-effect !
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
184
                    return 0, 0
185
                # RFC2616 14.35, end values should be truncated
186
                # to file_size -1 if they exceed it
187
                end = min(end, file_size - 1)
188
                return start, end
189
190
            ranges = map(check_range, ranges)
191
2182.2.2 by v.ladeuil+lp at free
Thanks again to Aaron, the http server RFC2616 compliance
192
        if not self._satisfiable_ranges:
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
193
            # RFC2616 14.16 and 14.35 says that when a server
194
            # encounters unsatisfiable range specifiers, it
195
            # SHOULD return a 416.
2000.3.9 by v.ladeuil+lp at free
The tests that would have help avoid bug #73948 and all that mess :)
196
            file.close()
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
197
            # FIXME: We SHOULD send a Content-Range header too,
198
            # but the implementation of send_error does not
199
            # allows that. So far.
2000.3.9 by v.ladeuil+lp at free
The tests that would have help avoid bug #73948 and all that mess :)
200
            self.send_error(416, "Requested range not satisfiable")
201
            return
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
202
203
        if len(ranges) == 1:
204
            (start, end) = ranges[0]
205
            self.get_single_range(file, file_size, start, end)
206
        else:
207
            self.get_multiple_ranges(file, file_size, ranges)
208
        file.close()
209
2420.1.9 by Vincent Ladeuil
Refactor proxy and auth test classes. Tests failing for digest auth.
210
    def translate_path(self, path):
211
        """Translate a /-separated PATH to the local filename syntax.
212
213
        If the server requires it, proxy the path before the usual translation
214
        """
215
        if self.server.test_case_server.proxy_requests:
216
            # We need to act as a proxy and accept absolute urls,
217
            # which SimpleHTTPRequestHandler (parent) is not
218
            # ready for. So we just drop the protocol://host:port
219
            # part in front of the request-url (because we know
220
            # we would not forward the request to *another*
221
            # proxy).
222
223
            # So we do what SimpleHTTPRequestHandler.translate_path
224
            # do beginning with python 2.4.3: abandon query
225
            # parameters, scheme, host port, etc (which ensure we
226
            # provide the right behaviour on all python versions).
227
            path = urlparse.urlparse(path)[2]
228
            # And now, we can apply *our* trick to proxy files
229
            path += '-proxied'
230
231
        return self._translate_path(path)
232
233
    def _translate_path(self, path):
234
        return SimpleHTTPRequestHandler.translate_path(self, path)
235
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
236
    if sys.platform == 'win32':
237
        # On win32 you cannot access non-ascii filenames without
238
        # decoding them into unicode first.
239
        # However, under Linux, you can access bytestream paths
240
        # without any problems. If this function was always active
241
        # it would probably break tests when LANG=C was set
2420.1.9 by Vincent Ladeuil
Refactor proxy and auth test classes. Tests failing for digest auth.
242
        def _translate_path(self, path):
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
243
            """Translate a /-separated PATH to the local filename syntax.
244
245
            For bzr, all url paths are considered to be utf8 paths.
246
            On Linux, you can access these paths directly over the bytestream
247
            request, but on win32, you must decode them, and access them
248
            as Unicode files.
249
            """
250
            # abandon query parameters
251
            path = urlparse.urlparse(path)[2]
252
            path = posixpath.normpath(urllib.unquote(path))
253
            path = path.decode('utf-8')
254
            words = path.split('/')
255
            words = filter(None, words)
256
            path = os.getcwdu()
257
            for word in words:
258
                drive, word = os.path.splitdrive(word)
259
                head, word = os.path.split(word)
260
                if word in (os.curdir, os.pardir): continue
261
                path = os.path.join(path, word)
262
            return path
263
264
265
class TestingHTTPServer(BaseHTTPServer.HTTPServer):
2164.2.28 by Vincent Ladeuil
TestingHTTPServer.test_case_server renamed from test_case to avoid confusions.
266
267
    def __init__(self, server_address, RequestHandlerClass,
268
                 test_case_server):
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
269
        BaseHTTPServer.HTTPServer.__init__(self, server_address,
2164.2.28 by Vincent Ladeuil
TestingHTTPServer.test_case_server renamed from test_case to avoid confusions.
270
                                           RequestHandlerClass)
271
        # test_case_server can be used to communicate between the
2164.2.29 by Vincent Ladeuil
Test the http redirection at the request level even if it's not
272
        # tests and the server (or the request handler and the
273
        # server), allowing dynamic behaviors to be defined from
274
        # the tests cases.
2164.2.28 by Vincent Ladeuil
TestingHTTPServer.test_case_server renamed from test_case to avoid confusions.
275
        self.test_case_server = test_case_server
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
276
2953.2.1 by Vincent Ladeuil
Fix #158972 by not using timeout for HttpServer.
277
    def server_close(self):
278
        """Called to clean-up the server.
279
280
        Since the server may be in a blocking read, we shutdown the socket
281
        before closing it.
282
        """
283
        self.socket.shutdown(socket.SHUT_RDWR)
284
        BaseHTTPServer.HTTPServer.server_close(self)
285
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
286
287
class HttpServer(Server):
288
    """A test server for http transports.
289
290
    Subclasses can provide a specific request handler.
291
    """
292
2420.1.9 by Vincent Ladeuil
Refactor proxy and auth test classes. Tests failing for digest auth.
293
    # Whether or not we proxy the requests (see
294
    # TestingHTTPRequestHandler.translate_path).
295
    proxy_requests = False
296
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
297
    # used to form the url that connects to this server
298
    _url_protocol = 'http'
299
300
    # Subclasses can provide a specific request handler
301
    def __init__(self, request_handler=TestingHTTPRequestHandler):
302
        Server.__init__(self)
303
        self.request_handler = request_handler
2164.2.13 by v.ladeuil+lp at free
Add tests for redirection. Preserve transport decorations.
304
        self.host = 'localhost'
305
        self.port = 0
306
        self._httpd = None
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
307
2004.1.28 by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code
308
    def _get_httpd(self):
2164.2.13 by v.ladeuil+lp at free
Add tests for redirection. Preserve transport decorations.
309
        if self._httpd is None:
310
            self._httpd = TestingHTTPServer((self.host, self.port),
311
                                            self.request_handler,
312
                                            self)
313
            host, self.port = self._httpd.socket.getsockname()
314
        return self._httpd
2004.1.28 by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code
315
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
316
    def _http_start(self):
2004.1.28 by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code
317
        httpd = self._get_httpd()
2164.2.13 by v.ladeuil+lp at free
Add tests for redirection. Preserve transport decorations.
318
        self._http_base_url = '%s://%s:%s/' % (self._url_protocol,
319
                                               self.host,
320
                                               self.port)
2831.2.1 by Vincent Ladeuil
Fix #140614 selftest _http_start noise by respecting semaphores.
321
        self._http_starting.release()
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
322
323
        while self._http_running:
324
            try:
325
                httpd.handle_request()
326
            except socket.timeout:
327
                pass
328
329
    def _get_remote_url(self, path):
330
        path_parts = path.split(os.path.sep)
331
        if os.path.isabs(path):
332
            if path_parts[:len(self._local_path_parts)] != \
333
                   self._local_path_parts:
334
                raise BadWebserverPath(path, self.test_dir)
335
            remote_path = '/'.join(path_parts[len(self._local_path_parts):])
336
        else:
337
            remote_path = '/'.join(path_parts)
338
339
        return self._http_base_url + remote_path
340
341
    def log(self, format, *args):
342
        """Capture Server log output."""
343
        self.logs.append(format % args)
344
2381.1.2 by Robert Collins
Fixup the test changes made for hpss to be clean and self contained.
345
    def setUp(self, backing_transport_server=None):
2018.5.42 by Robert Collins
Various hopefully improvements, but wsgi is broken, handing over to spiv :).
346
        """See bzrlib.transport.Server.setUp.
347
        
2381.1.2 by Robert Collins
Fixup the test changes made for hpss to be clean and self contained.
348
        :param backing_transport_server: The transport that requests over this
2018.5.42 by Robert Collins
Various hopefully improvements, but wsgi is broken, handing over to spiv :).
349
            protocol should be forwarded to. Note that this is currently not
2381.1.2 by Robert Collins
Fixup the test changes made for hpss to be clean and self contained.
350
            supported for HTTP.
2018.5.42 by Robert Collins
Various hopefully improvements, but wsgi is broken, handing over to spiv :).
351
        """
2381.1.2 by Robert Collins
Fixup the test changes made for hpss to be clean and self contained.
352
        # XXX: TODO: make the server back onto vfs_server rather than local
353
        # disk.
354
        assert backing_transport_server is None or \
355
            isinstance(backing_transport_server, LocalURLServer), \
356
            "HTTPServer currently assumes local transport, got %s" % \
357
            backing_transport_server
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
358
        self._home_dir = os.getcwdu()
359
        self._local_path_parts = self._home_dir.split(os.path.sep)
360
        self._http_starting = threading.Lock()
361
        self._http_starting.acquire()
362
        self._http_running = True
363
        self._http_base_url = None
364
        self._http_thread = threading.Thread(target=self._http_start)
365
        self._http_thread.setDaemon(True)
2167.3.5 by v.ladeuil+lp at free
Tests for proxies, covering #74759.
366
        self._http_thread.start()
2167.3.6 by v.ladeuil+lp at free
Take John's comments into account and add more tests.
367
        # Wait for the server thread to start (i.e release the lock)
368
        self._http_starting.acquire()
369
        self._http_starting.release()
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
370
        self.logs = []
371
372
    def tearDown(self):
373
        """See bzrlib.transport.Server.tearDown."""
2825.1.1 by Vincent Ladeuil
Fix #140055 by properly closing the http and ftp test servers.
374
        self._httpd.server_close()
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
375
        self._http_running = False
376
        self._http_thread.join()
377
378
    def get_url(self):
379
        """See bzrlib.transport.Server.get_url."""
380
        return self._get_remote_url(self._home_dir)
381
382
    def get_bogus_url(self):
383
        """See bzrlib.transport.Server.get_bogus_url."""
384
        # this is chosen to try to prevent trouble with proxies, weird dns,
385
        # etc
386
        return 'http://127.0.0.1:1/'
387
388
389
class HttpServer_urllib(HttpServer):
390
    """Subclass of HttpServer that gives http+urllib urls.
391
392
    This is for use in testing: connections to this server will always go
393
    through urllib where possible.
394
    """
395
396
    # urls returned by this server should require the urllib client impl
397
    _url_protocol = 'http+urllib'
398
399
400
class HttpServer_PyCurl(HttpServer):
401
    """Subclass of HttpServer that gives http+pycurl urls.
402
403
    This is for use in testing: connections to this server will always go
404
    through pycurl where possible.
405
    """
406
407
    # We don't care about checking the pycurl availability as
408
    # this server will be required only when pycurl is present
409
410
    # urls returned by this server should require the pycurl client impl
411
    _url_protocol = 'http+pycurl'