~bzr-pqm/bzr/bzr.dev

3059.2.2 by Vincent Ladeuil
Read http responses on demand without buffering the whole body
1
# Copyright (C) 2006, 2007 Canonical Ltd
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
import errno
3111.1.2 by Vincent Ladeuil
Preparatory cleanup.
18
import httplib
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
19
import os
2146.1.1 by Alexander Belchenko
fixes for test suite: forgotten imports in HttpServer.py
20
import posixpath
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
21
import random
22
import re
3734.2.8 by Vincent Ladeuil
Catch spurious exceptions (python-2.6) when SocketServer is shut down.
23
import select
3111.1.2 by Vincent Ladeuil
Preparatory cleanup.
24
import SimpleHTTPServer
25
import socket
3111.1.3 by Vincent Ladeuil
Rework http test servers classes. Both 1.0 and 1.1 passing tests (1.1 forced manually).
26
import SocketServer
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
27
import sys
28
import threading
29
import time
2146.1.1 by Alexander Belchenko
fixes for test suite: forgotten imports in HttpServer.py
30
import urllib
31
import urlparse
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
32
3111.1.2 by Vincent Ladeuil
Preparatory cleanup.
33
from bzrlib import transport
34
from bzrlib.transport import local
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
35
36
37
class WebserverNotAvailable(Exception):
38
    pass
39
40
41
class BadWebserverPath(ValueError):
42
    def __str__(self):
43
        return 'path %s is not in %s' % self.args
44
45
3111.1.2 by Vincent Ladeuil
Preparatory cleanup.
46
class TestingHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
2420.1.10 by Vincent Ladeuil
Doc fixes.
47
    """Handles one request.
48
3111.1.21 by Vincent Ladeuil
Add some comments.
49
    A TestingHTTPRequestHandler is instantiated for every request received by
50
    the associated server. Note that 'request' here is inherited from the base
51
    TCPServer class, for the HTTP server it is really a connection which itself
52
    will handle one or several HTTP requests.
2420.1.10 by Vincent Ladeuil
Doc fixes.
53
    """
3111.1.24 by Vincent Ladeuil
Cleanups.
54
    # Default protocol version
55
    protocol_version = 'HTTP/1.1'
56
3111.1.2 by Vincent Ladeuil
Preparatory cleanup.
57
    # The Message-like class used to parse the request headers
58
    MessageClass = httplib.HTTPMessage
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
59
3111.1.15 by Vincent Ladeuil
Provide a way to specify the protocol version at the server layer.
60
    def setup(self):
61
        SimpleHTTPServer.SimpleHTTPRequestHandler.setup(self)
3221.11.13 by Robert Collins
Allow push --shallow to just work, and fix the testing HTTPServer to not be affected by chdir() calls.
62
        self._cwd = self.server._home_dir
3111.1.15 by Vincent Ladeuil
Provide a way to specify the protocol version at the server layer.
63
        tcs = self.server.test_case_server
64
        if tcs.protocol_version is not None:
65
            # If the test server forced a protocol version, use it
66
            self.protocol_version = tcs.protocol_version
67
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
68
    def log_message(self, format, *args):
2164.2.28 by Vincent Ladeuil
TestingHTTPServer.test_case_server renamed from test_case to avoid confusions.
69
        tcs = self.server.test_case_server
70
        tcs.log('webserver - %s - - [%s] %s "%s" "%s"',
71
                self.address_string(),
72
                self.log_date_time_string(),
73
                format % args,
3111.1.19 by Vincent Ladeuil
Merge back test_http_implementations.pc into test_http.py.
74
                self.headers.get('referer', '-'),
2164.2.28 by Vincent Ladeuil
TestingHTTPServer.test_case_server renamed from test_case to avoid confusions.
75
                self.headers.get('user-agent', '-'))
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
76
77
    def handle_one_request(self):
78
        """Handle a single HTTP request.
79
2831.6.1 by Vincent Ladeuil
Remove some more noise from test suite.
80
        We catch all socket errors occurring when the client close the
81
        connection early to avoid polluting the test results.
82
        """
83
        try:
3111.1.2 by Vincent Ladeuil
Preparatory cleanup.
84
            SimpleHTTPServer.SimpleHTTPRequestHandler.handle_one_request(self)
2831.6.1 by Vincent Ladeuil
Remove some more noise from test suite.
85
        except socket.error, e:
3111.1.24 by Vincent Ladeuil
Cleanups.
86
            # Any socket error should close the connection, but some errors are
87
            # due to the client closing early and we don't want to pollute test
3111.1.20 by Vincent Ladeuil
Make all the test pass. Looks like we are HTTP/1.1 compliant.
88
            # results, so we raise only the others.
89
            self.close_connection = 1
90
            if (len(e.args) == 0
91
                or e.args[0] not in (errno.EPIPE, errno.ECONNRESET,
92
                                     errno.ECONNABORTED, errno.EBADF)):
2831.6.1 by Vincent Ladeuil
Remove some more noise from test suite.
93
                raise
94
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
95
    _range_regexp = re.compile(r'^(?P<start>\d+)-(?P<end>\d+)$')
96
    _tail_regexp = re.compile(r'^-(?P<tail>\d+)$')
97
98
    def parse_ranges(self, ranges_header):
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
99
        """Parse the range header value and returns ranges and tail.
100
101
        RFC2616 14.35 says that syntactically invalid range
102
        specifiers MUST be ignored. In that case, we return 0 for
103
        tail and [] for ranges.
104
        """
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
105
        tail = 0
106
        ranges = []
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
107
        if not ranges_header.startswith('bytes='):
108
            # Syntactically invalid header
109
            return 0, []
110
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
111
        ranges_header = ranges_header[len('bytes='):]
112
        for range_str in ranges_header.split(','):
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
113
            # FIXME: RFC2616 says end is optional and default to file_size
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
114
            range_match = self._range_regexp.match(range_str)
115
            if range_match is not None:
2182.2.2 by v.ladeuil+lp at free
Thanks again to Aaron, the http server RFC2616 compliance
116
                start = int(range_match.group('start'))
117
                end = int(range_match.group('end'))
118
                if start > end:
119
                    # Syntactically invalid range
120
                    return 0, []
121
                ranges.append((start, end))
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
122
            else:
123
                tail_match = self._tail_regexp.match(range_str)
124
                if tail_match is not None:
125
                    tail = int(tail_match.group('tail'))
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
126
                else:
127
                    # Syntactically invalid range
128
                    return 0, []
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
129
        return tail, ranges
130
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
131
    def _header_line_length(self, keyword, value):
132
        header_line = '%s: %s\r\n' % (keyword, value)
133
        return len(header_line)
134
3111.1.23 by Vincent Ladeuil
Make HTTP/1.1 the default implementation reveals one more bug.
135
    def send_head(self):
136
        """Overrides base implementation to work around a bug in python2.5."""
137
        path = self.translate_path(self.path)
138
        if os.path.isdir(path) and not self.path.endswith('/'):
139
            # redirect browser - doing basically what apache does when
140
            # DirectorySlash option is On which is quite common (braindead, but
141
            # common)
142
            self.send_response(301)
143
            self.send_header("Location", self.path + "/")
144
            # Indicates that the body is empty for HTTP/1.1 clients 
145
            self.send_header('Content-Length', '0')
146
            self.end_headers()
147
            return None
148
149
        return SimpleHTTPServer.SimpleHTTPRequestHandler.send_head(self)
150
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
151
    def send_range_content(self, file, start, length):
152
        file.seek(start)
153
        self.wfile.write(file.read(length))
154
155
    def get_single_range(self, file, file_size, start, end):
156
        self.send_response(206)
157
        length = end - start + 1
158
        self.send_header('Accept-Ranges', 'bytes')
159
        self.send_header("Content-Length", "%d" % length)
160
161
        self.send_header("Content-Type", 'application/octet-stream')
162
        self.send_header("Content-Range", "bytes %d-%d/%d" % (start,
163
                                                              end,
164
                                                              file_size))
165
        self.end_headers()
166
        self.send_range_content(file, start, length)
167
168
    def get_multiple_ranges(self, file, file_size, ranges):
169
        self.send_response(206)
170
        self.send_header('Accept-Ranges', 'bytes')
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
171
        boundary = '%d' % random.randint(0,0x7FFFFFFF)
172
        self.send_header('Content-Type',
173
                         'multipart/byteranges; boundary=%s' % boundary)
174
        boundary_line = '--%s\r\n' % boundary
175
        # Calculate the Content-Length
176
        content_length = 0
177
        for (start, end) in ranges:
178
            content_length += len(boundary_line)
179
            content_length += self._header_line_length(
180
                'Content-type', 'application/octet-stream')
181
            content_length += self._header_line_length(
182
                'Content-Range', 'bytes %d-%d/%d' % (start, end, file_size))
183
            content_length += len('\r\n') # end headers
184
            content_length += end - start # + 1
185
        content_length += len(boundary_line)
186
        self.send_header('Content-length', content_length)
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
187
        self.end_headers()
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
188
189
        # Send the multipart body
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
190
        for (start, end) in ranges:
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
191
            self.wfile.write(boundary_line)
192
            self.send_header('Content-type', 'application/octet-stream')
193
            self.send_header('Content-Range', 'bytes %d-%d/%d'
194
                             % (start, end, file_size))
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
195
            self.end_headers()
196
            self.send_range_content(file, start, end - start + 1)
3059.2.12 by Vincent Ladeuil
Spiv review feedback.
197
        # Final boundary
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
198
        self.wfile.write(boundary_line)
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
199
200
    def do_GET(self):
201
        """Serve a GET request.
202
203
        Handles the Range header.
204
        """
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
205
        # Update statistics
206
        self.server.test_case_server.GET_request_nb += 1
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
207
208
        path = self.translate_path(self.path)
209
        ranges_header_value = self.headers.get('Range')
210
        if ranges_header_value is None or os.path.isdir(path):
211
            # Let the mother class handle most cases
3111.1.2 by Vincent Ladeuil
Preparatory cleanup.
212
            return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
213
214
        try:
215
            # Always read in binary mode. Opening files in text
216
            # mode may cause newline translations, making the
217
            # actual size of the content transmitted *less* than
218
            # the content-length!
219
            file = open(path, 'rb')
220
        except IOError:
221
            self.send_error(404, "File not found")
2000.3.9 by v.ladeuil+lp at free
The tests that would have help avoid bug #73948 and all that mess :)
222
            return
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
223
224
        file_size = os.fstat(file.fileno())[6]
225
        tail, ranges = self.parse_ranges(ranges_header_value)
226
        # Normalize tail into ranges
227
        if tail != 0:
228
            ranges.append((file_size - tail, file_size))
229
2182.2.2 by v.ladeuil+lp at free
Thanks again to Aaron, the http server RFC2616 compliance
230
        self._satisfiable_ranges = True
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
231
        if len(ranges) == 0:
2182.2.2 by v.ladeuil+lp at free
Thanks again to Aaron, the http server RFC2616 compliance
232
            self._satisfiable_ranges = False
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
233
        else:
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
234
            def check_range(range_specifier):
235
                start, end = range_specifier
2182.2.2 by v.ladeuil+lp at free
Thanks again to Aaron, the http server RFC2616 compliance
236
                # RFC2616 14.35, ranges are invalid if start >= file_size
237
                if start >= file_size:
238
                    self._satisfiable_ranges = False # Side-effect !
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
239
                    return 0, 0
240
                # RFC2616 14.35, end values should be truncated
241
                # to file_size -1 if they exceed it
242
                end = min(end, file_size - 1)
243
                return start, end
244
245
            ranges = map(check_range, ranges)
246
2182.2.2 by v.ladeuil+lp at free
Thanks again to Aaron, the http server RFC2616 compliance
247
        if not self._satisfiable_ranges:
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
248
            # RFC2616 14.16 and 14.35 says that when a server
249
            # encounters unsatisfiable range specifiers, it
250
            # SHOULD return a 416.
2000.3.9 by v.ladeuil+lp at free
The tests that would have help avoid bug #73948 and all that mess :)
251
            file.close()
2182.2.1 by v.ladeuil+lp at free
Aaron was right. Thanks to him, the http server RFC2616 compliance
252
            # FIXME: We SHOULD send a Content-Range header too,
253
            # but the implementation of send_error does not
254
            # allows that. So far.
2000.3.9 by v.ladeuil+lp at free
The tests that would have help avoid bug #73948 and all that mess :)
255
            self.send_error(416, "Requested range not satisfiable")
256
            return
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
257
258
        if len(ranges) == 1:
259
            (start, end) = ranges[0]
260
            self.get_single_range(file, file_size, start, end)
261
        else:
262
            self.get_multiple_ranges(file, file_size, ranges)
263
        file.close()
264
2420.1.9 by Vincent Ladeuil
Refactor proxy and auth test classes. Tests failing for digest auth.
265
    def translate_path(self, path):
266
        """Translate a /-separated PATH to the local filename syntax.
267
268
        If the server requires it, proxy the path before the usual translation
269
        """
270
        if self.server.test_case_server.proxy_requests:
271
            # We need to act as a proxy and accept absolute urls,
272
            # which SimpleHTTPRequestHandler (parent) is not
273
            # ready for. So we just drop the protocol://host:port
274
            # part in front of the request-url (because we know
275
            # we would not forward the request to *another*
276
            # proxy).
277
278
            # So we do what SimpleHTTPRequestHandler.translate_path
279
            # do beginning with python 2.4.3: abandon query
280
            # parameters, scheme, host port, etc (which ensure we
281
            # provide the right behaviour on all python versions).
282
            path = urlparse.urlparse(path)[2]
283
            # And now, we can apply *our* trick to proxy files
284
            path += '-proxied'
285
286
        return self._translate_path(path)
287
288
    def _translate_path(self, path):
3221.11.13 by Robert Collins
Allow push --shallow to just work, and fix the testing HTTPServer to not be affected by chdir() calls.
289
        """Translate a /-separated PATH to the local filename syntax.
290
3221.9.5 by Ian Clatworthy
some tweaks from abentley's earlier review feedback
291
        Note that we're translating http URLs here, not file URLs.
292
        The URL root location is the server's startup directory.
3221.11.13 by Robert Collins
Allow push --shallow to just work, and fix the testing HTTPServer to not be affected by chdir() calls.
293
        Components that mean special things to the local file system
294
        (e.g. drive or directory names) are ignored.  (XXX They should
295
        probably be diagnosed.)
296
297
        Override from python standard library to stop it calling os.getcwd()
298
        """
299
        # abandon query parameters
300
        path = urlparse.urlparse(path)[2]
301
        path = posixpath.normpath(urllib.unquote(path))
3221.9.4 by Ian Clatworthy
fix failing unicode_paths test
302
        path = path.decode('utf-8')
3221.11.13 by Robert Collins
Allow push --shallow to just work, and fix the testing HTTPServer to not be affected by chdir() calls.
303
        words = path.split('/')
304
        words = filter(None, words)
305
        path = self._cwd
3221.9.5 by Ian Clatworthy
some tweaks from abentley's earlier review feedback
306
        for num, word in enumerate(words):
307
            if num == 0:
308
                drive, word = os.path.splitdrive(word)
3221.11.13 by Robert Collins
Allow push --shallow to just work, and fix the testing HTTPServer to not be affected by chdir() calls.
309
            head, word = os.path.split(word)
310
            if word in (os.curdir, os.pardir): continue
311
            path = os.path.join(path, word)
312
        return path
2420.1.9 by Vincent Ladeuil
Refactor proxy and auth test classes. Tests failing for digest auth.
313
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
314
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
315
class TestingHTTPServerMixin:
316
317
    def __init__(self, test_case_server):
2164.2.28 by Vincent Ladeuil
TestingHTTPServer.test_case_server renamed from test_case to avoid confusions.
318
        # test_case_server can be used to communicate between the
2164.2.29 by Vincent Ladeuil
Test the http redirection at the request level even if it's not
319
        # tests and the server (or the request handler and the
320
        # server), allowing dynamic behaviors to be defined from
321
        # the tests cases.
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
322
        self.test_case_server = test_case_server
3221.11.13 by Robert Collins
Allow push --shallow to just work, and fix the testing HTTPServer to not be affected by chdir() calls.
323
        self._home_dir = test_case_server._home_dir
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
324
325
    def tearDown(self):
3111.1.3 by Vincent Ladeuil
Rework http test servers classes. Both 1.0 and 1.1 passing tests (1.1 forced manually).
326
         """Called to clean-up the server.
327
 
328
         Since the server may be (surely is, even) in a blocking listen, we
329
         shutdown its socket before closing it.
330
         """
331
         # Note that is this executed as part of the implicit tear down in the
332
         # main thread while the server runs in its own thread. The clean way
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
333
         # to tear down the server is to instruct him to stop accepting
334
         # connections and wait for the current connection(s) to end
335
         # naturally. To end the connection naturally, the http transports
336
         # should close their socket when they do not need to talk to the
337
         # server anymore. This happens naturally during the garbage collection
338
         # phase of the test transport objetcs (the server clients), so we
339
         # don't have to worry about them.  So, for the server, we must tear
340
         # down here, from the main thread, when the test have ended.  Note
341
         # that since the server is in a blocking operation and since python
342
         # use select internally, shutting down the socket is reliable and
343
         # relatively clean.
3111.1.31 by Vincent Ladeuil
Review feeback.
344
         try:
345
             self.socket.shutdown(socket.SHUT_RDWR)
346
         except socket.error, e:
347
             # WSAENOTCONN (10057) 'Socket is not connected' is harmless on
348
             # windows (occurs before the first connection attempt
349
             # vila--20071230)
3815.2.3 by Martin Pool
merge fix for #293054, ssl on python2.6
350
351
             # 'Socket is not connected' can also occur on OSX, with a
352
             # "regular" ENOTCONN (when something went wrong during test case
353
             # setup leading to self.setUp() *not* being called but
354
             # self.tearDown() still being called -- vila20081106
355
             if not len(e.args) or e.args[0] not in (errno.ENOTCONN, 10057):
3111.1.31 by Vincent Ladeuil
Review feeback.
356
                 raise
3111.1.3 by Vincent Ladeuil
Rework http test servers classes. Both 1.0 and 1.1 passing tests (1.1 forced manually).
357
         # Let the server properly close the socket
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
358
         self.server_close()
3111.1.3 by Vincent Ladeuil
Rework http test servers classes. Both 1.0 and 1.1 passing tests (1.1 forced manually).
359
3221.11.13 by Robert Collins
Allow push --shallow to just work, and fix the testing HTTPServer to not be affected by chdir() calls.
360
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
361
class TestingHTTPServer(SocketServer.TCPServer, TestingHTTPServerMixin):
3111.1.3 by Vincent Ladeuil
Rework http test servers classes. Both 1.0 and 1.1 passing tests (1.1 forced manually).
362
3111.1.30 by Vincent Ladeuil
Update NEWS. Some cosmetic changes.
363
    def __init__(self, server_address, request_handler_class,
364
                 test_case_server):
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
365
        TestingHTTPServerMixin.__init__(self, test_case_server)
366
        SocketServer.TCPServer.__init__(self, server_address,
367
                                        request_handler_class)
368
369
370
class TestingThreadingHTTPServer(SocketServer.ThreadingTCPServer,
371
                                 TestingHTTPServerMixin):
3111.1.3 by Vincent Ladeuil
Rework http test servers classes. Both 1.0 and 1.1 passing tests (1.1 forced manually).
372
    """A threading HTTP test server for HTTP 1.1.
373
374
    Since tests can initiate several concurrent connections to the same http
375
    server, we need an independent connection for each of them. We achieve that
376
    by spawning a new thread for each connection.
377
    """
378
3111.1.30 by Vincent Ladeuil
Update NEWS. Some cosmetic changes.
379
    def __init__(self, server_address, request_handler_class,
380
                 test_case_server):
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
381
        TestingHTTPServerMixin.__init__(self, test_case_server)
382
        SocketServer.ThreadingTCPServer.__init__(self, server_address,
383
                                                 request_handler_class)
3111.1.3 by Vincent Ladeuil
Rework http test servers classes. Both 1.0 and 1.1 passing tests (1.1 forced manually).
384
        # Decides how threads will act upon termination of the main
385
        # process. This is prophylactic as we should not leave the threads
386
        # lying around.
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
387
        self.daemon_threads = True
2953.2.1 by Vincent Ladeuil
Fix #158972 by not using timeout for HttpServer.
388
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
389
3111.1.2 by Vincent Ladeuil
Preparatory cleanup.
390
class HttpServer(transport.Server):
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
391
    """A test server for http transports.
392
393
    Subclasses can provide a specific request handler.
394
    """
395
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
396
    # The real servers depending on the protocol
397
    http_server_class = {'HTTP/1.0': TestingHTTPServer,
398
                         'HTTP/1.1': TestingThreadingHTTPServer,
399
                         }
400
2420.1.9 by Vincent Ladeuil
Refactor proxy and auth test classes. Tests failing for digest auth.
401
    # Whether or not we proxy the requests (see
402
    # TestingHTTPRequestHandler.translate_path).
403
    proxy_requests = False
404
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
405
    # used to form the url that connects to this server
406
    _url_protocol = 'http'
407
3111.1.15 by Vincent Ladeuil
Provide a way to specify the protocol version at the server layer.
408
    def __init__(self, request_handler=TestingHTTPRequestHandler,
409
                 protocol_version=None):
410
        """Constructor.
411
412
        :param request_handler: a class that will be instantiated to handle an
413
            http connection (one or several requests).
414
415
        :param protocol_version: if specified, will override the protocol
416
            version of the request handler.
417
        """
3111.1.2 by Vincent Ladeuil
Preparatory cleanup.
418
        transport.Server.__init__(self)
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
419
        self.request_handler = request_handler
2164.2.13 by v.ladeuil+lp at free
Add tests for redirection. Preserve transport decorations.
420
        self.host = 'localhost'
421
        self.port = 0
422
        self._httpd = None
3111.1.15 by Vincent Ladeuil
Provide a way to specify the protocol version at the server layer.
423
        self.protocol_version = protocol_version
3052.3.2 by Vincent Ladeuil
Add tests and fix trivial bugs and other typos.
424
        # Allows tests to verify number of GET requests issued
425
        self.GET_request_nb = 0
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
426
3734.2.8 by Vincent Ladeuil
Catch spurious exceptions (python-2.6) when SocketServer is shut down.
427
    def __repr__(self):
428
        return "%s(%s:%s)" % \
429
            (self.__class__.__name__, self.host, self.port)
430
2004.1.28 by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code
431
    def _get_httpd(self):
2164.2.13 by v.ladeuil+lp at free
Add tests for redirection. Preserve transport decorations.
432
        if self._httpd is None:
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
433
            rhandler = self.request_handler
3111.1.21 by Vincent Ladeuil
Add some comments.
434
            # Depending on the protocol version, we will create the approriate
435
            # server
3111.1.15 by Vincent Ladeuil
Provide a way to specify the protocol version at the server layer.
436
            if self.protocol_version is None:
3111.1.21 by Vincent Ladeuil
Add some comments.
437
                # Use the request handler one
3111.1.15 by Vincent Ladeuil
Provide a way to specify the protocol version at the server layer.
438
                proto_vers = rhandler.protocol_version
439
            else:
3111.1.21 by Vincent Ladeuil
Add some comments.
440
                # Use our own, it will be used to override the request handler
441
                # one too.
3111.1.15 by Vincent Ladeuil
Provide a way to specify the protocol version at the server layer.
442
                proto_vers = self.protocol_version
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
443
            # Create the appropriate server for the required protocol
444
            serv_cls = self.http_server_class.get(proto_vers, None)
445
            if serv_cls is None:
446
                raise httplib.UnknownProtocol(proto_vers)
447
            else:
448
                self._httpd = serv_cls((self.host, self.port), rhandler, self)
2164.2.13 by v.ladeuil+lp at free
Add tests for redirection. Preserve transport decorations.
449
            host, self.port = self._httpd.socket.getsockname()
450
        return self._httpd
2004.1.28 by v.ladeuil+lp at free
Merge bzr.dev. Including http modifications by "smart" related code
451
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
452
    def _http_start(self):
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
453
        """Server thread main entry point. """
454
        self._http_running = False
455
        try:
456
            try:
457
                httpd = self._get_httpd()
458
                self._http_base_url = '%s://%s:%s/' % (self._url_protocol,
459
                                                       self.host, self.port)
460
                self._http_running = True
461
            except:
462
                # Whatever goes wrong, we save the exception for the main
463
                # thread. Note that since we are running in a thread, no signal
464
                # can be received, so we don't care about KeyboardInterrupt.
465
                self._http_exception = sys.exc_info()
466
        finally:
467
            # Release the lock or the main thread will block and the whole
468
            # process will hang.
469
            self._http_starting.release()
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
470
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
471
        # From now on, exceptions are taken care of by the
472
        # SocketServer.BaseServer or the request handler.
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
473
        while self._http_running:
474
            try:
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
475
                # Really an HTTP connection but the python framework is generic
476
                # and call them requests
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
477
                httpd.handle_request()
478
            except socket.timeout:
479
                pass
3734.2.8 by Vincent Ladeuil
Catch spurious exceptions (python-2.6) when SocketServer is shut down.
480
            except (socket.error, select.error), e:
481
               if e[0] == errno.EBADF:
482
                   # Starting with python-2.6, handle_request may raise socket
483
                   # or select exceptions when the server is shut down (as we
484
                   # do).
485
                   pass
486
               else:
487
                   raise
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
488
489
    def _get_remote_url(self, path):
490
        path_parts = path.split(os.path.sep)
491
        if os.path.isabs(path):
492
            if path_parts[:len(self._local_path_parts)] != \
493
                   self._local_path_parts:
494
                raise BadWebserverPath(path, self.test_dir)
495
            remote_path = '/'.join(path_parts[len(self._local_path_parts):])
496
        else:
497
            remote_path = '/'.join(path_parts)
498
499
        return self._http_base_url + remote_path
500
501
    def log(self, format, *args):
502
        """Capture Server log output."""
503
        self.logs.append(format % args)
504
2381.1.2 by Robert Collins
Fixup the test changes made for hpss to be clean and self contained.
505
    def setUp(self, backing_transport_server=None):
2018.5.42 by Robert Collins
Various hopefully improvements, but wsgi is broken, handing over to spiv :).
506
        """See bzrlib.transport.Server.setUp.
507
        
2381.1.2 by Robert Collins
Fixup the test changes made for hpss to be clean and self contained.
508
        :param backing_transport_server: The transport that requests over this
2018.5.42 by Robert Collins
Various hopefully improvements, but wsgi is broken, handing over to spiv :).
509
            protocol should be forwarded to. Note that this is currently not
2381.1.2 by Robert Collins
Fixup the test changes made for hpss to be clean and self contained.
510
            supported for HTTP.
2018.5.42 by Robert Collins
Various hopefully improvements, but wsgi is broken, handing over to spiv :).
511
        """
2381.1.2 by Robert Collins
Fixup the test changes made for hpss to be clean and self contained.
512
        # XXX: TODO: make the server back onto vfs_server rather than local
513
        # disk.
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
514
        if not (backing_transport_server is None or \
515
                isinstance(backing_transport_server, local.LocalURLServer)):
516
            raise AssertionError(
517
                "HTTPServer currently assumes local transport, got %s" % \
518
                backing_transport_server)
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
519
        self._home_dir = os.getcwdu()
520
        self._local_path_parts = self._home_dir.split(os.path.sep)
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
521
        self._http_base_url = None
522
3111.1.6 by Vincent Ladeuil
Begin refactoring test_http.py into parameterized tests.
523
        # Create the server thread
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
524
        self._http_starting = threading.Lock()
525
        self._http_starting.acquire()
526
        self._http_thread = threading.Thread(target=self._http_start)
527
        self._http_thread.setDaemon(True)
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
528
        self._http_exception = None
2167.3.5 by v.ladeuil+lp at free
Tests for proxies, covering #74759.
529
        self._http_thread.start()
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
530
2167.3.6 by v.ladeuil+lp at free
Take John's comments into account and add more tests.
531
        # Wait for the server thread to start (i.e release the lock)
532
        self._http_starting.acquire()
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
533
534
        if self._http_exception is not None:
3111.1.21 by Vincent Ladeuil
Add some comments.
535
            # Something went wrong during server start
3111.1.4 by Vincent Ladeuil
Select the server depending on the request handler protocol. Add tests.
536
            exc_class, exc_value, exc_tb = self._http_exception
537
            raise exc_class, exc_value, exc_tb
2167.3.6 by v.ladeuil+lp at free
Take John's comments into account and add more tests.
538
        self._http_starting.release()
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
539
        self.logs = []
540
541
    def tearDown(self):
542
        """See bzrlib.transport.Server.tearDown."""
3111.1.22 by Vincent Ladeuil
Rework TestingHTTPServer classes, fix test bug.
543
        self._httpd.tearDown()
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
544
        self._http_running = False
3111.1.28 by Vincent Ladeuil
Fix the multi-ranges http server and add tests.
545
        # We don't need to 'self._http_thread.join()' here since the thread is
546
        # a daemonic one and will be garbage collected anyway. Joining just
547
        # slows us down for no added benefit.
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
548
549
    def get_url(self):
550
        """See bzrlib.transport.Server.get_url."""
551
        return self._get_remote_url(self._home_dir)
552
553
    def get_bogus_url(self):
554
        """See bzrlib.transport.Server.get_bogus_url."""
555
        # this is chosen to try to prevent trouble with proxies, weird dns,
556
        # etc
3111.1.2 by Vincent Ladeuil
Preparatory cleanup.
557
        return self._url_protocol + '://127.0.0.1:1/'
2004.1.25 by v.ladeuil+lp at free
Shuffle http related test code. Hopefully it ends up at the right place :)
558
559
560
class HttpServer_urllib(HttpServer):
561
    """Subclass of HttpServer that gives http+urllib urls.
562
563
    This is for use in testing: connections to this server will always go
564
    through urllib where possible.
565
    """
566
567
    # urls returned by this server should require the urllib client impl
568
    _url_protocol = 'http+urllib'
569
570
571
class HttpServer_PyCurl(HttpServer):
572
    """Subclass of HttpServer that gives http+pycurl urls.
573
574
    This is for use in testing: connections to this server will always go
575
    through pycurl where possible.
576
    """
577
578
    # We don't care about checking the pycurl availability as
579
    # this server will be required only when pycurl is present
580
581
    # urls returned by this server should require the pycurl client impl
582
    _url_protocol = 'http+pycurl'