~bzr-pqm/bzr/bzr.dev

5557.1.7 by John Arbash Meinel
Merge in the bzr.dev 5582
1
# Copyright (C) 2010, 2011 Canonical Ltd
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
5247.3.10 by Vincent Ladeuil
Test errors during server life.
17
import errno
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
18
import socket
19
import SocketServer
5247.5.3 by Vincent Ladeuil
Fix exception raising only once for a given ThreadWithException.
20
import threading
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
21
6175.1.1 by John Arbash Meinel
Close the request when we don't process it.
22
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
23
from bzrlib import (
24
    osutils,
25
    tests,
26
    )
27
from bzrlib.tests import test_server
5559.2.2 by Martin Pool
Change to using standard load_tests_apply_scenarios.
28
from bzrlib.tests.scenarios import load_tests_apply_scenarios
29
30
31
load_tests = load_tests_apply_scenarios
5247.3.14 by Vincent Ladeuil
Use a proper load_tests.
32
33
6175.1.5 by John Arbash Meinel
Suppress ConnectionTimeout as a server-side exception.
34
def portable_socket_pair():
35
    """Return a pair of TCP sockets connected to each other.
36
37
    Unlike socket.socketpair, this should work on Windows.
38
    """
39
    listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
40
    listen_sock.bind(('127.0.0.1', 0))
41
    listen_sock.listen(1)
42
    client_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
43
    client_sock.connect(listen_sock.getsockname())
44
    server_sock, addr = listen_sock.accept()
45
    listen_sock.close()
46
    return server_sock, client_sock
47
48
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
49
class TCPClient(object):
50
51
    def __init__(self):
52
        self.sock = None
53
54
    def connect(self, addr):
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
55
        if self.sock is not None:
56
            raise AssertionError('Already connected to %r'
57
                                 % (self.sock.getsockname(),))
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
58
        self.sock = osutils.connect_socket(addr)
59
60
    def disconnect(self):
61
        if self.sock is not None:
5247.3.10 by Vincent Ladeuil
Test errors during server life.
62
            try:
63
                self.sock.shutdown(socket.SHUT_RDWR)
64
                self.sock.close()
65
            except socket.error, e:
66
                if e[0] in (errno.EBADF, errno.ENOTCONN):
67
                    # Right, the socket is already down
68
                    pass
69
                else:
70
                    raise
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
71
            self.sock = None
72
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
73
    def write(self, s):
74
        return self.sock.sendall(s)
75
76
    def read(self, bufsize=4096):
77
        return self.sock.recv(bufsize)
78
79
6175.1.2 by John Arbash Meinel
Change the TCPConnectionHandler to inherit from BaseRequestHandler.
80
class TCPConnectionHandler(SocketServer.BaseRequestHandler):
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
81
82
    def handle(self):
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
83
        self.done = False
84
        self.handle_connection()
85
        while not self.done:
86
            self.handle_connection()
87
6175.1.2 by John Arbash Meinel
Change the TCPConnectionHandler to inherit from BaseRequestHandler.
88
    def readline(self):
89
        # TODO: We should be buffering any extra data sent, etc. However, in
90
        #       practice, we don't send extra content, so we haven't bothered
91
        #       to implement it yet.
92
        req = self.request.recv(4096)
93
        # An empty string is allowed, to indicate the end of the connection
94
        if not req or (req.endswith('\n') and req.count('\n') == 1):
95
            return req
96
        raise ValueError('[%r] not a simple line' % (req,))
97
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
98
    def handle_connection(self):
6175.1.2 by John Arbash Meinel
Change the TCPConnectionHandler to inherit from BaseRequestHandler.
99
        req = self.readline()
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
100
        if not req:
101
            self.done = True
102
        elif req == 'ping\n':
6175.1.2 by John Arbash Meinel
Change the TCPConnectionHandler to inherit from BaseRequestHandler.
103
            self.request.sendall('pong\n')
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
104
        else:
105
            raise ValueError('[%s] not understood' % req)
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
106
5247.3.13 by Vincent Ladeuil
Really test against a threading server and properly shutdown socket and threads.
107
108
class TestTCPServerInAThread(tests.TestCase):
109
6133.4.23 by John Arbash Meinel
Setup the TestingSmartServer to set a default client timeout,
110
    scenarios = [
5559.2.2 by Martin Pool
Change to using standard load_tests_apply_scenarios.
111
        (name, {'server_class': getattr(test_server, name)})
112
        for name in
113
        ('TestingTCPServer', 'TestingThreadingTCPServer')]
114
5247.3.10 by Vincent Ladeuil
Test errors during server life.
115
    def get_server(self, server_class=None, connection_handler_class=None):
5247.3.13 by Vincent Ladeuil
Really test against a threading server and properly shutdown socket and threads.
116
        if server_class is not None:
117
            self.server_class = server_class
5247.3.10 by Vincent Ladeuil
Test errors during server life.
118
        if connection_handler_class is None:
119
            connection_handler_class = TCPConnectionHandler
6175.1.1 by John Arbash Meinel
Close the request when we don't process it.
120
        server = test_server.TestingTCPServerInAThread(
5247.3.13 by Vincent Ladeuil
Really test against a threading server and properly shutdown socket and threads.
121
            ('localhost', 0), self.server_class, connection_handler_class)
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
122
        server.start_server()
123
        self.addCleanup(server.stop_server)
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
124
        return server
125
126
    def get_client(self):
127
        client = TCPClient()
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
128
        self.addCleanup(client.disconnect)
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
129
        return client
130
5247.3.12 by Vincent Ladeuil
Spawn a thread for each connection from a client.
131
    def get_server_connection(self, server, conn_rank):
132
        return server.server.clients[conn_rank]
133
134
    def assertClientAddr(self, client, server, conn_rank):
135
        conn = self.get_server_connection(server, conn_rank)
136
        self.assertEquals(client.sock.getsockname(), conn[1])
137
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
138
    def test_start_stop(self):
139
        server = self.get_server()
140
        client = self.get_client()
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
141
        server.stop_server()
142
        # since the server doesn't accept connections anymore attempting to
143
        # connect should fail
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
144
        client = self.get_client()
5247.3.18 by Vincent Ladeuil
Fix some fallouts from previous fixes, all tests passing (no more http leaks).
145
        self.assertRaises(socket.error,
146
                          client.connect, (server.host, server.port))
5247.3.8 by Vincent Ladeuil
Start implementing a TCP server running in its own thread (using
147
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
148
    def test_client_talks_server_respond(self):
149
        server = self.get_server()
150
        client = self.get_client()
5247.3.18 by Vincent Ladeuil
Fix some fallouts from previous fixes, all tests passing (no more http leaks).
151
        client.connect((server.host, server.port))
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
152
        self.assertIs(None, client.write('ping\n'))
153
        resp = client.read()
5247.3.12 by Vincent Ladeuil
Spawn a thread for each connection from a client.
154
        self.assertClientAddr(client, server, 0)
5247.3.9 by Vincent Ladeuil
Ensure a simple dialog can occur between a client and a server.
155
        self.assertEquals('pong\n', resp)
5247.3.10 by Vincent Ladeuil
Test errors during server life.
156
157
    def test_server_fails_to_start(self):
158
        class CantStart(Exception):
159
            pass
160
161
        class CantStartServer(test_server.TestingTCPServer):
162
163
            def server_bind(self):
164
                raise CantStart()
165
166
        # The exception is raised in the main thread
167
        self.assertRaises(CantStart,
168
                          self.get_server, server_class=CantStartServer)
169
5247.5.10 by Vincent Ladeuil
Fix broken test.
170
    def test_server_fails_while_serving_or_stopping(self):
5247.5.4 by Vincent Ladeuil
Implement an execption handling mechanism that can be injected in ThreadWithException.
171
        class CantConnect(Exception):
5247.3.10 by Vincent Ladeuil
Test errors during server life.
172
            pass
173
6175.1.2 by John Arbash Meinel
Change the TCPConnectionHandler to inherit from BaseRequestHandler.
174
        class FailingConnectionHandler(TCPConnectionHandler):
5247.3.10 by Vincent Ladeuil
Test errors during server life.
175
176
            def handle(self):
5247.5.4 by Vincent Ladeuil
Implement an execption handling mechanism that can be injected in ThreadWithException.
177
                raise CantConnect()
5247.3.10 by Vincent Ladeuil
Test errors during server life.
178
179
        server = self.get_server(
180
            connection_handler_class=FailingConnectionHandler)
181
        # The server won't fail until a client connect
182
        client = self.get_client()
5247.3.18 by Vincent Ladeuil
Fix some fallouts from previous fixes, all tests passing (no more http leaks).
183
        client.connect((server.host, server.port))
6175.1.1 by John Arbash Meinel
Close the request when we don't process it.
184
        # We make sure the server wants to handle a request, but the request is
185
        # guaranteed to fail. However, the server should make sure that the
186
        # connection gets closed, and stop_server should then raise the
187
        # original exception.
188
        client.write('ping\n')
5247.3.10 by Vincent Ladeuil
Test errors during server life.
189
        try:
6175.1.1 by John Arbash Meinel
Close the request when we don't process it.
190
            self.assertEqual('', client.read())
191
        except socket.error, e:
6175.1.3 by John Arbash Meinel
Restrict the socket errors we will suppress.
192
            # On Windows, failing during 'handle' means we get
193
            # 'forced-close-of-connection'. Possibly because we haven't
194
            # processed the write request before we close the socket.
195
            WSAECONNRESET = 10054
196
            if e.errno in (WSAECONNRESET,):
197
                pass
5247.5.9 by Vincent Ladeuil
Use a better sync for test_exception_swallowed_while_serving test.
198
        # Now the server has raised the exception in its own thread
5247.5.4 by Vincent Ladeuil
Implement an execption handling mechanism that can be injected in ThreadWithException.
199
        self.assertRaises(CantConnect, server.stop_server)
5247.3.11 by Vincent Ladeuil
Start implementing the threading variants.
200
5247.5.3 by Vincent Ladeuil
Fix exception raising only once for a given ThreadWithException.
201
    def test_server_crash_while_responding(self):
6015.42.4 by Vincent Ladeuil
Less code, more explanations.
202
        # We want to ensure the exception has been caught
6015.42.2 by Vincent Ladeuil
One race can hide another... the exception may pass from the connection thread to the server thread so both need to be checked, with care.
203
        caught = threading.Event()
204
        caught.clear()
6015.42.4 by Vincent Ladeuil
Less code, more explanations.
205
        # The thread that will serve the client, this needs to be an attribute
206
        # so the handler below can modify it when it's executed (it's
207
        # instantiated when the request is processed)
6015.42.1 by Vincent Ladeuil
Fix a naughty race in test_server_crash_while_responding
208
        self.connection_thread = None
6015.42.4 by Vincent Ladeuil
Less code, more explanations.
209
5247.5.4 by Vincent Ladeuil
Implement an execption handling mechanism that can be injected in ThreadWithException.
210
        class FailToRespond(Exception):
5247.5.3 by Vincent Ladeuil
Fix exception raising only once for a given ThreadWithException.
211
            pass
212
213
        class FailingDuringResponseHandler(TCPConnectionHandler):
214
6015.42.5 by Vincent Ladeuil
More comments after review discussions.
215
            # We use 'request' instead of 'self' below because the test matters
216
            # more and we need a container to properly set connection_thread.
6015.42.2 by Vincent Ladeuil
One race can hide another... the exception may pass from the connection thread to the server thread so both need to be checked, with care.
217
            def handle_connection(request):
6205.1.1 by John Arbash Meinel
Merge bzr-2.4 into bzr.dev, resolve the small conflicts.
218
                req = request.readline()
6015.42.2 by Vincent Ladeuil
One race can hide another... the exception may pass from the connection thread to the server thread so both need to be checked, with care.
219
                # Capture the thread and make it use 'caught' so we can wait on
6015.42.5 by Vincent Ladeuil
More comments after review discussions.
220
                # the event that will be set when the exception is caught. We
6015.42.4 by Vincent Ladeuil
Less code, more explanations.
221
                # also capture the thread to know where to look.
6015.42.2 by Vincent Ladeuil
One race can hide another... the exception may pass from the connection thread to the server thread so both need to be checked, with care.
222
                self.connection_thread = threading.currentThread()
223
                self.connection_thread.set_sync_event(caught)
5247.5.4 by Vincent Ladeuil
Implement an execption handling mechanism that can be injected in ThreadWithException.
224
                raise FailToRespond()
5247.5.3 by Vincent Ladeuil
Fix exception raising only once for a given ThreadWithException.
225
226
        server = self.get_server(
227
            connection_handler_class=FailingDuringResponseHandler)
228
        client = self.get_client()
5247.3.21 by Vincent Ladeuil
Merge propagate-exceptions into http-leaks
229
        client.connect((server.host, server.port))
5247.5.3 by Vincent Ladeuil
Fix exception raising only once for a given ThreadWithException.
230
        client.write('ping\n')
6015.42.2 by Vincent Ladeuil
One race can hide another... the exception may pass from the connection thread to the server thread so both need to be checked, with care.
231
        # Wait for the exception to be caught
232
        caught.wait()
6175.1.4 by John Arbash Meinel
Add more checks that the server closes the connection properly.
233
        self.assertEqual('', client.read()) # connection closed
6015.42.1 by Vincent Ladeuil
Fix a naughty race in test_server_crash_while_responding
234
        # Check that the connection thread did catch the exception,
235
        # http://pad.lv/869366 was wrongly checking the server thread which
236
        # works for TestingTCPServer where the connection is handled in the
6015.42.6 by Vincent Ladeuil
Ensures that the connection thread is detached from the server thread before handling the connection.
237
        # same thread than the server one but was racy for
6015.42.7 by Vincent Ladeuil
Fix typo.
238
        # TestingThreadingTCPServer. Since the connection thread detaches
6015.42.6 by Vincent Ladeuil
Ensures that the connection thread is detached from the server thread before handling the connection.
239
        # itself before handling the request, we are guaranteed that the
240
        # exception won't leak into the server thread anymore.
241
        self.assertRaises(FailToRespond,
242
                          self.connection_thread.pending_exception)
5247.5.4 by Vincent Ladeuil
Implement an execption handling mechanism that can be injected in ThreadWithException.
243
244
    def test_exception_swallowed_while_serving(self):
6015.42.2 by Vincent Ladeuil
One race can hide another... the exception may pass from the connection thread to the server thread so both need to be checked, with care.
245
        # We need to ensure the exception has been caught
246
        caught = threading.Event()
247
        caught.clear()
6015.42.4 by Vincent Ladeuil
Less code, more explanations.
248
        # The thread that will serve the client, this needs to be an attribute
249
        # so the handler below can access it when it's executed (it's
250
        # instantiated when the request is processed)
6015.42.1 by Vincent Ladeuil
Fix a naughty race in test_server_crash_while_responding
251
        self.connection_thread = None
5247.5.4 by Vincent Ladeuil
Implement an execption handling mechanism that can be injected in ThreadWithException.
252
        class CantServe(Exception):
253
            pass
254
255
        class FailingWhileServingConnectionHandler(TCPConnectionHandler):
256
6015.42.5 by Vincent Ladeuil
More comments after review discussions.
257
            # We use 'request' instead of 'self' below because the test matters
258
            # more and we need a container to properly set connection_thread.
6015.42.1 by Vincent Ladeuil
Fix a naughty race in test_server_crash_while_responding
259
            def handle(request):
6015.42.2 by Vincent Ladeuil
One race can hide another... the exception may pass from the connection thread to the server thread so both need to be checked, with care.
260
                # Capture the thread and make it use 'caught' so we can wait on
6015.42.5 by Vincent Ladeuil
More comments after review discussions.
261
                # the event that will be set when the exception is caught. We
6015.42.4 by Vincent Ladeuil
Less code, more explanations.
262
                # also capture the thread to know where to look.
6015.42.2 by Vincent Ladeuil
One race can hide another... the exception may pass from the connection thread to the server thread so both need to be checked, with care.
263
                self.connection_thread = threading.currentThread()
264
                self.connection_thread.set_sync_event(caught)
5247.5.4 by Vincent Ladeuil
Implement an execption handling mechanism that can be injected in ThreadWithException.
265
                raise CantServe()
266
267
        server = self.get_server(
268
            connection_handler_class=FailingWhileServingConnectionHandler)
6015.42.5 by Vincent Ladeuil
More comments after review discussions.
269
        self.assertEquals(True, server.server.serving)
5247.5.4 by Vincent Ladeuil
Implement an execption handling mechanism that can be injected in ThreadWithException.
270
        # Install the exception swallower
271
        server.set_ignored_exceptions(CantServe)
272
        client = self.get_client()
5247.5.9 by Vincent Ladeuil
Use a better sync for test_exception_swallowed_while_serving test.
273
        # Connect to the server so the exception is raised there
5247.3.21 by Vincent Ladeuil
Merge propagate-exceptions into http-leaks
274
        client.connect((server.host, server.port))
6015.42.2 by Vincent Ladeuil
One race can hide another... the exception may pass from the connection thread to the server thread so both need to be checked, with care.
275
        # Wait for the exception to be caught
276
        caught.wait()
6175.1.4 by John Arbash Meinel
Add more checks that the server closes the connection properly.
277
        self.assertEqual('', client.read()) # connection closed
5247.5.4 by Vincent Ladeuil
Implement an execption handling mechanism that can be injected in ThreadWithException.
278
        # The connection wasn't served properly but the exception should have
6015.42.1 by Vincent Ladeuil
Fix a naughty race in test_server_crash_while_responding
279
        # been swallowed (see test_server_crash_while_responding remark about
280
        # http://pad.lv/869366 explaining why we can't check the server thread
6015.42.2 by Vincent Ladeuil
One race can hide another... the exception may pass from the connection thread to the server thread so both need to be checked, with care.
281
        # here). More precisely, the exception *has* been caught and captured
282
        # but it is cleared when joining the thread (or trying to acquire the
283
        # exception) and as such won't propagate to the server thread.
6015.42.5 by Vincent Ladeuil
More comments after review discussions.
284
        self.assertIs(None, self.connection_thread.pending_exception())
285
        self.assertIs(None, server.pending_exception())
6133.4.23 by John Arbash Meinel
Setup the TestingSmartServer to set a default client timeout,
286
6175.1.1 by John Arbash Meinel
Close the request when we don't process it.
287
    def test_handle_request_closes_if_it_doesnt_process(self):
288
        server = self.get_server()
289
        client = self.get_client()
290
        server.server.serving = False
291
        client.connect((server.host, server.port))
292
        self.assertEqual('', client.read())
293
6133.4.23 by John Arbash Meinel
Setup the TestingSmartServer to set a default client timeout,
294
295
class TestTestingSmartServer(tests.TestCase):
296
297
    def test_sets_client_timeout(self):
298
        server = test_server.TestingSmartServer(('localhost', 0), None, None,
299
            root_client_path='/no-such-client/path')
300
        self.assertEqual(test_server._DEFAULT_TESTING_CLIENT_TIMEOUT,
301
                         server._client_timeout)
302
        sock = socket.socket()
6133.4.24 by John Arbash Meinel
Change the name from _create_handler to _make_handler, to match similar code in the file.
303
        h = server._make_handler(sock)
6133.4.23 by John Arbash Meinel
Setup the TestingSmartServer to set a default client timeout,
304
        self.assertEqual(test_server._DEFAULT_TESTING_CLIENT_TIMEOUT,
305
                         h._client_timeout)
6175.1.5 by John Arbash Meinel
Suppress ConnectionTimeout as a server-side exception.
306
307
308
class FakeServer(object):
309
    """Minimal implementation to pass to TestingSmartConnectionHandler"""
310
    backing_transport = None
311
    root_client_path = '/'
312
313
314
class TestTestingSmartConnectionHandler(tests.TestCase):
315
316
    def test_connection_timeout_suppressed(self):
317
        self.overrideAttr(test_server, '_DEFAULT_TESTING_CLIENT_TIMEOUT', 0.01)
318
        s = FakeServer()
319
        server_sock, client_sock = portable_socket_pair()
320
        # This should timeout quickly, but not generate an exception.
321
        handler = test_server.TestingSmartConnectionHandler(server_sock,
322
            server_sock.getpeername(), s)
323
324
    def test_connection_shutdown_while_serving_no_error(self):
325
        s = FakeServer()
326
        server_sock, client_sock = portable_socket_pair()
327
        class ShutdownConnectionHandler(
328
            test_server.TestingSmartConnectionHandler):
329
330
            def _build_protocol(self):
331
                self.finished = True
332
                return super(ShutdownConnectionHandler, self)._build_protocol()
333
        # This should trigger shutdown after the entering _build_protocol, and
334
        # we should exit cleanly, without raising an exception.
335
        handler = ShutdownConnectionHandler(server_sock,
336
            server_sock.getpeername(), s)