~bzr-pqm/bzr/bzr.dev

5752.3.8 by John Arbash Meinel
Merge bzr.dev 5764 to resolve release-notes (aka NEWS) conflicts
1
# Copyright (C) 2006-2011 Canonical Ltd
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
16
2018.5.19 by Andrew Bennetts
Add docstrings to all the new modules, and a few other places.
17
"""The 'medium' layer for the smart servers and clients.
18
19
"Medium" here is the noun meaning "a means of transmission", not the adjective
20
for "the quality between big and small."
21
22
Media carry the bytes of the requests somehow (e.g. via TCP, wrapped in HTTP, or
23
over SSH), and pass them to and from the protocol logic.  See the overview in
24
bzrlib/transport/smart/__init__.py.
25
"""
26
6379.6.7 by Jelmer Vernooij
Move importing from future until after doc string, otherwise the doc string will disappear.
27
from __future__ import absolute_import
28
6133.4.10 by John Arbash Meinel
Handle the win32 not-a-socket errors. I think this is ~ready. At least for feedback.
29
import errno
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
30
import os
2018.5.162 by Andrew Bennetts
Add some missing _ensure_real calls, and a missing import.
31
import sys
6133.4.8 by John Arbash Meinel
Refactor a bit. Use a common _wait_for_descriptor code.
32
import time
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
33
5222.2.9 by Robert Collins
Write up some doc about bzrlib.initialize.
34
import bzrlib
3530.1.1 by John Arbash Meinel
Make bzrlib.smart use lazy imports.
35
from bzrlib.lazy_import import lazy_import
36
lazy_import(globals(), """
6133.4.7 by John Arbash Meinel
Currently causes 5-10 failures in bt.per_branch.test_branch Remote out of 150 tests.
37
import select
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
38
import socket
4913.1.1 by John Arbash Meinel
Switch to using thread.get_ident() which is available on all python versions.
39
import thread
3731.2.4 by Andrew Bennetts
Minor tweaks.
40
import weakref
4889.2.2 by John Arbash Meinel
Add a -Dhpssthread debug flag to include thread.ident info.
41
1551.18.17 by Aaron Bentley
Introduce bzr_remote_path configuration variable
42
from bzrlib import (
3731.2.1 by Andrew Bennetts
Show total HPSS calls (if any) on stderr when -Dhpss is active.
43
    debug,
1551.18.17 by Aaron Bentley
Introduce bzr_remote_path configuration variable
44
    errors,
3731.2.5 by Andrew Bennetts
Rework hpss call counter.
45
    trace,
5436.3.9 by Martin Packman
Move post_connect callback for RemoteTransports to smart.medium
46
    transport,
3958.1.1 by Andrew Bennetts
Report traffic on smart media as transport activity.
47
    ui,
3431.3.11 by Andrew Bennetts
Push remote_path_from_transport logic into SmartClientMedium, removing special-casing of bzr+http from _SmartClient.
48
    urlutils,
1551.18.17 by Aaron Bentley
Introduce bzr_remote_path configuration variable
49
    )
6138.3.4 by Jonathan Riddell
add gettext() to uses of trace.note()
50
from bzrlib.i18n import gettext
6133.4.61 by John Arbash Meinel
Expose infrastructure so that we can test the Inet portion of the server.
51
from bzrlib.smart import client, protocol, request, signals, vfs
3066.2.1 by John Arbash Meinel
We don't require paramiko for bzr+ssh.
52
from bzrlib.transport import ssh
3530.1.1 by John Arbash Meinel
Make bzrlib.smart use lazy imports.
53
""")
6133.4.55 by John Arbash Meinel
Now we've implemented the basic structure.
54
from bzrlib import osutils
2018.5.17 by Andrew Bennetts
Paramaterise the commands handled by SmartServerRequestHandler.
55
5011.3.3 by Martin
Reintroduce EINTR handling only for socket object functions and general cleanup
56
# Throughout this module buffer size parameters are either limited to be at
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
57
# most _MAX_READ_SIZE, or are ignored and _MAX_READ_SIZE is used instead.
58
# For this module's purposes, MAX_SOCKET_CHUNK is a reasonable size for reads
59
# from non-sockets as well.
60
_MAX_READ_SIZE = osutils.MAX_SOCKET_CHUNK
3565.1.3 by Andrew Bennetts
Define a _MAX_READ_SIZE constant as suggested by John's review.
61
3245.4.16 by Andrew Bennetts
Remove duplication of request version identification logic in wsgi.py
62
def _get_protocol_factory_for_bytes(bytes):
63
    """Determine the right protocol factory for 'bytes'.
64
65
    This will return an appropriate protocol factory depending on the version
66
    of the protocol being used, as determined by inspecting the given bytes.
67
    The bytes should have at least one newline byte (i.e. be a whole line),
68
    otherwise it's possible that a request will be incorrectly identified as
69
    version 1.
70
71
    Typical use would be::
72
73
         factory, unused_bytes = _get_protocol_factory_for_bytes(bytes)
74
         server_protocol = factory(transport, write_func, root_client_path)
75
         server_protocol.accept_bytes(unused_bytes)
76
77
    :param bytes: a str of bytes of the start of the request.
78
    :returns: 2-tuple of (protocol_factory, unused_bytes).  protocol_factory is
79
        a callable that takes three args: transport, write_func,
80
        root_client_path.  unused_bytes are any bytes that were not part of a
81
        protocol version marker.
82
    """
3530.1.1 by John Arbash Meinel
Make bzrlib.smart use lazy imports.
83
    if bytes.startswith(protocol.MESSAGE_VERSION_THREE):
84
        protocol_factory = protocol.build_server_protocol_three
85
        bytes = bytes[len(protocol.MESSAGE_VERSION_THREE):]
86
    elif bytes.startswith(protocol.REQUEST_VERSION_TWO):
87
        protocol_factory = protocol.SmartServerRequestProtocolTwo
88
        bytes = bytes[len(protocol.REQUEST_VERSION_TWO):]
3245.4.16 by Andrew Bennetts
Remove duplication of request version identification logic in wsgi.py
89
    else:
3530.1.1 by John Arbash Meinel
Make bzrlib.smart use lazy imports.
90
        protocol_factory = protocol.SmartServerRequestProtocolOne
3245.4.16 by Andrew Bennetts
Remove duplication of request version identification logic in wsgi.py
91
    return protocol_factory, bytes
92
93
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
94
def _get_line(read_bytes_func):
95
    """Read bytes using read_bytes_func until a newline byte.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
96
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
97
    This isn't particularly efficient, so should only be used when the
98
    expected size of the line is quite short.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
99
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
100
    :returns: a tuple of two strs: (line, excess)
101
    """
102
    newline_pos = -1
103
    bytes = ''
104
    while newline_pos == -1:
105
        new_bytes = read_bytes_func(1)
106
        bytes += new_bytes
107
        if new_bytes == '':
108
            # Ran out of bytes before receiving a complete line.
109
            return bytes, ''
110
        newline_pos = bytes.find('\n')
111
    line = bytes[:newline_pos+1]
112
    excess = bytes[newline_pos+1:]
113
    return line, excess
114
115
3565.1.1 by Andrew Bennetts
Read no more then 64k at a time in the smart protocol code.
116
class SmartMedium(object):
117
    """Base class for smart protocol media, both client- and server-side."""
118
119
    def __init__(self):
120
        self._push_back_buffer = None
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
121
3565.1.1 by Andrew Bennetts
Read no more then 64k at a time in the smart protocol code.
122
    def _push_back(self, bytes):
123
        """Return unused bytes to the medium, because they belong to the next
124
        request(s).
125
126
        This sets the _push_back_buffer to the given bytes.
127
        """
128
        if self._push_back_buffer is not None:
129
            raise AssertionError(
130
                "_push_back called when self._push_back_buffer is %r"
131
                % (self._push_back_buffer,))
132
        if bytes == '':
133
            return
134
        self._push_back_buffer = bytes
135
136
    def _get_push_back_buffer(self):
137
        if self._push_back_buffer == '':
138
            raise AssertionError(
139
                '%s._push_back_buffer should never be the empty string, '
140
                'which can be confused with EOF' % (self,))
141
        bytes = self._push_back_buffer
142
        self._push_back_buffer = None
143
        return bytes
144
145
    def read_bytes(self, desired_count):
3565.1.2 by Andrew Bennetts
Delete some more code, fix some bugs, add more comments.
146
        """Read some bytes from this medium.
147
148
        :returns: some bytes, possibly more or less than the number requested
149
            in 'desired_count' depending on the medium.
150
        """
151
        if self._push_back_buffer is not None:
152
            return self._get_push_back_buffer()
3565.1.3 by Andrew Bennetts
Define a _MAX_READ_SIZE constant as suggested by John's review.
153
        bytes_to_read = min(desired_count, _MAX_READ_SIZE)
3565.1.1 by Andrew Bennetts
Read no more then 64k at a time in the smart protocol code.
154
        return self._read_bytes(bytes_to_read)
155
156
    def _read_bytes(self, count):
157
        raise NotImplementedError(self._read_bytes)
158
159
    def _get_line(self):
160
        """Read bytes from this request's response until a newline byte.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
161
3565.1.1 by Andrew Bennetts
Read no more then 64k at a time in the smart protocol code.
162
        This isn't particularly efficient, so should only be used when the
163
        expected size of the line is quite short.
164
165
        :returns: a string of bytes ending in a newline (byte 0x0A).
166
        """
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
167
        line, excess = _get_line(self.read_bytes)
168
        self._push_back(excess)
3565.1.1 by Andrew Bennetts
Read no more then 64k at a time in the smart protocol code.
169
        return line
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
170
3958.1.1 by Andrew Bennetts
Report traffic on smart media as transport activity.
171
    def _report_activity(self, bytes, direction):
172
        """Notify that this medium has activity.
173
174
        Implementations should call this from all methods that actually do IO.
175
        Be careful that it's not called twice, if one method is implemented on
176
        top of another.
177
178
        :param bytes: Number of bytes read or written.
179
        :param direction: 'read' or 'write' or None.
180
        """
181
        ui.ui_factory.report_transport_activity(self, bytes, direction)
182
3565.1.1 by Andrew Bennetts
Read no more then 64k at a time in the smart protocol code.
183
6133.4.10 by John Arbash Meinel
Handle the win32 not-a-socket errors. I think this is ~ready. At least for feedback.
184
_bad_file_descriptor = (errno.EBADF,)
185
if sys.platform == 'win32':
186
    # Given on Windows if you pass a closed socket to select.select. Probably
187
    # also given if you pass a file handle to select.
188
    WSAENOTSOCK = 10038
189
    _bad_file_descriptor += (WSAENOTSOCK,)
190
191
3565.1.1 by Andrew Bennetts
Read no more then 64k at a time in the smart protocol code.
192
class SmartServerStreamMedium(SmartMedium):
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
193
    """Handles smart commands coming over a stream.
194
195
    The stream may be a pipe connected to sshd, or a tcp socket, or an
196
    in-process fifo for testing.
197
198
    One instance is created for each connected client; it can serve multiple
199
    requests in the lifetime of the connection.
200
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
201
    The server passes requests through to an underlying backing transport,
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
202
    which will typically be a LocalTransport looking at the server's filesystem.
3236.3.4 by Andrew Bennetts
Rename 'push_back' attribute to '_push_back_buffer', add some docstrings, and remove a little bit of redundant code from SmartServerSocketStreamMedium._serve_one_request_unguarded.
203
204
    :ivar _push_back_buffer: a str of bytes that have been read from the stream
205
        but not used yet, or None if there are no buffered bytes.  Subclasses
206
        should make sure to exhaust this buffer before reading more bytes from
207
        the stream.  See also the _push_back method.
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
208
    """
209
6133.4.43 by John Arbash Meinel
Initial implementation of SIGHUP for bzr serve. bug #795025.
210
    _timer = time.time
211
6133.4.15 by John Arbash Meinel
Start working on exposing timeout as a configuration item.
212
    def __init__(self, backing_transport, root_client_path='/', timeout=None):
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
213
        """Construct new server.
214
215
        :param backing_transport: Transport for the directory served.
216
        """
217
        # backing_transport could be passed to serve instead of __init__
218
        self.backing_transport = backing_transport
2692.1.11 by Andrew Bennetts
Improve test coverage by making SmartTCPServer_for_testing by default create a server that does not serve the backing transport's root at its own root. This mirrors the way most HTTP smart servers are configured.
219
        self.root_client_path = root_client_path
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
220
        self.finished = False
6133.4.15 by John Arbash Meinel
Start working on exposing timeout as a configuration item.
221
        if timeout is None:
6133.4.26 by John Arbash Meinel
get rid of the default timeout parameters.
222
            raise AssertionError('You must supply a timeout.')
6133.4.15 by John Arbash Meinel
Start working on exposing timeout as a configuration item.
223
        self._client_timeout = timeout
224
        self._client_poll_timeout = min(timeout / 10.0, 1.0)
3565.1.1 by Andrew Bennetts
Read no more then 64k at a time in the smart protocol code.
225
        SmartMedium.__init__(self)
3236.3.5 by Andrew Bennetts
Add _get_push_back_buffer helper.
226
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
227
    def serve(self):
228
        """Serve requests until the client disconnects."""
229
        # Keep a reference to stderr because the sys module's globals get set to
230
        # None during interpreter shutdown.
231
        from sys import stderr
232
        try:
233
            while not self.finished:
2432.2.3 by Andrew Bennetts
Merge from bzr.dev.
234
                server_protocol = self._build_protocol()
2018.5.14 by Andrew Bennetts
Move SmartTCPServer to smart/server.py, and SmartServerRequestHandler to smart/request.py.
235
                self._serve_one_request(server_protocol)
6133.4.11 by John Arbash Meinel
It turns out that if we don't explicitly close the socket, it hangs around somewhere.
236
        except errors.ConnectionTimeout, e:
237
            trace.note('%s' % (e,))
6133.4.12 by John Arbash Meinel
Don't traceback for timeouts, and give better documentation about why we don't just select.
238
            trace.log_exception_quietly()
6133.5.2 by John Arbash Meinel
Rename _close to _disconnect_client. This makes it more obvious.
239
            self._disconnect_client()
6133.4.12 by John Arbash Meinel
Don't traceback for timeouts, and give better documentation about why we don't just select.
240
            # We reported it, no reason to make a big fuss.
241
            return
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
242
        except Exception, e:
243
            stderr.write("%s terminating on exception %s\n" % (self, e))
244
            raise
6133.4.61 by John Arbash Meinel
Expose infrastructure so that we can test the Inet portion of the server.
245
        self._disconnect_client()
6133.4.43 by John Arbash Meinel
Initial implementation of SIGHUP for bzr serve. bug #795025.
246
6133.4.44 by John Arbash Meinel
Move the code into bzrlib.smart.signals.
247
    def _stop_gracefully(self):
248
        """When we finish this message, stop looking for more."""
249
        trace.mutter('Stopping %s' % (self,))
6133.4.43 by John Arbash Meinel
Initial implementation of SIGHUP for bzr serve. bug #795025.
250
        self.finished = True
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
251
6133.5.2 by John Arbash Meinel
Rename _close to _disconnect_client. This makes it more obvious.
252
    def _disconnect_client(self):
6133.4.11 by John Arbash Meinel
It turns out that if we don't explicitly close the socket, it hangs around somewhere.
253
        """Close the current connection. We stopped due to a timeout/etc."""
254
        # The default implementation is a no-op, because that is all we used to
255
        # do when disconnecting from a client. I suppose we never had the
256
        # *server* initiate a disconnect, before
257
258
    def _wait_for_bytes_with_timeout(self, timeout_seconds):
259
        """Wait for more bytes to be read, but timeout if none available.
260
261
        This allows us to detect idle connections, and stop trying to read from
262
        them, without setting the socket itself to non-blocking. This also
263
        allows us to specify when we watch for idle timeouts.
264
265
        :return: Did we timeout? (True if we timed out, False if there is data
266
            to be read)
267
        """
268
        raise NotImplementedError(self._wait_for_bytes_with_timeout)
269
2432.2.2 by Andrew Bennetts
Smart server mediums now detect which protocol version a request is and dispatch accordingly.
270
    def _build_protocol(self):
2432.2.8 by Andrew Bennetts
NEWS entry, greatly improved docstring in bzrlib.smart.
271
        """Identifies the version of the incoming request, and returns an
272
        a protocol object that can interpret it.
273
274
        If more bytes than the version prefix of the request are read, they will
275
        be fed into the protocol before it is returned.
276
277
        :returns: a SmartServerRequestProtocol.
278
        """
6133.4.42 by John Arbash Meinel
Change the code to just raise ConnectionTimeout.
279
        self._wait_for_bytes_with_timeout(self._client_timeout)
6133.4.43 by John Arbash Meinel
Initial implementation of SIGHUP for bzr serve. bug #795025.
280
        if self.finished:
281
            # We're stopping, so don't try to do any more work
282
            return None
2432.2.7 by Andrew Bennetts
Use less confusing version strings, and define REQUEST_VERSION_TWO/RESPONSE_VERSION_TWO constants for them.
283
        bytes = self._get_line()
3245.4.16 by Andrew Bennetts
Remove duplication of request version identification logic in wsgi.py
284
        protocol_factory, unused_bytes = _get_protocol_factory_for_bytes(bytes)
3245.4.14 by Andrew Bennetts
Merge from bzr.dev (via loom thread).
285
        protocol = protocol_factory(
2692.1.11 by Andrew Bennetts
Improve test coverage by making SmartTCPServer_for_testing by default create a server that does not serve the backing transport's root at its own root. This mirrors the way most HTTP smart servers are configured.
286
            self.backing_transport, self._write_out, self.root_client_path)
3245.4.16 by Andrew Bennetts
Remove duplication of request version identification logic in wsgi.py
287
        protocol.accept_bytes(unused_bytes)
2432.2.2 by Andrew Bennetts
Smart server mediums now detect which protocol version a request is and dispatch accordingly.
288
        return protocol
289
6133.4.8 by John Arbash Meinel
Refactor a bit. Use a common _wait_for_descriptor code.
290
    def _wait_on_descriptor(self, fd, timeout_seconds):
6133.4.41 by John Arbash Meinel
Re-order the loop a bit. Move the exception handling into the loop, so that we can handle EINTR.
291
        """select() on a file descriptor, waiting for nonblocking read()
292
6133.4.42 by John Arbash Meinel
Change the code to just raise ConnectionTimeout.
293
        This will raise a ConnectionTimeout exception if we do not get a
294
        readable handle before timeout_seconds.
295
        :return: None
6133.4.41 by John Arbash Meinel
Re-order the loop a bit. Move the exception handling into the loop, so that we can handle EINTR.
296
        """
6133.4.43 by John Arbash Meinel
Initial implementation of SIGHUP for bzr serve. bug #795025.
297
        t_end = self._timer() + timeout_seconds
6133.4.41 by John Arbash Meinel
Re-order the loop a bit. Move the exception handling into the loop, so that we can handle EINTR.
298
        poll_timeout = min(timeout_seconds, self._client_poll_timeout)
299
        rs = xs = None
6133.4.43 by John Arbash Meinel
Initial implementation of SIGHUP for bzr serve. bug #795025.
300
        while not rs and not xs and self._timer() < t_end:
301
            if self.finished:
302
                return
6133.4.41 by John Arbash Meinel
Re-order the loop a bit. Move the exception handling into the loop, so that we can handle EINTR.
303
            try:
6133.4.39 by John Arbash Meinel
pass fd in the 'check for errors' section of select.
304
                rs, _, xs = select.select([fd], [], [fd], poll_timeout)
6133.4.41 by John Arbash Meinel
Re-order the loop a bit. Move the exception handling into the loop, so that we can handle EINTR.
305
            except (select.error, socket.error) as e:
306
                err = getattr(e, 'errno', None)
307
                if err is None and getattr(e, 'args', None) is not None:
308
                    # select.error doesn't have 'errno', it just has args[0]
6133.4.8 by John Arbash Meinel
Refactor a bit. Use a common _wait_for_descriptor code.
309
                    err = e.args[0]
6133.4.41 by John Arbash Meinel
Re-order the loop a bit. Move the exception handling into the loop, so that we can handle EINTR.
310
                if err in _bad_file_descriptor:
6133.4.42 by John Arbash Meinel
Change the code to just raise ConnectionTimeout.
311
                    return # Not a socket indicates read() will fail
6133.4.41 by John Arbash Meinel
Re-order the loop a bit. Move the exception handling into the loop, so that we can handle EINTR.
312
                elif err == errno.EINTR:
313
                    # Interrupted, keep looping.
314
                    continue
315
                raise
6133.4.40 by John Arbash Meinel
Treat an 'errfds' as just something we want to try and read from later.
316
        if rs or xs:
6133.4.42 by John Arbash Meinel
Change the code to just raise ConnectionTimeout.
317
            return
318
        raise errors.ConnectionTimeout('disconnecting client after %.1f seconds'
319
                                       % (timeout_seconds,))
6133.4.8 by John Arbash Meinel
Refactor a bit. Use a common _wait_for_descriptor code.
320
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
321
    def _serve_one_request(self, protocol):
322
        """Read one request from input, process, send back a response.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
323
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
324
        :param protocol: a SmartServerRequestProtocol.
325
        """
6175.1.5 by John Arbash Meinel
Suppress ConnectionTimeout as a server-side exception.
326
        if protocol is None:
327
            return
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
328
        try:
329
            self._serve_one_request_unguarded(protocol)
330
        except KeyboardInterrupt:
331
            raise
332
        except Exception, e:
333
            self.terminate_due_to_error()
334
335
    def terminate_due_to_error(self):
336
        """Called when an unhandled exception from the protocol occurs."""
337
        raise NotImplementedError(self.terminate_due_to_error)
338
3565.1.2 by Andrew Bennetts
Delete some more code, fix some bugs, add more comments.
339
    def _read_bytes(self, desired_count):
2432.2.2 by Andrew Bennetts
Smart server mediums now detect which protocol version a request is and dispatch accordingly.
340
        """Get some bytes from the medium.
341
342
        :param desired_count: number of bytes we want to read.
343
        """
3565.1.2 by Andrew Bennetts
Delete some more code, fix some bugs, add more comments.
344
        raise NotImplementedError(self._read_bytes)
2432.2.2 by Andrew Bennetts
Smart server mediums now detect which protocol version a request is and dispatch accordingly.
345
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
346
347
class SmartServerSocketStreamMedium(SmartServerStreamMedium):
348
6133.4.15 by John Arbash Meinel
Start working on exposing timeout as a configuration item.
349
    def __init__(self, sock, backing_transport, root_client_path='/',
350
                 timeout=None):
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
351
        """Constructor.
352
353
        :param sock: the socket the server will read from.  It will be put
354
            into blocking mode.
355
        """
2692.1.11 by Andrew Bennetts
Improve test coverage by making SmartTCPServer_for_testing by default create a server that does not serve the backing transport's root at its own root. This mirrors the way most HTTP smart servers are configured.
356
        SmartServerStreamMedium.__init__(
6133.4.15 by John Arbash Meinel
Start working on exposing timeout as a configuration item.
357
            self, backing_transport, root_client_path=root_client_path,
358
            timeout=timeout)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
359
        sock.setblocking(True)
360
        self.socket = sock
6133.4.62 by John Arbash Meinel
Add a nicer repr for shutting down.
361
        # Get the getpeername now, as we might be closed later when we care.
362
        try:
363
            self._client_info = sock.getpeername()
364
        except socket.error:
365
            self._client_info = '<unknown>'
366
367
    def __str__(self):
368
        return '%s(client=%s)' % (self.__class__.__name__, self._client_info)
369
370
    def __repr__(self):
371
        return '%s.%s(client=%s)' % (self.__module__, self.__class__.__name__,
372
            self._client_info)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
373
374
    def _serve_one_request_unguarded(self, protocol):
375
        while protocol.next_read_size():
3565.1.3 by Andrew Bennetts
Define a _MAX_READ_SIZE constant as suggested by John's review.
376
            # We can safely try to read large chunks.  If there is less data
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
377
            # than MAX_SOCKET_CHUNK ready, the socket will just return a
378
            # short read immediately rather than block.
379
            bytes = self.read_bytes(osutils.MAX_SOCKET_CHUNK)
3236.3.4 by Andrew Bennetts
Rename 'push_back' attribute to '_push_back_buffer', add some docstrings, and remove a little bit of redundant code from SmartServerSocketStreamMedium._serve_one_request_unguarded.
380
            if bytes == '':
381
                self.finished = True
382
                return
383
            protocol.accept_bytes(bytes)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
384
3245.4.21 by Andrew Bennetts
Remove 'excess_buffer' attribute and another crufty comment.
385
        self._push_back(protocol.unused_data)
3195.3.18 by Andrew Bennetts
call_with_body_bytes now works with v3 (e.g. test_copy_content_remote_to_local passes). Lots of debugging cruft, though.
386
6133.5.2 by John Arbash Meinel
Rename _close to _disconnect_client. This makes it more obvious.
387
    def _disconnect_client(self):
6133.4.11 by John Arbash Meinel
It turns out that if we don't explicitly close the socket, it hangs around somewhere.
388
        """Close the current connection. We stopped due to a timeout/etc."""
389
        self.socket.close()
390
6133.4.1 by John Arbash Meinel
Add a _wait_for_bytes_with_timeout helper function.
391
    def _wait_for_bytes_with_timeout(self, timeout_seconds):
392
        """Wait for more bytes to be read, but timeout if none available.
393
394
        This allows us to detect idle connections, and stop trying to read from
395
        them, without setting the socket itself to non-blocking. This also
396
        allows us to specify when we watch for idle timeouts.
6133.4.2 by John Arbash Meinel
Implement waiting for Pipe mediums, including handling ones
397
6133.4.42 by John Arbash Meinel
Change the code to just raise ConnectionTimeout.
398
        :return: None, this will raise ConnectionTimeout if we time out before
399
            data is available.
6133.4.1 by John Arbash Meinel
Add a _wait_for_bytes_with_timeout helper function.
400
        """
6133.4.8 by John Arbash Meinel
Refactor a bit. Use a common _wait_for_descriptor code.
401
        return self._wait_on_descriptor(self.socket, timeout_seconds)
6133.4.1 by John Arbash Meinel
Add a _wait_for_bytes_with_timeout helper function.
402
3565.1.2 by Andrew Bennetts
Delete some more code, fix some bugs, add more comments.
403
    def _read_bytes(self, desired_count):
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
404
        return osutils.read_bytes_from_socket(
405
            self.socket, self._report_activity)
3565.1.1 by Andrew Bennetts
Read no more then 64k at a time in the smart protocol code.
406
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
407
    def terminate_due_to_error(self):
3245.4.59 by Andrew Bennetts
Various tweaks in response to Martin's review.
408
        # TODO: This should log to a server log file, but no such thing
409
        # exists yet.  Andrew Bennetts 2006-09-29.
5011.3.1 by Martin
Revert second-phase EINTR changes
410
        self.socket.close()
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
411
        self.finished = True
412
413
    def _write_out(self, bytes):
4889.2.1 by John Arbash Meinel
Make -Dhpss log debug information for the server process.
414
        tstart = osutils.timer_func()
5011.3.9 by Andrew Bennetts
Remove _send_bytes_chunked.
415
        osutils.send_all(self.socket, bytes, self._report_activity)
4889.2.1 by John Arbash Meinel
Make -Dhpss log debug information for the server process.
416
        if 'hpss' in debug.debug_flags:
4913.1.1 by John Arbash Meinel
Switch to using thread.get_ident() which is available on all python versions.
417
            thread_id = thread.get_ident()
4889.2.3 by John Arbash Meinel
Get rid of -Dhpssthread, just always include it.
418
            trace.mutter('%12s: [%s] %d bytes to the socket in %.3fs'
419
                         % ('wrote', thread_id, len(bytes),
4889.2.2 by John Arbash Meinel
Add a -Dhpssthread debug flag to include thread.ident info.
420
                            osutils.timer_func() - tstart))
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
421
422
423
class SmartServerPipeStreamMedium(SmartServerStreamMedium):
424
6133.4.15 by John Arbash Meinel
Start working on exposing timeout as a configuration item.
425
    def __init__(self, in_file, out_file, backing_transport, timeout=None):
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
426
        """Construct new server.
427
428
        :param in_file: Python file from which requests can be read.
429
        :param out_file: Python file to write responses.
430
        :param backing_transport: Transport for the directory served.
431
        """
6133.4.15 by John Arbash Meinel
Start working on exposing timeout as a configuration item.
432
        SmartServerStreamMedium.__init__(self, backing_transport,
433
            timeout=timeout)
2018.5.161 by Andrew Bennetts
Reinstate forcing binary mode on windows in SmartServerStreamMedium.
434
        if sys.platform == 'win32':
435
            # force binary mode for files
436
            import msvcrt
437
            for f in (in_file, out_file):
438
                fileno = getattr(f, 'fileno', None)
439
                if fileno:
440
                    msvcrt.setmode(fileno(), os.O_BINARY)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
441
        self._in = in_file
442
        self._out = out_file
443
6133.4.61 by John Arbash Meinel
Expose infrastructure so that we can test the Inet portion of the server.
444
    def serve(self):
445
        """See SmartServerStreamMedium.serve"""
446
        # This is the regular serve, except it adds signal trapping for soft
447
        # shutdown.
448
        stop_gracefully = self._stop_gracefully
449
        signals.register_on_hangup(id(self), stop_gracefully)
450
        try:
451
            return super(SmartServerPipeStreamMedium, self).serve()
452
        finally:
453
            signals.unregister_on_hangup(id(self))
454
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
455
    def _serve_one_request_unguarded(self, protocol):
456
        while True:
3565.1.2 by Andrew Bennetts
Delete some more code, fix some bugs, add more comments.
457
            # We need to be careful not to read past the end of the current
458
            # request, or else the read from the pipe will block, so we use
459
            # protocol.next_read_size().
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
460
            bytes_to_read = protocol.next_read_size()
461
            if bytes_to_read == 0:
462
                # Finished serving this request.
5011.3.1 by Martin
Revert second-phase EINTR changes
463
                self._out.flush()
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
464
                return
3565.1.2 by Andrew Bennetts
Delete some more code, fix some bugs, add more comments.
465
            bytes = self.read_bytes(bytes_to_read)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
466
            if bytes == '':
467
                # Connection has been closed.
468
                self.finished = True
5011.3.1 by Martin
Revert second-phase EINTR changes
469
                self._out.flush()
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
470
                return
471
            protocol.accept_bytes(bytes)
472
6133.5.2 by John Arbash Meinel
Rename _close to _disconnect_client. This makes it more obvious.
473
    def _disconnect_client(self):
6133.4.11 by John Arbash Meinel
It turns out that if we don't explicitly close the socket, it hangs around somewhere.
474
        self._in.close()
6133.4.63 by John Arbash Meinel
Flush before we close _out.
475
        self._out.flush()
6133.4.11 by John Arbash Meinel
It turns out that if we don't explicitly close the socket, it hangs around somewhere.
476
        self._out.close()
477
6133.4.2 by John Arbash Meinel
Implement waiting for Pipe mediums, including handling ones
478
    def _wait_for_bytes_with_timeout(self, timeout_seconds):
479
        """Wait for more bytes to be read, but timeout if none available.
480
481
        This allows us to detect idle connections, and stop trying to read from
482
        them, without setting the socket itself to non-blocking. This also
483
        allows us to specify when we watch for idle timeouts.
484
6133.4.42 by John Arbash Meinel
Change the code to just raise ConnectionTimeout.
485
        :return: None, this will raise ConnectionTimeout if we time out before
486
            data is available.
6133.4.2 by John Arbash Meinel
Implement waiting for Pipe mediums, including handling ones
487
        """
6133.4.10 by John Arbash Meinel
Handle the win32 not-a-socket errors. I think this is ~ready. At least for feedback.
488
        if (getattr(self._in, 'fileno', None) is None
489
            or sys.platform == 'win32'):
490
            # You can't select() file descriptors on Windows.
6133.4.42 by John Arbash Meinel
Change the code to just raise ConnectionTimeout.
491
            return
6133.4.8 by John Arbash Meinel
Refactor a bit. Use a common _wait_for_descriptor code.
492
        return self._wait_on_descriptor(self._in, timeout_seconds)
6133.4.2 by John Arbash Meinel
Implement waiting for Pipe mediums, including handling ones
493
3565.1.2 by Andrew Bennetts
Delete some more code, fix some bugs, add more comments.
494
    def _read_bytes(self, desired_count):
5011.3.1 by Martin
Revert second-phase EINTR changes
495
        return self._in.read(desired_count)
2432.2.2 by Andrew Bennetts
Smart server mediums now detect which protocol version a request is and dispatch accordingly.
496
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
497
    def terminate_due_to_error(self):
498
        # TODO: This should log to a server log file, but no such thing
499
        # exists yet.  Andrew Bennetts 2006-09-29.
5011.3.1 by Martin
Revert second-phase EINTR changes
500
        self._out.close()
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
501
        self.finished = True
502
503
    def _write_out(self, bytes):
5011.3.1 by Martin
Revert second-phase EINTR changes
504
        self._out.write(bytes)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
505
506
3565.1.2 by Andrew Bennetts
Delete some more code, fix some bugs, add more comments.
507
class SmartClientMediumRequest(object):
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
508
    """A request on a SmartClientMedium.
509
510
    Each request allows bytes to be provided to it via accept_bytes, and then
511
    the response bytes to be read via read_bytes.
512
513
    For instance:
514
    request.accept_bytes('123')
515
    request.finished_writing()
516
    result = request.read_bytes(3)
517
    request.finished_reading()
518
519
    It is up to the individual SmartClientMedium whether multiple concurrent
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
520
    requests can exist. See SmartClientMedium.get_request to obtain instances
521
    of SmartClientMediumRequest, and the concrete Medium you are using for
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
522
    details on concurrency and pipelining.
523
    """
524
525
    def __init__(self, medium):
526
        """Construct a SmartClientMediumRequest for the medium medium."""
527
        self._medium = medium
528
        # we track state by constants - we may want to use the same
529
        # pattern as BodyReader if it gets more complex.
530
        # valid states are: "writing", "reading", "done"
531
        self._state = "writing"
532
533
    def accept_bytes(self, bytes):
534
        """Accept bytes for inclusion in this request.
535
4031.3.1 by Frank Aspell
Fixing various typos
536
        This method may not be called after finished_writing() has been
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
537
        called.  It depends upon the Medium whether or not the bytes will be
538
        immediately transmitted. Message based Mediums will tend to buffer the
539
        bytes until finished_writing() is called.
540
541
        :param bytes: A bytestring.
542
        """
543
        if self._state != "writing":
544
            raise errors.WritingCompleted(self)
545
        self._accept_bytes(bytes)
546
547
    def _accept_bytes(self, bytes):
548
        """Helper for accept_bytes.
549
550
        Accept_bytes checks the state of the request to determing if bytes
551
        should be accepted. After that it hands off to _accept_bytes to do the
552
        actual acceptance.
553
        """
554
        raise NotImplementedError(self._accept_bytes)
555
556
    def finished_reading(self):
557
        """Inform the request that all desired data has been read.
558
559
        This will remove the request from the pipeline for its medium (if the
560
        medium supports pipelining) and any further calls to methods on the
561
        request will raise ReadingCompleted.
562
        """
563
        if self._state == "writing":
564
            raise errors.WritingNotComplete(self)
565
        if self._state != "reading":
566
            raise errors.ReadingCompleted(self)
567
        self._state = "done"
568
        self._finished_reading()
569
570
    def _finished_reading(self):
571
        """Helper for finished_reading.
572
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
573
        finished_reading checks the state of the request to determine if
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
574
        finished_reading is allowed, and if it is hands off to _finished_reading
575
        to perform the action.
576
        """
577
        raise NotImplementedError(self._finished_reading)
578
579
    def finished_writing(self):
580
        """Finish the writing phase of this request.
581
582
        This will flush all pending data for this request along the medium.
583
        After calling finished_writing, you may not call accept_bytes anymore.
584
        """
585
        if self._state != "writing":
586
            raise errors.WritingCompleted(self)
587
        self._state = "reading"
588
        self._finished_writing()
589
590
    def _finished_writing(self):
591
        """Helper for finished_writing.
592
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
593
        finished_writing checks the state of the request to determine if
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
594
        finished_writing is allowed, and if it is hands off to _finished_writing
595
        to perform the action.
596
        """
597
        raise NotImplementedError(self._finished_writing)
598
599
    def read_bytes(self, count):
600
        """Read bytes from this requests response.
601
602
        This method will block and wait for count bytes to be read. It may not
603
        be invoked until finished_writing() has been called - this is to ensure
2432.2.7 by Andrew Bennetts
Use less confusing version strings, and define REQUEST_VERSION_TWO/RESPONSE_VERSION_TWO constants for them.
604
        a message-based approach to requests, for compatibility with message
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
605
        based mediums like HTTP.
606
        """
607
        if self._state == "writing":
608
            raise errors.WritingNotComplete(self)
609
        if self._state != "reading":
610
            raise errors.ReadingCompleted(self)
611
        return self._read_bytes(count)
612
613
    def _read_bytes(self, count):
3565.1.2 by Andrew Bennetts
Delete some more code, fix some bugs, add more comments.
614
        """Helper for SmartClientMediumRequest.read_bytes.
615
616
        read_bytes checks the state of the request to determing if bytes
617
        should be read. After that it hands off to _read_bytes to do the
618
        actual read.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
619
3565.1.2 by Andrew Bennetts
Delete some more code, fix some bugs, add more comments.
620
        By default this forwards to self._medium.read_bytes because we are
621
        operating on the medium's stream.
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
622
        """
3565.1.2 by Andrew Bennetts
Delete some more code, fix some bugs, add more comments.
623
        return self._medium.read_bytes(count)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
624
2432.2.7 by Andrew Bennetts
Use less confusing version strings, and define REQUEST_VERSION_TWO/RESPONSE_VERSION_TWO constants for them.
625
    def read_line(self):
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
626
        line = self._read_line()
3565.1.1 by Andrew Bennetts
Read no more then 64k at a time in the smart protocol code.
627
        if not line.endswith('\n'):
628
            # end of file encountered reading from server
629
            raise errors.ConnectionReset(
4509.2.3 by Martin Pool
Test tweaks for ConnectionReset message change
630
                "Unexpected end of message. Please check connectivity "
631
                "and permissions, and report a bug if problems persist.")
2432.2.7 by Andrew Bennetts
Use less confusing version strings, and define REQUEST_VERSION_TWO/RESPONSE_VERSION_TWO constants for them.
632
        return line
633
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
634
    def _read_line(self):
635
        """Helper for SmartClientMediumRequest.read_line.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
636
3606.4.1 by Andrew Bennetts
Fix NotImplementedError when probing for smart protocol via HTTP.
637
        By default this forwards to self._medium._get_line because we are
638
        operating on the medium's stream.
639
        """
640
        return self._medium._get_line()
641
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
642
6060.7.1 by Jelmer Vernooij
Add vfs refuser.
643
class _VfsRefuser(object):
644
    """An object that refuses all VFS requests.
645
646
    """
647
648
    def __init__(self):
649
        client._SmartClient.hooks.install_named_hook(
6060.7.2 by Jelmer Vernooij
Fix hook installation
650
            'call', self.check_vfs, 'vfs refuser')
6060.7.1 by Jelmer Vernooij
Add vfs refuser.
651
652
    def check_vfs(self, params):
653
        try:
654
            request_method = request.request_handlers.get(params.method)
655
        except KeyError:
656
            # A method we don't know about doesn't count as a VFS method.
657
            return
6060.7.4 by Jelmer Vernooij
Update NEWS
658
        if issubclass(request_method, vfs.VfsRequest):
659
            raise errors.HpssVfsRequestNotAllowed(params.method, params.args)
6060.7.1 by Jelmer Vernooij
Add vfs refuser.
660
661
3731.2.5 by Andrew Bennetts
Rework hpss call counter.
662
class _DebugCounter(object):
663
    """An object that counts the HPSS calls made to each client medium.
664
5222.2.9 by Robert Collins
Write up some doc about bzrlib.initialize.
665
    When a medium is garbage-collected, or failing that when
666
    bzrlib.global_state exits, the total number of calls made on that medium
667
    are reported via trace.note.
3731.2.1 by Andrew Bennetts
Show total HPSS calls (if any) on stderr when -Dhpss is active.
668
    """
669
3731.2.5 by Andrew Bennetts
Rework hpss call counter.
670
    def __init__(self):
671
        self.counts = weakref.WeakKeyDictionary()
672
        client._SmartClient.hooks.install_named_hook(
673
            'call', self.increment_call_count, 'hpss call counter')
5310.1.1 by Vincent Ladeuil
Fix typo but we may want to use addCleanup instead indeed.
674
        bzrlib.global_state.cleanups.add_cleanup(self.flush_all)
3731.2.5 by Andrew Bennetts
Rework hpss call counter.
675
676
    def track(self, medium):
677
        """Start tracking calls made to a medium.
678
679
        This only keeps a weakref to the medium, so shouldn't affect the
680
        medium's lifetime.
681
        """
682
        medium_repr = repr(medium)
683
        # Add this medium to the WeakKeyDictionary
4326.2.3 by Jonathan Lange
Use as a dict.
684
        self.counts[medium] = dict(count=0, vfs_count=0,
685
                                   medium_repr=medium_repr)
3731.2.5 by Andrew Bennetts
Rework hpss call counter.
686
        # Weakref callbacks are fired in reverse order of their association
687
        # with the referenced object.  So we add a weakref *after* adding to
688
        # the WeakKeyDict so that we can report the value from it before the
689
        # entry is removed by the WeakKeyDict's own callback.
690
        ref = weakref.ref(medium, self.done)
691
692
    def increment_call_count(self, params):
693
        # Increment the count in the WeakKeyDictionary
694
        value = self.counts[params.medium]
4326.2.3 by Jonathan Lange
Use as a dict.
695
        value['count'] += 1
4476.3.15 by Andrew Bennetts
Partially working fallback for pre-1.17 servers.
696
        try:
697
            request_method = request.request_handlers.get(params.method)
698
        except KeyError:
4547.3.1 by Andrew Bennetts
Fix minor bug in -Dhpss that would cause a KeyError when issuing a request for a method not registered in request_handlers.
699
            # A method we don't know about doesn't count as a VFS method.
4476.3.15 by Andrew Bennetts
Partially working fallback for pre-1.17 servers.
700
            return
4326.2.1 by Jonathan Lange
Show the number of VFS calls in -Dhpss output.
701
        if issubclass(request_method, vfs.VfsRequest):
4326.2.3 by Jonathan Lange
Use as a dict.
702
            value['vfs_count'] += 1
3731.2.1 by Andrew Bennetts
Show total HPSS calls (if any) on stderr when -Dhpss is active.
703
704
    def done(self, ref):
3731.2.5 by Andrew Bennetts
Rework hpss call counter.
705
        value = self.counts[ref]
4326.2.3 by Jonathan Lange
Use as a dict.
706
        count, vfs_count, medium_repr = (
707
            value['count'], value['vfs_count'], value['medium_repr'])
3731.2.5 by Andrew Bennetts
Rework hpss call counter.
708
        # In case this callback is invoked for the same ref twice (by the
709
        # weakref callback and by the atexit function), set the call count back
710
        # to 0 so this item won't be reported twice.
4326.2.3 by Jonathan Lange
Use as a dict.
711
        value['count'] = 0
712
        value['vfs_count'] = 0
3731.2.5 by Andrew Bennetts
Rework hpss call counter.
713
        if count != 0:
6138.3.4 by Jonathan Riddell
add gettext() to uses of trace.note()
714
            trace.note(gettext('HPSS calls: {0} ({1} vfs) {2}').format(
715
                       count, vfs_count, medium_repr))
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
716
3731.2.5 by Andrew Bennetts
Rework hpss call counter.
717
    def flush_all(self):
718
        for ref in list(self.counts.keys()):
719
            self.done(ref)
720
721
_debug_counter = None
6060.7.2 by Jelmer Vernooij
Fix hook installation
722
_vfs_refuser = None
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
723
724
3565.1.1 by Andrew Bennetts
Read no more then 64k at a time in the smart protocol code.
725
class SmartClientMedium(SmartMedium):
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
726
    """Smart client is a medium for sending smart protocol requests over."""
727
3431.3.3 by Andrew Bennetts
Set 'base' in SmartClientMedium base class.
728
    def __init__(self, base):
3241.1.1 by Andrew Bennetts
Shift protocol version querying from RemoteBzrDirFormat into SmartClientMedium.
729
        super(SmartClientMedium, self).__init__()
3431.3.3 by Andrew Bennetts
Set 'base' in SmartClientMedium base class.
730
        self.base = base
3241.1.4 by Andrew Bennetts
Use get_smart_medium as suggested by Robert, and deal with the fallout.
731
        self._protocol_version_error = None
3241.1.1 by Andrew Bennetts
Shift protocol version querying from RemoteBzrDirFormat into SmartClientMedium.
732
        self._protocol_version = None
3245.4.47 by Andrew Bennetts
Don't automatically send 'hello' requests from RemoteBzrDirFormat.probe_transport unless we have to (i.e. the transport is HTTP).
733
        self._done_hello = False
3435.1.1 by Andrew Bennetts
Define _remote_is_at_least_1_2 on SmartClientMedium base class, rather than just SmartClientStreamMedium.
734
        # Be optimistic: we assume the remote end can accept new remote
3453.4.1 by Andrew Bennetts
Better infrastructure on SmartClientMedium for tracking the remote version.
735
        # requests until we get an error saying otherwise.
3453.4.10 by Andrew Bennetts
Change _is_remote_at_least to _is_remote_before.
736
        # _remote_version_is_before tracks the bzr version the remote side
3453.4.1 by Andrew Bennetts
Better infrastructure on SmartClientMedium for tracking the remote version.
737
        # can be based on what we've seen so far.
3453.4.10 by Andrew Bennetts
Change _is_remote_at_least to _is_remote_before.
738
        self._remote_version_is_before = None
3731.2.5 by Andrew Bennetts
Rework hpss call counter.
739
        # Install debug hook function if debug flag is set.
3731.2.1 by Andrew Bennetts
Show total HPSS calls (if any) on stderr when -Dhpss is active.
740
        if 'hpss' in debug.debug_flags:
3731.2.5 by Andrew Bennetts
Rework hpss call counter.
741
            global _debug_counter
742
            if _debug_counter is None:
743
                _debug_counter = _DebugCounter()
744
            _debug_counter.track(self)
6060.7.5 by Jelmer Vernooij
Rename hpss_no_vfs to hpss_client_no_vfs.
745
        if 'hpss_client_no_vfs' in debug.debug_flags:
6060.7.1 by Jelmer Vernooij
Add vfs refuser.
746
            global _vfs_refuser
747
            if _vfs_refuser is None:
748
                _vfs_refuser = _VfsRefuser()
3453.4.1 by Andrew Bennetts
Better infrastructure on SmartClientMedium for tracking the remote version.
749
3453.4.10 by Andrew Bennetts
Change _is_remote_at_least to _is_remote_before.
750
    def _is_remote_before(self, version_tuple):
3502.1.1 by Matt Nordhoff
Fix a docstring typo, and a two-expression ``raise`` statement
751
        """Is it possible the remote side supports RPCs for a given version?
3453.4.1 by Andrew Bennetts
Better infrastructure on SmartClientMedium for tracking the remote version.
752
753
        Typical use::
754
755
            needed_version = (1, 2)
3453.4.10 by Andrew Bennetts
Change _is_remote_at_least to _is_remote_before.
756
            if medium._is_remote_before(needed_version):
3453.4.1 by Andrew Bennetts
Better infrastructure on SmartClientMedium for tracking the remote version.
757
                fallback_to_pre_1_2_rpc()
758
            else:
759
                try:
760
                    do_1_2_rpc()
761
                except UnknownSmartMethod:
3453.4.9 by Andrew Bennetts
Rename _remote_is_not to _remember_remote_is_before.
762
                    medium._remember_remote_is_before(needed_version)
3453.4.1 by Andrew Bennetts
Better infrastructure on SmartClientMedium for tracking the remote version.
763
                    fallback_to_pre_1_2_rpc()
764
3453.4.9 by Andrew Bennetts
Rename _remote_is_not to _remember_remote_is_before.
765
        :seealso: _remember_remote_is_before
3453.4.1 by Andrew Bennetts
Better infrastructure on SmartClientMedium for tracking the remote version.
766
        """
3453.4.10 by Andrew Bennetts
Change _is_remote_at_least to _is_remote_before.
767
        if self._remote_version_is_before is None:
3453.4.1 by Andrew Bennetts
Better infrastructure on SmartClientMedium for tracking the remote version.
768
            # So far, the remote side seems to support everything
3453.4.10 by Andrew Bennetts
Change _is_remote_at_least to _is_remote_before.
769
            return False
770
        return version_tuple >= self._remote_version_is_before
3453.4.1 by Andrew Bennetts
Better infrastructure on SmartClientMedium for tracking the remote version.
771
3453.4.9 by Andrew Bennetts
Rename _remote_is_not to _remember_remote_is_before.
772
    def _remember_remote_is_before(self, version_tuple):
3453.4.1 by Andrew Bennetts
Better infrastructure on SmartClientMedium for tracking the remote version.
773
        """Tell this medium that the remote side is older the given version.
774
3453.4.10 by Andrew Bennetts
Change _is_remote_at_least to _is_remote_before.
775
        :seealso: _is_remote_before
3453.4.1 by Andrew Bennetts
Better infrastructure on SmartClientMedium for tracking the remote version.
776
        """
3453.4.10 by Andrew Bennetts
Change _is_remote_at_least to _is_remote_before.
777
        if (self._remote_version_is_before is not None and
778
            version_tuple > self._remote_version_is_before):
4017.3.3 by Robert Collins
Review feedback - make RemoteRepository.initialize use helpers, and version-lock the new method to not attempt the method on older servers.
779
            # We have been told that the remote side is older than some version
780
            # which is newer than a previously supplied older-than version.
781
            # This indicates that some smart verb call is not guarded
782
            # appropriately (it should simply not have been tried).
4797.49.1 by Andrew Bennetts
First, fix _remember_remote_is_before to never raise AssertionError for what is a very minor bug.
783
            trace.mutter(
3453.4.9 by Andrew Bennetts
Rename _remote_is_not to _remember_remote_is_before.
784
                "_remember_remote_is_before(%r) called, but "
785
                "_remember_remote_is_before(%r) was called previously."
4797.49.1 by Andrew Bennetts
First, fix _remember_remote_is_before to never raise AssertionError for what is a very minor bug.
786
                , version_tuple, self._remote_version_is_before)
787
            if 'hpss' in debug.debug_flags:
788
                ui.ui_factory.show_warning(
789
                    "_remember_remote_is_before(%r) called, but "
790
                    "_remember_remote_is_before(%r) was called previously."
791
                    % (version_tuple, self._remote_version_is_before))
792
            return
3453.4.10 by Andrew Bennetts
Change _is_remote_at_least to _is_remote_before.
793
        self._remote_version_is_before = version_tuple
3241.1.1 by Andrew Bennetts
Shift protocol version querying from RemoteBzrDirFormat into SmartClientMedium.
794
795
    def protocol_version(self):
3245.4.47 by Andrew Bennetts
Don't automatically send 'hello' requests from RemoteBzrDirFormat.probe_transport unless we have to (i.e. the transport is HTTP).
796
        """Find out if 'hello' smart request works."""
3241.1.4 by Andrew Bennetts
Use get_smart_medium as suggested by Robert, and deal with the fallout.
797
        if self._protocol_version_error is not None:
798
            raise self._protocol_version_error
3245.4.47 by Andrew Bennetts
Don't automatically send 'hello' requests from RemoteBzrDirFormat.probe_transport unless we have to (i.e. the transport is HTTP).
799
        if not self._done_hello:
3241.1.4 by Andrew Bennetts
Use get_smart_medium as suggested by Robert, and deal with the fallout.
800
            try:
801
                medium_request = self.get_request()
802
                # Send a 'hello' request in protocol version one, for maximum
803
                # backwards compatibility.
3530.1.2 by John Arbash Meinel
missed one of the imports
804
                client_protocol = protocol.SmartClientRequestProtocolOne(medium_request)
3245.4.47 by Andrew Bennetts
Don't automatically send 'hello' requests from RemoteBzrDirFormat.probe_transport unless we have to (i.e. the transport is HTTP).
805
                client_protocol.query_version()
806
                self._done_hello = True
3241.1.4 by Andrew Bennetts
Use get_smart_medium as suggested by Robert, and deal with the fallout.
807
            except errors.SmartProtocolError, e:
808
                # Cache the error, just like we would cache a successful
809
                # result.
810
                self._protocol_version_error = e
811
                raise
3245.4.47 by Andrew Bennetts
Don't automatically send 'hello' requests from RemoteBzrDirFormat.probe_transport unless we have to (i.e. the transport is HTTP).
812
        return '2'
813
814
    def should_probe(self):
815
        """Should RemoteBzrDirFormat.probe_transport send a smart request on
816
        this medium?
817
818
        Some transports are unambiguously smart-only; there's no need to check
819
        if the transport is able to carry smart requests, because that's all
820
        it is for.  In those cases, this method should return False.
821
822
        But some HTTP transports can sometimes fail to carry smart requests,
823
        but still be usuable for accessing remote bzrdirs via plain file
824
        accesses.  So for those transports, their media should return True here
825
        so that RemoteBzrDirFormat can determine if it is appropriate for that
826
        transport.
827
        """
828
        return False
3241.1.1 by Andrew Bennetts
Shift protocol version querying from RemoteBzrDirFormat into SmartClientMedium.
829
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
830
    def disconnect(self):
831
        """If this medium maintains a persistent connection, close it.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
832
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
833
        The default implementation does nothing.
834
        """
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
835
3431.3.11 by Andrew Bennetts
Push remote_path_from_transport logic into SmartClientMedium, removing special-casing of bzr+http from _SmartClient.
836
    def remote_path_from_transport(self, transport):
837
        """Convert transport into a path suitable for using in a request.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
838
3431.3.11 by Andrew Bennetts
Push remote_path_from_transport logic into SmartClientMedium, removing special-casing of bzr+http from _SmartClient.
839
        Note that the resulting remote path doesn't encode the host name or
840
        anything but path, so it is only safe to use it in requests sent over
841
        the medium from the matching transport.
842
        """
843
        medium_base = urlutils.join(self.base, '/')
844
        rel_url = urlutils.relative_url(medium_base, transport.base)
6379.4.2 by Jelmer Vernooij
Add urlutils.quote / urlutils.unquote.
845
        return urlutils.unquote(rel_url)
3431.3.11 by Andrew Bennetts
Push remote_path_from_transport logic into SmartClientMedium, removing special-casing of bzr+http from _SmartClient.
846
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
847
848
class SmartClientStreamMedium(SmartClientMedium):
849
    """Stream based medium common class.
850
851
    SmartClientStreamMediums operate on a stream. All subclasses use a common
852
    SmartClientStreamMediumRequest for their requests, and should implement
853
    _accept_bytes and _read_bytes to allow the request objects to send and
854
    receive bytes.
855
    """
856
3431.3.3 by Andrew Bennetts
Set 'base' in SmartClientMedium base class.
857
    def __init__(self, base):
858
        SmartClientMedium.__init__(self, base)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
859
        self._current_request = None
860
861
    def accept_bytes(self, bytes):
862
        self._accept_bytes(bytes)
863
864
    def __del__(self):
865
        """The SmartClientStreamMedium knows how to close the stream when it is
866
        finished with it.
867
        """
868
        self.disconnect()
869
870
    def _flush(self):
871
        """Flush the output stream.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
872
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
873
        This method is used by the SmartClientStreamMediumRequest to ensure that
874
        all data for a request is sent, to avoid long timeouts or deadlocks.
875
        """
876
        raise NotImplementedError(self._flush)
877
878
    def get_request(self):
879
        """See SmartClientMedium.get_request().
880
881
        SmartClientStreamMedium always returns a SmartClientStreamMediumRequest
882
        for get_request.
883
        """
884
        return SmartClientStreamMediumRequest(self)
885
4797.85.21 by John Arbash Meinel
Finally, we have something that can at least handle simple reconnects.
886
    def reset(self):
887
        """We have been disconnected, reset current state.
888
889
        This resets things like _current_request and connected state.
890
        """
891
        self.disconnect()
892
        self._current_request = None
893
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
894
895
class SmartSimplePipesClientMedium(SmartClientStreamMedium):
896
    """A client medium using simple pipes.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
897
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
898
    This client does not manage the pipes: it assumes they will always be open.
899
    """
900
3431.3.1 by Andrew Bennetts
First rough cut of a fix for bug #230550, by adding .base to SmartClientMedia rather than relying on other objects to track this accurately while reusing client media.
901
    def __init__(self, readable_pipe, writeable_pipe, base):
3431.3.3 by Andrew Bennetts
Set 'base' in SmartClientMedium base class.
902
        SmartClientStreamMedium.__init__(self, base)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
903
        self._readable_pipe = readable_pipe
904
        self._writeable_pipe = writeable_pipe
905
906
    def _accept_bytes(self, bytes):
907
        """See SmartClientStreamMedium.accept_bytes."""
4797.85.10 by John Arbash Meinel
Wrap the call to .write() to raise ConnectionReset on failure.
908
        try:
5050.78.5 by John Arbash Meinel
Merge the 2.1-client-read-reconnect-819604 (bug #819604) to bzr-2.2
909
            self._writeable_pipe.write(bytes)
4797.85.10 by John Arbash Meinel
Wrap the call to .write() to raise ConnectionReset on failure.
910
        except IOError, e:
911
            if e.errno in (errno.EINVAL, errno.EPIPE):
912
                raise errors.ConnectionReset(
6437.74.2 by John Arbash Meinel
Push the change down into osutils.send_all. Needs tests at that level.
913
                    "Error trying to write to subprocess", e)
4797.85.11 by John Arbash Meinel
Maybe we can just use a regular pipe for testing, rather than spawning a subprocess.
914
            raise
3958.1.2 by Andrew Bennetts
Report network activity from more client medium implementations.
915
        self._report_activity(len(bytes), 'write')
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
916
917
    def _flush(self):
918
        """See SmartClientStreamMedium._flush()."""
4797.85.13 by John Arbash Meinel
Document the behavior of flush.
919
        # Note: If flush were to fail, we'd like to raise ConnectionReset, etc.
920
        #       However, testing shows that even when the child process is
921
        #       gone, this doesn't error.
5011.3.1 by Martin
Revert second-phase EINTR changes
922
        self._writeable_pipe.flush()
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
923
924
    def _read_bytes(self, count):
925
        """See SmartClientStreamMedium._read_bytes."""
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
926
        bytes_to_read = min(count, _MAX_READ_SIZE)
927
        bytes = self._readable_pipe.read(bytes_to_read)
3958.1.2 by Andrew Bennetts
Report network activity from more client medium implementations.
928
        self._report_activity(len(bytes), 'read')
929
        return bytes
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
930
931
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
932
class SSHParams(object):
5284.5.5 by Andrew Bennetts
Fix SSHParams docs.
933
    """A set of parameters for starting a remote bzr via SSH."""
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
934
935
    def __init__(self, host, port=None, username=None, password=None,
936
            bzr_remote_path='bzr'):
937
        self.host = host
938
        self.port = port
939
        self.username = username
940
        self.password = password
941
        self.bzr_remote_path = bzr_remote_path
942
943
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
944
class SmartSSHClientMedium(SmartClientStreamMedium):
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
945
    """A client medium using SSH.
4797.85.20 by John Arbash Meinel
Pull in the code from bzr.dev that changes SmartSSHClientMedium
946
5050.78.5 by John Arbash Meinel
Merge the 2.1-client-read-reconnect-819604 (bug #819604) to bzr-2.2
947
    It delegates IO to a SmartSimplePipesClientMedium or
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
948
    SmartClientAlreadyConnectedSocketMedium (depending on platform).
949
    """
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
950
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
951
    def __init__(self, base, ssh_params, vendor=None):
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
952
        """Creates a client that will connect on the first use.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
953
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
954
        :param ssh_params: A SSHParams instance.
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
955
        :param vendor: An optional override for the ssh vendor to use. See
956
            bzrlib.transport.ssh for details on ssh vendors.
957
        """
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
958
        self._real_medium = None
959
        self._ssh_params = ssh_params
4964.2.5 by Martin Pool
Make sure variables used in repr are set early in initialization
960
        # for the benefit of progress making a short description of this
961
        # transport
962
        self._scheme = 'bzr+ssh'
4100.1.5 by Martin Pool
Fix crash in SSHSmartClientStreamMedium repr.
963
        # SmartClientStreamMedium stores the repr of this object in its
964
        # _DebugCounter so we have to store all the values used in our repr
965
        # method before calling the super init.
966
        SmartClientStreamMedium.__init__(self, base)
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
967
        self._vendor = vendor
4797.96.6 by John Arbash Meinel
Move read_bytes_from_socket into osutils like it is in bzr-2.5.
968
        self._ssh_connection = None
4100.1.1 by Martin Pool
Cleanup and add SmartSSHClientMedium repr
969
970
    def __repr__(self):
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
971
        if self._ssh_params.port is None:
4964.2.4 by Martin Pool
Tweak SSHSmartClientMedium to look better when there's no port
972
            maybe_port = ''
973
        else:
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
974
            maybe_port = ':%s' % self._ssh_params.port
6242.1.1 by Jelmer Vernooij
Cope with username being None in SmartSSHMedium.__repr__.
975
        if self._ssh_params.username is None:
976
            maybe_user = ''
977
        else:
978
            maybe_user = '%s@' % self._ssh_params.username
979
        return "%s(%s://%s%s%s/)" % (
4100.1.1 by Martin Pool
Cleanup and add SmartSSHClientMedium repr
980
            self.__class__.__name__,
4964.2.3 by Martin Pool
Tweak SmartSSHClientMedium repr
981
            self._scheme,
6242.1.1 by Jelmer Vernooij
Cope with username being None in SmartSSHMedium.__repr__.
982
            maybe_user,
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
983
            self._ssh_params.host,
4964.2.4 by Martin Pool
Tweak SSHSmartClientMedium to look better when there's no port
984
            maybe_port)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
985
986
    def _accept_bytes(self, bytes):
987
        """See SmartClientStreamMedium.accept_bytes."""
988
        self._ensure_connection()
5284.5.6 by Andrew Bennetts
Tweaks prompted by Robert's review.
989
        self._real_medium.accept_bytes(bytes)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
990
991
    def disconnect(self):
992
        """See SmartClientMedium.disconnect()."""
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
993
        if self._real_medium is not None:
994
            self._real_medium.disconnect()
995
            self._real_medium = None
996
        if self._ssh_connection is not None:
997
            self._ssh_connection.close()
998
            self._ssh_connection = None
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
999
1000
    def _ensure_connection(self):
1001
        """Connect this medium if not already connected."""
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1002
        if self._real_medium is not None:
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1003
            return
1004
        if self._vendor is None:
1005
            vendor = ssh._get_ssh_vendor()
1006
        else:
1007
            vendor = self._vendor
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1008
        self._ssh_connection = vendor.connect_ssh(self._ssh_params.username,
1009
                self._ssh_params.password, self._ssh_params.host,
1010
                self._ssh_params.port,
1011
                command=[self._ssh_params.bzr_remote_path, 'serve', '--inet',
1551.18.17 by Aaron Bentley
Introduce bzr_remote_path configuration variable
1012
                         '--directory=/', '--allow-writes'])
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1013
        io_kind, io_object = self._ssh_connection.get_sock_or_pipes()
1014
        if io_kind == 'socket':
1015
            self._real_medium = SmartClientAlreadyConnectedSocketMedium(
1016
                self.base, io_object)
1017
        elif io_kind == 'pipes':
1018
            read_from, write_to = io_object
1019
            self._real_medium = SmartSimplePipesClientMedium(
1020
                read_from, write_to, self.base)
1021
        else:
1022
            raise AssertionError(
1023
                "Unexpected io_kind %r from %r"
1024
                % (io_kind, self._ssh_connection))
5436.3.9 by Martin Packman
Move post_connect callback for RemoteTransports to smart.medium
1025
        for hook in transport.Transport.hooks["post_connect"]:
1026
            hook(self)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1027
1028
    def _flush(self):
1029
        """See SmartClientStreamMedium._flush()."""
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1030
        self._real_medium._flush()
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1031
1032
    def _read_bytes(self, count):
1033
        """See SmartClientStreamMedium.read_bytes."""
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1034
        if self._real_medium is None:
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1035
            raise errors.MediumNotConnected(self)
5284.5.6 by Andrew Bennetts
Tweaks prompted by Robert's review.
1036
        return self._real_medium.read_bytes(count)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1037
1038
3004.2.1 by Vincent Ladeuil
Fix 150860 by leaving port as user specified it.
1039
# Port 4155 is the default port for bzr://, registered with IANA.
3665.4.1 by Jelmer Vernooij
Support IPv6 in the smart server.
1040
BZR_DEFAULT_INTERFACE = None
3004.2.1 by Vincent Ladeuil
Fix 150860 by leaving port as user specified it.
1041
BZR_DEFAULT_PORT = 4155
1042
1043
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1044
class SmartClientSocketMedium(SmartClientStreamMedium):
5284.5.6 by Andrew Bennetts
Tweaks prompted by Robert's review.
1045
    """A client medium using a socket.
6437.74.2 by John Arbash Meinel
Push the change down into osutils.send_all. Needs tests at that level.
1046
5284.5.6 by Andrew Bennetts
Tweaks prompted by Robert's review.
1047
    This class isn't usable directly.  Use one of its subclasses instead.
1048
    """
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1049
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1050
    def __init__(self, base):
3431.3.3 by Andrew Bennetts
Set 'base' in SmartClientMedium base class.
1051
        SmartClientStreamMedium.__init__(self, base)
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1052
        self._socket = None
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1053
        self._connected = False
1054
1055
    def _accept_bytes(self, bytes):
1056
        """See SmartClientMedium.accept_bytes."""
1057
        self._ensure_connection()
6437.74.2 by John Arbash Meinel
Push the change down into osutils.send_all. Needs tests at that level.
1058
        osutils.send_all(self._socket, bytes, self._report_activity)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1059
5284.5.6 by Andrew Bennetts
Tweaks prompted by Robert's review.
1060
    def _ensure_connection(self):
1061
        """Connect this medium if not already connected."""
1062
        raise NotImplementedError(self._ensure_connection)
1063
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1064
    def _flush(self):
1065
        """See SmartClientStreamMedium._flush().
1066
5284.5.6 by Andrew Bennetts
Tweaks prompted by Robert's review.
1067
        For sockets we do no flushing. For TCP sockets we may want to turn off
1068
        TCP_NODELAY and add a means to do a flush, but that can be done in the
1069
        future.
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1070
        """
1071
1072
    def _read_bytes(self, count):
1073
        """See SmartClientMedium.read_bytes."""
1074
        if not self._connected:
1075
            raise errors.MediumNotConnected(self)
1076
        return osutils.read_bytes_from_socket(
1077
            self._socket, self._report_activity)
1078
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1079
    def disconnect(self):
1080
        """See SmartClientMedium.disconnect()."""
1081
        if not self._connected:
1082
            return
5011.3.1 by Martin
Revert second-phase EINTR changes
1083
        self._socket.close()
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1084
        self._socket = None
1085
        self._connected = False
1086
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1087
1088
class SmartTCPClientMedium(SmartClientSocketMedium):
5284.5.6 by Andrew Bennetts
Tweaks prompted by Robert's review.
1089
    """A client medium that creates a TCP connection."""
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1090
1091
    def __init__(self, host, port, base):
1092
        """Creates a client that will connect on the first use."""
1093
        SmartClientSocketMedium.__init__(self, base)
1094
        self._host = host
1095
        self._port = port
1096
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1097
    def _ensure_connection(self):
1098
        """Connect this medium if not already connected."""
1099
        if self._connected:
1100
            return
3004.2.1 by Vincent Ladeuil
Fix 150860 by leaving port as user specified it.
1101
        if self._port is None:
1102
            port = BZR_DEFAULT_PORT
1103
        else:
1104
            port = int(self._port)
3711.2.2 by Jelmer Vernooij
Avoid using AI_ADDRCONFIG since it's not portable.
1105
        try:
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1106
            sockaddrs = socket.getaddrinfo(self._host, port, socket.AF_UNSPEC,
3711.2.2 by Jelmer Vernooij
Avoid using AI_ADDRCONFIG since it's not portable.
1107
                socket.SOCK_STREAM, 0, 0)
1108
        except socket.gaierror, (err_num, err_msg):
1109
            raise errors.ConnectionError("failed to lookup %s:%d: %s" %
1110
                    (self._host, port, err_msg))
3711.2.3 by Jelmer Vernooij
Add comment.
1111
        # Initialize err in case there are no addresses returned:
3665.4.2 by Jelmer Vernooij
Fall through to next available address if previous fails.
1112
        err = socket.error("no address found for %s" % self._host)
3665.4.1 by Jelmer Vernooij
Support IPv6 in the smart server.
1113
        for (family, socktype, proto, canonname, sockaddr) in sockaddrs:
1114
            try:
3665.4.2 by Jelmer Vernooij
Fall through to next available address if previous fails.
1115
                self._socket = socket.socket(family, socktype, proto)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1116
                self._socket.setsockopt(socket.IPPROTO_TCP,
3665.4.2 by Jelmer Vernooij
Fall through to next available address if previous fails.
1117
                                        socket.TCP_NODELAY, 1)
3665.4.1 by Jelmer Vernooij
Support IPv6 in the smart server.
1118
                self._socket.connect(sockaddr)
1119
            except socket.error, err:
3665.4.2 by Jelmer Vernooij
Fall through to next available address if previous fails.
1120
                if self._socket is not None:
1121
                    self._socket.close()
1122
                self._socket = None
1123
                continue
1124
            break
1125
        if self._socket is None:
1126
            # socket errors either have a (string) or (errno, string) as their
1127
            # args.
1128
            if type(err.args) is str:
1129
                err_msg = err.args
1130
            else:
1131
                err_msg = err.args[1]
1132
            raise errors.ConnectionError("failed to connect to %s:%d: %s" %
1133
                    (self._host, port, err_msg))
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1134
        self._connected = True
5436.3.9 by Martin Packman
Move post_connect callback for RemoteTransports to smart.medium
1135
        for hook in transport.Transport.hooks["post_connect"]:
1136
            hook(self)
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1137
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1138
1139
class SmartClientAlreadyConnectedSocketMedium(SmartClientSocketMedium):
5284.5.3 by Andrew Bennetts
Docstring tweaks.
1140
    """A client medium for an already connected socket.
5284.5.1 by Andrew Bennetts
Use socketpairs (rather than pipes) for SSH subprocesses where possible, and formalise some internal APIs a little more.
1141
    
1142
    Note that this class will assume it "owns" the socket, so it will close it
1143
    when its disconnect method is called.
1144
    """
1145
1146
    def __init__(self, base, sock):
1147
        SmartClientSocketMedium.__init__(self, base)
1148
        self._socket = sock
1149
        self._connected = True
1150
1151
    def _ensure_connection(self):
1152
        # Already connected, by definition!  So nothing to do.
1153
        pass
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1154
1155
1156
class SmartClientStreamMediumRequest(SmartClientMediumRequest):
1157
    """A SmartClientMediumRequest that works with an SmartClientStreamMedium."""
1158
1159
    def __init__(self, medium):
1160
        SmartClientMediumRequest.__init__(self, medium)
1161
        # check that we are safe concurrency wise. If some streams start
1162
        # allowing concurrent requests - i.e. via multiplexing - then this
1163
        # assert should be moved to SmartClientStreamMedium.get_request,
1164
        # and the setting/unsetting of _current_request likewise moved into
1165
        # that class : but its unneeded overhead for now. RBC 20060922
1166
        if self._medium._current_request is not None:
1167
            raise errors.TooManyConcurrentRequests(self._medium)
1168
        self._medium._current_request = self
1169
1170
    def _accept_bytes(self, bytes):
1171
        """See SmartClientMediumRequest._accept_bytes.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1172
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1173
        This forwards to self._medium._accept_bytes because we are operating
1174
        on the mediums stream.
1175
        """
1176
        self._medium._accept_bytes(bytes)
1177
1178
    def _finished_reading(self):
1179
        """See SmartClientMediumRequest._finished_reading.
1180
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1181
        This clears the _current_request on self._medium to allow a new
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1182
        request to be created.
1183
        """
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
1184
        if self._medium._current_request is not self:
1185
            raise AssertionError()
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1186
        self._medium._current_request = None
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1187
2018.5.2 by Andrew Bennetts
Start splitting bzrlib/transport/smart.py into a package.
1188
    def _finished_writing(self):
1189
        """See SmartClientMediumRequest._finished_writing.
1190
1191
        This invokes self._medium._flush to ensure all bytes are transmitted.
1192
        """
1193
        self._medium._flush()