80
67
root_client_path. unused_bytes are any bytes that were not part of a
81
68
protocol version marker.
83
if bytes.startswith(protocol.MESSAGE_VERSION_THREE):
84
protocol_factory = protocol.build_server_protocol_three
85
bytes = bytes[len(protocol.MESSAGE_VERSION_THREE):]
86
elif bytes.startswith(protocol.REQUEST_VERSION_TWO):
87
protocol_factory = protocol.SmartServerRequestProtocolTwo
88
bytes = bytes[len(protocol.REQUEST_VERSION_TWO):]
70
if bytes.startswith(MESSAGE_VERSION_THREE):
71
protocol_factory = build_server_protocol_three
72
bytes = bytes[len(MESSAGE_VERSION_THREE):]
73
elif bytes.startswith(REQUEST_VERSION_TWO):
74
protocol_factory = SmartServerRequestProtocolTwo
75
bytes = bytes[len(REQUEST_VERSION_TWO):]
90
protocol_factory = protocol.SmartServerRequestProtocolOne
77
protocol_factory = SmartServerRequestProtocolOne
91
78
return protocol_factory, bytes
94
def _get_line(read_bytes_func):
95
"""Read bytes using read_bytes_func until a newline byte.
97
This isn't particularly efficient, so should only be used when the
98
expected size of the line is quite short.
100
:returns: a tuple of two strs: (line, excess)
104
while newline_pos == -1:
105
new_bytes = read_bytes_func(1)
108
# Ran out of bytes before receiving a complete line.
110
newline_pos = bytes.find('\n')
111
line = bytes[:newline_pos+1]
112
excess = bytes[newline_pos+1:]
116
class SmartMedium(object):
117
"""Base class for smart protocol media, both client- and server-side."""
120
self._push_back_buffer = None
122
def _push_back(self, bytes):
123
"""Return unused bytes to the medium, because they belong to the next
126
This sets the _push_back_buffer to the given bytes.
128
if self._push_back_buffer is not None:
129
raise AssertionError(
130
"_push_back called when self._push_back_buffer is %r"
131
% (self._push_back_buffer,))
134
self._push_back_buffer = bytes
136
def _get_push_back_buffer(self):
137
if self._push_back_buffer == '':
138
raise AssertionError(
139
'%s._push_back_buffer should never be the empty string, '
140
'which can be confused with EOF' % (self,))
141
bytes = self._push_back_buffer
142
self._push_back_buffer = None
145
def read_bytes(self, desired_count):
146
"""Read some bytes from this medium.
148
:returns: some bytes, possibly more or less than the number requested
149
in 'desired_count' depending on the medium.
151
if self._push_back_buffer is not None:
152
return self._get_push_back_buffer()
153
bytes_to_read = min(desired_count, _MAX_READ_SIZE)
154
return self._read_bytes(bytes_to_read)
156
def _read_bytes(self, count):
157
raise NotImplementedError(self._read_bytes)
160
"""Read bytes from this request's response until a newline byte.
162
This isn't particularly efficient, so should only be used when the
163
expected size of the line is quite short.
165
:returns: a string of bytes ending in a newline (byte 0x0A).
167
line, excess = _get_line(self.read_bytes)
168
self._push_back(excess)
171
def _report_activity(self, bytes, direction):
172
"""Notify that this medium has activity.
174
Implementations should call this from all methods that actually do IO.
175
Be careful that it's not called twice, if one method is implemented on
178
:param bytes: Number of bytes read or written.
179
:param direction: 'read' or 'write' or None.
181
ui.ui_factory.report_transport_activity(self, bytes, direction)
184
_bad_file_descriptor = (errno.EBADF,)
185
if sys.platform == 'win32':
186
# Given on Windows if you pass a closed socket to select.select. Probably
187
# also given if you pass a file handle to select.
189
_bad_file_descriptor += (WSAENOTSOCK,)
192
class SmartServerStreamMedium(SmartMedium):
81
class SmartServerStreamMedium(object):
193
82
"""Handles smart commands coming over a stream.
195
84
The stream may be a pipe connected to sshd, or a tcp socket, or an
233
139
while not self.finished:
234
140
server_protocol = self._build_protocol()
235
141
self._serve_one_request(server_protocol)
236
except errors.ConnectionTimeout, e:
237
trace.note('%s' % (e,))
238
trace.log_exception_quietly()
239
self._disconnect_client()
240
# We reported it, no reason to make a big fuss.
242
142
except Exception, e:
243
143
stderr.write("%s terminating on exception %s\n" % (self, e))
245
self._disconnect_client()
247
def _stop_gracefully(self):
248
"""When we finish this message, stop looking for more."""
249
trace.mutter('Stopping %s' % (self,))
252
def _disconnect_client(self):
253
"""Close the current connection. We stopped due to a timeout/etc."""
254
# The default implementation is a no-op, because that is all we used to
255
# do when disconnecting from a client. I suppose we never had the
256
# *server* initiate a disconnect, before
258
def _wait_for_bytes_with_timeout(self, timeout_seconds):
259
"""Wait for more bytes to be read, but timeout if none available.
261
This allows us to detect idle connections, and stop trying to read from
262
them, without setting the socket itself to non-blocking. This also
263
allows us to specify when we watch for idle timeouts.
265
:return: Did we timeout? (True if we timed out, False if there is data
268
raise NotImplementedError(self._wait_for_bytes_with_timeout)
270
146
def _build_protocol(self):
271
147
"""Identifies the version of the incoming request, and returns an
287
159
protocol.accept_bytes(unused_bytes)
290
def _wait_on_descriptor(self, fd, timeout_seconds):
291
"""select() on a file descriptor, waiting for nonblocking read()
293
This will raise a ConnectionTimeout exception if we do not get a
294
readable handle before timeout_seconds.
297
t_end = self._timer() + timeout_seconds
298
poll_timeout = min(timeout_seconds, self._client_poll_timeout)
300
while not rs and not xs and self._timer() < t_end:
304
rs, _, xs = select.select([fd], [], [fd], poll_timeout)
305
except (select.error, socket.error) as e:
306
err = getattr(e, 'errno', None)
307
if err is None and getattr(e, 'args', None) is not None:
308
# select.error doesn't have 'errno', it just has args[0]
310
if err in _bad_file_descriptor:
311
return # Not a socket indicates read() will fail
312
elif err == errno.EINTR:
313
# Interrupted, keep looping.
318
raise errors.ConnectionTimeout('disconnecting client after %.1f seconds'
319
% (timeout_seconds,))
321
162
def _serve_one_request(self, protocol):
322
163
"""Read one request from input, process, send back a response.
324
165
:param protocol: a SmartServerRequestProtocol.
329
168
self._serve_one_request_unguarded(protocol)
330
169
except KeyboardInterrupt:
336
175
"""Called when an unhandled exception from the protocol occurs."""
337
176
raise NotImplementedError(self.terminate_due_to_error)
339
def _read_bytes(self, desired_count):
178
def _get_bytes(self, desired_count):
340
179
"""Get some bytes from the medium.
342
181
:param desired_count: number of bytes we want to read.
344
raise NotImplementedError(self._read_bytes)
183
raise NotImplementedError(self._get_bytes)
186
"""Read bytes from this request's response until a newline byte.
188
This isn't particularly efficient, so should only be used when the
189
expected size of the line is quite short.
191
:returns: a string of bytes ending in a newline (byte 0x0A).
195
while newline_pos == -1:
196
new_bytes = self._get_bytes(1)
199
# Ran out of bytes before receiving a complete line.
201
newline_pos = bytes.find('\n')
202
line = bytes[:newline_pos+1]
203
self._push_back(bytes[newline_pos+1:])
347
207
class SmartServerSocketStreamMedium(SmartServerStreamMedium):
349
def __init__(self, sock, backing_transport, root_client_path='/',
209
def __init__(self, sock, backing_transport, root_client_path='/'):
353
212
:param sock: the socket the server will read from. It will be put
354
213
into blocking mode.
356
215
SmartServerStreamMedium.__init__(
357
self, backing_transport, root_client_path=root_client_path,
216
self, backing_transport, root_client_path=root_client_path)
359
217
sock.setblocking(True)
360
218
self.socket = sock
361
# Get the getpeername now, as we might be closed later when we care.
363
self._client_info = sock.getpeername()
365
self._client_info = '<unknown>'
368
return '%s(client=%s)' % (self.__class__.__name__, self._client_info)
371
return '%s.%s(client=%s)' % (self.__module__, self.__class__.__name__,
374
220
def _serve_one_request_unguarded(self, protocol):
375
221
while protocol.next_read_size():
376
# We can safely try to read large chunks. If there is less data
377
# than MAX_SOCKET_CHUNK ready, the socket will just return a
378
# short read immediately rather than block.
379
bytes = self.read_bytes(osutils.MAX_SOCKET_CHUNK)
222
bytes = self._get_bytes(4096)
381
224
self.finished = True
383
226
protocol.accept_bytes(bytes)
385
228
self._push_back(protocol.unused_data)
387
def _disconnect_client(self):
388
"""Close the current connection. We stopped due to a timeout/etc."""
391
def _wait_for_bytes_with_timeout(self, timeout_seconds):
392
"""Wait for more bytes to be read, but timeout if none available.
394
This allows us to detect idle connections, and stop trying to read from
395
them, without setting the socket itself to non-blocking. This also
396
allows us to specify when we watch for idle timeouts.
398
:return: None, this will raise ConnectionTimeout if we time out before
401
return self._wait_on_descriptor(self.socket, timeout_seconds)
403
def _read_bytes(self, desired_count):
404
return osutils.read_bytes_from_socket(
405
self.socket, self._report_activity)
230
def _get_bytes(self, desired_count):
231
if self._push_back_buffer is not None:
232
return self._get_push_back_buffer()
233
# We ignore the desired_count because on sockets it's more efficient to
235
return self.socket.recv(4096)
407
237
def terminate_due_to_error(self):
408
238
# TODO: This should log to a server log file, but no such thing
409
239
# exists yet. Andrew Bennetts 2006-09-29.
611
401
return self._read_bytes(count)
613
403
def _read_bytes(self, count):
614
"""Helper for SmartClientMediumRequest.read_bytes.
404
"""Helper for read_bytes.
616
406
read_bytes checks the state of the request to determing if bytes
617
407
should be read. After that it hands off to _read_bytes to do the
620
By default this forwards to self._medium.read_bytes because we are
621
operating on the medium's stream.
623
return self._medium.read_bytes(count)
410
raise NotImplementedError(self._read_bytes)
625
412
def read_line(self):
626
line = self._read_line()
627
if not line.endswith('\n'):
628
# end of file encountered reading from server
629
raise errors.ConnectionReset(
630
"Unexpected end of message. Please check connectivity "
631
"and permissions, and report a bug if problems persist.")
413
"""Read bytes from this request's response until a newline byte.
415
This isn't particularly efficient, so should only be used when the
416
expected size of the line is quite short.
418
:returns: a string of bytes ending in a newline (byte 0x0A).
420
# XXX: this duplicates SmartClientRequestProtocolOne._recv_tuple
422
while not line or line[-1] != '\n':
423
new_char = self.read_bytes(1)
426
# end of file encountered reading from server
427
raise errors.ConnectionReset(
428
"please check connectivity and permissions",
429
"(and try -Dhpss if further diagnosis is required)")
634
def _read_line(self):
635
"""Helper for SmartClientMediumRequest.read_line.
637
By default this forwards to self._medium._get_line because we are
638
operating on the medium's stream.
640
return self._medium._get_line()
643
class _VfsRefuser(object):
644
"""An object that refuses all VFS requests.
649
client._SmartClient.hooks.install_named_hook(
650
'call', self.check_vfs, 'vfs refuser')
652
def check_vfs(self, params):
654
request_method = request.request_handlers.get(params.method)
656
# A method we don't know about doesn't count as a VFS method.
658
if issubclass(request_method, vfs.VfsRequest):
659
raise errors.HpssVfsRequestNotAllowed(params.method, params.args)
662
class _DebugCounter(object):
663
"""An object that counts the HPSS calls made to each client medium.
665
When a medium is garbage-collected, or failing that when
666
bzrlib.global_state exits, the total number of calls made on that medium
667
are reported via trace.note.
671
self.counts = weakref.WeakKeyDictionary()
672
client._SmartClient.hooks.install_named_hook(
673
'call', self.increment_call_count, 'hpss call counter')
674
bzrlib.global_state.cleanups.add_cleanup(self.flush_all)
676
def track(self, medium):
677
"""Start tracking calls made to a medium.
679
This only keeps a weakref to the medium, so shouldn't affect the
682
medium_repr = repr(medium)
683
# Add this medium to the WeakKeyDictionary
684
self.counts[medium] = dict(count=0, vfs_count=0,
685
medium_repr=medium_repr)
686
# Weakref callbacks are fired in reverse order of their association
687
# with the referenced object. So we add a weakref *after* adding to
688
# the WeakKeyDict so that we can report the value from it before the
689
# entry is removed by the WeakKeyDict's own callback.
690
ref = weakref.ref(medium, self.done)
692
def increment_call_count(self, params):
693
# Increment the count in the WeakKeyDictionary
694
value = self.counts[params.medium]
697
request_method = request.request_handlers.get(params.method)
699
# A method we don't know about doesn't count as a VFS method.
701
if issubclass(request_method, vfs.VfsRequest):
702
value['vfs_count'] += 1
705
value = self.counts[ref]
706
count, vfs_count, medium_repr = (
707
value['count'], value['vfs_count'], value['medium_repr'])
708
# In case this callback is invoked for the same ref twice (by the
709
# weakref callback and by the atexit function), set the call count back
710
# to 0 so this item won't be reported twice.
712
value['vfs_count'] = 0
714
trace.note(gettext('HPSS calls: {0} ({1} vfs) {2}').format(
715
count, vfs_count, medium_repr))
718
for ref in list(self.counts.keys()):
721
_debug_counter = None
725
class SmartClientMedium(SmartMedium):
433
class SmartClientMedium(object):
726
434
"""Smart client is a medium for sending smart protocol requests over."""
728
436
def __init__(self, base):
906
589
def _accept_bytes(self, bytes):
907
590
"""See SmartClientStreamMedium.accept_bytes."""
909
self._writeable_pipe.write(bytes)
911
if e.errno in (errno.EINVAL, errno.EPIPE):
912
raise errors.ConnectionReset(
913
"Error trying to write to subprocess", e)
915
self._report_activity(len(bytes), 'write')
591
self._writeable_pipe.write(bytes)
917
593
def _flush(self):
918
594
"""See SmartClientStreamMedium._flush()."""
919
# Note: If flush were to fail, we'd like to raise ConnectionReset, etc.
920
# However, testing shows that even when the child process is
921
# gone, this doesn't error.
922
595
self._writeable_pipe.flush()
924
597
def _read_bytes(self, count):
925
598
"""See SmartClientStreamMedium._read_bytes."""
926
bytes_to_read = min(count, _MAX_READ_SIZE)
927
bytes = self._readable_pipe.read(bytes_to_read)
928
self._report_activity(len(bytes), 'read')
932
class SSHParams(object):
933
"""A set of parameters for starting a remote bzr via SSH."""
599
return self._readable_pipe.read(count)
602
class SmartSSHClientMedium(SmartClientStreamMedium):
603
"""A client medium using SSH."""
935
605
def __init__(self, host, port=None, username=None, password=None,
936
bzr_remote_path='bzr'):
939
self.username = username
940
self.password = password
941
self.bzr_remote_path = bzr_remote_path
944
class SmartSSHClientMedium(SmartClientStreamMedium):
945
"""A client medium using SSH.
947
It delegates IO to a SmartSimplePipesClientMedium or
948
SmartClientAlreadyConnectedSocketMedium (depending on platform).
951
def __init__(self, base, ssh_params, vendor=None):
606
base=None, vendor=None, bzr_remote_path=None):
952
607
"""Creates a client that will connect on the first use.
954
:param ssh_params: A SSHParams instance.
955
609
:param vendor: An optional override for the ssh vendor to use. See
956
610
bzrlib.transport.ssh for details on ssh vendors.
958
self._real_medium = None
959
self._ssh_params = ssh_params
960
# for the benefit of progress making a short description of this
962
self._scheme = 'bzr+ssh'
963
# SmartClientStreamMedium stores the repr of this object in its
964
# _DebugCounter so we have to store all the values used in our repr
965
# method before calling the super init.
966
612
SmartClientStreamMedium.__init__(self, base)
613
self._connected = False
615
self._password = password
617
self._username = username
618
self._read_from = None
619
self._ssh_connection = None
967
620
self._vendor = vendor
968
self._ssh_connection = None
971
if self._ssh_params.port is None:
974
maybe_port = ':%s' % self._ssh_params.port
975
if self._ssh_params.username is None:
978
maybe_user = '%s@' % self._ssh_params.username
979
return "%s(%s://%s%s%s/)" % (
980
self.__class__.__name__,
983
self._ssh_params.host,
621
self._write_to = None
622
self._bzr_remote_path = bzr_remote_path
623
if self._bzr_remote_path is None:
624
symbol_versioning.warn(
625
'bzr_remote_path is required as of bzr 0.92',
626
DeprecationWarning, stacklevel=2)
627
self._bzr_remote_path = os.environ.get('BZR_REMOTE_PATH', 'bzr')
986
629
def _accept_bytes(self, bytes):
987
630
"""See SmartClientStreamMedium.accept_bytes."""
988
631
self._ensure_connection()
989
self._real_medium.accept_bytes(bytes)
632
self._write_to.write(bytes)
991
634
def disconnect(self):
992
635
"""See SmartClientMedium.disconnect()."""
993
if self._real_medium is not None:
994
self._real_medium.disconnect()
995
self._real_medium = None
996
if self._ssh_connection is not None:
997
self._ssh_connection.close()
998
self._ssh_connection = None
636
if not self._connected:
638
self._read_from.close()
639
self._write_to.close()
640
self._ssh_connection.close()
641
self._connected = False
1000
643
def _ensure_connection(self):
1001
644
"""Connect this medium if not already connected."""
1002
if self._real_medium is not None:
1004
647
if self._vendor is None:
1005
648
vendor = ssh._get_ssh_vendor()
1007
650
vendor = self._vendor
1008
self._ssh_connection = vendor.connect_ssh(self._ssh_params.username,
1009
self._ssh_params.password, self._ssh_params.host,
1010
self._ssh_params.port,
1011
command=[self._ssh_params.bzr_remote_path, 'serve', '--inet',
651
self._ssh_connection = vendor.connect_ssh(self._username,
652
self._password, self._host, self._port,
653
command=[self._bzr_remote_path, 'serve', '--inet',
1012
654
'--directory=/', '--allow-writes'])
1013
io_kind, io_object = self._ssh_connection.get_sock_or_pipes()
1014
if io_kind == 'socket':
1015
self._real_medium = SmartClientAlreadyConnectedSocketMedium(
1016
self.base, io_object)
1017
elif io_kind == 'pipes':
1018
read_from, write_to = io_object
1019
self._real_medium = SmartSimplePipesClientMedium(
1020
read_from, write_to, self.base)
1022
raise AssertionError(
1023
"Unexpected io_kind %r from %r"
1024
% (io_kind, self._ssh_connection))
1025
for hook in transport.Transport.hooks["post_connect"]:
655
self._read_from, self._write_to = \
656
self._ssh_connection.get_filelike_channels()
657
self._connected = True
1028
659
def _flush(self):
1029
660
"""See SmartClientStreamMedium._flush()."""
1030
self._real_medium._flush()
661
self._write_to.flush()
1032
663
def _read_bytes(self, count):
1033
664
"""See SmartClientStreamMedium.read_bytes."""
1034
if self._real_medium is None:
665
if not self._connected:
1035
666
raise errors.MediumNotConnected(self)
1036
return self._real_medium.read_bytes(count)
667
return self._read_from.read(count)
1039
670
# Port 4155 is the default port for bzr://, registered with IANA.
1040
BZR_DEFAULT_INTERFACE = None
671
BZR_DEFAULT_INTERFACE = '0.0.0.0'
1041
672
BZR_DEFAULT_PORT = 4155
1044
class SmartClientSocketMedium(SmartClientStreamMedium):
1045
"""A client medium using a socket.
1047
This class isn't usable directly. Use one of its subclasses instead.
1050
def __init__(self, base):
675
class SmartTCPClientMedium(SmartClientStreamMedium):
676
"""A client medium using TCP."""
678
def __init__(self, host, port, base):
679
"""Creates a client that will connect on the first use."""
1051
680
SmartClientStreamMedium.__init__(self, base)
681
self._connected = False
1052
684
self._socket = None
1053
self._connected = False
1055
686
def _accept_bytes(self, bytes):
1056
687
"""See SmartClientMedium.accept_bytes."""
1057
688
self._ensure_connection()
1058
osutils.send_all(self._socket, bytes, self._report_activity)
1060
def _ensure_connection(self):
1061
"""Connect this medium if not already connected."""
1062
raise NotImplementedError(self._ensure_connection)
1065
"""See SmartClientStreamMedium._flush().
1067
For sockets we do no flushing. For TCP sockets we may want to turn off
1068
TCP_NODELAY and add a means to do a flush, but that can be done in the
1072
def _read_bytes(self, count):
1073
"""See SmartClientMedium.read_bytes."""
1074
if not self._connected:
1075
raise errors.MediumNotConnected(self)
1076
return osutils.read_bytes_from_socket(
1077
self._socket, self._report_activity)
689
osutils.send_all(self._socket, bytes)
1079
691
def disconnect(self):
1080
692
"""See SmartClientMedium.disconnect()."""
1084
696
self._socket = None
1085
697
self._connected = False
1088
class SmartTCPClientMedium(SmartClientSocketMedium):
1089
"""A client medium that creates a TCP connection."""
1091
def __init__(self, host, port, base):
1092
"""Creates a client that will connect on the first use."""
1093
SmartClientSocketMedium.__init__(self, base)
1097
699
def _ensure_connection(self):
1098
700
"""Connect this medium if not already connected."""
1099
701
if self._connected:
703
self._socket = socket.socket()
704
self._socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
1101
705
if self._port is None:
1102
706
port = BZR_DEFAULT_PORT
1104
708
port = int(self._port)
1106
sockaddrs = socket.getaddrinfo(self._host, port, socket.AF_UNSPEC,
1107
socket.SOCK_STREAM, 0, 0)
1108
except socket.gaierror, (err_num, err_msg):
1109
raise errors.ConnectionError("failed to lookup %s:%d: %s" %
1110
(self._host, port, err_msg))
1111
# Initialize err in case there are no addresses returned:
1112
err = socket.error("no address found for %s" % self._host)
1113
for (family, socktype, proto, canonname, sockaddr) in sockaddrs:
1115
self._socket = socket.socket(family, socktype, proto)
1116
self._socket.setsockopt(socket.IPPROTO_TCP,
1117
socket.TCP_NODELAY, 1)
1118
self._socket.connect(sockaddr)
1119
except socket.error, err:
1120
if self._socket is not None:
1121
self._socket.close()
1125
if self._socket is None:
710
self._socket.connect((self._host, port))
711
except socket.error, err:
1126
712
# socket errors either have a (string) or (errno, string) as their
1128
714
if type(err.args) is str:
added
removed
bzrlib/smart/medium.py
