33
34
from bzrlib.lazy_import import lazy_import
34
35
lazy_import(globals(), """
37
41
from bzrlib import (
45
from bzrlib.smart import client, protocol
48
from bzrlib.i18n import gettext
49
from bzrlib.smart import client, protocol, request, signals, vfs
46
50
from bzrlib.transport import ssh
50
# We must not read any more than 64k at a time so we don't risk "no buffer
51
# space available" errors on some platforms. Windows in particular is likely
52
# to give error 10053 or 10055 if we read more than 64k from a socket.
53
_MAX_READ_SIZE = 64 * 1024
52
from bzrlib import osutils
54
# Throughout this module buffer size parameters are either limited to be at
55
# most _MAX_READ_SIZE, or are ignored and _MAX_READ_SIZE is used instead.
56
# For this module's purposes, MAX_SOCKET_CHUNK is a reasonable size for reads
57
# from non-sockets as well.
58
_MAX_READ_SIZE = osutils.MAX_SOCKET_CHUNK
56
60
def _get_protocol_factory_for_bytes(bytes):
57
61
"""Determine the right protocol factory for 'bytes'.
161
165
line, excess = _get_line(self.read_bytes)
162
166
self._push_back(excess)
169
def _report_activity(self, bytes, direction):
170
"""Notify that this medium has activity.
172
Implementations should call this from all methods that actually do IO.
173
Be careful that it's not called twice, if one method is implemented on
176
:param bytes: Number of bytes read or written.
177
:param direction: 'read' or 'write' or None.
179
ui.ui_factory.report_transport_activity(self, bytes, direction)
182
_bad_file_descriptor = (errno.EBADF,)
183
if sys.platform == 'win32':
184
# Given on Windows if you pass a closed socket to select.select. Probably
185
# also given if you pass a file handle to select.
187
_bad_file_descriptor += (WSAENOTSOCK,)
166
190
class SmartServerStreamMedium(SmartMedium):
167
191
"""Handles smart commands coming over a stream.
201
231
while not self.finished:
202
232
server_protocol = self._build_protocol()
233
# TODO: This seems inelegant:
234
if server_protocol is None:
235
# We could 'continue' only to notice that self.finished is
203
238
self._serve_one_request(server_protocol)
239
except errors.ConnectionTimeout, e:
240
trace.note('%s' % (e,))
241
trace.log_exception_quietly()
242
self._disconnect_client()
243
# We reported it, no reason to make a big fuss.
204
245
except Exception, e:
205
246
stderr.write("%s terminating on exception %s\n" % (self, e))
248
self._disconnect_client()
250
def _stop_gracefully(self):
251
"""When we finish this message, stop looking for more."""
252
trace.mutter('Stopping %s' % (self,))
255
def _disconnect_client(self):
256
"""Close the current connection. We stopped due to a timeout/etc."""
257
# The default implementation is a no-op, because that is all we used to
258
# do when disconnecting from a client. I suppose we never had the
259
# *server* initiate a disconnect, before
261
def _wait_for_bytes_with_timeout(self, timeout_seconds):
262
"""Wait for more bytes to be read, but timeout if none available.
264
This allows us to detect idle connections, and stop trying to read from
265
them, without setting the socket itself to non-blocking. This also
266
allows us to specify when we watch for idle timeouts.
268
:return: Did we timeout? (True if we timed out, False if there is data
271
raise NotImplementedError(self._wait_for_bytes_with_timeout)
208
273
def _build_protocol(self):
209
274
"""Identifies the version of the incoming request, and returns an
221
290
protocol.accept_bytes(unused_bytes)
293
def _wait_on_descriptor(self, fd, timeout_seconds):
294
"""select() on a file descriptor, waiting for nonblocking read()
296
This will raise a ConnectionTimeout exception if we do not get a
297
readable handle before timeout_seconds.
300
t_end = self._timer() + timeout_seconds
301
poll_timeout = min(timeout_seconds, self._client_poll_timeout)
303
while not rs and not xs and self._timer() < t_end:
307
rs, _, xs = select.select([fd], [], [fd], poll_timeout)
308
except (select.error, socket.error) as e:
309
err = getattr(e, 'errno', None)
310
if err is None and getattr(e, 'args', None) is not None:
311
# select.error doesn't have 'errno', it just has args[0]
313
if err in _bad_file_descriptor:
314
return # Not a socket indicates read() will fail
315
elif err == errno.EINTR:
316
# Interrupted, keep looping.
321
raise errors.ConnectionTimeout('disconnecting client after %.1f seconds'
322
% (timeout_seconds,))
224
324
def _serve_one_request(self, protocol):
225
325
"""Read one request from input, process, send back a response.
227
327
:param protocol: a SmartServerRequestProtocol.
248
348
class SmartServerSocketStreamMedium(SmartServerStreamMedium):
250
def __init__(self, sock, backing_transport, root_client_path='/'):
350
def __init__(self, sock, backing_transport, root_client_path='/',
253
354
:param sock: the socket the server will read from. It will be put
254
355
into blocking mode.
256
357
SmartServerStreamMedium.__init__(
257
self, backing_transport, root_client_path=root_client_path)
358
self, backing_transport, root_client_path=root_client_path,
258
360
sock.setblocking(True)
259
361
self.socket = sock
362
# Get the getpeername now, as we might be closed later when we care.
364
self._client_info = sock.getpeername()
366
self._client_info = '<unknown>'
369
return '%s(client=%s)' % (self.__class__.__name__, self._client_info)
372
return '%s.%s(client=%s)' % (self.__module__, self.__class__.__name__,
261
375
def _serve_one_request_unguarded(self, protocol):
262
376
while protocol.next_read_size():
263
377
# We can safely try to read large chunks. If there is less data
264
# than _MAX_READ_SIZE ready, the socket wil just return a short
265
# read immediately rather than block.
266
bytes = self.read_bytes(_MAX_READ_SIZE)
378
# than MAX_SOCKET_CHUNK ready, the socket will just return a
379
# short read immediately rather than block.
380
bytes = self.read_bytes(osutils.MAX_SOCKET_CHUNK)
268
382
self.finished = True
270
384
protocol.accept_bytes(bytes)
272
386
self._push_back(protocol.unused_data)
388
def _disconnect_client(self):
389
"""Close the current connection. We stopped due to a timeout/etc."""
392
def _wait_for_bytes_with_timeout(self, timeout_seconds):
393
"""Wait for more bytes to be read, but timeout if none available.
395
This allows us to detect idle connections, and stop trying to read from
396
them, without setting the socket itself to non-blocking. This also
397
allows us to specify when we watch for idle timeouts.
399
:return: None, this will raise ConnectionTimeout if we time out before
402
return self._wait_on_descriptor(self.socket, timeout_seconds)
274
404
def _read_bytes(self, desired_count):
275
# We ignore the desired_count because on sockets it's more efficient to
276
# read large chunks (of _MAX_READ_SIZE bytes) at a time.
277
return osutils.until_no_eintr(self.socket.recv, _MAX_READ_SIZE)
405
return osutils.read_bytes_from_socket(
406
self.socket, self._report_activity)
279
408
def terminate_due_to_error(self):
280
409
# TODO: This should log to a server log file, but no such thing
283
412
self.finished = True
285
414
def _write_out(self, bytes):
286
osutils.send_all(self.socket, bytes)
415
tstart = osutils.timer_func()
416
osutils.send_all(self.socket, bytes, self._report_activity)
417
if 'hpss' in debug.debug_flags:
418
thread_id = thread.get_ident()
419
trace.mutter('%12s: [%s] %d bytes to the socket in %.3fs'
420
% ('wrote', thread_id, len(bytes),
421
osutils.timer_func() - tstart))
289
424
class SmartServerPipeStreamMedium(SmartServerStreamMedium):
291
def __init__(self, in_file, out_file, backing_transport):
426
def __init__(self, in_file, out_file, backing_transport, timeout=None):
292
427
"""Construct new server.
294
429
:param in_file: Python file from which requests can be read.
295
430
:param out_file: Python file to write responses.
296
431
:param backing_transport: Transport for the directory served.
298
SmartServerStreamMedium.__init__(self, backing_transport)
433
SmartServerStreamMedium.__init__(self, backing_transport,
299
435
if sys.platform == 'win32':
300
436
# force binary mode for files
325
472
protocol.accept_bytes(bytes)
474
def _disconnect_client(self):
479
def _wait_for_bytes_with_timeout(self, timeout_seconds):
480
"""Wait for more bytes to be read, but timeout if none available.
482
This allows us to detect idle connections, and stop trying to read from
483
them, without setting the socket itself to non-blocking. This also
484
allows us to specify when we watch for idle timeouts.
486
:return: None, this will raise ConnectionTimeout if we time out before
489
if (getattr(self._in, 'fileno', None) is None
490
or sys.platform == 'win32'):
491
# You can't select() file descriptors on Windows.
493
return self._wait_on_descriptor(self._in, timeout_seconds)
327
495
def _read_bytes(self, desired_count):
328
496
return self._in.read(desired_count)
460
628
if not line.endswith('\n'):
461
629
# end of file encountered reading from server
462
630
raise errors.ConnectionReset(
463
"please check connectivity and permissions",
464
"(and try -Dhpss if further diagnosis is required)")
631
"Unexpected end of message. Please check connectivity "
632
"and permissions, and report a bug if problems persist.")
467
635
def _read_line(self):
468
636
"""Helper for SmartClientMediumRequest.read_line.
470
638
By default this forwards to self._medium._get_line because we are
471
639
operating on the medium's stream.
473
641
return self._medium._get_line()
644
class _VfsRefuser(object):
645
"""An object that refuses all VFS requests.
650
client._SmartClient.hooks.install_named_hook(
651
'call', self.check_vfs, 'vfs refuser')
653
def check_vfs(self, params):
655
request_method = request.request_handlers.get(params.method)
657
# A method we don't know about doesn't count as a VFS method.
659
if issubclass(request_method, vfs.VfsRequest):
660
raise errors.HpssVfsRequestNotAllowed(params.method, params.args)
476
663
class _DebugCounter(object):
477
664
"""An object that counts the HPSS calls made to each client medium.
479
When a medium is garbage-collected, or failing that when atexit functions
480
are run, the total number of calls made on that medium are reported via
666
When a medium is garbage-collected, or failing that when
667
bzrlib.global_state exits, the total number of calls made on that medium
668
are reported via trace.note.
484
671
def __init__(self):
485
672
self.counts = weakref.WeakKeyDictionary()
486
673
client._SmartClient.hooks.install_named_hook(
487
674
'call', self.increment_call_count, 'hpss call counter')
488
atexit.register(self.flush_all)
675
bzrlib.global_state.cleanups.add_cleanup(self.flush_all)
490
677
def track(self, medium):
491
678
"""Start tracking calls made to a medium.
505
693
def increment_call_count(self, params):
506
694
# Increment the count in the WeakKeyDictionary
507
695
value = self.counts[params.medium]
698
request_method = request.request_handlers.get(params.method)
700
# A method we don't know about doesn't count as a VFS method.
702
if issubclass(request_method, vfs.VfsRequest):
703
value['vfs_count'] += 1
510
705
def done(self, ref):
511
706
value = self.counts[ref]
512
count, medium_repr = value
707
count, vfs_count, medium_repr = (
708
value['count'], value['vfs_count'], value['medium_repr'])
513
709
# In case this callback is invoked for the same ref twice (by the
514
710
# weakref callback and by the atexit function), set the call count back
515
711
# to 0 so this item won't be reported twice.
713
value['vfs_count'] = 0
518
trace.note('HPSS calls: %d %s', count, medium_repr)
715
trace.note(gettext('HPSS calls: {0} ({1} vfs) {2}').format(
716
count, vfs_count, medium_repr))
520
718
def flush_all(self):
521
719
for ref in list(self.counts.keys()):
524
722
_debug_counter = None
527
726
class SmartClientMedium(SmartMedium):
528
727
"""Smart client is a medium for sending smart protocol requests over."""
575
778
if (self._remote_version_is_before is not None and
576
779
version_tuple > self._remote_version_is_before):
577
raise AssertionError(
780
# We have been told that the remote side is older than some version
781
# which is newer than a previously supplied older-than version.
782
# This indicates that some smart verb call is not guarded
783
# appropriately (it should simply not have been tried).
578
785
"_remember_remote_is_before(%r) called, but "
579
786
"_remember_remote_is_before(%r) was called previously."
580
% (version_tuple, self._remote_version_is_before))
787
, version_tuple, self._remote_version_is_before)
788
if 'hpss' in debug.debug_flags:
789
ui.ui_factory.show_warning(
790
"_remember_remote_is_before(%r) called, but "
791
"_remember_remote_is_before(%r) was called previously."
792
% (version_tuple, self._remote_version_is_before))
581
794
self._remote_version_is_before = version_tuple
583
796
def protocol_version(self):
694
908
def _read_bytes(self, count):
695
909
"""See SmartClientStreamMedium._read_bytes."""
696
return self._readable_pipe.read(count)
910
bytes_to_read = min(count, _MAX_READ_SIZE)
911
bytes = self._readable_pipe.read(bytes_to_read)
912
self._report_activity(len(bytes), 'read')
916
class SSHParams(object):
917
"""A set of parameters for starting a remote bzr via SSH."""
919
def __init__(self, host, port=None, username=None, password=None,
920
bzr_remote_path='bzr'):
923
self.username = username
924
self.password = password
925
self.bzr_remote_path = bzr_remote_path
699
928
class SmartSSHClientMedium(SmartClientStreamMedium):
700
"""A client medium using SSH."""
929
"""A client medium using SSH.
702
def __init__(self, host, port=None, username=None, password=None,
703
base=None, vendor=None, bzr_remote_path=None):
931
It delegates IO to a SmartClientSocketMedium or
932
SmartClientAlreadyConnectedSocketMedium (depending on platform).
935
def __init__(self, base, ssh_params, vendor=None):
704
936
"""Creates a client that will connect on the first use.
938
:param ssh_params: A SSHParams instance.
706
939
:param vendor: An optional override for the ssh vendor to use. See
707
940
bzrlib.transport.ssh for details on ssh vendors.
942
self._real_medium = None
943
self._ssh_params = ssh_params
944
# for the benefit of progress making a short description of this
946
self._scheme = 'bzr+ssh'
947
# SmartClientStreamMedium stores the repr of this object in its
948
# _DebugCounter so we have to store all the values used in our repr
949
# method before calling the super init.
709
950
SmartClientStreamMedium.__init__(self, base)
710
self._connected = False
712
self._password = password
714
self._username = username
715
self._read_from = None
951
self._vendor = vendor
716
952
self._ssh_connection = None
717
self._vendor = vendor
718
self._write_to = None
719
self._bzr_remote_path = bzr_remote_path
720
if self._bzr_remote_path is None:
721
symbol_versioning.warn(
722
'bzr_remote_path is required as of bzr 0.92',
723
DeprecationWarning, stacklevel=2)
724
self._bzr_remote_path = os.environ.get('BZR_REMOTE_PATH', 'bzr')
955
if self._ssh_params.port is None:
958
maybe_port = ':%s' % self._ssh_params.port
959
return "%s(%s://%s@%s%s/)" % (
960
self.__class__.__name__,
962
self._ssh_params.username,
963
self._ssh_params.host,
726
966
def _accept_bytes(self, bytes):
727
967
"""See SmartClientStreamMedium.accept_bytes."""
728
968
self._ensure_connection()
729
self._write_to.write(bytes)
969
self._real_medium.accept_bytes(bytes)
731
971
def disconnect(self):
732
972
"""See SmartClientMedium.disconnect()."""
733
if not self._connected:
735
self._read_from.close()
736
self._write_to.close()
737
self._ssh_connection.close()
738
self._connected = False
973
if self._real_medium is not None:
974
self._real_medium.disconnect()
975
self._real_medium = None
976
if self._ssh_connection is not None:
977
self._ssh_connection.close()
978
self._ssh_connection = None
740
980
def _ensure_connection(self):
741
981
"""Connect this medium if not already connected."""
982
if self._real_medium is not None:
744
984
if self._vendor is None:
745
985
vendor = ssh._get_ssh_vendor()
747
987
vendor = self._vendor
748
self._ssh_connection = vendor.connect_ssh(self._username,
749
self._password, self._host, self._port,
750
command=[self._bzr_remote_path, 'serve', '--inet',
988
self._ssh_connection = vendor.connect_ssh(self._ssh_params.username,
989
self._ssh_params.password, self._ssh_params.host,
990
self._ssh_params.port,
991
command=[self._ssh_params.bzr_remote_path, 'serve', '--inet',
751
992
'--directory=/', '--allow-writes'])
752
self._read_from, self._write_to = \
753
self._ssh_connection.get_filelike_channels()
754
self._connected = True
993
io_kind, io_object = self._ssh_connection.get_sock_or_pipes()
994
if io_kind == 'socket':
995
self._real_medium = SmartClientAlreadyConnectedSocketMedium(
996
self.base, io_object)
997
elif io_kind == 'pipes':
998
read_from, write_to = io_object
999
self._real_medium = SmartSimplePipesClientMedium(
1000
read_from, write_to, self.base)
1002
raise AssertionError(
1003
"Unexpected io_kind %r from %r"
1004
% (io_kind, self._ssh_connection))
756
1006
def _flush(self):
757
1007
"""See SmartClientStreamMedium._flush()."""
758
self._write_to.flush()
1008
self._real_medium._flush()
760
1010
def _read_bytes(self, count):
761
1011
"""See SmartClientStreamMedium.read_bytes."""
762
if not self._connected:
1012
if self._real_medium is None:
763
1013
raise errors.MediumNotConnected(self)
764
bytes_to_read = min(count, _MAX_READ_SIZE)
765
return self._read_from.read(bytes_to_read)
1014
return self._real_medium.read_bytes(count)
768
1017
# Port 4155 is the default port for bzr://, registered with IANA.
770
1019
BZR_DEFAULT_PORT = 4155
773
class SmartTCPClientMedium(SmartClientStreamMedium):
774
"""A client medium using TCP."""
1022
class SmartClientSocketMedium(SmartClientStreamMedium):
1023
"""A client medium using a socket.
1025
This class isn't usable directly. Use one of its subclasses instead.
1028
def __init__(self, base):
1029
SmartClientStreamMedium.__init__(self, base)
1031
self._connected = False
1033
def _accept_bytes(self, bytes):
1034
"""See SmartClientMedium.accept_bytes."""
1035
self._ensure_connection()
1036
osutils.send_all(self._socket, bytes, self._report_activity)
1038
def _ensure_connection(self):
1039
"""Connect this medium if not already connected."""
1040
raise NotImplementedError(self._ensure_connection)
1043
"""See SmartClientStreamMedium._flush().
1045
For sockets we do no flushing. For TCP sockets we may want to turn off
1046
TCP_NODELAY and add a means to do a flush, but that can be done in the
1050
def _read_bytes(self, count):
1051
"""See SmartClientMedium.read_bytes."""
1052
if not self._connected:
1053
raise errors.MediumNotConnected(self)
1054
return osutils.read_bytes_from_socket(
1055
self._socket, self._report_activity)
1057
def disconnect(self):
1058
"""See SmartClientMedium.disconnect()."""
1059
if not self._connected:
1061
self._socket.close()
1063
self._connected = False
1066
class SmartTCPClientMedium(SmartClientSocketMedium):
1067
"""A client medium that creates a TCP connection."""
776
1069
def __init__(self, host, port, base):
777
1070
"""Creates a client that will connect on the first use."""
778
SmartClientStreamMedium.__init__(self, base)
779
self._connected = False
1071
SmartClientSocketMedium.__init__(self, base)
780
1072
self._host = host
781
1073
self._port = port
784
def _accept_bytes(self, bytes):
785
"""See SmartClientMedium.accept_bytes."""
786
self._ensure_connection()
787
osutils.send_all(self._socket, bytes)
789
def disconnect(self):
790
"""See SmartClientMedium.disconnect()."""
791
if not self._connected:
795
self._connected = False
797
1075
def _ensure_connection(self):
798
1076
"""Connect this medium if not already connected."""
833
1111
(self._host, port, err_msg))
834
1112
self._connected = True
837
"""See SmartClientStreamMedium._flush().
839
For TCP we do no flushing. We may want to turn off TCP_NODELAY and
840
add a means to do a flush, but that can be done in the future.
843
def _read_bytes(self, count):
844
"""See SmartClientMedium.read_bytes."""
845
if not self._connected:
846
raise errors.MediumNotConnected(self)
847
# We ignore the desired_count because on sockets it's more efficient to
848
# read large chunks (of _MAX_READ_SIZE bytes) at a time.
850
return self._socket.recv(_MAX_READ_SIZE)
851
except socket.error, e:
852
if len(e.args) and e.args[0] == errno.ECONNRESET:
853
# Callers expect an empty string in that case
1115
class SmartClientAlreadyConnectedSocketMedium(SmartClientSocketMedium):
1116
"""A client medium for an already connected socket.
1118
Note that this class will assume it "owns" the socket, so it will close it
1119
when its disconnect method is called.
1122
def __init__(self, base, sock):
1123
SmartClientSocketMedium.__init__(self, base)
1125
self._connected = True
1127
def _ensure_connection(self):
1128
# Already connected, by definition! So nothing to do.
859
1132
class SmartClientStreamMediumRequest(SmartClientMediumRequest):
added
removed
bzrlib/smart/medium.py
