33
34
from bzrlib.lazy_import import lazy_import
34
35
lazy_import(globals(), """
39
41
from bzrlib import (
47
from bzrlib.smart import client, protocol, request, vfs
48
from bzrlib.i18n import gettext
49
from bzrlib.smart import client, protocol, request, signals, vfs
48
50
from bzrlib.transport import ssh
50
#usually already imported, and getting IllegalScoperReplacer on it here.
51
52
from bzrlib import osutils
53
# We must not read any more than 64k at a time so we don't risk "no buffer
54
# space available" errors on some platforms. Windows in particular is likely
55
# to give error 10053 or 10055 if we read more than 64k from a socket.
56
_MAX_READ_SIZE = 64 * 1024
54
# Throughout this module buffer size parameters are either limited to be at
55
# most _MAX_READ_SIZE, or are ignored and _MAX_READ_SIZE is used instead.
56
# For this module's purposes, MAX_SOCKET_CHUNK is a reasonable size for reads
57
# from non-sockets as well.
58
_MAX_READ_SIZE = osutils.MAX_SOCKET_CHUNK
59
60
def _get_protocol_factory_for_bytes(bytes):
60
61
"""Determine the right protocol factory for 'bytes'.
216
231
while not self.finished:
217
232
server_protocol = self._build_protocol()
218
233
self._serve_one_request(server_protocol)
234
except errors.ConnectionTimeout, e:
235
trace.note('%s' % (e,))
236
trace.log_exception_quietly()
237
self._disconnect_client()
238
# We reported it, no reason to make a big fuss.
219
240
except Exception, e:
220
241
stderr.write("%s terminating on exception %s\n" % (self, e))
243
self._disconnect_client()
245
def _stop_gracefully(self):
246
"""When we finish this message, stop looking for more."""
247
trace.mutter('Stopping %s' % (self,))
250
def _disconnect_client(self):
251
"""Close the current connection. We stopped due to a timeout/etc."""
252
# The default implementation is a no-op, because that is all we used to
253
# do when disconnecting from a client. I suppose we never had the
254
# *server* initiate a disconnect, before
256
def _wait_for_bytes_with_timeout(self, timeout_seconds):
257
"""Wait for more bytes to be read, but timeout if none available.
259
This allows us to detect idle connections, and stop trying to read from
260
them, without setting the socket itself to non-blocking. This also
261
allows us to specify when we watch for idle timeouts.
263
:return: Did we timeout? (True if we timed out, False if there is data
266
raise NotImplementedError(self._wait_for_bytes_with_timeout)
223
268
def _build_protocol(self):
224
269
"""Identifies the version of the incoming request, and returns an
236
285
protocol.accept_bytes(unused_bytes)
288
def _wait_on_descriptor(self, fd, timeout_seconds):
289
"""select() on a file descriptor, waiting for nonblocking read()
291
This will raise a ConnectionTimeout exception if we do not get a
292
readable handle before timeout_seconds.
295
t_end = self._timer() + timeout_seconds
296
poll_timeout = min(timeout_seconds, self._client_poll_timeout)
298
while not rs and not xs and self._timer() < t_end:
302
rs, _, xs = select.select([fd], [], [fd], poll_timeout)
303
except (select.error, socket.error) as e:
304
err = getattr(e, 'errno', None)
305
if err is None and getattr(e, 'args', None) is not None:
306
# select.error doesn't have 'errno', it just has args[0]
308
if err in _bad_file_descriptor:
309
return # Not a socket indicates read() will fail
310
elif err == errno.EINTR:
311
# Interrupted, keep looping.
316
raise errors.ConnectionTimeout('disconnecting client after %.1f seconds'
317
% (timeout_seconds,))
239
319
def _serve_one_request(self, protocol):
240
320
"""Read one request from input, process, send back a response.
242
322
:param protocol: a SmartServerRequestProtocol.
245
327
self._serve_one_request_unguarded(protocol)
246
328
except KeyboardInterrupt:
263
345
class SmartServerSocketStreamMedium(SmartServerStreamMedium):
265
def __init__(self, sock, backing_transport, root_client_path='/'):
347
def __init__(self, sock, backing_transport, root_client_path='/',
268
351
:param sock: the socket the server will read from. It will be put
269
352
into blocking mode.
271
354
SmartServerStreamMedium.__init__(
272
self, backing_transport, root_client_path=root_client_path)
355
self, backing_transport, root_client_path=root_client_path,
273
357
sock.setblocking(True)
274
358
self.socket = sock
359
# Get the getpeername now, as we might be closed later when we care.
361
self._client_info = sock.getpeername()
363
self._client_info = '<unknown>'
366
return '%s(client=%s)' % (self.__class__.__name__, self._client_info)
369
return '%s.%s(client=%s)' % (self.__module__, self.__class__.__name__,
276
372
def _serve_one_request_unguarded(self, protocol):
277
373
while protocol.next_read_size():
278
374
# We can safely try to read large chunks. If there is less data
279
# than _MAX_READ_SIZE ready, the socket wil just return a short
280
# read immediately rather than block.
281
bytes = self.read_bytes(_MAX_READ_SIZE)
375
# than MAX_SOCKET_CHUNK ready, the socket will just return a
376
# short read immediately rather than block.
377
bytes = self.read_bytes(osutils.MAX_SOCKET_CHUNK)
283
379
self.finished = True
287
383
self._push_back(protocol.unused_data)
385
def _disconnect_client(self):
386
"""Close the current connection. We stopped due to a timeout/etc."""
389
def _wait_for_bytes_with_timeout(self, timeout_seconds):
390
"""Wait for more bytes to be read, but timeout if none available.
392
This allows us to detect idle connections, and stop trying to read from
393
them, without setting the socket itself to non-blocking. This also
394
allows us to specify when we watch for idle timeouts.
396
:return: None, this will raise ConnectionTimeout if we time out before
399
return self._wait_on_descriptor(self.socket, timeout_seconds)
289
401
def _read_bytes(self, desired_count):
290
return _read_bytes_from_socket(
291
self.socket.recv, desired_count, self._report_activity)
402
return osutils.read_bytes_from_socket(
403
self.socket, self._report_activity)
293
405
def terminate_due_to_error(self):
294
406
# TODO: This should log to a server log file, but no such thing
295
407
# exists yet. Andrew Bennetts 2006-09-29.
296
osutils.until_no_eintr(self.socket.close)
297
409
self.finished = True
299
411
def _write_out(self, bytes):
309
421
class SmartServerPipeStreamMedium(SmartServerStreamMedium):
311
def __init__(self, in_file, out_file, backing_transport):
423
def __init__(self, in_file, out_file, backing_transport, timeout=None):
312
424
"""Construct new server.
314
426
:param in_file: Python file from which requests can be read.
315
427
:param out_file: Python file to write responses.
316
428
:param backing_transport: Transport for the directory served.
318
SmartServerStreamMedium.__init__(self, backing_transport)
430
SmartServerStreamMedium.__init__(self, backing_transport,
319
432
if sys.platform == 'win32':
320
433
# force binary mode for files
334
458
bytes_to_read = protocol.next_read_size()
335
459
if bytes_to_read == 0:
336
460
# Finished serving this request.
337
osutils.until_no_eintr(self._out.flush)
339
463
bytes = self.read_bytes(bytes_to_read)
341
465
# Connection has been closed.
342
466
self.finished = True
343
osutils.until_no_eintr(self._out.flush)
345
469
protocol.accept_bytes(bytes)
471
def _disconnect_client(self):
476
def _wait_for_bytes_with_timeout(self, timeout_seconds):
477
"""Wait for more bytes to be read, but timeout if none available.
479
This allows us to detect idle connections, and stop trying to read from
480
them, without setting the socket itself to non-blocking. This also
481
allows us to specify when we watch for idle timeouts.
483
:return: None, this will raise ConnectionTimeout if we time out before
486
if (getattr(self._in, 'fileno', None) is None
487
or sys.platform == 'win32'):
488
# You can't select() file descriptors on Windows.
490
return self._wait_on_descriptor(self._in, timeout_seconds)
347
492
def _read_bytes(self, desired_count):
348
return osutils.until_no_eintr(self._in.read, desired_count)
493
return self._in.read(desired_count)
350
495
def terminate_due_to_error(self):
351
496
# TODO: This should log to a server log file, but no such thing
352
497
# exists yet. Andrew Bennetts 2006-09-29.
353
osutils.until_no_eintr(self._out.close)
354
499
self.finished = True
356
501
def _write_out(self, bytes):
357
osutils.until_no_eintr(self._out.write, bytes)
502
self._out.write(bytes)
360
505
class SmartClientMediumRequest(object):
493
638
return self._medium._get_line()
641
class _VfsRefuser(object):
642
"""An object that refuses all VFS requests.
647
client._SmartClient.hooks.install_named_hook(
648
'call', self.check_vfs, 'vfs refuser')
650
def check_vfs(self, params):
652
request_method = request.request_handlers.get(params.method)
654
# A method we don't know about doesn't count as a VFS method.
656
if issubclass(request_method, vfs.VfsRequest):
657
raise errors.HpssVfsRequestNotAllowed(params.method, params.args)
496
660
class _DebugCounter(object):
497
661
"""An object that counts the HPSS calls made to each client medium.
499
When a medium is garbage-collected, or failing that when atexit functions
500
are run, the total number of calls made on that medium are reported via
663
When a medium is garbage-collected, or failing that when
664
bzrlib.global_state exits, the total number of calls made on that medium
665
are reported via trace.note.
504
668
def __init__(self):
505
669
self.counts = weakref.WeakKeyDictionary()
506
670
client._SmartClient.hooks.install_named_hook(
507
671
'call', self.increment_call_count, 'hpss call counter')
508
atexit.register(self.flush_all)
672
bzrlib.global_state.cleanups.add_cleanup(self.flush_all)
510
674
def track(self, medium):
511
675
"""Start tracking calls made to a medium.
609
778
# which is newer than a previously supplied older-than version.
610
779
# This indicates that some smart verb call is not guarded
611
780
# appropriately (it should simply not have been tried).
612
raise AssertionError(
613
782
"_remember_remote_is_before(%r) called, but "
614
783
"_remember_remote_is_before(%r) was called previously."
615
% (version_tuple, self._remote_version_is_before))
784
, version_tuple, self._remote_version_is_before)
785
if 'hpss' in debug.debug_flags:
786
ui.ui_factory.show_warning(
787
"_remember_remote_is_before(%r) called, but "
788
"_remember_remote_is_before(%r) was called previously."
789
% (version_tuple, self._remote_version_is_before))
616
791
self._remote_version_is_before = version_tuple
618
793
def protocol_version(self):
721
904
def _accept_bytes(self, bytes):
722
905
"""See SmartClientStreamMedium.accept_bytes."""
723
osutils.until_no_eintr(self._writeable_pipe.write, bytes)
907
self._writeable_pipe.write(bytes)
909
if e.errno in (errno.EINVAL, errno.EPIPE):
910
raise errors.ConnectionReset(
911
"Error trying to write to subprocess:\n%s" % (e,))
724
913
self._report_activity(len(bytes), 'write')
726
915
def _flush(self):
727
916
"""See SmartClientStreamMedium._flush()."""
728
osutils.until_no_eintr(self._writeable_pipe.flush)
917
# Note: If flush were to fail, we'd like to raise ConnectionReset, etc.
918
# However, testing shows that even when the child process is
919
# gone, this doesn't error.
920
self._writeable_pipe.flush()
730
922
def _read_bytes(self, count):
731
923
"""See SmartClientStreamMedium._read_bytes."""
732
bytes = osutils.until_no_eintr(self._readable_pipe.read, count)
924
bytes_to_read = min(count, _MAX_READ_SIZE)
925
bytes = self._readable_pipe.read(bytes_to_read)
733
926
self._report_activity(len(bytes), 'read')
930
class SSHParams(object):
931
"""A set of parameters for starting a remote bzr via SSH."""
933
def __init__(self, host, port=None, username=None, password=None,
934
bzr_remote_path='bzr'):
937
self.username = username
938
self.password = password
939
self.bzr_remote_path = bzr_remote_path
737
942
class SmartSSHClientMedium(SmartClientStreamMedium):
738
"""A client medium using SSH."""
740
def __init__(self, host, port=None, username=None, password=None,
741
base=None, vendor=None, bzr_remote_path=None):
943
"""A client medium using SSH.
945
It delegates IO to a SmartSimplePipesClientMedium or
946
SmartClientAlreadyConnectedSocketMedium (depending on platform).
949
def __init__(self, base, ssh_params, vendor=None):
742
950
"""Creates a client that will connect on the first use.
952
:param ssh_params: A SSHParams instance.
744
953
:param vendor: An optional override for the ssh vendor to use. See
745
954
bzrlib.transport.ssh for details on ssh vendors.
747
self._connected = False
749
self._password = password
751
self._username = username
956
self._real_medium = None
957
self._ssh_params = ssh_params
752
958
# for the benefit of progress making a short description of this
754
960
self._scheme = 'bzr+ssh'
756
962
# _DebugCounter so we have to store all the values used in our repr
757
963
# method before calling the super init.
758
964
SmartClientStreamMedium.__init__(self, base)
759
self._read_from = None
965
self._vendor = vendor
760
966
self._ssh_connection = None
761
self._vendor = vendor
762
self._write_to = None
763
self._bzr_remote_path = bzr_remote_path
765
968
def __repr__(self):
766
if self._port is None:
969
if self._ssh_params.port is None:
769
maybe_port = ':%s' % self._port
770
return "%s(%s://%s@%s%s/)" % (
972
maybe_port = ':%s' % self._ssh_params.port
973
if self._ssh_params.username is None:
976
maybe_user = '%s@' % self._ssh_params.username
977
return "%s(%s://%s%s%s/)" % (
771
978
self.__class__.__name__,
981
self._ssh_params.host,
777
984
def _accept_bytes(self, bytes):
778
985
"""See SmartClientStreamMedium.accept_bytes."""
779
986
self._ensure_connection()
780
osutils.until_no_eintr(self._write_to.write, bytes)
781
self._report_activity(len(bytes), 'write')
987
self._real_medium.accept_bytes(bytes)
783
989
def disconnect(self):
784
990
"""See SmartClientMedium.disconnect()."""
785
if not self._connected:
787
osutils.until_no_eintr(self._read_from.close)
788
osutils.until_no_eintr(self._write_to.close)
789
self._ssh_connection.close()
790
self._connected = False
991
if self._real_medium is not None:
992
self._real_medium.disconnect()
993
self._real_medium = None
994
if self._ssh_connection is not None:
995
self._ssh_connection.close()
996
self._ssh_connection = None
792
998
def _ensure_connection(self):
793
999
"""Connect this medium if not already connected."""
1000
if self._real_medium is not None:
796
1002
if self._vendor is None:
797
1003
vendor = ssh._get_ssh_vendor()
799
1005
vendor = self._vendor
800
self._ssh_connection = vendor.connect_ssh(self._username,
801
self._password, self._host, self._port,
802
command=[self._bzr_remote_path, 'serve', '--inet',
1006
self._ssh_connection = vendor.connect_ssh(self._ssh_params.username,
1007
self._ssh_params.password, self._ssh_params.host,
1008
self._ssh_params.port,
1009
command=[self._ssh_params.bzr_remote_path, 'serve', '--inet',
803
1010
'--directory=/', '--allow-writes'])
804
self._read_from, self._write_to = \
805
self._ssh_connection.get_filelike_channels()
806
self._connected = True
1011
io_kind, io_object = self._ssh_connection.get_sock_or_pipes()
1012
if io_kind == 'socket':
1013
self._real_medium = SmartClientAlreadyConnectedSocketMedium(
1014
self.base, io_object)
1015
elif io_kind == 'pipes':
1016
read_from, write_to = io_object
1017
self._real_medium = SmartSimplePipesClientMedium(
1018
read_from, write_to, self.base)
1020
raise AssertionError(
1021
"Unexpected io_kind %r from %r"
1022
% (io_kind, self._ssh_connection))
808
1024
def _flush(self):
809
1025
"""See SmartClientStreamMedium._flush()."""
810
self._write_to.flush()
1026
self._real_medium._flush()
812
1028
def _read_bytes(self, count):
813
1029
"""See SmartClientStreamMedium.read_bytes."""
814
if not self._connected:
1030
if self._real_medium is None:
815
1031
raise errors.MediumNotConnected(self)
816
bytes_to_read = min(count, _MAX_READ_SIZE)
817
bytes = osutils.until_no_eintr(self._read_from.read, bytes_to_read)
818
self._report_activity(len(bytes), 'read')
1032
return self._real_medium.read_bytes(count)
822
1035
# Port 4155 is the default port for bzr://, registered with IANA.
824
1037
BZR_DEFAULT_PORT = 4155
827
class SmartTCPClientMedium(SmartClientStreamMedium):
828
"""A client medium using TCP."""
1040
class SmartClientSocketMedium(SmartClientStreamMedium):
1041
"""A client medium using a socket.
1043
This class isn't usable directly. Use one of its subclasses instead.
1046
def __init__(self, base):
1047
SmartClientStreamMedium.__init__(self, base)
1049
self._connected = False
1051
def _accept_bytes(self, bytes):
1052
"""See SmartClientMedium.accept_bytes."""
1053
self._ensure_connection()
1054
osutils.send_all(self._socket, bytes, self._report_activity)
1056
def _ensure_connection(self):
1057
"""Connect this medium if not already connected."""
1058
raise NotImplementedError(self._ensure_connection)
1061
"""See SmartClientStreamMedium._flush().
1063
For sockets we do no flushing. For TCP sockets we may want to turn off
1064
TCP_NODELAY and add a means to do a flush, but that can be done in the
1068
def _read_bytes(self, count):
1069
"""See SmartClientMedium.read_bytes."""
1070
if not self._connected:
1071
raise errors.MediumNotConnected(self)
1072
return osutils.read_bytes_from_socket(
1073
self._socket, self._report_activity)
1075
def disconnect(self):
1076
"""See SmartClientMedium.disconnect()."""
1077
if not self._connected:
1079
self._socket.close()
1081
self._connected = False
1084
class SmartTCPClientMedium(SmartClientSocketMedium):
1085
"""A client medium that creates a TCP connection."""
830
1087
def __init__(self, host, port, base):
831
1088
"""Creates a client that will connect on the first use."""
832
SmartClientStreamMedium.__init__(self, base)
833
self._connected = False
1089
SmartClientSocketMedium.__init__(self, base)
834
1090
self._host = host
835
1091
self._port = port
838
def _accept_bytes(self, bytes):
839
"""See SmartClientMedium.accept_bytes."""
840
self._ensure_connection()
841
osutils.send_all(self._socket, bytes, self._report_activity)
843
def disconnect(self):
844
"""See SmartClientMedium.disconnect()."""
845
if not self._connected:
847
osutils.until_no_eintr(self._socket.close)
849
self._connected = False
851
1093
def _ensure_connection(self):
852
1094
"""Connect this medium if not already connected."""
887
1129
(self._host, port, err_msg))
888
1130
self._connected = True
891
"""See SmartClientStreamMedium._flush().
893
For TCP we do no flushing. We may want to turn off TCP_NODELAY and
894
add a means to do a flush, but that can be done in the future.
897
def _read_bytes(self, count):
898
"""See SmartClientMedium.read_bytes."""
899
if not self._connected:
900
raise errors.MediumNotConnected(self)
901
return _read_bytes_from_socket(
902
self._socket.recv, count, self._report_activity)
1133
class SmartClientAlreadyConnectedSocketMedium(SmartClientSocketMedium):
1134
"""A client medium for an already connected socket.
1136
Note that this class will assume it "owns" the socket, so it will close it
1137
when its disconnect method is called.
1140
def __init__(self, base, sock):
1141
SmartClientSocketMedium.__init__(self, base)
1143
self._connected = True
1145
def _ensure_connection(self):
1146
# Already connected, by definition! So nothing to do.
905
1150
class SmartClientStreamMediumRequest(SmartClientMediumRequest):
940
1185
This invokes self._medium._flush to ensure all bytes are transmitted.
942
1187
self._medium._flush()
945
def _read_bytes_from_socket(sock, desired_count, report_activity):
946
# We ignore the desired_count because on sockets it's more efficient to
947
# read large chunks (of _MAX_READ_SIZE bytes) at a time.
949
bytes = osutils.until_no_eintr(sock, _MAX_READ_SIZE)
950
except socket.error, e:
951
if len(e.args) and e.args[0] in (errno.ECONNRESET, 10054):
952
# The connection was closed by the other side. Callers expect an
953
# empty string to signal end-of-stream.
958
report_activity(len(bytes), 'read')
added
removed
bzrlib/smart/medium.py
