24
24
bzrlib/transport/smart/__init__.py.
27
from __future__ import absolute_import
35
33
from bzrlib.lazy_import import lazy_import
36
34
lazy_import(globals(), """
42
39
from bzrlib import (
50
from bzrlib.i18n import gettext
51
from bzrlib.smart import client, protocol, request, signals, vfs
47
from bzrlib.smart import client, protocol, request, vfs
52
48
from bzrlib.transport import ssh
50
#usually already imported, and getting IllegalScoperReplacer on it here.
54
51
from bzrlib import osutils
56
# Throughout this module buffer size parameters are either limited to be at
57
# most _MAX_READ_SIZE, or are ignored and _MAX_READ_SIZE is used instead.
58
# For this module's purposes, MAX_SOCKET_CHUNK is a reasonable size for reads
59
# from non-sockets as well.
60
_MAX_READ_SIZE = osutils.MAX_SOCKET_CHUNK
53
# We must not read any more than 64k at a time so we don't risk "no buffer
54
# space available" errors on some platforms. Windows in particular is likely
55
# to give error 10053 or 10055 if we read more than 64k from a socket.
56
_MAX_READ_SIZE = 64 * 1024
62
59
def _get_protocol_factory_for_bytes(bytes):
63
60
"""Determine the right protocol factory for 'bytes'.
233
216
while not self.finished:
234
217
server_protocol = self._build_protocol()
235
218
self._serve_one_request(server_protocol)
236
except errors.ConnectionTimeout, e:
237
trace.note('%s' % (e,))
238
trace.log_exception_quietly()
239
self._disconnect_client()
240
# We reported it, no reason to make a big fuss.
242
219
except Exception, e:
243
220
stderr.write("%s terminating on exception %s\n" % (self, e))
245
self._disconnect_client()
247
def _stop_gracefully(self):
248
"""When we finish this message, stop looking for more."""
249
trace.mutter('Stopping %s' % (self,))
252
def _disconnect_client(self):
253
"""Close the current connection. We stopped due to a timeout/etc."""
254
# The default implementation is a no-op, because that is all we used to
255
# do when disconnecting from a client. I suppose we never had the
256
# *server* initiate a disconnect, before
258
def _wait_for_bytes_with_timeout(self, timeout_seconds):
259
"""Wait for more bytes to be read, but timeout if none available.
261
This allows us to detect idle connections, and stop trying to read from
262
them, without setting the socket itself to non-blocking. This also
263
allows us to specify when we watch for idle timeouts.
265
:return: Did we timeout? (True if we timed out, False if there is data
268
raise NotImplementedError(self._wait_for_bytes_with_timeout)
270
223
def _build_protocol(self):
271
224
"""Identifies the version of the incoming request, and returns an
287
236
protocol.accept_bytes(unused_bytes)
290
def _wait_on_descriptor(self, fd, timeout_seconds):
291
"""select() on a file descriptor, waiting for nonblocking read()
293
This will raise a ConnectionTimeout exception if we do not get a
294
readable handle before timeout_seconds.
297
t_end = self._timer() + timeout_seconds
298
poll_timeout = min(timeout_seconds, self._client_poll_timeout)
300
while not rs and not xs and self._timer() < t_end:
304
rs, _, xs = select.select([fd], [], [fd], poll_timeout)
305
except (select.error, socket.error) as e:
306
err = getattr(e, 'errno', None)
307
if err is None and getattr(e, 'args', None) is not None:
308
# select.error doesn't have 'errno', it just has args[0]
310
if err in _bad_file_descriptor:
311
return # Not a socket indicates read() will fail
312
elif err == errno.EINTR:
313
# Interrupted, keep looping.
318
raise errors.ConnectionTimeout('disconnecting client after %.1f seconds'
319
% (timeout_seconds,))
321
239
def _serve_one_request(self, protocol):
322
240
"""Read one request from input, process, send back a response.
324
242
:param protocol: a SmartServerRequestProtocol.
329
245
self._serve_one_request_unguarded(protocol)
330
246
except KeyboardInterrupt:
347
263
class SmartServerSocketStreamMedium(SmartServerStreamMedium):
349
def __init__(self, sock, backing_transport, root_client_path='/',
265
def __init__(self, sock, backing_transport, root_client_path='/'):
353
268
:param sock: the socket the server will read from. It will be put
354
269
into blocking mode.
356
271
SmartServerStreamMedium.__init__(
357
self, backing_transport, root_client_path=root_client_path,
272
self, backing_transport, root_client_path=root_client_path)
359
273
sock.setblocking(True)
360
274
self.socket = sock
361
# Get the getpeername now, as we might be closed later when we care.
363
self._client_info = sock.getpeername()
365
self._client_info = '<unknown>'
368
return '%s(client=%s)' % (self.__class__.__name__, self._client_info)
371
return '%s.%s(client=%s)' % (self.__module__, self.__class__.__name__,
374
276
def _serve_one_request_unguarded(self, protocol):
375
277
while protocol.next_read_size():
376
278
# We can safely try to read large chunks. If there is less data
377
# than MAX_SOCKET_CHUNK ready, the socket will just return a
378
# short read immediately rather than block.
379
bytes = self.read_bytes(osutils.MAX_SOCKET_CHUNK)
279
# than _MAX_READ_SIZE ready, the socket wil just return a short
280
# read immediately rather than block.
281
bytes = self.read_bytes(_MAX_READ_SIZE)
381
283
self.finished = True
385
287
self._push_back(protocol.unused_data)
387
def _disconnect_client(self):
388
"""Close the current connection. We stopped due to a timeout/etc."""
391
def _wait_for_bytes_with_timeout(self, timeout_seconds):
392
"""Wait for more bytes to be read, but timeout if none available.
394
This allows us to detect idle connections, and stop trying to read from
395
them, without setting the socket itself to non-blocking. This also
396
allows us to specify when we watch for idle timeouts.
398
:return: None, this will raise ConnectionTimeout if we time out before
401
return self._wait_on_descriptor(self.socket, timeout_seconds)
403
289
def _read_bytes(self, desired_count):
404
return osutils.read_bytes_from_socket(
405
self.socket, self._report_activity)
290
return _read_bytes_from_socket(
291
self.socket.recv, desired_count, self._report_activity)
407
293
def terminate_due_to_error(self):
408
294
# TODO: This should log to a server log file, but no such thing
409
295
# exists yet. Andrew Bennetts 2006-09-29.
296
osutils.until_no_eintr(self.socket.close)
411
297
self.finished = True
413
299
def _write_out(self, bytes):
423
309
class SmartServerPipeStreamMedium(SmartServerStreamMedium):
425
def __init__(self, in_file, out_file, backing_transport, timeout=None):
311
def __init__(self, in_file, out_file, backing_transport):
426
312
"""Construct new server.
428
314
:param in_file: Python file from which requests can be read.
429
315
:param out_file: Python file to write responses.
430
316
:param backing_transport: Transport for the directory served.
432
SmartServerStreamMedium.__init__(self, backing_transport,
318
SmartServerStreamMedium.__init__(self, backing_transport)
434
319
if sys.platform == 'win32':
435
320
# force binary mode for files
460
334
bytes_to_read = protocol.next_read_size()
461
335
if bytes_to_read == 0:
462
336
# Finished serving this request.
337
osutils.until_no_eintr(self._out.flush)
465
339
bytes = self.read_bytes(bytes_to_read)
467
341
# Connection has been closed.
468
342
self.finished = True
343
osutils.until_no_eintr(self._out.flush)
471
345
protocol.accept_bytes(bytes)
473
def _disconnect_client(self):
478
def _wait_for_bytes_with_timeout(self, timeout_seconds):
479
"""Wait for more bytes to be read, but timeout if none available.
481
This allows us to detect idle connections, and stop trying to read from
482
them, without setting the socket itself to non-blocking. This also
483
allows us to specify when we watch for idle timeouts.
485
:return: None, this will raise ConnectionTimeout if we time out before
488
if (getattr(self._in, 'fileno', None) is None
489
or sys.platform == 'win32'):
490
# You can't select() file descriptors on Windows.
492
return self._wait_on_descriptor(self._in, timeout_seconds)
494
347
def _read_bytes(self, desired_count):
495
return self._in.read(desired_count)
348
return osutils.until_no_eintr(self._in.read, desired_count)
497
350
def terminate_due_to_error(self):
498
351
# TODO: This should log to a server log file, but no such thing
499
352
# exists yet. Andrew Bennetts 2006-09-29.
353
osutils.until_no_eintr(self._out.close)
501
354
self.finished = True
503
356
def _write_out(self, bytes):
504
self._out.write(bytes)
357
osutils.until_no_eintr(self._out.write, bytes)
507
360
class SmartClientMediumRequest(object):
640
493
return self._medium._get_line()
643
class _VfsRefuser(object):
644
"""An object that refuses all VFS requests.
649
client._SmartClient.hooks.install_named_hook(
650
'call', self.check_vfs, 'vfs refuser')
652
def check_vfs(self, params):
654
request_method = request.request_handlers.get(params.method)
656
# A method we don't know about doesn't count as a VFS method.
658
if issubclass(request_method, vfs.VfsRequest):
659
raise errors.HpssVfsRequestNotAllowed(params.method, params.args)
662
496
class _DebugCounter(object):
663
497
"""An object that counts the HPSS calls made to each client medium.
665
When a medium is garbage-collected, or failing that when
666
bzrlib.global_state exits, the total number of calls made on that medium
667
are reported via trace.note.
499
When a medium is garbage-collected, or failing that when atexit functions
500
are run, the total number of calls made on that medium are reported via
670
504
def __init__(self):
671
505
self.counts = weakref.WeakKeyDictionary()
672
506
client._SmartClient.hooks.install_named_hook(
673
507
'call', self.increment_call_count, 'hpss call counter')
674
bzrlib.global_state.cleanups.add_cleanup(self.flush_all)
508
atexit.register(self.flush_all)
676
510
def track(self, medium):
677
511
"""Start tracking calls made to a medium.
780
609
# which is newer than a previously supplied older-than version.
781
610
# This indicates that some smart verb call is not guarded
782
611
# appropriately (it should simply not have been tried).
612
raise AssertionError(
784
613
"_remember_remote_is_before(%r) called, but "
785
614
"_remember_remote_is_before(%r) was called previously."
786
, version_tuple, self._remote_version_is_before)
787
if 'hpss' in debug.debug_flags:
788
ui.ui_factory.show_warning(
789
"_remember_remote_is_before(%r) called, but "
790
"_remember_remote_is_before(%r) was called previously."
791
% (version_tuple, self._remote_version_is_before))
615
% (version_tuple, self._remote_version_is_before))
793
616
self._remote_version_is_before = version_tuple
795
618
def protocol_version(self):
906
721
def _accept_bytes(self, bytes):
907
722
"""See SmartClientStreamMedium.accept_bytes."""
909
self._writeable_pipe.write(bytes)
911
if e.errno in (errno.EINVAL, errno.EPIPE):
912
raise errors.ConnectionReset(
913
"Error trying to write to subprocess", e)
723
osutils.until_no_eintr(self._writeable_pipe.write, bytes)
915
724
self._report_activity(len(bytes), 'write')
917
726
def _flush(self):
918
727
"""See SmartClientStreamMedium._flush()."""
919
# Note: If flush were to fail, we'd like to raise ConnectionReset, etc.
920
# However, testing shows that even when the child process is
921
# gone, this doesn't error.
922
self._writeable_pipe.flush()
728
osutils.until_no_eintr(self._writeable_pipe.flush)
924
730
def _read_bytes(self, count):
925
731
"""See SmartClientStreamMedium._read_bytes."""
926
bytes_to_read = min(count, _MAX_READ_SIZE)
927
bytes = self._readable_pipe.read(bytes_to_read)
732
bytes = osutils.until_no_eintr(self._readable_pipe.read, count)
928
733
self._report_activity(len(bytes), 'read')
932
class SSHParams(object):
933
"""A set of parameters for starting a remote bzr via SSH."""
737
class SmartSSHClientMedium(SmartClientStreamMedium):
738
"""A client medium using SSH."""
935
740
def __init__(self, host, port=None, username=None, password=None,
936
bzr_remote_path='bzr'):
939
self.username = username
940
self.password = password
941
self.bzr_remote_path = bzr_remote_path
944
class SmartSSHClientMedium(SmartClientStreamMedium):
945
"""A client medium using SSH.
947
It delegates IO to a SmartSimplePipesClientMedium or
948
SmartClientAlreadyConnectedSocketMedium (depending on platform).
951
def __init__(self, base, ssh_params, vendor=None):
741
base=None, vendor=None, bzr_remote_path=None):
952
742
"""Creates a client that will connect on the first use.
954
:param ssh_params: A SSHParams instance.
955
744
:param vendor: An optional override for the ssh vendor to use. See
956
745
bzrlib.transport.ssh for details on ssh vendors.
958
self._real_medium = None
959
self._ssh_params = ssh_params
747
self._connected = False
749
self._password = password
751
self._username = username
960
752
# for the benefit of progress making a short description of this
962
754
self._scheme = 'bzr+ssh'
964
756
# _DebugCounter so we have to store all the values used in our repr
965
757
# method before calling the super init.
966
758
SmartClientStreamMedium.__init__(self, base)
759
self._read_from = None
760
self._ssh_connection = None
967
761
self._vendor = vendor
968
self._ssh_connection = None
762
self._write_to = None
763
self._bzr_remote_path = bzr_remote_path
970
765
def __repr__(self):
971
if self._ssh_params.port is None:
766
if self._port is None:
974
maybe_port = ':%s' % self._ssh_params.port
975
if self._ssh_params.username is None:
978
maybe_user = '%s@' % self._ssh_params.username
979
return "%s(%s://%s%s%s/)" % (
769
maybe_port = ':%s' % self._port
770
return "%s(%s://%s@%s%s/)" % (
980
771
self.__class__.__name__,
983
self._ssh_params.host,
986
777
def _accept_bytes(self, bytes):
987
778
"""See SmartClientStreamMedium.accept_bytes."""
988
779
self._ensure_connection()
989
self._real_medium.accept_bytes(bytes)
780
osutils.until_no_eintr(self._write_to.write, bytes)
781
self._report_activity(len(bytes), 'write')
991
783
def disconnect(self):
992
784
"""See SmartClientMedium.disconnect()."""
993
if self._real_medium is not None:
994
self._real_medium.disconnect()
995
self._real_medium = None
996
if self._ssh_connection is not None:
997
self._ssh_connection.close()
998
self._ssh_connection = None
785
if not self._connected:
787
osutils.until_no_eintr(self._read_from.close)
788
osutils.until_no_eintr(self._write_to.close)
789
self._ssh_connection.close()
790
self._connected = False
1000
792
def _ensure_connection(self):
1001
793
"""Connect this medium if not already connected."""
1002
if self._real_medium is not None:
1004
796
if self._vendor is None:
1005
797
vendor = ssh._get_ssh_vendor()
1007
799
vendor = self._vendor
1008
self._ssh_connection = vendor.connect_ssh(self._ssh_params.username,
1009
self._ssh_params.password, self._ssh_params.host,
1010
self._ssh_params.port,
1011
command=[self._ssh_params.bzr_remote_path, 'serve', '--inet',
800
self._ssh_connection = vendor.connect_ssh(self._username,
801
self._password, self._host, self._port,
802
command=[self._bzr_remote_path, 'serve', '--inet',
1012
803
'--directory=/', '--allow-writes'])
1013
io_kind, io_object = self._ssh_connection.get_sock_or_pipes()
1014
if io_kind == 'socket':
1015
self._real_medium = SmartClientAlreadyConnectedSocketMedium(
1016
self.base, io_object)
1017
elif io_kind == 'pipes':
1018
read_from, write_to = io_object
1019
self._real_medium = SmartSimplePipesClientMedium(
1020
read_from, write_to, self.base)
1022
raise AssertionError(
1023
"Unexpected io_kind %r from %r"
1024
% (io_kind, self._ssh_connection))
1025
for hook in transport.Transport.hooks["post_connect"]:
804
self._read_from, self._write_to = \
805
self._ssh_connection.get_filelike_channels()
806
self._connected = True
1028
808
def _flush(self):
1029
809
"""See SmartClientStreamMedium._flush()."""
1030
self._real_medium._flush()
810
self._write_to.flush()
1032
812
def _read_bytes(self, count):
1033
813
"""See SmartClientStreamMedium.read_bytes."""
1034
if self._real_medium is None:
814
if not self._connected:
1035
815
raise errors.MediumNotConnected(self)
1036
return self._real_medium.read_bytes(count)
816
bytes_to_read = min(count, _MAX_READ_SIZE)
817
bytes = osutils.until_no_eintr(self._read_from.read, bytes_to_read)
818
self._report_activity(len(bytes), 'read')
1039
822
# Port 4155 is the default port for bzr://, registered with IANA.
1041
824
BZR_DEFAULT_PORT = 4155
1044
class SmartClientSocketMedium(SmartClientStreamMedium):
1045
"""A client medium using a socket.
1047
This class isn't usable directly. Use one of its subclasses instead.
1050
def __init__(self, base):
827
class SmartTCPClientMedium(SmartClientStreamMedium):
828
"""A client medium using TCP."""
830
def __init__(self, host, port, base):
831
"""Creates a client that will connect on the first use."""
1051
832
SmartClientStreamMedium.__init__(self, base)
833
self._connected = False
1052
836
self._socket = None
1053
self._connected = False
1055
838
def _accept_bytes(self, bytes):
1056
839
"""See SmartClientMedium.accept_bytes."""
1057
840
self._ensure_connection()
1058
841
osutils.send_all(self._socket, bytes, self._report_activity)
1060
def _ensure_connection(self):
1061
"""Connect this medium if not already connected."""
1062
raise NotImplementedError(self._ensure_connection)
1065
"""See SmartClientStreamMedium._flush().
1067
For sockets we do no flushing. For TCP sockets we may want to turn off
1068
TCP_NODELAY and add a means to do a flush, but that can be done in the
1072
def _read_bytes(self, count):
1073
"""See SmartClientMedium.read_bytes."""
1074
if not self._connected:
1075
raise errors.MediumNotConnected(self)
1076
return osutils.read_bytes_from_socket(
1077
self._socket, self._report_activity)
1079
843
def disconnect(self):
1080
844
"""See SmartClientMedium.disconnect()."""
1081
845
if not self._connected:
1083
self._socket.close()
847
osutils.until_no_eintr(self._socket.close)
1084
848
self._socket = None
1085
849
self._connected = False
1088
class SmartTCPClientMedium(SmartClientSocketMedium):
1089
"""A client medium that creates a TCP connection."""
1091
def __init__(self, host, port, base):
1092
"""Creates a client that will connect on the first use."""
1093
SmartClientSocketMedium.__init__(self, base)
1097
851
def _ensure_connection(self):
1098
852
"""Connect this medium if not already connected."""
1099
853
if self._connected:
1132
886
raise errors.ConnectionError("failed to connect to %s:%d: %s" %
1133
887
(self._host, port, err_msg))
1134
888
self._connected = True
1135
for hook in transport.Transport.hooks["post_connect"]:
1139
class SmartClientAlreadyConnectedSocketMedium(SmartClientSocketMedium):
1140
"""A client medium for an already connected socket.
1142
Note that this class will assume it "owns" the socket, so it will close it
1143
when its disconnect method is called.
1146
def __init__(self, base, sock):
1147
SmartClientSocketMedium.__init__(self, base)
1149
self._connected = True
1151
def _ensure_connection(self):
1152
# Already connected, by definition! So nothing to do.
891
"""See SmartClientStreamMedium._flush().
893
For TCP we do no flushing. We may want to turn off TCP_NODELAY and
894
add a means to do a flush, but that can be done in the future.
897
def _read_bytes(self, count):
898
"""See SmartClientMedium.read_bytes."""
899
if not self._connected:
900
raise errors.MediumNotConnected(self)
901
return _read_bytes_from_socket(
902
self._socket.recv, count, self._report_activity)
1156
905
class SmartClientStreamMediumRequest(SmartClientMediumRequest):
1191
940
This invokes self._medium._flush to ensure all bytes are transmitted.
1193
942
self._medium._flush()
945
def _read_bytes_from_socket(sock, desired_count, report_activity):
946
# We ignore the desired_count because on sockets it's more efficient to
947
# read large chunks (of _MAX_READ_SIZE bytes) at a time.
949
bytes = osutils.until_no_eintr(sock, _MAX_READ_SIZE)
950
except socket.error, e:
951
if len(e.args) and e.args[0] in (errno.ECONNRESET, 10054):
952
# The connection was closed by the other side. Callers expect an
953
# empty string to signal end-of-stream.
958
report_activity(len(bytes), 'read')
added
removed
bzrlib/smart/medium.py
