33
34
from bzrlib.lazy_import import lazy_import
34
35
lazy_import(globals(), """
37
41
from bzrlib import (
46
from bzrlib.smart import client, protocol, request, vfs
48
from bzrlib.i18n import gettext
49
from bzrlib.smart import client, protocol, request, signals, vfs
47
50
from bzrlib.transport import ssh
51
# We must not read any more than 64k at a time so we don't risk "no buffer
52
# space available" errors on some platforms. Windows in particular is likely
53
# to give error 10053 or 10055 if we read more than 64k from a socket.
54
_MAX_READ_SIZE = 64 * 1024
52
from bzrlib import osutils
54
# Throughout this module buffer size parameters are either limited to be at
55
# most _MAX_READ_SIZE, or are ignored and _MAX_READ_SIZE is used instead.
56
# For this module's purposes, MAX_SOCKET_CHUNK is a reasonable size for reads
57
# from non-sockets as well.
58
_MAX_READ_SIZE = osutils.MAX_SOCKET_CHUNK
57
60
def _get_protocol_factory_for_bytes(bytes):
58
61
"""Determine the right protocol factory for 'bytes'.
214
231
while not self.finished:
215
232
server_protocol = self._build_protocol()
216
233
self._serve_one_request(server_protocol)
234
except errors.ConnectionTimeout, e:
235
trace.note('%s' % (e,))
236
trace.log_exception_quietly()
237
self._disconnect_client()
238
# We reported it, no reason to make a big fuss.
217
240
except Exception, e:
218
241
stderr.write("%s terminating on exception %s\n" % (self, e))
243
self._disconnect_client()
245
def _stop_gracefully(self):
246
"""When we finish this message, stop looking for more."""
247
trace.mutter('Stopping %s' % (self,))
250
def _disconnect_client(self):
251
"""Close the current connection. We stopped due to a timeout/etc."""
252
# The default implementation is a no-op, because that is all we used to
253
# do when disconnecting from a client. I suppose we never had the
254
# *server* initiate a disconnect, before
256
def _wait_for_bytes_with_timeout(self, timeout_seconds):
257
"""Wait for more bytes to be read, but timeout if none available.
259
This allows us to detect idle connections, and stop trying to read from
260
them, without setting the socket itself to non-blocking. This also
261
allows us to specify when we watch for idle timeouts.
263
:return: Did we timeout? (True if we timed out, False if there is data
266
raise NotImplementedError(self._wait_for_bytes_with_timeout)
221
268
def _build_protocol(self):
222
269
"""Identifies the version of the incoming request, and returns an
234
285
protocol.accept_bytes(unused_bytes)
288
def _wait_on_descriptor(self, fd, timeout_seconds):
289
"""select() on a file descriptor, waiting for nonblocking read()
291
This will raise a ConnectionTimeout exception if we do not get a
292
readable handle before timeout_seconds.
295
t_end = self._timer() + timeout_seconds
296
poll_timeout = min(timeout_seconds, self._client_poll_timeout)
298
while not rs and not xs and self._timer() < t_end:
302
rs, _, xs = select.select([fd], [], [fd], poll_timeout)
303
except (select.error, socket.error) as e:
304
err = getattr(e, 'errno', None)
305
if err is None and getattr(e, 'args', None) is not None:
306
# select.error doesn't have 'errno', it just has args[0]
308
if err in _bad_file_descriptor:
309
return # Not a socket indicates read() will fail
310
elif err == errno.EINTR:
311
# Interrupted, keep looping.
316
raise errors.ConnectionTimeout('disconnecting client after %.1f seconds'
317
% (timeout_seconds,))
237
319
def _serve_one_request(self, protocol):
238
320
"""Read one request from input, process, send back a response.
240
322
:param protocol: a SmartServerRequestProtocol.
243
327
self._serve_one_request_unguarded(protocol)
244
328
except KeyboardInterrupt:
261
345
class SmartServerSocketStreamMedium(SmartServerStreamMedium):
263
def __init__(self, sock, backing_transport, root_client_path='/'):
347
def __init__(self, sock, backing_transport, root_client_path='/',
266
351
:param sock: the socket the server will read from. It will be put
267
352
into blocking mode.
269
354
SmartServerStreamMedium.__init__(
270
self, backing_transport, root_client_path=root_client_path)
355
self, backing_transport, root_client_path=root_client_path,
271
357
sock.setblocking(True)
272
358
self.socket = sock
359
# Get the getpeername now, as we might be closed later when we care.
361
self._client_info = sock.getpeername()
363
self._client_info = '<unknown>'
366
return '%s(client=%s)' % (self.__class__.__name__, self._client_info)
369
return '%s.%s(client=%s)' % (self.__module__, self.__class__.__name__,
274
372
def _serve_one_request_unguarded(self, protocol):
275
373
while protocol.next_read_size():
276
374
# We can safely try to read large chunks. If there is less data
277
# than _MAX_READ_SIZE ready, the socket wil just return a short
278
# read immediately rather than block.
279
bytes = self.read_bytes(_MAX_READ_SIZE)
375
# than MAX_SOCKET_CHUNK ready, the socket will just return a
376
# short read immediately rather than block.
377
bytes = self.read_bytes(osutils.MAX_SOCKET_CHUNK)
281
379
self.finished = True
285
383
self._push_back(protocol.unused_data)
385
def _disconnect_client(self):
386
"""Close the current connection. We stopped due to a timeout/etc."""
389
def _wait_for_bytes_with_timeout(self, timeout_seconds):
390
"""Wait for more bytes to be read, but timeout if none available.
392
This allows us to detect idle connections, and stop trying to read from
393
them, without setting the socket itself to non-blocking. This also
394
allows us to specify when we watch for idle timeouts.
396
:return: None, this will raise ConnectionTimeout if we time out before
399
return self._wait_on_descriptor(self.socket, timeout_seconds)
287
401
def _read_bytes(self, desired_count):
288
return _read_bytes_from_socket(
289
self.socket.recv, desired_count, self._report_activity)
402
return osutils.read_bytes_from_socket(
403
self.socket, self._report_activity)
291
405
def terminate_due_to_error(self):
292
406
# TODO: This should log to a server log file, but no such thing
295
409
self.finished = True
297
411
def _write_out(self, bytes):
412
tstart = osutils.timer_func()
298
413
osutils.send_all(self.socket, bytes, self._report_activity)
414
if 'hpss' in debug.debug_flags:
415
thread_id = thread.get_ident()
416
trace.mutter('%12s: [%s] %d bytes to the socket in %.3fs'
417
% ('wrote', thread_id, len(bytes),
418
osutils.timer_func() - tstart))
301
421
class SmartServerPipeStreamMedium(SmartServerStreamMedium):
303
def __init__(self, in_file, out_file, backing_transport):
423
def __init__(self, in_file, out_file, backing_transport, timeout=None):
304
424
"""Construct new server.
306
426
:param in_file: Python file from which requests can be read.
307
427
:param out_file: Python file to write responses.
308
428
:param backing_transport: Transport for the directory served.
310
SmartServerStreamMedium.__init__(self, backing_transport)
430
SmartServerStreamMedium.__init__(self, backing_transport,
311
432
if sys.platform == 'win32':
312
433
# force binary mode for files
337
469
protocol.accept_bytes(bytes)
471
def _disconnect_client(self):
476
def _wait_for_bytes_with_timeout(self, timeout_seconds):
477
"""Wait for more bytes to be read, but timeout if none available.
479
This allows us to detect idle connections, and stop trying to read from
480
them, without setting the socket itself to non-blocking. This also
481
allows us to specify when we watch for idle timeouts.
483
:return: None, this will raise ConnectionTimeout if we time out before
486
if (getattr(self._in, 'fileno', None) is None
487
or sys.platform == 'win32'):
488
# You can't select() file descriptors on Windows.
490
return self._wait_on_descriptor(self._in, timeout_seconds)
339
492
def _read_bytes(self, desired_count):
340
493
return self._in.read(desired_count)
485
638
return self._medium._get_line()
641
class _VfsRefuser(object):
642
"""An object that refuses all VFS requests.
647
client._SmartClient.hooks.install_named_hook(
648
'call', self.check_vfs, 'vfs refuser')
650
def check_vfs(self, params):
652
request_method = request.request_handlers.get(params.method)
654
# A method we don't know about doesn't count as a VFS method.
656
if issubclass(request_method, vfs.VfsRequest):
657
raise errors.HpssVfsRequestNotAllowed(params.method, params.args)
488
660
class _DebugCounter(object):
489
661
"""An object that counts the HPSS calls made to each client medium.
491
When a medium is garbage-collected, or failing that when atexit functions
492
are run, the total number of calls made on that medium are reported via
663
When a medium is garbage-collected, or failing that when
664
bzrlib.global_state exits, the total number of calls made on that medium
665
are reported via trace.note.
496
668
def __init__(self):
497
669
self.counts = weakref.WeakKeyDictionary()
498
670
client._SmartClient.hooks.install_named_hook(
499
671
'call', self.increment_call_count, 'hpss call counter')
500
atexit.register(self.flush_all)
672
bzrlib.global_state.cleanups.add_cleanup(self.flush_all)
502
674
def track(self, medium):
503
675
"""Start tracking calls made to a medium.
722
905
def _read_bytes(self, count):
723
906
"""See SmartClientStreamMedium._read_bytes."""
724
bytes = self._readable_pipe.read(count)
907
bytes_to_read = min(count, _MAX_READ_SIZE)
908
bytes = self._readable_pipe.read(bytes_to_read)
725
909
self._report_activity(len(bytes), 'read')
913
class SSHParams(object):
914
"""A set of parameters for starting a remote bzr via SSH."""
916
def __init__(self, host, port=None, username=None, password=None,
917
bzr_remote_path='bzr'):
920
self.username = username
921
self.password = password
922
self.bzr_remote_path = bzr_remote_path
729
925
class SmartSSHClientMedium(SmartClientStreamMedium):
730
"""A client medium using SSH."""
926
"""A client medium using SSH.
928
It delegates IO to a SmartClientSocketMedium or
929
SmartClientAlreadyConnectedSocketMedium (depending on platform).
732
def __init__(self, host, port=None, username=None, password=None,
733
base=None, vendor=None, bzr_remote_path=None):
932
def __init__(self, base, ssh_params, vendor=None):
734
933
"""Creates a client that will connect on the first use.
935
:param ssh_params: A SSHParams instance.
736
936
:param vendor: An optional override for the ssh vendor to use. See
737
937
bzrlib.transport.ssh for details on ssh vendors.
739
self._connected = False
741
self._password = password
743
self._username = username
939
self._real_medium = None
940
self._ssh_params = ssh_params
941
# for the benefit of progress making a short description of this
943
self._scheme = 'bzr+ssh'
744
944
# SmartClientStreamMedium stores the repr of this object in its
745
945
# _DebugCounter so we have to store all the values used in our repr
746
946
# method before calling the super init.
747
947
SmartClientStreamMedium.__init__(self, base)
748
self._read_from = None
948
self._vendor = vendor
749
949
self._ssh_connection = None
750
self._vendor = vendor
751
self._write_to = None
752
self._bzr_remote_path = bzr_remote_path
753
# for the benefit of progress making a short description of this
755
self._scheme = 'bzr+ssh'
757
951
def __repr__(self):
758
return "%s(connected=%r, username=%r, host=%r, port=%r)" % (
952
if self._ssh_params.port is None:
955
maybe_port = ':%s' % self._ssh_params.port
956
return "%s(%s://%s@%s%s/)" % (
759
957
self.__class__.__name__,
959
self._ssh_params.username,
960
self._ssh_params.host,
765
963
def _accept_bytes(self, bytes):
766
964
"""See SmartClientStreamMedium.accept_bytes."""
767
965
self._ensure_connection()
768
self._write_to.write(bytes)
769
self._report_activity(len(bytes), 'write')
966
self._real_medium.accept_bytes(bytes)
771
968
def disconnect(self):
772
969
"""See SmartClientMedium.disconnect()."""
773
if not self._connected:
775
self._read_from.close()
776
self._write_to.close()
777
self._ssh_connection.close()
778
self._connected = False
970
if self._real_medium is not None:
971
self._real_medium.disconnect()
972
self._real_medium = None
973
if self._ssh_connection is not None:
974
self._ssh_connection.close()
975
self._ssh_connection = None
780
977
def _ensure_connection(self):
781
978
"""Connect this medium if not already connected."""
979
if self._real_medium is not None:
784
981
if self._vendor is None:
785
982
vendor = ssh._get_ssh_vendor()
787
984
vendor = self._vendor
788
self._ssh_connection = vendor.connect_ssh(self._username,
789
self._password, self._host, self._port,
790
command=[self._bzr_remote_path, 'serve', '--inet',
985
self._ssh_connection = vendor.connect_ssh(self._ssh_params.username,
986
self._ssh_params.password, self._ssh_params.host,
987
self._ssh_params.port,
988
command=[self._ssh_params.bzr_remote_path, 'serve', '--inet',
791
989
'--directory=/', '--allow-writes'])
792
self._read_from, self._write_to = \
793
self._ssh_connection.get_filelike_channels()
794
self._connected = True
990
io_kind, io_object = self._ssh_connection.get_sock_or_pipes()
991
if io_kind == 'socket':
992
self._real_medium = SmartClientAlreadyConnectedSocketMedium(
993
self.base, io_object)
994
elif io_kind == 'pipes':
995
read_from, write_to = io_object
996
self._real_medium = SmartSimplePipesClientMedium(
997
read_from, write_to, self.base)
999
raise AssertionError(
1000
"Unexpected io_kind %r from %r"
1001
% (io_kind, self._ssh_connection))
796
1003
def _flush(self):
797
1004
"""See SmartClientStreamMedium._flush()."""
798
self._write_to.flush()
1005
self._real_medium._flush()
800
1007
def _read_bytes(self, count):
801
1008
"""See SmartClientStreamMedium.read_bytes."""
802
if not self._connected:
1009
if self._real_medium is None:
803
1010
raise errors.MediumNotConnected(self)
804
bytes_to_read = min(count, _MAX_READ_SIZE)
805
bytes = self._read_from.read(bytes_to_read)
806
self._report_activity(len(bytes), 'read')
1011
return self._real_medium.read_bytes(count)
810
1014
# Port 4155 is the default port for bzr://, registered with IANA.
812
1016
BZR_DEFAULT_PORT = 4155
815
class SmartTCPClientMedium(SmartClientStreamMedium):
816
"""A client medium using TCP."""
1019
class SmartClientSocketMedium(SmartClientStreamMedium):
1020
"""A client medium using a socket.
1022
This class isn't usable directly. Use one of its subclasses instead.
818
def __init__(self, host, port, base):
819
"""Creates a client that will connect on the first use."""
1025
def __init__(self, base):
820
1026
SmartClientStreamMedium.__init__(self, base)
821
1028
self._connected = False
826
1030
def _accept_bytes(self, bytes):
827
1031
"""See SmartClientMedium.accept_bytes."""
828
1032
self._ensure_connection()
829
1033
osutils.send_all(self._socket, bytes, self._report_activity)
1035
def _ensure_connection(self):
1036
"""Connect this medium if not already connected."""
1037
raise NotImplementedError(self._ensure_connection)
1040
"""See SmartClientStreamMedium._flush().
1042
For sockets we do no flushing. For TCP sockets we may want to turn off
1043
TCP_NODELAY and add a means to do a flush, but that can be done in the
1047
def _read_bytes(self, count):
1048
"""See SmartClientMedium.read_bytes."""
1049
if not self._connected:
1050
raise errors.MediumNotConnected(self)
1051
return osutils.read_bytes_from_socket(
1052
self._socket, self._report_activity)
831
1054
def disconnect(self):
832
1055
"""See SmartClientMedium.disconnect()."""
833
1056
if not self._connected:
875
1108
(self._host, port, err_msg))
876
1109
self._connected = True
879
"""See SmartClientStreamMedium._flush().
881
For TCP we do no flushing. We may want to turn off TCP_NODELAY and
882
add a means to do a flush, but that can be done in the future.
885
def _read_bytes(self, count):
886
"""See SmartClientMedium.read_bytes."""
887
if not self._connected:
888
raise errors.MediumNotConnected(self)
889
return _read_bytes_from_socket(
890
self._socket.recv, count, self._report_activity)
1112
class SmartClientAlreadyConnectedSocketMedium(SmartClientSocketMedium):
1113
"""A client medium for an already connected socket.
1115
Note that this class will assume it "owns" the socket, so it will close it
1116
when its disconnect method is called.
1119
def __init__(self, base, sock):
1120
SmartClientSocketMedium.__init__(self, base)
1122
self._connected = True
1124
def _ensure_connection(self):
1125
# Already connected, by definition! So nothing to do.
893
1129
class SmartClientStreamMediumRequest(SmartClientMediumRequest):
added
removed
bzrlib/smart/medium.py
