24
24
bzrlib/transport/smart/__init__.py.
27
from __future__ import absolute_import
35
33
from bzrlib.lazy_import import lazy_import
36
34
lazy_import(globals(), """
42
37
from bzrlib import (
49
from bzrlib.i18n import gettext
50
from bzrlib.smart import client, protocol, request, signals, vfs
45
from bzrlib.smart import client, protocol, request, vfs
51
46
from bzrlib.transport import ssh
48
#usually already imported, and getting IllegalScoperReplacer on it here.
53
49
from bzrlib import osutils
55
# Throughout this module buffer size parameters are either limited to be at
56
# most _MAX_READ_SIZE, or are ignored and _MAX_READ_SIZE is used instead.
57
# For this module's purposes, MAX_SOCKET_CHUNK is a reasonable size for reads
58
# from non-sockets as well.
59
_MAX_READ_SIZE = osutils.MAX_SOCKET_CHUNK
51
# We must not read any more than 64k at a time so we don't risk "no buffer
52
# space available" errors on some platforms. Windows in particular is likely
53
# to give error 10053 or 10055 if we read more than 64k from a socket.
54
_MAX_READ_SIZE = 64 * 1024
61
57
def _get_protocol_factory_for_bytes(bytes):
62
58
"""Determine the right protocol factory for 'bytes'.
232
214
while not self.finished:
233
215
server_protocol = self._build_protocol()
234
216
self._serve_one_request(server_protocol)
235
except errors.ConnectionTimeout, e:
236
trace.note('%s' % (e,))
237
trace.log_exception_quietly()
238
self._disconnect_client()
239
# We reported it, no reason to make a big fuss.
241
217
except Exception, e:
242
218
stderr.write("%s terminating on exception %s\n" % (self, e))
244
self._disconnect_client()
246
def _stop_gracefully(self):
247
"""When we finish this message, stop looking for more."""
248
trace.mutter('Stopping %s' % (self,))
251
def _disconnect_client(self):
252
"""Close the current connection. We stopped due to a timeout/etc."""
253
# The default implementation is a no-op, because that is all we used to
254
# do when disconnecting from a client. I suppose we never had the
255
# *server* initiate a disconnect, before
257
def _wait_for_bytes_with_timeout(self, timeout_seconds):
258
"""Wait for more bytes to be read, but timeout if none available.
260
This allows us to detect idle connections, and stop trying to read from
261
them, without setting the socket itself to non-blocking. This also
262
allows us to specify when we watch for idle timeouts.
264
:return: Did we timeout? (True if we timed out, False if there is data
267
raise NotImplementedError(self._wait_for_bytes_with_timeout)
269
221
def _build_protocol(self):
270
222
"""Identifies the version of the incoming request, and returns an
286
234
protocol.accept_bytes(unused_bytes)
289
def _wait_on_descriptor(self, fd, timeout_seconds):
290
"""select() on a file descriptor, waiting for nonblocking read()
292
This will raise a ConnectionTimeout exception if we do not get a
293
readable handle before timeout_seconds.
296
t_end = self._timer() + timeout_seconds
297
poll_timeout = min(timeout_seconds, self._client_poll_timeout)
299
while not rs and not xs and self._timer() < t_end:
303
rs, _, xs = select.select([fd], [], [fd], poll_timeout)
304
except (select.error, socket.error) as e:
305
err = getattr(e, 'errno', None)
306
if err is None and getattr(e, 'args', None) is not None:
307
# select.error doesn't have 'errno', it just has args[0]
309
if err in _bad_file_descriptor:
310
return # Not a socket indicates read() will fail
311
elif err == errno.EINTR:
312
# Interrupted, keep looping.
317
raise errors.ConnectionTimeout('disconnecting client after %.1f seconds'
318
% (timeout_seconds,))
320
237
def _serve_one_request(self, protocol):
321
238
"""Read one request from input, process, send back a response.
323
240
:param protocol: a SmartServerRequestProtocol.
328
243
self._serve_one_request_unguarded(protocol)
329
244
except KeyboardInterrupt:
346
261
class SmartServerSocketStreamMedium(SmartServerStreamMedium):
348
def __init__(self, sock, backing_transport, root_client_path='/',
263
def __init__(self, sock, backing_transport, root_client_path='/'):
352
266
:param sock: the socket the server will read from. It will be put
353
267
into blocking mode.
355
269
SmartServerStreamMedium.__init__(
356
self, backing_transport, root_client_path=root_client_path,
270
self, backing_transport, root_client_path=root_client_path)
358
271
sock.setblocking(True)
359
272
self.socket = sock
360
# Get the getpeername now, as we might be closed later when we care.
362
self._client_info = sock.getpeername()
364
self._client_info = '<unknown>'
367
return '%s(client=%s)' % (self.__class__.__name__, self._client_info)
370
return '%s.%s(client=%s)' % (self.__module__, self.__class__.__name__,
373
274
def _serve_one_request_unguarded(self, protocol):
374
275
while protocol.next_read_size():
375
276
# We can safely try to read large chunks. If there is less data
376
# than MAX_SOCKET_CHUNK ready, the socket will just return a
377
# short read immediately rather than block.
378
bytes = self.read_bytes(osutils.MAX_SOCKET_CHUNK)
277
# than _MAX_READ_SIZE ready, the socket wil just return a short
278
# read immediately rather than block.
279
bytes = self.read_bytes(_MAX_READ_SIZE)
380
281
self.finished = True
384
285
self._push_back(protocol.unused_data)
386
def _disconnect_client(self):
387
"""Close the current connection. We stopped due to a timeout/etc."""
390
def _wait_for_bytes_with_timeout(self, timeout_seconds):
391
"""Wait for more bytes to be read, but timeout if none available.
393
This allows us to detect idle connections, and stop trying to read from
394
them, without setting the socket itself to non-blocking. This also
395
allows us to specify when we watch for idle timeouts.
397
:return: None, this will raise ConnectionTimeout if we time out before
400
return self._wait_on_descriptor(self.socket, timeout_seconds)
402
287
def _read_bytes(self, desired_count):
403
return osutils.read_bytes_from_socket(
404
self.socket, self._report_activity)
288
return _read_bytes_from_socket(
289
self.socket.recv, desired_count, self._report_activity)
406
291
def terminate_due_to_error(self):
407
292
# TODO: This should log to a server log file, but no such thing
408
293
# exists yet. Andrew Bennetts 2006-09-29.
294
osutils.until_no_eintr(self.socket.close)
410
295
self.finished = True
412
297
def _write_out(self, bytes):
413
tstart = osutils.timer_func()
414
298
osutils.send_all(self.socket, bytes, self._report_activity)
415
if 'hpss' in debug.debug_flags:
416
thread_id = thread.get_ident()
417
trace.mutter('%12s: [%s] %d bytes to the socket in %.3fs'
418
% ('wrote', thread_id, len(bytes),
419
osutils.timer_func() - tstart))
422
301
class SmartServerPipeStreamMedium(SmartServerStreamMedium):
424
def __init__(self, in_file, out_file, backing_transport, timeout=None):
303
def __init__(self, in_file, out_file, backing_transport):
425
304
"""Construct new server.
427
306
:param in_file: Python file from which requests can be read.
428
307
:param out_file: Python file to write responses.
429
308
:param backing_transport: Transport for the directory served.
431
SmartServerStreamMedium.__init__(self, backing_transport,
310
SmartServerStreamMedium.__init__(self, backing_transport)
433
311
if sys.platform == 'win32':
434
312
# force binary mode for files
459
326
bytes_to_read = protocol.next_read_size()
460
327
if bytes_to_read == 0:
461
328
# Finished serving this request.
329
osutils.until_no_eintr(self._out.flush)
464
331
bytes = self.read_bytes(bytes_to_read)
466
333
# Connection has been closed.
467
334
self.finished = True
335
osutils.until_no_eintr(self._out.flush)
470
337
protocol.accept_bytes(bytes)
472
def _disconnect_client(self):
477
def _wait_for_bytes_with_timeout(self, timeout_seconds):
478
"""Wait for more bytes to be read, but timeout if none available.
480
This allows us to detect idle connections, and stop trying to read from
481
them, without setting the socket itself to non-blocking. This also
482
allows us to specify when we watch for idle timeouts.
484
:return: None, this will raise ConnectionTimeout if we time out before
487
if (getattr(self._in, 'fileno', None) is None
488
or sys.platform == 'win32'):
489
# You can't select() file descriptors on Windows.
491
return self._wait_on_descriptor(self._in, timeout_seconds)
493
339
def _read_bytes(self, desired_count):
494
return self._in.read(desired_count)
340
return osutils.until_no_eintr(self._in.read, desired_count)
496
342
def terminate_due_to_error(self):
497
343
# TODO: This should log to a server log file, but no such thing
498
344
# exists yet. Andrew Bennetts 2006-09-29.
345
osutils.until_no_eintr(self._out.close)
500
346
self.finished = True
502
348
def _write_out(self, bytes):
503
self._out.write(bytes)
349
osutils.until_no_eintr(self._out.write, bytes)
506
352
class SmartClientMediumRequest(object):
639
485
return self._medium._get_line()
642
class _VfsRefuser(object):
643
"""An object that refuses all VFS requests.
648
client._SmartClient.hooks.install_named_hook(
649
'call', self.check_vfs, 'vfs refuser')
651
def check_vfs(self, params):
653
request_method = request.request_handlers.get(params.method)
655
# A method we don't know about doesn't count as a VFS method.
657
if issubclass(request_method, vfs.VfsRequest):
658
raise errors.HpssVfsRequestNotAllowed(params.method, params.args)
661
488
class _DebugCounter(object):
662
489
"""An object that counts the HPSS calls made to each client medium.
664
When a medium is garbage-collected, or failing that when
665
bzrlib.global_state exits, the total number of calls made on that medium
666
are reported via trace.note.
491
When a medium is garbage-collected, or failing that when atexit functions
492
are run, the total number of calls made on that medium are reported via
669
496
def __init__(self):
670
497
self.counts = weakref.WeakKeyDictionary()
671
498
client._SmartClient.hooks.install_named_hook(
672
499
'call', self.increment_call_count, 'hpss call counter')
673
bzrlib.global_state.cleanups.add_cleanup(self.flush_all)
500
atexit.register(self.flush_all)
675
502
def track(self, medium):
676
503
"""Start tracking calls made to a medium.
905
713
def _accept_bytes(self, bytes):
906
714
"""See SmartClientStreamMedium.accept_bytes."""
908
self._writeable_pipe.write(bytes)
910
if e.errno in (errno.EINVAL, errno.EPIPE):
911
raise errors.ConnectionReset(
912
"Error trying to write to subprocess:\n%s" % (e,))
715
osutils.until_no_eintr(self._writeable_pipe.write, bytes)
914
716
self._report_activity(len(bytes), 'write')
916
718
def _flush(self):
917
719
"""See SmartClientStreamMedium._flush()."""
918
# Note: If flush were to fail, we'd like to raise ConnectionReset, etc.
919
# However, testing shows that even when the child process is
920
# gone, this doesn't error.
921
self._writeable_pipe.flush()
720
osutils.until_no_eintr(self._writeable_pipe.flush)
923
722
def _read_bytes(self, count):
924
723
"""See SmartClientStreamMedium._read_bytes."""
925
bytes_to_read = min(count, _MAX_READ_SIZE)
926
bytes = self._readable_pipe.read(bytes_to_read)
724
bytes = osutils.until_no_eintr(self._readable_pipe.read, count)
927
725
self._report_activity(len(bytes), 'read')
931
class SSHParams(object):
932
"""A set of parameters for starting a remote bzr via SSH."""
729
class SmartSSHClientMedium(SmartClientStreamMedium):
730
"""A client medium using SSH."""
934
732
def __init__(self, host, port=None, username=None, password=None,
935
bzr_remote_path='bzr'):
938
self.username = username
939
self.password = password
940
self.bzr_remote_path = bzr_remote_path
943
class SmartSSHClientMedium(SmartClientStreamMedium):
944
"""A client medium using SSH.
946
It delegates IO to a SmartSimplePipesClientMedium or
947
SmartClientAlreadyConnectedSocketMedium (depending on platform).
950
def __init__(self, base, ssh_params, vendor=None):
733
base=None, vendor=None, bzr_remote_path=None):
951
734
"""Creates a client that will connect on the first use.
953
:param ssh_params: A SSHParams instance.
954
736
:param vendor: An optional override for the ssh vendor to use. See
955
737
bzrlib.transport.ssh for details on ssh vendors.
957
self._real_medium = None
958
self._ssh_params = ssh_params
959
# for the benefit of progress making a short description of this
961
self._scheme = 'bzr+ssh'
739
self._connected = False
741
self._password = password
743
self._username = username
962
744
# SmartClientStreamMedium stores the repr of this object in its
963
745
# _DebugCounter so we have to store all the values used in our repr
964
746
# method before calling the super init.
965
747
SmartClientStreamMedium.__init__(self, base)
748
self._read_from = None
749
self._ssh_connection = None
966
750
self._vendor = vendor
967
self._ssh_connection = None
751
self._write_to = None
752
self._bzr_remote_path = bzr_remote_path
753
# for the benefit of progress making a short description of this
755
self._scheme = 'bzr+ssh'
969
757
def __repr__(self):
970
if self._ssh_params.port is None:
973
maybe_port = ':%s' % self._ssh_params.port
974
if self._ssh_params.username is None:
977
maybe_user = '%s@' % self._ssh_params.username
978
return "%s(%s://%s%s%s/)" % (
758
return "%s(connected=%r, username=%r, host=%r, port=%r)" % (
979
759
self.__class__.__name__,
982
self._ssh_params.host,
985
765
def _accept_bytes(self, bytes):
986
766
"""See SmartClientStreamMedium.accept_bytes."""
987
767
self._ensure_connection()
988
self._real_medium.accept_bytes(bytes)
768
osutils.until_no_eintr(self._write_to.write, bytes)
769
self._report_activity(len(bytes), 'write')
990
771
def disconnect(self):
991
772
"""See SmartClientMedium.disconnect()."""
992
if self._real_medium is not None:
993
self._real_medium.disconnect()
994
self._real_medium = None
995
if self._ssh_connection is not None:
996
self._ssh_connection.close()
997
self._ssh_connection = None
773
if not self._connected:
775
osutils.until_no_eintr(self._read_from.close)
776
osutils.until_no_eintr(self._write_to.close)
777
self._ssh_connection.close()
778
self._connected = False
999
780
def _ensure_connection(self):
1000
781
"""Connect this medium if not already connected."""
1001
if self._real_medium is not None:
1003
784
if self._vendor is None:
1004
785
vendor = ssh._get_ssh_vendor()
1006
787
vendor = self._vendor
1007
self._ssh_connection = vendor.connect_ssh(self._ssh_params.username,
1008
self._ssh_params.password, self._ssh_params.host,
1009
self._ssh_params.port,
1010
command=[self._ssh_params.bzr_remote_path, 'serve', '--inet',
788
self._ssh_connection = vendor.connect_ssh(self._username,
789
self._password, self._host, self._port,
790
command=[self._bzr_remote_path, 'serve', '--inet',
1011
791
'--directory=/', '--allow-writes'])
1012
io_kind, io_object = self._ssh_connection.get_sock_or_pipes()
1013
if io_kind == 'socket':
1014
self._real_medium = SmartClientAlreadyConnectedSocketMedium(
1015
self.base, io_object)
1016
elif io_kind == 'pipes':
1017
read_from, write_to = io_object
1018
self._real_medium = SmartSimplePipesClientMedium(
1019
read_from, write_to, self.base)
1021
raise AssertionError(
1022
"Unexpected io_kind %r from %r"
1023
% (io_kind, self._ssh_connection))
792
self._read_from, self._write_to = \
793
self._ssh_connection.get_filelike_channels()
794
self._connected = True
1025
796
def _flush(self):
1026
797
"""See SmartClientStreamMedium._flush()."""
1027
self._real_medium._flush()
798
self._write_to.flush()
1029
800
def _read_bytes(self, count):
1030
801
"""See SmartClientStreamMedium.read_bytes."""
1031
if self._real_medium is None:
802
if not self._connected:
1032
803
raise errors.MediumNotConnected(self)
1033
return self._real_medium.read_bytes(count)
804
bytes_to_read = min(count, _MAX_READ_SIZE)
805
bytes = osutils.until_no_eintr(self._read_from.read, bytes_to_read)
806
self._report_activity(len(bytes), 'read')
1036
810
# Port 4155 is the default port for bzr://, registered with IANA.
1038
812
BZR_DEFAULT_PORT = 4155
1041
class SmartClientSocketMedium(SmartClientStreamMedium):
1042
"""A client medium using a socket.
1044
This class isn't usable directly. Use one of its subclasses instead.
815
class SmartTCPClientMedium(SmartClientStreamMedium):
816
"""A client medium using TCP."""
1047
def __init__(self, base):
818
def __init__(self, host, port, base):
819
"""Creates a client that will connect on the first use."""
1048
820
SmartClientStreamMedium.__init__(self, base)
821
self._connected = False
1049
824
self._socket = None
1050
self._connected = False
1052
826
def _accept_bytes(self, bytes):
1053
827
"""See SmartClientMedium.accept_bytes."""
1054
828
self._ensure_connection()
1055
829
osutils.send_all(self._socket, bytes, self._report_activity)
1057
def _ensure_connection(self):
1058
"""Connect this medium if not already connected."""
1059
raise NotImplementedError(self._ensure_connection)
1062
"""See SmartClientStreamMedium._flush().
1064
For sockets we do no flushing. For TCP sockets we may want to turn off
1065
TCP_NODELAY and add a means to do a flush, but that can be done in the
1069
def _read_bytes(self, count):
1070
"""See SmartClientMedium.read_bytes."""
1071
if not self._connected:
1072
raise errors.MediumNotConnected(self)
1073
return osutils.read_bytes_from_socket(
1074
self._socket, self._report_activity)
1076
831
def disconnect(self):
1077
832
"""See SmartClientMedium.disconnect()."""
1078
833
if not self._connected:
1080
self._socket.close()
835
osutils.until_no_eintr(self._socket.close)
1081
836
self._socket = None
1082
837
self._connected = False
1085
class SmartTCPClientMedium(SmartClientSocketMedium):
1086
"""A client medium that creates a TCP connection."""
1088
def __init__(self, host, port, base):
1089
"""Creates a client that will connect on the first use."""
1090
SmartClientSocketMedium.__init__(self, base)
1094
839
def _ensure_connection(self):
1095
840
"""Connect this medium if not already connected."""
1096
841
if self._connected:
1130
875
(self._host, port, err_msg))
1131
876
self._connected = True
1134
class SmartClientAlreadyConnectedSocketMedium(SmartClientSocketMedium):
1135
"""A client medium for an already connected socket.
1137
Note that this class will assume it "owns" the socket, so it will close it
1138
when its disconnect method is called.
1141
def __init__(self, base, sock):
1142
SmartClientSocketMedium.__init__(self, base)
1144
self._connected = True
1146
def _ensure_connection(self):
1147
# Already connected, by definition! So nothing to do.
879
"""See SmartClientStreamMedium._flush().
881
For TCP we do no flushing. We may want to turn off TCP_NODELAY and
882
add a means to do a flush, but that can be done in the future.
885
def _read_bytes(self, count):
886
"""See SmartClientMedium.read_bytes."""
887
if not self._connected:
888
raise errors.MediumNotConnected(self)
889
return _read_bytes_from_socket(
890
self._socket.recv, count, self._report_activity)
1151
893
class SmartClientStreamMediumRequest(SmartClientMediumRequest):
added
removed
bzrlib/smart/medium.py
