« back to all changes in this revision
Viewing changes to bzrlib/transport/sftp.py
- browse files at revision 1969
- compare with another revision
- compare with revision 2054.2.15
- download diff
- download tarball
- view history from revision 1969
- Committer: Canonical.com Patch Queue Manager
- Date: 2006-08-29 20:29:23 UTC
- mfrom: (1711.9.8 jam-integration)
- Revision ID: pqm@pqm.ubuntu.com-20060829202923-fb8340be7d4adadb
(spiv) Refactor sftp vendor support
- files added:
- bzrlib/transport/ssh.py
- files modified:
- NEWS
added
removed
bzrlib/transport/sftp.py
18
18
"""Implementation of Transport over SFTP, using paramiko."""
19
19
20
20
import errno
21
import getpass
22
import itertools
23
21
import os
24
22
import random
25
import re
26
23
import select
27
24
import socket
28
25
import stat
33
30
import urlparse
34
31
import weakref
35
32
36
from bzrlib.config import config_dir, ensure_config_dir_exists
37
from bzrlib.errors import (ConnectionError,
38
FileExists,
39
TransportNotPossible, NoSuchFile, PathNotChild,
33
from bzrlib.errors import (FileExists,
34
NoSuchFile, PathNotChild,
40
35
TransportError,
41
36
LockError,
42
37
PathError,
43
38
ParamikoNotPresent,
39
UnknownSSH,
44
40
)
45
41
from bzrlib.osutils import pathjoin, fancy_rename, getcwd
46
from bzrlib.trace import mutter, warning, error
42
from bzrlib.trace import mutter, warning
47
43
from bzrlib.transport import (
48
44
register_urlparse_netloc_protocol,
49
45
Server,
50
46
split_url,
47
ssh,
51
48
Transport,
52
49
)
53
import bzrlib.ui
54
50
import bzrlib.urlutils as urlutils
55
51
56
52
try:
63
59
CMD_HANDLE, CMD_OPEN)
64
60
from paramiko.sftp_attr import SFTPAttributes
65
61
from paramiko.sftp_file import SFTPFile
66
from paramiko.sftp_client import SFTPClient
67
62
68
63
69
64
register_urlparse_netloc_protocol('sftp')
70
65
71
66
72
def _ignore_sigint():
73
# TODO: This should possibly ignore SIGHUP as well, but bzr currently
74
# doesn't handle it itself.
75
# <https://launchpad.net/products/bzr/+bug/41433/+index>
76
import signal
77
signal.signal(signal.SIGINT, signal.SIG_IGN)
78
79
80
def os_specific_subprocess_params():
81
"""Get O/S specific subprocess parameters."""
82
if sys.platform == 'win32':
83
# setting the process group and closing fds is not supported on
84
# win32
85
return {}
86
else:
87
# We close fds other than the pipes as the child process does not need
88
# them to be open.
89
#
90
# We also set the child process to ignore SIGINT. Normally the signal
91
# would be sent to every process in the foreground process group, but
92
# this causes it to be seen only by bzr and not by ssh. Python will
93
# generate a KeyboardInterrupt in bzr, and we will then have a chance
94
# to release locks or do other cleanup over ssh before the connection
95
# goes away.
96
# <https://launchpad.net/products/bzr/+bug/5987>
97
#
98
# Running it in a separate process group is not good because then it
99
# can't get non-echoed input of a password or passphrase.
100
# <https://launchpad.net/products/bzr/+bug/40508>
101
return {'preexec_fn': _ignore_sigint,
102
'close_fds': True,
103
}
104
105
106
_paramiko_version = getattr(paramiko, '__version_info__', (0, 0, 0))
107
# don't use prefetch unless paramiko version >= 1.5.5 (there were bugs earlier)
108
_default_do_prefetch = (_paramiko_version >= (1, 5, 5))
109
110
# Paramiko 1.5 tries to open a socket.AF_UNIX in order to connect
111
# to ssh-agent. That attribute doesn't exist on win32 (it does in cygwin)
112
# so we get an AttributeError exception. So we will not try to
113
# connect to an agent if we are on win32 and using Paramiko older than 1.6
114
_use_ssh_agent = (sys.platform != 'win32' or _paramiko_version >= (1, 6, 0))
115
116
117
_ssh_vendor = None
118
def _get_ssh_vendor():
119
"""Find out what version of SSH is on the system."""
120
global _ssh_vendor
121
if _ssh_vendor is not None:
122
return _ssh_vendor
123
124
_ssh_vendor = 'none'
125
126
if 'BZR_SSH' in os.environ:
127
_ssh_vendor = os.environ['BZR_SSH']
128
if _ssh_vendor == 'paramiko':
129
_ssh_vendor = 'none'
130
return _ssh_vendor
131
132
try:
133
p = subprocess.Popen(['ssh', '-V'],
134
stdin=subprocess.PIPE,
135
stdout=subprocess.PIPE,
136
stderr=subprocess.PIPE,
137
**os_specific_subprocess_params())
138
returncode = p.returncode
139
stdout, stderr = p.communicate()
140
except OSError:
141
returncode = -1
142
stdout = stderr = ''
143
if 'OpenSSH' in stderr:
144
mutter('ssh implementation is OpenSSH')
145
_ssh_vendor = 'openssh'
146
elif 'SSH Secure Shell' in stderr:
147
mutter('ssh implementation is SSH Corp.')
148
_ssh_vendor = 'ssh'
149
150
if _ssh_vendor != 'none':
151
return _ssh_vendor
152
153
# XXX: 20051123 jamesh
154
# A check for putty's plink or lsh would go here.
155
156
mutter('falling back to paramiko implementation')
157
return _ssh_vendor
158
159
160
class SFTPSubprocess:
161
"""A socket-like object that talks to an ssh subprocess via pipes."""
162
def __init__(self, hostname, vendor, port=None, user=None):
163
assert vendor in ['openssh', 'ssh']
164
if vendor == 'openssh':
165
args = ['ssh',
166
'-oForwardX11=no', '-oForwardAgent=no',
167
'-oClearAllForwardings=yes', '-oProtocol=2',
168
'-oNoHostAuthenticationForLocalhost=yes']
169
if port is not None:
170
args.extend(['-p', str(port)])
171
if user is not None:
172
args.extend(['-l', user])
173
args.extend(['-s', hostname, 'sftp'])
174
elif vendor == 'ssh':
175
args = ['ssh', '-x']
176
if port is not None:
177
args.extend(['-p', str(port)])
178
if user is not None:
179
args.extend(['-l', user])
180
args.extend(['-s', 'sftp', hostname])
181
182
self.proc = subprocess.Popen(args,
183
stdin=subprocess.PIPE,
184
stdout=subprocess.PIPE,
185
**os_specific_subprocess_params())
186
187
def send(self, data):
188
return os.write(self.proc.stdin.fileno(), data)
189
190
def recv_ready(self):
191
# TODO: jam 20051215 this function is necessary to support the
192
# pipelined() function. In reality, it probably should use
193
# poll() or select() to actually return if there is data
194
# available, otherwise we probably don't get any benefit
195
return True
196
197
def recv(self, count):
198
return os.read(self.proc.stdout.fileno(), count)
199
200
def close(self):
201
self.proc.stdin.close()
202
self.proc.stdout.close()
203
self.proc.wait()
204
205
206
class LoopbackSFTP(object):
207
"""Simple wrapper for a socket that pretends to be a paramiko Channel."""
208
209
def __init__(self, sock):
210
self.__socket = sock
211
212
def send(self, data):
213
return self.__socket.send(data)
214
215
def recv(self, n):
216
return self.__socket.recv(n)
217
218
def recv_ready(self):
219
return True
220
221
def close(self):
222
self.__socket.close()
223
224
225
SYSTEM_HOSTKEYS = {}
226
BZR_HOSTKEYS = {}
227
228
67
# This is a weakref dictionary, so that we can reuse connections
229
68
# that are still active. Long term, it might be nice to have some
230
69
# sort of expiration policy, such as disconnect if inactive for
231
70
# X seconds. But that requires a lot more fanciness.
232
71
_connected_hosts = weakref.WeakValueDictionary()
233
72
73
74
_paramiko_version = getattr(paramiko, '__version_info__', (0, 0, 0))
75
# don't use prefetch unless paramiko version >= 1.5.5 (there were bugs earlier)
76
_default_do_prefetch = (_paramiko_version >= (1, 5, 5))
77
78
234
79
def clear_connection_cache():
235
80
"""Remove all hosts from the SFTP connection cache.
236
81
239
84
_connected_hosts.clear()
240
85
241
86
242
def load_host_keys():
243
"""
244
Load system host keys (probably doesn't work on windows) and any
245
"discovered" keys from previous sessions.
246
"""
247
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
248
try:
249
SYSTEM_HOSTKEYS = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
250
except Exception, e:
251
mutter('failed to load system host keys: ' + str(e))
252
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
253
try:
254
BZR_HOSTKEYS = paramiko.util.load_host_keys(bzr_hostkey_path)
255
except Exception, e:
256
mutter('failed to load bzr host keys: ' + str(e))
257
save_host_keys()
258
259
260
def save_host_keys():
261
"""
262
Save "discovered" host keys in $(config)/ssh_host_keys/.
263
"""
264
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
265
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
266
ensure_config_dir_exists()
267
268
try:
269
f = open(bzr_hostkey_path, 'w')
270
f.write('# SSH host keys collected by bzr\n')
271
for hostname, keys in BZR_HOSTKEYS.iteritems():
272
for keytype, key in keys.iteritems():
273
f.write('%s %s %s\n' % (hostname, keytype, key.get_base64()))
274
f.close()
275
except IOError, e:
276
mutter('failed to save bzr host keys: ' + str(e))
277
278
279
87
class SFTPLock(object):
280
88
"""This fakes a lock in a remote location."""
281
89
__slots__ = ['path', 'lock_path', 'lock_file', 'transport']
816
624
817
625
TODO: Raise a more reasonable ConnectionFailed exception
818
626
"""
819
global _connected_hosts
820
821
idx = (self._host, self._port, self._username)
822
try:
823
self._sftp = _connected_hosts[idx]
824
return
825
except KeyError:
826
pass
827
828
vendor = _get_ssh_vendor()
829
if vendor == 'loopback':
830
sock = socket.socket()
831
try:
832
sock.connect((self._host, self._port))
833
except socket.error, e:
834
raise ConnectionError('Unable to connect to SSH host %s:%s: %s'
835
% (self._host, self._port, e))
836
self._sftp = SFTPClient(LoopbackSFTP(sock))
837
elif vendor != 'none':
838
try:
839
sock = SFTPSubprocess(self._host, vendor, self._port,
840
self._username)
841
self._sftp = SFTPClient(sock)
842
except (EOFError, paramiko.SSHException), e:
843
raise ConnectionError('Unable to connect to SSH host %s:%s: %s'
844
% (self._host, self._port, e))
845
except (OSError, IOError), e:
846
# If the machine is fast enough, ssh can actually exit
847
# before we try and send it the sftp request, which
848
# raises a Broken Pipe
849
if e.errno not in (errno.EPIPE,):
850
raise
851
raise ConnectionError('Unable to connect to SSH host %s:%s: %s'
852
% (self._host, self._port, e))
853
else:
854
self._paramiko_connect()
855
856
_connected_hosts[idx] = self._sftp
857
858
def _paramiko_connect(self):
859
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
860
861
load_host_keys()
862
863
try:
864
t = paramiko.Transport((self._host, self._port or 22))
865
t.set_log_channel('bzr.paramiko')
866
t.start_client()
867
except (paramiko.SSHException, socket.error), e:
868
raise ConnectionError('Unable to reach SSH host %s:%s: %s'
869
% (self._host, self._port, e))
870
871
server_key = t.get_remote_server_key()
872
server_key_hex = paramiko.util.hexify(server_key.get_fingerprint())
873
keytype = server_key.get_name()
874
if SYSTEM_HOSTKEYS.has_key(self._host) and SYSTEM_HOSTKEYS[self._host].has_key(keytype):
875
our_server_key = SYSTEM_HOSTKEYS[self._host][keytype]
876
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
877
elif BZR_HOSTKEYS.has_key(self._host) and BZR_HOSTKEYS[self._host].has_key(keytype):
878
our_server_key = BZR_HOSTKEYS[self._host][keytype]
879
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
880
else:
881
warning('Adding %s host key for %s: %s' % (keytype, self._host, server_key_hex))
882
if not BZR_HOSTKEYS.has_key(self._host):
883
BZR_HOSTKEYS[self._host] = {}
884
BZR_HOSTKEYS[self._host][keytype] = server_key
885
our_server_key = server_key
886
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
887
save_host_keys()
888
if server_key != our_server_key:
889
filename1 = os.path.expanduser('~/.ssh/known_hosts')
890
filename2 = pathjoin(config_dir(), 'ssh_host_keys')
891
raise TransportError('Host keys for %s do not match! %s != %s' % \
892
(self._host, our_server_key_hex, server_key_hex),
893
['Try editing %s or %s' % (filename1, filename2)])
894
895
self._sftp_auth(t)
896
897
try:
898
self._sftp = t.open_sftp_client()
899
except paramiko.SSHException, e:
900
raise ConnectionError('Unable to start sftp client %s:%d' %
901
(self._host, self._port), e)
902
903
def _sftp_auth(self, transport):
904
# paramiko requires a username, but it might be none if nothing was supplied
905
# use the local username, just in case.
906
# We don't override self._username, because if we aren't using paramiko,
907
# the username might be specified in ~/.ssh/config and we don't want to
908
# force it to something else
909
# Also, it would mess up the self.relpath() functionality
910
username = self._username or getpass.getuser()
911
912
if _use_ssh_agent:
913
agent = paramiko.Agent()
914
for key in agent.get_keys():
915
mutter('Trying SSH agent key %s' % paramiko.util.hexify(key.get_fingerprint()))
916
try:
917
transport.auth_publickey(username, key)
918
return
919
except paramiko.SSHException, e:
920
pass
921
922
# okay, try finding id_rsa or id_dss? (posix only)
923
if self._try_pkey_auth(transport, paramiko.RSAKey, username, 'id_rsa'):
924
return
925
if self._try_pkey_auth(transport, paramiko.DSSKey, username, 'id_dsa'):
926
return
927
928
if self._password:
929
try:
930
transport.auth_password(username, self._password)
931
return
932
except paramiko.SSHException, e:
933
pass
934
935
# FIXME: Don't keep a password held in memory if you can help it
936
#self._password = None
937
938
# give up and ask for a password
939
password = bzrlib.ui.ui_factory.get_password(
940
prompt='SSH %(user)s@%(host)s password',
941
user=username, host=self._host)
942
try:
943
transport.auth_password(username, password)
944
except paramiko.SSHException, e:
945
raise ConnectionError('Unable to authenticate to SSH host as %s@%s' %
946
(username, self._host), e)
947
948
def _try_pkey_auth(self, transport, pkey_class, username, filename):
949
filename = os.path.expanduser('~/.ssh/' + filename)
950
try:
951
key = pkey_class.from_private_key_file(filename)
952
transport.auth_publickey(username, key)
953
return True
954
except paramiko.PasswordRequiredException:
955
password = bzrlib.ui.ui_factory.get_password(
956
prompt='SSH %(filename)s password',
957
filename=filename)
958
try:
959
key = pkey_class.from_private_key_file(filename, password)
960
transport.auth_publickey(username, key)
961
return True
962
except paramiko.SSHException:
963
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
964
except paramiko.SSHException:
965
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
966
except IOError:
967
pass
968
return False
627
self._sftp = _sftp_connect(self._host, self._port, self._username,
628
self._password)
969
629
970
630
def _sftp_open_exclusive(self, abspath, mode=None):
971
631
"""Open a remote path exclusively.
1000
660
1001
661
1002
662
# ------------- server test implementation --------------
1003
import socket
1004
663
import threading
1005
664
1006
665
from bzrlib.tests.stub_sftp import StubServer, StubSFTPServer
1153
812
self._server_homedir = None
1154
813
self._listener = None
1155
814
self._root = None
1156
self._vendor = 'none'
815
self._vendor = ssh.ParamikoVendor()
1157
816
# sftp server logs
1158
817
self.logs = []
1159
818
self.add_latency = 0
1193
852
event.wait(5.0)
1194
853
1195
854
def setUp(self):
1196
global _ssh_vendor
1197
self._original_vendor = _ssh_vendor
1198
_ssh_vendor = self._vendor
855
self._original_vendor = ssh._ssh_vendor
856
ssh._ssh_vendor = self._vendor
1199
857
if sys.platform == 'win32':
1200
858
# Win32 needs to use the UNICODE api
1201
859
self._homedir = getcwd()
1213
871
1214
872
def tearDown(self):
1215
873
"""See bzrlib.transport.Server.tearDown."""
1216
global _ssh_vendor
1217
874
self._listener.stop()
1218
_ssh_vendor = self._original_vendor
875
ssh._ssh_vendor = self._original_vendor
1219
876
1220
877
def get_bogus_url(self):
1221
878
"""See bzrlib.transport.Server.get_bogus_url."""
1240
897
1241
898
def __init__(self):
1242
899
super(SFTPServerWithoutSSH, self).__init__()
1243
self._vendor = 'loopback'
900
self._vendor = ssh.LoopbackVendor()
1244
901
1245
902
def _run_server(self, sock):
903
# Re-import these as locals, so that they're still accessible during
904
# interpreter shutdown (when all module globals get set to None, leading
905
# to confusing errors like "'NoneType' object has no attribute 'error'".
906
import socket, errno
1246
907
class FakeChannel(object):
1247
908
def get_transport(self):
1248
909
return self
1298
959
super(SFTPSiblingAbsoluteServer, self).setUp()
1299
960
1300
961
962
def _sftp_connect(host, port, username, password):
963
"""Connect to the remote sftp server.
964
965
:raises: a TransportError 'could not connect'.
966
967
:returns: an paramiko.sftp_client.SFTPClient
968
969
TODO: Raise a more reasonable ConnectionFailed exception
970
"""
971
idx = (host, port, username)
972
try:
973
return _connected_hosts[idx]
974
except KeyError:
975
pass
976
977
sftp = _sftp_connect_uncached(host, port, username, password)
978
_connected_hosts[idx] = sftp
979
return sftp
980
981
def _sftp_connect_uncached(host, port, username, password):
982
vendor = ssh._get_ssh_vendor()
983
sftp = vendor.connect_sftp(username, password, host, port)
984
return sftp
985
986
1301
987
def get_test_permutations():
1302
988
"""Return the permutations to be used in testing."""
1303
989
return [(SFTPTransport, SFTPAbsoluteServer),
Loggerhead is a web-based interface for Breezy
Version: 2.0.1