1
# Copyright (C) 2005 Robey Pointer <robey@lag.net>, Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Implementation of Transport over SFTP, using paramiko."""
32
from bzrlib.errors import (FileExists,
33
TransportNotPossible, NoSuchFile, PathNotChild,
36
from bzrlib.config import config_dir, ensure_config_dir_exists
37
from bzrlib.trace import mutter, warning, error
38
from bzrlib.transport import Transport, register_transport
39
from bzrlib.osutils import pathjoin, fancy_rename
45
error('The SFTP transport requires paramiko.')
48
from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE,
49
SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC,
51
from paramiko.sftp_attr import SFTPAttributes
52
from paramiko.sftp_file import SFTPFile
53
from paramiko.sftp_client import SFTPClient
55
if 'sftp' not in urlparse.uses_netloc: urlparse.uses_netloc.append('sftp')
59
if sys.platform == 'win32':
60
# close_fds not supported on win32
64
def _get_ssh_vendor():
65
"""Find out what version of SSH is on the system."""
67
if _ssh_vendor is not None:
73
p = subprocess.Popen(['ssh', '-V'],
75
stdin=subprocess.PIPE,
76
stdout=subprocess.PIPE,
77
stderr=subprocess.PIPE)
78
returncode = p.returncode
79
stdout, stderr = p.communicate()
83
if 'OpenSSH' in stderr:
84
mutter('ssh implementation is OpenSSH')
85
_ssh_vendor = 'openssh'
86
elif 'SSH Secure Shell' in stderr:
87
mutter('ssh implementation is SSH Corp.')
90
if _ssh_vendor != 'none':
93
# XXX: 20051123 jamesh
94
# A check for putty's plink or lsh would go here.
96
mutter('falling back to paramiko implementation')
100
class SFTPSubprocess:
101
"""A socket-like object that talks to an ssh subprocess via pipes."""
102
def __init__(self, hostname, port=None, user=None):
103
vendor = _get_ssh_vendor()
104
assert vendor in ['openssh', 'ssh']
105
if vendor == 'openssh':
107
'-oForwardX11=no', '-oForwardAgent=no',
108
'-oClearAllForwardings=yes', '-oProtocol=2',
109
'-oNoHostAuthenticationForLocalhost=yes']
111
args.extend(['-p', str(port)])
113
args.extend(['-l', user])
114
args.extend(['-s', hostname, 'sftp'])
115
elif vendor == 'ssh':
118
args.extend(['-p', str(port)])
120
args.extend(['-l', user])
121
args.extend(['-s', 'sftp', hostname])
123
self.proc = subprocess.Popen(args, close_fds=_close_fds,
124
stdin=subprocess.PIPE,
125
stdout=subprocess.PIPE)
127
def send(self, data):
128
return os.write(self.proc.stdin.fileno(), data)
130
def recv_ready(self):
131
# TODO: jam 20051215 this function is necessary to support the
132
# pipelined() function. In reality, it probably should use
133
# poll() or select() to actually return if there is data
134
# available, otherwise we probably don't get any benefit
137
def recv(self, count):
138
return os.read(self.proc.stdout.fileno(), count)
141
self.proc.stdin.close()
142
self.proc.stdout.close()
149
# This is a weakref dictionary, so that we can reuse connections
150
# that are still active. Long term, it might be nice to have some
151
# sort of expiration policy, such as disconnect if inactive for
152
# X seconds. But that requires a lot more fanciness.
153
_connected_hosts = weakref.WeakValueDictionary()
155
def load_host_keys():
157
Load system host keys (probably doesn't work on windows) and any
158
"discovered" keys from previous sessions.
160
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
162
SYSTEM_HOSTKEYS = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
164
mutter('failed to load system host keys: ' + str(e))
165
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
167
BZR_HOSTKEYS = paramiko.util.load_host_keys(bzr_hostkey_path)
169
mutter('failed to load bzr host keys: ' + str(e))
172
def save_host_keys():
174
Save "discovered" host keys in $(config)/ssh_host_keys/.
176
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
177
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
178
ensure_config_dir_exists()
181
f = open(bzr_hostkey_path, 'w')
182
f.write('# SSH host keys collected by bzr\n')
183
for hostname, keys in BZR_HOSTKEYS.iteritems():
184
for keytype, key in keys.iteritems():
185
f.write('%s %s %s\n' % (hostname, keytype, key.get_base64()))
188
mutter('failed to save bzr host keys: ' + str(e))
191
class SFTPLock(object):
192
"""This fakes a lock in a remote location."""
193
__slots__ = ['path', 'lock_path', 'lock_file', 'transport']
194
def __init__(self, path, transport):
195
assert isinstance(transport, SFTPTransport)
197
self.lock_file = None
199
self.lock_path = path + '.write-lock'
200
self.transport = transport
202
abspath = transport._abspath(self.lock_path)
203
self.lock_file = transport._sftp_open_exclusive(abspath)
205
raise LockError('File %r already locked' % (self.path,))
208
"""Should this warn, or actually try to cleanup?"""
210
warn("SFTPLock %r not explicitly unlocked" % (self.path,))
214
if not self.lock_file:
216
self.lock_file.close()
217
self.lock_file = None
219
self.transport.delete(self.lock_path)
220
except (NoSuchFile,):
221
# What specific errors should we catch here?
224
class SFTPTransport (Transport):
226
Transport implementation for SFTP access.
228
_do_prefetch = False # Right now Paramiko's prefetch support causes things to hang
230
def __init__(self, base, clone_from=None):
231
assert base.startswith('sftp://')
232
self._parse_url(base)
233
base = self._unparse_url()
234
super(SFTPTransport, self).__init__(base)
235
if clone_from is None:
238
# use the same ssh connection, etc
239
self._sftp = clone_from._sftp
240
# super saves 'self.base'
242
def should_cache(self):
244
Return True if the data pulled across should be cached locally.
248
def clone(self, offset=None):
250
Return a new SFTPTransport with root at self.base + offset.
251
We share the same SFTP session between such transports, because it's
252
fairly expensive to set them up.
255
return SFTPTransport(self.base, self)
257
return SFTPTransport(self.abspath(offset), self)
259
def abspath(self, relpath):
261
Return the full url to the given relative path.
263
@param relpath: the relative path or path components
264
@type relpath: str or list
266
return self._unparse_url(self._abspath(relpath))
268
def _abspath(self, relpath):
269
"""Return the absolute path segment without the SFTP URL."""
270
# FIXME: share the common code across transports
271
assert isinstance(relpath, basestring)
272
relpath = [urllib.unquote(relpath)]
273
basepath = self._path.split('/')
274
if len(basepath) > 0 and basepath[-1] == '':
275
basepath = basepath[:-1]
279
if len(basepath) == 0:
280
# In most filesystems, a request for the parent
281
# of root, just returns root.
289
path = '/'.join(basepath)
290
# could still be a "relative" path here, but relative on the sftp server
293
def relpath(self, abspath):
294
username, password, host, port, path = self._split_url(abspath)
296
if (username != self._username):
297
error.append('username mismatch')
298
if (host != self._host):
299
error.append('host mismatch')
300
if (port != self._port):
301
error.append('port mismatch')
302
if (not path.startswith(self._path)):
303
error.append('path mismatch')
305
extra = ': ' + ', '.join(error)
306
raise PathNotChild(abspath, self.base, extra=extra)
308
return path[pl:].lstrip('/')
310
def has(self, relpath):
312
Does the target location exist?
315
self._sftp.stat(self._abspath(relpath))
320
def get(self, relpath, decode=False):
322
Get the file at the given relative path.
324
:param relpath: The relative path to the file
327
path = self._abspath(relpath)
328
f = self._sftp.file(path, mode='rb')
329
if self._do_prefetch and hasattr(f, 'prefetch'):
332
except (IOError, paramiko.SSHException), e:
333
self._translate_io_exception(e, path, ': error retrieving')
335
def get_partial(self, relpath, start, length=None):
337
Get just part of a file.
339
:param relpath: Path to the file, relative to base
340
:param start: The starting position to read from
341
:param length: The length to read. A length of None indicates
342
read to the end of the file.
343
:return: A file-like object containing at least the specified bytes.
344
Some implementations may return objects which can be read
345
past this length, but this is not guaranteed.
347
# TODO: implement get_partial_multi to help with knit support
348
f = self.get(relpath)
350
if self._do_prefetch and hasattr(f, 'prefetch'):
354
def put(self, relpath, f, mode=None):
356
Copy the file-like or string object into the location.
358
:param relpath: Location to put the contents, relative to base.
359
:param f: File-like or string object.
360
:param mode: The final mode for the file
362
final_path = self._abspath(relpath)
363
self._put(final_path, f, mode=mode)
365
def _put(self, abspath, f, mode=None):
366
"""Helper function so both put() and copy_abspaths can reuse the code"""
367
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
368
os.getpid(), random.randint(0,0x7FFFFFFF))
369
fout = self._sftp_open_exclusive(tmp_abspath, mode=mode)
373
fout.set_pipelined(True)
375
except (IOError, paramiko.SSHException), e:
376
self._translate_io_exception(e, tmp_abspath)
378
self._sftp.chmod(tmp_abspath, mode)
381
self._rename(tmp_abspath, abspath)
383
# If we fail, try to clean up the temporary file
384
# before we throw the exception
385
# but don't let another exception mess things up
386
# Write out the traceback, because otherwise
387
# the catch and throw destroys it
389
mutter(traceback.format_exc())
393
self._sftp.remove(tmp_abspath)
395
# raise the saved except
397
# raise the original with its traceback if we can.
400
def iter_files_recursive(self):
401
"""Walk the relative paths of all files in this transport."""
402
queue = list(self.list_dir('.'))
404
relpath = urllib.quote(queue.pop(0))
405
st = self.stat(relpath)
406
if stat.S_ISDIR(st.st_mode):
407
for i, basename in enumerate(self.list_dir(relpath)):
408
queue.insert(i, relpath+'/'+basename)
412
def mkdir(self, relpath, mode=None):
413
"""Create a directory at the given path."""
415
path = self._abspath(relpath)
416
# In the paramiko documentation, it says that passing a mode flag
417
# will filtered against the server umask.
418
# StubSFTPServer does not do this, which would be nice, because it is
419
# what we really want :)
420
# However, real servers do use umask, so we really should do it that way
421
self._sftp.mkdir(path)
423
self._sftp.chmod(path, mode=mode)
424
except (paramiko.SSHException, IOError), e:
425
self._translate_io_exception(e, path, ': unable to mkdir',
426
failure_exc=FileExists)
428
def _translate_io_exception(self, e, path, more_info='', failure_exc=NoSuchFile):
429
"""Translate a paramiko or IOError into a friendlier exception.
431
:param e: The original exception
432
:param path: The path in question when the error is raised
433
:param more_info: Extra information that can be included,
434
such as what was going on
435
:param failure_exc: Paramiko has the super fun ability to raise completely
436
opaque errors that just set "e.args = ('Failure',)" with
438
This sometimes means FileExists, but it also sometimes
441
# paramiko seems to generate detailless errors.
442
self._translate_error(e, path, raise_generic=False)
443
if hasattr(e, 'args'):
444
if (e.args == ('No such file or directory',) or
445
e.args == ('No such file',)):
446
raise NoSuchFile(path, str(e) + more_info)
447
if (e.args == ('mkdir failed',)):
448
raise FileExists(path, str(e) + more_info)
449
# strange but true, for the paramiko server.
450
if (e.args == ('Failure',)):
451
raise failure_exc(path, str(e) + more_info)
452
mutter('Raising exception with args %s', e.args)
453
if hasattr(e, 'errno'):
454
mutter('Raising exception with errno %s', e.errno)
457
def append(self, relpath, f):
459
Append the text in the file-like object into the final
463
path = self._abspath(relpath)
464
fout = self._sftp.file(path, 'ab')
466
except (IOError, paramiko.SSHException), e:
467
self._translate_io_exception(e, relpath, ': unable to append')
469
def copy(self, rel_from, rel_to):
470
"""Copy the item at rel_from to the location at rel_to"""
471
path_from = self._abspath(rel_from)
472
path_to = self._abspath(rel_to)
473
self._copy_abspaths(path_from, path_to)
475
def _copy_abspaths(self, path_from, path_to, mode=None):
476
"""Copy files given an absolute path
478
:param path_from: Path on remote server to read
479
:param path_to: Path on remote server to write
482
TODO: Should the destination location be atomically created?
483
This has not been specified
484
TODO: This should use some sort of remote copy, rather than
485
pulling the data locally, and then writing it remotely
488
fin = self._sftp.file(path_from, 'rb')
490
self._put(path_to, fin, mode=mode)
493
except (IOError, paramiko.SSHException), e:
494
self._translate_io_exception(e, path_from, ': unable copy to: %r' % path_to)
496
def copy_to(self, relpaths, other, mode=None, pb=None):
497
"""Copy a set of entries from self into another Transport.
499
:param relpaths: A list/generator of entries to be copied.
501
if isinstance(other, SFTPTransport) and other._sftp is self._sftp:
502
# Both from & to are on the same remote filesystem
503
# We can use a remote copy, instead of pulling locally, and pushing
505
total = self._get_total(relpaths)
507
for path in relpaths:
508
path_from = self._abspath(relpath)
509
path_to = other._abspath(relpath)
510
self._update_pb(pb, 'copy-to', count, total)
511
self._copy_abspaths(path_from, path_to, mode=mode)
515
return super(SFTPTransport, self).copy_to(relpaths, other, mode=mode, pb=pb)
517
def _rename(self, abs_from, abs_to):
518
"""Do a fancy rename on the remote server.
520
Using the implementation provided by osutils.
523
fancy_rename(abs_from, abs_to,
524
rename_func=self._sftp.rename,
525
unlink_func=self._sftp.remove)
526
except (IOError, paramiko.SSHException), e:
527
self._translate_io_exception(e, abs_from, ': unable to rename to %r' % (abs_to))
529
def move(self, rel_from, rel_to):
530
"""Move the item at rel_from to the location at rel_to"""
531
path_from = self._abspath(rel_from)
532
path_to = self._abspath(rel_to)
533
self._rename(path_from, path_to)
535
def delete(self, relpath):
536
"""Delete the item at relpath"""
537
path = self._abspath(relpath)
539
self._sftp.remove(path)
540
except (IOError, paramiko.SSHException), e:
541
self._translate_io_exception(e, path, ': unable to delete')
544
"""Return True if this store supports listing."""
547
def list_dir(self, relpath):
549
Return a list of all files at the given location.
551
# does anything actually use this?
552
path = self._abspath(relpath)
554
return self._sftp.listdir(path)
555
except (IOError, paramiko.SSHException), e:
556
self._translate_io_exception(e, path, ': failed to list_dir')
558
def stat(self, relpath):
559
"""Return the stat information for a file."""
560
path = self._abspath(relpath)
562
return self._sftp.stat(path)
563
except (IOError, paramiko.SSHException), e:
564
self._translate_io_exception(e, path, ': unable to stat')
566
def lock_read(self, relpath):
568
Lock the given file for shared (read) access.
569
:return: A lock object, which has an unlock() member function
571
# FIXME: there should be something clever i can do here...
572
class BogusLock(object):
573
def __init__(self, path):
577
return BogusLock(relpath)
579
def lock_write(self, relpath):
581
Lock the given file for exclusive (write) access.
582
WARNING: many transports do not support this, so trying avoid using it
584
:return: A lock object, which has an unlock() member function
586
# This is a little bit bogus, but basically, we create a file
587
# which should not already exist, and if it does, we assume
588
# that there is a lock, and if it doesn't, the we assume
589
# that we have taken the lock.
590
return SFTPLock(relpath, self)
593
def _unparse_url(self, path=None):
596
path = urllib.quote(path)
597
if path.startswith('/'):
598
path = '/%2F' + path[1:]
601
netloc = urllib.quote(self._host)
602
if self._username is not None:
603
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
604
if self._port is not None:
605
netloc = '%s:%d' % (netloc, self._port)
607
return urlparse.urlunparse(('sftp', netloc, path, '', '', ''))
609
def _split_url(self, url):
610
if isinstance(url, unicode):
611
url = url.encode('utf-8')
612
(scheme, netloc, path, params,
613
query, fragment) = urlparse.urlparse(url, allow_fragments=False)
614
assert scheme == 'sftp'
615
username = password = host = port = None
617
username, host = netloc.split('@', 1)
619
username, password = username.split(':', 1)
620
password = urllib.unquote(password)
621
username = urllib.unquote(username)
626
host, port = host.rsplit(':', 1)
630
# TODO: Should this be ConnectionError?
631
raise TransportError('%s: invalid port number' % port)
632
host = urllib.unquote(host)
634
path = urllib.unquote(path)
636
# the initial slash should be removed from the path, and treated
637
# as a homedir relative path (the path begins with a double slash
638
# if it is absolute).
639
# see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt
640
if path.startswith('/'):
643
return (username, password, host, port, path)
645
def _parse_url(self, url):
646
(self._username, self._password,
647
self._host, self._port, self._path) = self._split_url(url)
649
def _sftp_connect(self):
650
"""Connect to the remote sftp server.
651
After this, self._sftp should have a valid connection (or
652
we raise an TransportError 'could not connect').
654
TODO: Raise a more reasonable ConnectionFailed exception
656
global _connected_hosts
658
idx = (self._host, self._port, self._username)
660
self._sftp = _connected_hosts[idx]
665
vendor = _get_ssh_vendor()
667
sock = SFTPSubprocess(self._host, self._port, self._username)
668
self._sftp = SFTPClient(sock)
670
self._paramiko_connect()
672
_connected_hosts[idx] = self._sftp
674
def _paramiko_connect(self):
675
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
680
t = paramiko.Transport((self._host, self._port or 22))
682
except paramiko.SSHException, e:
683
raise ConnectionError('Unable to reach SSH host %s:%d' %
684
(self._host, self._port), e)
686
server_key = t.get_remote_server_key()
687
server_key_hex = paramiko.util.hexify(server_key.get_fingerprint())
688
keytype = server_key.get_name()
689
if SYSTEM_HOSTKEYS.has_key(self._host) and SYSTEM_HOSTKEYS[self._host].has_key(keytype):
690
our_server_key = SYSTEM_HOSTKEYS[self._host][keytype]
691
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
692
elif BZR_HOSTKEYS.has_key(self._host) and BZR_HOSTKEYS[self._host].has_key(keytype):
693
our_server_key = BZR_HOSTKEYS[self._host][keytype]
694
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
696
warning('Adding %s host key for %s: %s' % (keytype, self._host, server_key_hex))
697
if not BZR_HOSTKEYS.has_key(self._host):
698
BZR_HOSTKEYS[self._host] = {}
699
BZR_HOSTKEYS[self._host][keytype] = server_key
700
our_server_key = server_key
701
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
703
if server_key != our_server_key:
704
filename1 = os.path.expanduser('~/.ssh/known_hosts')
705
filename2 = pathjoin(config_dir(), 'ssh_host_keys')
706
raise TransportError('Host keys for %s do not match! %s != %s' % \
707
(self._host, our_server_key_hex, server_key_hex),
708
['Try editing %s or %s' % (filename1, filename2)])
713
self._sftp = t.open_sftp_client()
714
except paramiko.SSHException, e:
715
raise ConnectionError('Unable to start sftp client %s:%d' %
716
(self._host, self._port), e)
718
def _sftp_auth(self, transport):
719
# paramiko requires a username, but it might be none if nothing was supplied
720
# use the local username, just in case.
721
# We don't override self._username, because if we aren't using paramiko,
722
# the username might be specified in ~/.ssh/config and we don't want to
723
# force it to something else
724
# Also, it would mess up the self.relpath() functionality
725
username = self._username or getpass.getuser()
727
# Paramiko tries to open a socket.AF_UNIX in order to connect
728
# to ssh-agent. That attribute doesn't exist on win32 (it does in cygwin)
729
# so we get an AttributeError exception. For now, just don't try to
730
# connect to an agent if we are on win32
731
if sys.platform != 'win32':
732
agent = paramiko.Agent()
733
for key in agent.get_keys():
734
mutter('Trying SSH agent key %s' % paramiko.util.hexify(key.get_fingerprint()))
736
transport.auth_publickey(username, key)
738
except paramiko.SSHException, e:
741
# okay, try finding id_rsa or id_dss? (posix only)
742
if self._try_pkey_auth(transport, paramiko.RSAKey, username, 'id_rsa'):
744
if self._try_pkey_auth(transport, paramiko.DSSKey, username, 'id_dsa'):
750
transport.auth_password(username, self._password)
752
except paramiko.SSHException, e:
755
# FIXME: Don't keep a password held in memory if you can help it
756
#self._password = None
758
# give up and ask for a password
759
password = bzrlib.ui.ui_factory.get_password(
760
prompt='SSH %(user)s@%(host)s password',
761
user=username, host=self._host)
763
transport.auth_password(username, password)
764
except paramiko.SSHException, e:
765
raise ConnectionError('Unable to authenticate to SSH host as %s@%s' %
766
(username, self._host), e)
768
def _try_pkey_auth(self, transport, pkey_class, username, filename):
769
filename = os.path.expanduser('~/.ssh/' + filename)
771
key = pkey_class.from_private_key_file(filename)
772
transport.auth_publickey(username, key)
774
except paramiko.PasswordRequiredException:
775
password = bzrlib.ui.ui_factory.get_password(
776
prompt='SSH %(filename)s password',
779
key = pkey_class.from_private_key_file(filename, password)
780
transport.auth_publickey(username, key)
782
except paramiko.SSHException:
783
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
784
except paramiko.SSHException:
785
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
790
def _sftp_open_exclusive(self, abspath, mode=None):
791
"""Open a remote path exclusively.
793
SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if
794
the file already exists. However it does not expose this
795
at the higher level of SFTPClient.open(), so we have to
798
WARNING: This breaks the SFTPClient abstraction, so it
799
could easily break against an updated version of paramiko.
801
:param abspath: The remote absolute path where the file should be opened
802
:param mode: The mode permissions bits for the new file
804
path = self._sftp._adjust_cwd(abspath)
805
attr = SFTPAttributes()
808
omode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
809
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
811
t, msg = self._sftp._request(CMD_OPEN, path, omode, attr)
813
raise TransportError('Expected an SFTP handle')
814
handle = msg.get_string()
815
return SFTPFile(self._sftp, handle, 'wb', -1)
816
except (paramiko.SSHException, IOError), e:
817
self._translate_io_exception(e, abspath, ': unable to open',
818
failure_exc=FileExists)
added
removed
bzrlib/transport/sftp.py
