1
# Copyright (C) 2005 Robey Pointer <robey@lag.net>, Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Implementation of Transport over SFTP, using paramiko."""
32
from bzrlib.errors import (FileExists,
33
TransportNotPossible, NoSuchFile, PathNotChild,
36
from bzrlib.config import config_dir, ensure_config_dir_exists
37
from bzrlib.trace import mutter, warning, error
38
from bzrlib.transport import Transport, register_transport
39
from bzrlib.osutils import pathjoin, fancy_rename
45
error('The SFTP transport requires paramiko.')
48
from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE,
49
SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC,
51
from paramiko.sftp_attr import SFTPAttributes
52
from paramiko.sftp_file import SFTPFile
53
from paramiko.sftp_client import SFTPClient
55
if 'sftp' not in urlparse.uses_netloc: urlparse.uses_netloc.append('sftp')
59
if sys.platform == 'win32':
60
# close_fds not supported on win32
64
def _get_ssh_vendor():
65
"""Find out what version of SSH is on the system."""
67
if _ssh_vendor is not None:
73
p = subprocess.Popen(['ssh', '-V'],
75
stdin=subprocess.PIPE,
76
stdout=subprocess.PIPE,
77
stderr=subprocess.PIPE)
78
returncode = p.returncode
79
stdout, stderr = p.communicate()
83
if 'OpenSSH' in stderr:
84
mutter('ssh implementation is OpenSSH')
85
_ssh_vendor = 'openssh'
86
elif 'SSH Secure Shell' in stderr:
87
mutter('ssh implementation is SSH Corp.')
90
if _ssh_vendor != 'none':
93
# XXX: 20051123 jamesh
94
# A check for putty's plink or lsh would go here.
96
mutter('falling back to paramiko implementation')
100
class SFTPSubprocess:
101
"""A socket-like object that talks to an ssh subprocess via pipes."""
102
def __init__(self, hostname, port=None, user=None):
103
vendor = _get_ssh_vendor()
104
assert vendor in ['openssh', 'ssh']
105
if vendor == 'openssh':
107
'-oForwardX11=no', '-oForwardAgent=no',
108
'-oClearAllForwardings=yes', '-oProtocol=2',
109
'-oNoHostAuthenticationForLocalhost=yes']
111
args.extend(['-p', str(port)])
113
args.extend(['-l', user])
114
args.extend(['-s', hostname, 'sftp'])
115
elif vendor == 'ssh':
118
args.extend(['-p', str(port)])
120
args.extend(['-l', user])
121
args.extend(['-s', 'sftp', hostname])
123
self.proc = subprocess.Popen(args, close_fds=_close_fds,
124
stdin=subprocess.PIPE,
125
stdout=subprocess.PIPE)
127
def send(self, data):
128
return os.write(self.proc.stdin.fileno(), data)
130
def recv_ready(self):
131
# TODO: jam 20051215 this function is necessary to support the
132
# pipelined() function. In reality, it probably should use
133
# poll() or select() to actually return if there is data
134
# available, otherwise we probably don't get any benefit
137
def recv(self, count):
138
return os.read(self.proc.stdout.fileno(), count)
141
self.proc.stdin.close()
142
self.proc.stdout.close()
149
# This is a weakref dictionary, so that we can reuse connections
150
# that are still active. Long term, it might be nice to have some
151
# sort of expiration policy, such as disconnect if inactive for
152
# X seconds. But that requires a lot more fanciness.
153
_connected_hosts = weakref.WeakValueDictionary()
155
def load_host_keys():
157
Load system host keys (probably doesn't work on windows) and any
158
"discovered" keys from previous sessions.
160
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
162
SYSTEM_HOSTKEYS = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
164
mutter('failed to load system host keys: ' + str(e))
165
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
167
BZR_HOSTKEYS = paramiko.util.load_host_keys(bzr_hostkey_path)
169
mutter('failed to load bzr host keys: ' + str(e))
172
def save_host_keys():
174
Save "discovered" host keys in $(config)/ssh_host_keys/.
176
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
177
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
178
ensure_config_dir_exists()
181
f = open(bzr_hostkey_path, 'w')
182
f.write('# SSH host keys collected by bzr\n')
183
for hostname, keys in BZR_HOSTKEYS.iteritems():
184
for keytype, key in keys.iteritems():
185
f.write('%s %s %s\n' % (hostname, keytype, key.get_base64()))
188
mutter('failed to save bzr host keys: ' + str(e))
191
class SFTPLock(object):
192
"""This fakes a lock in a remote location."""
193
__slots__ = ['path', 'lock_path', 'lock_file', 'transport']
194
def __init__(self, path, transport):
195
assert isinstance(transport, SFTPTransport)
197
self.lock_file = None
199
self.lock_path = path + '.write-lock'
200
self.transport = transport
202
abspath = transport._abspath(self.lock_path)
203
self.lock_file = transport._sftp_open_exclusive(abspath)
205
raise LockError('File %r already locked' % (self.path,))
208
"""Should this warn, or actually try to cleanup?"""
210
warn("SFTPLock %r not explicitly unlocked" % (self.path,))
214
if not self.lock_file:
216
self.lock_file.close()
217
self.lock_file = None
219
self.transport.delete(self.lock_path)
220
except (NoSuchFile,):
221
# What specific errors should we catch here?
224
class SFTPTransport (Transport):
226
Transport implementation for SFTP access.
228
_do_prefetch = False # Right now Paramiko's prefetch support causes things to hang
230
def __init__(self, base, clone_from=None):
231
assert base.startswith('sftp://')
232
self._parse_url(base)
233
base = self._unparse_url()
234
super(SFTPTransport, self).__init__(base)
235
if clone_from is None:
238
# use the same ssh connection, etc
239
self._sftp = clone_from._sftp
240
# super saves 'self.base'
242
def should_cache(self):
244
Return True if the data pulled across should be cached locally.
248
def clone(self, offset=None):
250
Return a new SFTPTransport with root at self.base + offset.
251
We share the same SFTP session between such transports, because it's
252
fairly expensive to set them up.
255
return SFTPTransport(self.base, self)
257
return SFTPTransport(self.abspath(offset), self)
259
def abspath(self, relpath):
261
Return the full url to the given relative path.
263
@param relpath: the relative path or path components
264
@type relpath: str or list
266
return self._unparse_url(self._abspath(relpath))
268
def _abspath(self, relpath):
269
"""Return the absolute path segment without the SFTP URL."""
270
# FIXME: share the common code across transports
271
assert isinstance(relpath, basestring)
272
relpath = [urllib.unquote(relpath)]
273
basepath = self._path.split('/')
274
if len(basepath) > 0 and basepath[-1] == '':
275
basepath = basepath[:-1]
279
if len(basepath) == 0:
280
# In most filesystems, a request for the parent
281
# of root, just returns root.
289
path = '/'.join(basepath)
290
# could still be a "relative" path here, but relative on the sftp server
293
def relpath(self, abspath):
294
username, password, host, port, path = self._split_url(abspath)
296
if (username != self._username):
297
error.append('username mismatch')
298
if (host != self._host):
299
error.append('host mismatch')
300
if (port != self._port):
301
error.append('port mismatch')
302
if (not path.startswith(self._path)):
303
error.append('path mismatch')
305
extra = ': ' + ', '.join(error)
306
raise PathNotChild(abspath, self.base, extra=extra)
308
return path[pl:].lstrip('/')
310
def has(self, relpath):
312
Does the target location exist?
315
self._sftp.stat(self._abspath(relpath))
320
def get(self, relpath, decode=False):
322
Get the file at the given relative path.
324
:param relpath: The relative path to the file
327
path = self._abspath(relpath)
328
f = self._sftp.file(path, mode='rb')
329
if self._do_prefetch and hasattr(f, 'prefetch'):
332
except (IOError, paramiko.SSHException), e:
333
self._translate_io_exception(e, path, ': error retrieving')
335
def get_partial(self, relpath, start, length=None):
337
Get just part of a file.
339
:param relpath: Path to the file, relative to base
340
:param start: The starting position to read from
341
:param length: The length to read. A length of None indicates
342
read to the end of the file.
343
:return: A file-like object containing at least the specified bytes.
344
Some implementations may return objects which can be read
345
past this length, but this is not guaranteed.
347
# TODO: implement get_partial_multi to help with knit support
348
f = self.get(relpath)
350
if self._do_prefetch and hasattr(f, 'prefetch'):
354
def put(self, relpath, f, mode=None):
356
Copy the file-like or string object into the location.
358
:param relpath: Location to put the contents, relative to base.
359
:param f: File-like or string object.
360
:param mode: The final mode for the file
362
final_path = self._abspath(relpath)
363
self._put(final_path, f, mode=mode)
365
def _put(self, abspath, f, mode=None):
366
"""Helper function so both put() and copy_abspaths can reuse the code"""
367
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
368
os.getpid(), random.randint(0,0x7FFFFFFF))
369
fout = self._sftp_open_exclusive(tmp_abspath, mode=mode)
373
fout.set_pipelined(True)
375
except (IOError, paramiko.SSHException), e:
376
self._translate_io_exception(e, tmp_abspath)
378
self._sftp.chmod(tmp_abspath, mode)
381
self._rename(tmp_abspath, abspath)
383
# If we fail, try to clean up the temporary file
384
# before we throw the exception
385
# but don't let another exception mess things up
386
# Write out the traceback, because otherwise
387
# the catch and throw destroys it
389
mutter(traceback.format_exc())
393
self._sftp.remove(tmp_abspath)
398
def iter_files_recursive(self):
399
"""Walk the relative paths of all files in this transport."""
400
queue = list(self.list_dir('.'))
402
relpath = urllib.quote(queue.pop(0))
403
st = self.stat(relpath)
404
if stat.S_ISDIR(st.st_mode):
405
for i, basename in enumerate(self.list_dir(relpath)):
406
queue.insert(i, relpath+'/'+basename)
410
def mkdir(self, relpath, mode=None):
411
"""Create a directory at the given path."""
413
path = self._abspath(relpath)
414
# In the paramiko documentation, it says that passing a mode flag
415
# will filtered against the server umask.
416
# StubSFTPServer does not do this, which would be nice, because it is
417
# what we really want :)
418
# However, real servers do use umask, so we really should do it that way
419
self._sftp.mkdir(path)
421
self._sftp.chmod(path, mode=mode)
422
except (paramiko.SSHException, IOError), e:
423
self._translate_io_exception(e, path, ': unable to mkdir',
424
failure_exc=FileExists)
426
def _translate_io_exception(self, e, path, more_info='', failure_exc=NoSuchFile):
427
"""Translate a paramiko or IOError into a friendlier exception.
429
:param e: The original exception
430
:param path: The path in question when the error is raised
431
:param more_info: Extra information that can be included,
432
such as what was going on
433
:param failure_exc: Paramiko has the super fun ability to raise completely
434
opaque errors that just set "e.args = ('Failure',)" with
436
This sometimes means FileExists, but it also sometimes
439
# paramiko seems to generate detailless errors.
440
self._translate_error(e, path, raise_generic=False)
441
if hasattr(e, 'args'):
442
if (e.args == ('No such file or directory',) or
443
e.args == ('No such file',)):
444
raise NoSuchFile(path, str(e) + more_info)
445
if (e.args == ('mkdir failed',)):
446
raise FileExists(path, str(e) + more_info)
447
# strange but true, for the paramiko server.
448
if (e.args == ('Failure',)):
449
raise failure_exc(path, str(e) + more_info)
450
mutter('Raising exception with args %s', e.args)
451
if hasattr(e, 'errno'):
452
mutter('Raising exception with errno %s', e.errno)
455
def append(self, relpath, f):
457
Append the text in the file-like object into the final
461
path = self._abspath(relpath)
462
fout = self._sftp.file(path, 'ab')
464
except (IOError, paramiko.SSHException), e:
465
self._translate_io_exception(e, relpath, ': unable to append')
467
def copy(self, rel_from, rel_to):
468
"""Copy the item at rel_from to the location at rel_to"""
469
path_from = self._abspath(rel_from)
470
path_to = self._abspath(rel_to)
471
self._copy_abspaths(path_from, path_to)
473
def _copy_abspaths(self, path_from, path_to, mode=None):
474
"""Copy files given an absolute path
476
:param path_from: Path on remote server to read
477
:param path_to: Path on remote server to write
480
TODO: Should the destination location be atomically created?
481
This has not been specified
482
TODO: This should use some sort of remote copy, rather than
483
pulling the data locally, and then writing it remotely
486
fin = self._sftp.file(path_from, 'rb')
488
self._put(path_to, fin, mode=mode)
491
except (IOError, paramiko.SSHException), e:
492
self._translate_io_exception(e, path_from, ': unable copy to: %r' % path_to)
494
def copy_to(self, relpaths, other, mode=None, pb=None):
495
"""Copy a set of entries from self into another Transport.
497
:param relpaths: A list/generator of entries to be copied.
499
if isinstance(other, SFTPTransport) and other._sftp is self._sftp:
500
# Both from & to are on the same remote filesystem
501
# We can use a remote copy, instead of pulling locally, and pushing
503
total = self._get_total(relpaths)
505
for path in relpaths:
506
path_from = self._abspath(relpath)
507
path_to = other._abspath(relpath)
508
self._update_pb(pb, 'copy-to', count, total)
509
self._copy_abspaths(path_from, path_to, mode=mode)
513
return super(SFTPTransport, self).copy_to(relpaths, other, mode=mode, pb=pb)
515
def _rename(self, abs_from, abs_to):
516
"""Do a fancy rename on the remote server.
518
Using the implementation provided by osutils.
521
fancy_rename(abs_from, abs_to,
522
rename_func=self._sftp.rename,
523
unlink_func=self._sftp.remove)
524
except (IOError, paramiko.SSHException), e:
525
self._translate_io_exception(e, abs_from, ': unable to rename to %r' % (abs_to))
527
def move(self, rel_from, rel_to):
528
"""Move the item at rel_from to the location at rel_to"""
529
path_from = self._abspath(rel_from)
530
path_to = self._abspath(rel_to)
531
self._rename(path_from, path_to)
533
def delete(self, relpath):
534
"""Delete the item at relpath"""
535
path = self._abspath(relpath)
537
self._sftp.remove(path)
538
except (IOError, paramiko.SSHException), e:
539
self._translate_io_exception(e, path, ': unable to delete')
542
"""Return True if this store supports listing."""
545
def list_dir(self, relpath):
547
Return a list of all files at the given location.
549
# does anything actually use this?
550
path = self._abspath(relpath)
552
return self._sftp.listdir(path)
553
except (IOError, paramiko.SSHException), e:
554
self._translate_io_exception(e, path, ': failed to list_dir')
556
def stat(self, relpath):
557
"""Return the stat information for a file."""
558
path = self._abspath(relpath)
560
return self._sftp.stat(path)
561
except (IOError, paramiko.SSHException), e:
562
self._translate_io_exception(e, path, ': unable to stat')
564
def lock_read(self, relpath):
566
Lock the given file for shared (read) access.
567
:return: A lock object, which has an unlock() member function
569
# FIXME: there should be something clever i can do here...
570
class BogusLock(object):
571
def __init__(self, path):
575
return BogusLock(relpath)
577
def lock_write(self, relpath):
579
Lock the given file for exclusive (write) access.
580
WARNING: many transports do not support this, so trying avoid using it
582
:return: A lock object, which has an unlock() member function
584
# This is a little bit bogus, but basically, we create a file
585
# which should not already exist, and if it does, we assume
586
# that there is a lock, and if it doesn't, the we assume
587
# that we have taken the lock.
588
return SFTPLock(relpath, self)
591
def _unparse_url(self, path=None):
594
path = urllib.quote(path)
595
if path.startswith('/'):
596
path = '/%2F' + path[1:]
599
netloc = urllib.quote(self._host)
600
if self._username is not None:
601
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
602
if self._port is not None:
603
netloc = '%s:%d' % (netloc, self._port)
605
return urlparse.urlunparse(('sftp', netloc, path, '', '', ''))
607
def _split_url(self, url):
608
if isinstance(url, unicode):
609
url = url.encode('utf-8')
610
(scheme, netloc, path, params,
611
query, fragment) = urlparse.urlparse(url, allow_fragments=False)
612
assert scheme == 'sftp'
613
username = password = host = port = None
615
username, host = netloc.split('@', 1)
617
username, password = username.split(':', 1)
618
password = urllib.unquote(password)
619
username = urllib.unquote(username)
624
host, port = host.rsplit(':', 1)
628
# TODO: Should this be ConnectionError?
629
raise TransportError('%s: invalid port number' % port)
630
host = urllib.unquote(host)
632
path = urllib.unquote(path)
634
# the initial slash should be removed from the path, and treated
635
# as a homedir relative path (the path begins with a double slash
636
# if it is absolute).
637
# see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt
638
if path.startswith('/'):
641
return (username, password, host, port, path)
643
def _parse_url(self, url):
644
(self._username, self._password,
645
self._host, self._port, self._path) = self._split_url(url)
647
def _sftp_connect(self):
648
"""Connect to the remote sftp server.
649
After this, self._sftp should have a valid connection (or
650
we raise an TransportError 'could not connect').
652
TODO: Raise a more reasonable ConnectionFailed exception
654
global _connected_hosts
656
idx = (self._host, self._port, self._username)
658
self._sftp = _connected_hosts[idx]
663
vendor = _get_ssh_vendor()
665
sock = SFTPSubprocess(self._host, self._port, self._username)
666
self._sftp = SFTPClient(sock)
668
self._paramiko_connect()
670
_connected_hosts[idx] = self._sftp
672
def _paramiko_connect(self):
673
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
678
t = paramiko.Transport((self._host, self._port or 22))
680
except paramiko.SSHException, e:
681
raise ConnectionError('Unable to reach SSH host %s:%d' %
682
(self._host, self._port), e)
684
server_key = t.get_remote_server_key()
685
server_key_hex = paramiko.util.hexify(server_key.get_fingerprint())
686
keytype = server_key.get_name()
687
if SYSTEM_HOSTKEYS.has_key(self._host) and SYSTEM_HOSTKEYS[self._host].has_key(keytype):
688
our_server_key = SYSTEM_HOSTKEYS[self._host][keytype]
689
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
690
elif BZR_HOSTKEYS.has_key(self._host) and BZR_HOSTKEYS[self._host].has_key(keytype):
691
our_server_key = BZR_HOSTKEYS[self._host][keytype]
692
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
694
warning('Adding %s host key for %s: %s' % (keytype, self._host, server_key_hex))
695
if not BZR_HOSTKEYS.has_key(self._host):
696
BZR_HOSTKEYS[self._host] = {}
697
BZR_HOSTKEYS[self._host][keytype] = server_key
698
our_server_key = server_key
699
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
701
if server_key != our_server_key:
702
filename1 = os.path.expanduser('~/.ssh/known_hosts')
703
filename2 = pathjoin(config_dir(), 'ssh_host_keys')
704
raise TransportError('Host keys for %s do not match! %s != %s' % \
705
(self._host, our_server_key_hex, server_key_hex),
706
['Try editing %s or %s' % (filename1, filename2)])
711
self._sftp = t.open_sftp_client()
712
except paramiko.SSHException, e:
713
raise ConnectionError('Unable to start sftp client %s:%d' %
714
(self._host, self._port), e)
716
def _sftp_auth(self, transport):
717
# paramiko requires a username, but it might be none if nothing was supplied
718
# use the local username, just in case.
719
# We don't override self._username, because if we aren't using paramiko,
720
# the username might be specified in ~/.ssh/config and we don't want to
721
# force it to something else
722
# Also, it would mess up the self.relpath() functionality
723
username = self._username or getpass.getuser()
725
# Paramiko tries to open a socket.AF_UNIX in order to connect
726
# to ssh-agent. That attribute doesn't exist on win32 (it does in cygwin)
727
# so we get an AttributeError exception. For now, just don't try to
728
# connect to an agent if we are on win32
729
if sys.platform != 'win32':
730
agent = paramiko.Agent()
731
for key in agent.get_keys():
732
mutter('Trying SSH agent key %s' % paramiko.util.hexify(key.get_fingerprint()))
734
transport.auth_publickey(username, key)
736
except paramiko.SSHException, e:
739
# okay, try finding id_rsa or id_dss? (posix only)
740
if self._try_pkey_auth(transport, paramiko.RSAKey, username, 'id_rsa'):
742
if self._try_pkey_auth(transport, paramiko.DSSKey, username, 'id_dsa'):
748
transport.auth_password(username, self._password)
750
except paramiko.SSHException, e:
753
# FIXME: Don't keep a password held in memory if you can help it
754
#self._password = None
756
# give up and ask for a password
757
password = bzrlib.ui.ui_factory.get_password(
758
prompt='SSH %(user)s@%(host)s password',
759
user=username, host=self._host)
761
transport.auth_password(username, password)
762
except paramiko.SSHException, e:
763
raise ConnectionError('Unable to authenticate to SSH host as %s@%s' %
764
(username, self._host), e)
766
def _try_pkey_auth(self, transport, pkey_class, username, filename):
767
filename = os.path.expanduser('~/.ssh/' + filename)
769
key = pkey_class.from_private_key_file(filename)
770
transport.auth_publickey(username, key)
772
except paramiko.PasswordRequiredException:
773
password = bzrlib.ui.ui_factory.get_password(
774
prompt='SSH %(filename)s password',
777
key = pkey_class.from_private_key_file(filename, password)
778
transport.auth_publickey(username, key)
780
except paramiko.SSHException:
781
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
782
except paramiko.SSHException:
783
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
788
def _sftp_open_exclusive(self, abspath, mode=None):
789
"""Open a remote path exclusively.
791
SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if
792
the file already exists. However it does not expose this
793
at the higher level of SFTPClient.open(), so we have to
796
WARNING: This breaks the SFTPClient abstraction, so it
797
could easily break against an updated version of paramiko.
799
:param abspath: The remote absolute path where the file should be opened
800
:param mode: The mode permissions bits for the new file
802
path = self._sftp._adjust_cwd(abspath)
803
attr = SFTPAttributes()
806
omode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
807
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
809
t, msg = self._sftp._request(CMD_OPEN, path, omode, attr)
811
raise TransportError('Expected an SFTP handle')
812
handle = msg.get_string()
813
return SFTPFile(self._sftp, handle, 'wb', -1)
814
except (paramiko.SSHException, IOError), e:
815
self._translate_io_exception(e, abspath, ': unable to open',
816
failure_exc=FileExists)
added
removed
bzrlib/transport/sftp.py
