1
# Copyright (C) 2005 Robey Pointer <robey@lag.net>, Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Implementation of Transport over SFTP, using paramiko."""
32
from bzrlib.errors import (FileExists,
33
TransportNotPossible, NoSuchFile, NonRelativePath,
36
from bzrlib.config import config_dir
37
from bzrlib.trace import mutter, warning, error
38
from bzrlib.transport import Transport, register_transport
39
from bzrlib.osutils import pathjoin
40
from bzrlib.ui import ui_factory
45
error('The SFTP transport requires paramiko.')
48
from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE,
49
SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC,
51
from paramiko.sftp_attr import SFTPAttributes
52
from paramiko.sftp_file import SFTPFile
53
from paramiko.sftp_client import SFTPClient
55
if 'sftp' not in urlparse.uses_netloc: urlparse.uses_netloc.append('sftp')
59
if sys.platform == 'win32':
60
# close_fds not supported on win32
64
def _get_ssh_vendor():
65
"""Find out what version of SSH is on the system."""
67
if _ssh_vendor is not None:
73
p = subprocess.Popen(['ssh', '-V'],
75
stdin=subprocess.PIPE,
76
stdout=subprocess.PIPE,
77
stderr=subprocess.PIPE)
78
returncode = p.returncode
79
stdout, stderr = p.communicate()
83
if 'OpenSSH' in stderr:
84
mutter('ssh implementation is OpenSSH')
85
_ssh_vendor = 'openssh'
86
elif 'SSH Secure Shell' in stderr:
87
mutter('ssh implementation is SSH Corp.')
90
if _ssh_vendor != 'none':
93
# XXX: 20051123 jamesh
94
# A check for putty's plink or lsh would go here.
96
mutter('falling back to paramiko implementation')
100
class SFTPSubprocess:
101
"""A socket-like object that talks to an ssh subprocess via pipes."""
102
def __init__(self, hostname, port=None, user=None):
103
vendor = _get_ssh_vendor()
104
assert vendor in ['openssh', 'ssh']
105
if vendor == 'openssh':
107
'-oForwardX11=no', '-oForwardAgent=no',
108
'-oClearAllForwardings=yes', '-oProtocol=2',
109
'-oNoHostAuthenticationForLocalhost=yes']
111
args.extend(['-p', str(port)])
113
args.extend(['-l', user])
114
args.extend(['-s', hostname, 'sftp'])
115
elif vendor == 'ssh':
118
args.extend(['-p', str(port)])
120
args.extend(['-l', user])
121
args.extend(['-s', 'sftp', hostname])
123
self.proc = subprocess.Popen(args, close_fds=_close_fds,
124
stdin=subprocess.PIPE,
125
stdout=subprocess.PIPE)
127
def send(self, data):
128
return os.write(self.proc.stdin.fileno(), data)
130
def recv(self, count):
131
return os.read(self.proc.stdout.fileno(), count)
134
self.proc.stdin.close()
135
self.proc.stdout.close()
142
# This is a weakref dictionary, so that we can reuse connections
143
# that are still active. Long term, it might be nice to have some
144
# sort of expiration policy, such as disconnect if inactive for
145
# X seconds. But that requires a lot more fanciness.
146
_connected_hosts = weakref.WeakValueDictionary()
148
def load_host_keys():
150
Load system host keys (probably doesn't work on windows) and any
151
"discovered" keys from previous sessions.
153
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
155
SYSTEM_HOSTKEYS = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
157
mutter('failed to load system host keys: ' + str(e))
158
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
160
BZR_HOSTKEYS = paramiko.util.load_host_keys(bzr_hostkey_path)
162
mutter('failed to load bzr host keys: ' + str(e))
165
def save_host_keys():
167
Save "discovered" host keys in $(config)/ssh_host_keys/.
169
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
170
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
171
if not os.path.isdir(config_dir()):
172
os.mkdir(config_dir())
174
f = open(bzr_hostkey_path, 'w')
175
f.write('# SSH host keys collected by bzr\n')
176
for hostname, keys in BZR_HOSTKEYS.iteritems():
177
for keytype, key in keys.iteritems():
178
f.write('%s %s %s\n' % (hostname, keytype, key.get_base64()))
181
mutter('failed to save bzr host keys: ' + str(e))
185
class SFTPTransportError (TransportError):
188
class SFTPLock(object):
189
"""This fakes a lock in a remote location."""
190
__slots__ = ['path', 'lock_path', 'lock_file', 'transport']
191
def __init__(self, path, transport):
192
assert isinstance(transport, SFTPTransport)
194
self.lock_file = None
196
self.lock_path = path + '.write-lock'
197
self.transport = transport
199
self.lock_file = transport._sftp_open_exclusive(self.lock_path)
201
raise LockError('File %r already locked' % (self.path,))
204
"""Should this warn, or actually try to cleanup?"""
206
warn("SFTPLock %r not explicitly unlocked" % (self.path,))
210
if not self.lock_file:
212
self.lock_file.close()
213
self.lock_file = None
215
self.transport.delete(self.lock_path)
216
except (NoSuchFile,):
217
# What specific errors should we catch here?
220
class SFTPTransport (Transport):
222
Transport implementation for SFTP access.
224
_do_prefetch = False # Right now Paramiko's prefetch support causes things to hang
226
def __init__(self, base, clone_from=None):
227
assert base.startswith('sftp://')
228
self._parse_url(base)
229
base = self._unparse_url()
230
super(SFTPTransport, self).__init__(base)
231
if clone_from is None:
234
# use the same ssh connection, etc
235
self._sftp = clone_from._sftp
236
# super saves 'self.base'
238
def should_cache(self):
240
Return True if the data pulled across should be cached locally.
244
def clone(self, offset=None):
246
Return a new SFTPTransport with root at self.base + offset.
247
We share the same SFTP session between such transports, because it's
248
fairly expensive to set them up.
251
return SFTPTransport(self.base, self)
253
return SFTPTransport(self.abspath(offset), self)
255
def abspath(self, relpath):
257
Return the full url to the given relative path.
259
@param relpath: the relative path or path components
260
@type relpath: str or list
262
return self._unparse_url(self._abspath(relpath))
264
def _abspath(self, relpath):
265
"""Return the absolute path segment without the SFTP URL."""
266
# FIXME: share the common code across transports
267
assert isinstance(relpath, basestring)
268
relpath = [urllib.unquote(relpath)]
269
basepath = self._path.split('/')
270
if len(basepath) > 0 and basepath[-1] == '':
271
basepath = basepath[:-1]
275
if len(basepath) == 0:
276
# In most filesystems, a request for the parent
277
# of root, just returns root.
285
path = '/'.join(basepath)
286
# could still be a "relative" path here, but relative on the sftp server
289
def relpath(self, abspath):
290
username, password, host, port, path = self._split_url(abspath)
292
if (username != self._username):
293
error.append('username mismatch')
294
if (host != self._host):
295
error.append('host mismatch')
296
if (port != self._port):
297
error.append('port mismatch')
298
if (not path.startswith(self._path)):
299
error.append('path mismatch')
301
raise NonRelativePath('path %r is not under base URL %r: %s'
302
% (abspath, self.base, ', '.join(error)))
304
return path[pl:].lstrip('/')
306
def has(self, relpath):
308
Does the target location exist?
311
self._sftp.stat(self._abspath(relpath))
316
def get(self, relpath, decode=False):
318
Get the file at the given relative path.
320
:param relpath: The relative path to the file
323
path = self._abspath(relpath)
324
f = self._sftp.file(path)
325
if self._do_prefetch and hasattr(f, 'prefetch'):
328
except (IOError, paramiko.SSHException), x:
329
raise NoSuchFile('Error retrieving %s: %s' % (path, str(x)), x)
331
def get_partial(self, relpath, start, length=None):
333
Get just part of a file.
335
:param relpath: Path to the file, relative to base
336
:param start: The starting position to read from
337
:param length: The length to read. A length of None indicates
338
read to the end of the file.
339
:return: A file-like object containing at least the specified bytes.
340
Some implementations may return objects which can be read
341
past this length, but this is not guaranteed.
343
# TODO: implement get_partial_multi to help with knit support
344
f = self.get(relpath)
346
if self._do_prefetch and hasattr(f, 'prefetch'):
350
def put(self, relpath, f):
352
Copy the file-like or string object into the location.
354
:param relpath: Location to put the contents, relative to base.
355
:param f: File-like or string object.
357
final_path = self._abspath(relpath)
358
tmp_relpath = '%s.tmp.%.9f.%d.%d' % (relpath, time.time(),
359
os.getpid(), random.randint(0,0x7FFFFFFF))
360
tmp_abspath = self._abspath(tmp_relpath)
361
fout = self._sftp_open_exclusive(tmp_relpath)
367
self._translate_io_exception(e, relpath)
368
except paramiko.SSHException, x:
369
raise SFTPTransportError('Unable to write file %r' % (relpath,), x)
371
# If we fail, try to clean up the temporary file
372
# before we throw the exception
373
# but don't let another exception mess things up
376
self._sftp.remove(tmp_abspath)
381
# sftp rename doesn't allow overwriting, so play tricks:
382
tmp_safety = 'bzr.tmp.%.9f.%d.%d' % (time.time(), os.getpid(), random.randint(0, 0x7FFFFFFF))
383
tmp_safety = self._abspath(tmp_safety)
385
self._sftp.rename(final_path, tmp_safety)
392
self._sftp.rename(tmp_abspath, final_path)
394
self._translate_io_exception(e, relpath)
395
except paramiko.SSHException, x:
396
raise SFTPTransportError('Unable to rename into file %r' % (path,), x)
402
self._sftp.unlink(tmp_safety)
404
self._sftp.rename(tmp_safety, final_path)
406
def iter_files_recursive(self):
407
"""Walk the relative paths of all files in this transport."""
408
queue = list(self.list_dir('.'))
410
relpath = urllib.quote(queue.pop(0))
411
st = self.stat(relpath)
412
if stat.S_ISDIR(st.st_mode):
413
for i, basename in enumerate(self.list_dir(relpath)):
414
queue.insert(i, relpath+'/'+basename)
418
def mkdir(self, relpath):
419
"""Create a directory at the given path."""
421
path = self._abspath(relpath)
422
self._sftp.mkdir(path)
424
self._translate_io_exception(e, relpath)
425
except (IOError, paramiko.SSHException), x:
426
raise SFTPTransportError('Unable to mkdir %r' % (path,), x)
428
def _translate_io_exception(self, e, relpath):
429
# paramiko seems to generate detailless errors.
430
if (e.errno == errno.ENOENT or
431
e.args == ('No such file or directory',) or
432
e.args == ('No such file',)):
433
raise NoSuchFile(relpath)
434
if (e.args == ('mkdir failed',)):
435
raise FileExists(relpath)
436
# strange but true, for the paramiko server.
437
if (e.args == ('Failure',)):
438
raise FileExists(relpath)
441
def append(self, relpath, f):
443
Append the text in the file-like object into the final
447
path = self._abspath(relpath)
448
fout = self._sftp.file(path, 'ab')
450
except (IOError, paramiko.SSHException), x:
451
raise SFTPTransportError('Unable to append file %r' % (path,), x)
453
def copy(self, rel_from, rel_to):
454
"""Copy the item at rel_from to the location at rel_to"""
455
path_from = self._abspath(rel_from)
456
path_to = self._abspath(rel_to)
457
self._copy_abspaths(path_from, path_to)
459
def _copy_abspaths(self, path_from, path_to):
460
"""Copy files given an absolute path
462
:param path_from: Path on remote server to read
463
:param path_to: Path on remote server to write
466
TODO: Should the destination location be atomically created?
467
This has not been specified
468
TODO: This should use some sort of remote copy, rather than
469
pulling the data locally, and then writing it remotely
472
fin = self._sftp.file(path_from, 'rb')
474
fout = self._sftp.file(path_to, 'wb')
476
fout.set_pipelined(True)
477
self._pump(fin, fout)
482
except (IOError, paramiko.SSHException), x:
483
raise SFTPTransportError('Unable to copy %r to %r' % (path_from, path_to), x)
485
def copy_to(self, relpaths, other, pb=None):
486
"""Copy a set of entries from self into another Transport.
488
:param relpaths: A list/generator of entries to be copied.
490
if isinstance(other, SFTPTransport) and other._sftp is self._sftp:
491
# Both from & to are on the same remote filesystem
492
# We can use a remote copy, instead of pulling locally, and pushing
494
total = self._get_total(relpaths)
496
for path in relpaths:
497
path_from = self._abspath(relpath)
498
path_to = other._abspath(relpath)
499
self._update_pb(pb, 'copy-to', count, total)
500
self._copy_abspaths(path_from, path_to)
504
return super(SFTPTransport, self).copy_to(relpaths, other, pb=pb)
506
# The dummy implementation just does a simple get + put
507
def copy_entry(path):
508
other.put(path, self.get(path))
510
return self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False)
512
def move(self, rel_from, rel_to):
513
"""Move the item at rel_from to the location at rel_to"""
514
path_from = self._abspath(rel_from)
515
path_to = self._abspath(rel_to)
517
self._sftp.rename(path_from, path_to)
518
except (IOError, paramiko.SSHException), x:
519
raise SFTPTransportError('Unable to move %r to %r' % (path_from, path_to), x)
521
def delete(self, relpath):
522
"""Delete the item at relpath"""
523
path = self._abspath(relpath)
525
self._sftp.remove(path)
526
except (IOError, paramiko.SSHException), x:
527
raise SFTPTransportError('Unable to delete %r' % (path,), x)
530
"""Return True if this store supports listing."""
533
def list_dir(self, relpath):
535
Return a list of all files at the given location.
537
# does anything actually use this?
538
path = self._abspath(relpath)
540
return self._sftp.listdir(path)
541
except (IOError, paramiko.SSHException), x:
542
raise SFTPTransportError('Unable to list folder %r' % (path,), x)
544
def stat(self, relpath):
545
"""Return the stat information for a file."""
546
path = self._abspath(relpath)
548
return self._sftp.stat(path)
549
except (IOError, paramiko.SSHException), x:
550
raise SFTPTransportError('Unable to stat %r' % (path,), x)
552
def lock_read(self, relpath):
554
Lock the given file for shared (read) access.
555
:return: A lock object, which has an unlock() member function
557
# FIXME: there should be something clever i can do here...
558
class BogusLock(object):
559
def __init__(self, path):
563
return BogusLock(relpath)
565
def lock_write(self, relpath):
567
Lock the given file for exclusive (write) access.
568
WARNING: many transports do not support this, so trying avoid using it
570
:return: A lock object, which has an unlock() member function
572
# This is a little bit bogus, but basically, we create a file
573
# which should not already exist, and if it does, we assume
574
# that there is a lock, and if it doesn't, the we assume
575
# that we have taken the lock.
576
return SFTPLock(relpath, self)
579
def _unparse_url(self, path=None):
582
path = urllib.quote(path)
583
if path.startswith('/'):
584
path = '/%2F' + path[1:]
587
netloc = urllib.quote(self._host)
588
if self._username is not None:
589
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
590
if self._port is not None:
591
netloc = '%s:%d' % (netloc, self._port)
593
return urlparse.urlunparse(('sftp', netloc, path, '', '', ''))
595
def _split_url(self, url):
596
if isinstance(url, unicode):
597
url = url.encode('utf-8')
598
(scheme, netloc, path, params,
599
query, fragment) = urlparse.urlparse(url, allow_fragments=False)
600
assert scheme == 'sftp'
601
username = password = host = port = None
603
username, host = netloc.split('@', 1)
605
username, password = username.split(':', 1)
606
password = urllib.unquote(password)
607
username = urllib.unquote(username)
612
host, port = host.rsplit(':', 1)
616
raise SFTPTransportError('%s: invalid port number' % port)
617
host = urllib.unquote(host)
619
path = urllib.unquote(path)
621
# the initial slash should be removed from the path, and treated
622
# as a homedir relative path (the path begins with a double slash
623
# if it is absolute).
624
# see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt
625
if path.startswith('/'):
628
return (username, password, host, port, path)
630
def _parse_url(self, url):
631
(self._username, self._password,
632
self._host, self._port, self._path) = self._split_url(url)
634
def _sftp_connect(self):
635
"""Connect to the remote sftp server.
636
After this, self._sftp should have a valid connection (or
637
we raise an SFTPTransportError 'could not connect').
639
TODO: Raise a more reasonable ConnectionFailed exception
641
global _connected_hosts
643
idx = (self._host, self._port, self._username)
645
self._sftp = _connected_hosts[idx]
650
vendor = _get_ssh_vendor()
652
sock = SFTPSubprocess(self._host, self._port, self._username)
653
self._sftp = SFTPClient(sock)
655
self._paramiko_connect()
657
_connected_hosts[idx] = self._sftp
659
def _paramiko_connect(self):
660
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
665
t = paramiko.Transport((self._host, self._port))
667
except paramiko.SSHException:
668
raise SFTPTransportError('Unable to reach SSH host %s:%d' % (self._host, self._port))
670
server_key = t.get_remote_server_key()
671
server_key_hex = paramiko.util.hexify(server_key.get_fingerprint())
672
keytype = server_key.get_name()
673
if SYSTEM_HOSTKEYS.has_key(self._host) and SYSTEM_HOSTKEYS[self._host].has_key(keytype):
674
our_server_key = SYSTEM_HOSTKEYS[self._host][keytype]
675
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
676
elif BZR_HOSTKEYS.has_key(self._host) and BZR_HOSTKEYS[self._host].has_key(keytype):
677
our_server_key = BZR_HOSTKEYS[self._host][keytype]
678
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
680
warning('Adding %s host key for %s: %s' % (keytype, self._host, server_key_hex))
681
if not BZR_HOSTKEYS.has_key(self._host):
682
BZR_HOSTKEYS[self._host] = {}
683
BZR_HOSTKEYS[self._host][keytype] = server_key
684
our_server_key = server_key
685
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
687
if server_key != our_server_key:
688
filename1 = os.path.expanduser('~/.ssh/known_hosts')
689
filename2 = pathjoin(config_dir(), 'ssh_host_keys')
690
raise SFTPTransportError('Host keys for %s do not match! %s != %s' % \
691
(self._host, our_server_key_hex, server_key_hex),
692
['Try editing %s or %s' % (filename1, filename2)])
697
self._sftp = t.open_sftp_client()
698
except paramiko.SSHException:
699
raise BzrError('Unable to find path %s on SFTP server %s' % \
700
(self._path, self._host))
702
def _sftp_auth(self, transport):
703
# paramiko requires a username, but it might be none if nothing was supplied
704
# use the local username, just in case.
705
# We don't override self._username, because if we aren't using paramiko,
706
# the username might be specified in ~/.ssh/config and we don't want to
707
# force it to something else
708
# Also, it would mess up the self.relpath() functionality
709
username = self._username or getpass.getuser()
711
agent = paramiko.Agent()
712
for key in agent.get_keys():
713
mutter('Trying SSH agent key %s' % paramiko.util.hexify(key.get_fingerprint()))
715
transport.auth_publickey(username, key)
717
except paramiko.SSHException, e:
720
# okay, try finding id_rsa or id_dss? (posix only)
721
if self._try_pkey_auth(transport, paramiko.RSAKey, username, 'id_rsa'):
723
if self._try_pkey_auth(transport, paramiko.DSSKey, username, 'id_dsa'):
729
transport.auth_password(username, self._password)
731
except paramiko.SSHException, e:
734
# FIXME: Don't keep a password held in memory if you can help it
735
#self._password = None
737
# give up and ask for a password
738
password = ui_factory.get_password(prompt='SSH %(user)s@%(host)s password',
739
user=username, host=self._host)
741
transport.auth_password(username, password)
742
except paramiko.SSHException:
743
raise SFTPTransportError('Unable to authenticate to SSH host as %s@%s' % \
744
(username, self._host))
746
def _try_pkey_auth(self, transport, pkey_class, username, filename):
747
filename = os.path.expanduser('~/.ssh/' + filename)
749
key = pkey_class.from_private_key_file(filename)
750
transport.auth_publickey(username, key)
752
except paramiko.PasswordRequiredException:
753
password = ui_factory.get_password(prompt='SSH %(filename)s password',
756
key = pkey_class.from_private_key_file(filename, password)
757
transport.auth_publickey(username, key)
759
except paramiko.SSHException:
760
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
761
except paramiko.SSHException:
762
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
767
def _sftp_open_exclusive(self, relpath):
768
"""Open a remote path exclusively.
770
SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if
771
the file already exists. However it does not expose this
772
at the higher level of SFTPClient.open(), so we have to
775
WARNING: This breaks the SFTPClient abstraction, so it
776
could easily break against an updated version of paramiko.
778
:param relpath: The relative path, where the file should be opened
780
path = self._sftp._adjust_cwd(self._abspath(relpath))
781
attr = SFTPAttributes()
782
mode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
783
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
785
t, msg = self._sftp._request(CMD_OPEN, path, mode, attr)
787
raise SFTPTransportError('Expected an SFTP handle')
788
handle = msg.get_string()
789
return SFTPFile(self._sftp, handle, 'w', -1)
791
self._translate_io_exception(e, relpath)
792
except paramiko.SSHException, x:
793
raise SFTPTransportError('Unable to open file %r' % (path,), x)
added
removed
bzrlib/transport/sftp.py
