1
# Copyright (C) 2005 Robey Pointer <robey@lag.net>, Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Implementation of Transport over SFTP, using paramiko."""
32
from bzrlib.errors import (FileExists,
33
TransportNotPossible, NoSuchFile, PathNotChild,
36
from bzrlib.config import config_dir
37
from bzrlib.trace import mutter, warning, error
38
from bzrlib.transport import Transport, register_transport
44
error('The SFTP transport requires paramiko.')
47
from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE,
48
SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC,
50
from paramiko.sftp_attr import SFTPAttributes
51
from paramiko.sftp_file import SFTPFile
52
from paramiko.sftp_client import SFTPClient
54
if 'sftp' not in urlparse.uses_netloc: urlparse.uses_netloc.append('sftp')
58
if sys.platform == 'win32':
59
# close_fds not supported on win32
63
def _get_ssh_vendor():
64
"""Find out what version of SSH is on the system."""
66
if _ssh_vendor is not None:
72
p = subprocess.Popen(['ssh', '-V'],
74
stdin=subprocess.PIPE,
75
stdout=subprocess.PIPE,
76
stderr=subprocess.PIPE)
77
returncode = p.returncode
78
stdout, stderr = p.communicate()
82
if 'OpenSSH' in stderr:
83
mutter('ssh implementation is OpenSSH')
84
_ssh_vendor = 'openssh'
85
elif 'SSH Secure Shell' in stderr:
86
mutter('ssh implementation is SSH Corp.')
89
if _ssh_vendor != 'none':
92
# XXX: 20051123 jamesh
93
# A check for putty's plink or lsh would go here.
95
mutter('falling back to paramiko implementation')
100
"""A socket-like object that talks to an ssh subprocess via pipes."""
101
def __init__(self, hostname, port=None, user=None):
102
vendor = _get_ssh_vendor()
103
assert vendor in ['openssh', 'ssh']
104
if vendor == 'openssh':
106
'-oForwardX11=no', '-oForwardAgent=no',
107
'-oClearAllForwardings=yes', '-oProtocol=2',
108
'-oNoHostAuthenticationForLocalhost=yes']
110
args.extend(['-p', str(port)])
112
args.extend(['-l', user])
113
args.extend(['-s', hostname, 'sftp'])
114
elif vendor == 'ssh':
117
args.extend(['-p', str(port)])
119
args.extend(['-l', user])
120
args.extend(['-s', 'sftp', hostname])
122
self.proc = subprocess.Popen(args, close_fds=_close_fds,
123
stdin=subprocess.PIPE,
124
stdout=subprocess.PIPE)
126
def send(self, data):
127
return os.write(self.proc.stdin.fileno(), data)
129
def recv(self, count):
130
return os.read(self.proc.stdout.fileno(), count)
133
self.proc.stdin.close()
134
self.proc.stdout.close()
141
# This is a weakref dictionary, so that we can reuse connections
142
# that are still active. Long term, it might be nice to have some
143
# sort of expiration policy, such as disconnect if inactive for
144
# X seconds. But that requires a lot more fanciness.
145
_connected_hosts = weakref.WeakValueDictionary()
147
def load_host_keys():
149
Load system host keys (probably doesn't work on windows) and any
150
"discovered" keys from previous sessions.
152
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
154
SYSTEM_HOSTKEYS = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
156
mutter('failed to load system host keys: ' + str(e))
157
bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')
159
BZR_HOSTKEYS = paramiko.util.load_host_keys(bzr_hostkey_path)
161
mutter('failed to load bzr host keys: ' + str(e))
164
def save_host_keys():
166
Save "discovered" host keys in $(config)/ssh_host_keys/.
168
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
169
bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')
170
if not os.path.isdir(config_dir()):
171
os.mkdir(config_dir())
173
f = open(bzr_hostkey_path, 'w')
174
f.write('# SSH host keys collected by bzr\n')
175
for hostname, keys in BZR_HOSTKEYS.iteritems():
176
for keytype, key in keys.iteritems():
177
f.write('%s %s %s\n' % (hostname, keytype, key.get_base64()))
180
mutter('failed to save bzr host keys: ' + str(e))
183
class SFTPLock(object):
184
"""This fakes a lock in a remote location."""
185
__slots__ = ['path', 'lock_path', 'lock_file', 'transport']
186
def __init__(self, path, transport):
187
assert isinstance(transport, SFTPTransport)
189
self.lock_file = None
191
self.lock_path = path + '.write-lock'
192
self.transport = transport
194
self.lock_file = transport._sftp_open_exclusive(self.lock_path)
196
raise LockError('File %r already locked' % (self.path,))
199
"""Should this warn, or actually try to cleanup?"""
201
warn("SFTPLock %r not explicitly unlocked" % (self.path,))
205
if not self.lock_file:
207
self.lock_file.close()
208
self.lock_file = None
210
self.transport.delete(self.lock_path)
211
except (NoSuchFile,):
212
# What specific errors should we catch here?
215
class SFTPTransport (Transport):
217
Transport implementation for SFTP access.
219
_do_prefetch = False # Right now Paramiko's prefetch support causes things to hang
221
def __init__(self, base, clone_from=None):
222
assert base.startswith('sftp://')
223
self._parse_url(base)
224
base = self._unparse_url()
225
super(SFTPTransport, self).__init__(base)
226
if clone_from is None:
229
# use the same ssh connection, etc
230
self._sftp = clone_from._sftp
231
# super saves 'self.base'
233
def should_cache(self):
235
Return True if the data pulled across should be cached locally.
239
def clone(self, offset=None):
241
Return a new SFTPTransport with root at self.base + offset.
242
We share the same SFTP session between such transports, because it's
243
fairly expensive to set them up.
246
return SFTPTransport(self.base, self)
248
return SFTPTransport(self.abspath(offset), self)
250
def abspath(self, relpath):
252
Return the full url to the given relative path.
254
@param relpath: the relative path or path components
255
@type relpath: str or list
257
return self._unparse_url(self._abspath(relpath))
259
def _abspath(self, relpath):
260
"""Return the absolute path segment without the SFTP URL."""
261
# FIXME: share the common code across transports
262
assert isinstance(relpath, basestring)
263
relpath = [urllib.unquote(relpath)]
264
basepath = self._path.split('/')
265
if len(basepath) > 0 and basepath[-1] == '':
266
basepath = basepath[:-1]
270
if len(basepath) == 0:
271
# In most filesystems, a request for the parent
272
# of root, just returns root.
280
path = '/'.join(basepath)
281
# could still be a "relative" path here, but relative on the sftp server
284
def relpath(self, abspath):
285
username, password, host, port, path = self._split_url(abspath)
287
if (username != self._username):
288
error.append('username mismatch')
289
if (host != self._host):
290
error.append('host mismatch')
291
if (port != self._port):
292
error.append('port mismatch')
293
if (not path.startswith(self._path)):
294
error.append('path mismatch')
296
extra = ': ' + ', '.join(error)
297
raise PathNotChild(abspath, self.base, extra=extra)
299
return path[pl:].lstrip('/')
301
def has(self, relpath):
303
Does the target location exist?
306
self._sftp.stat(self._abspath(relpath))
311
def get(self, relpath, decode=False):
313
Get the file at the given relative path.
315
:param relpath: The relative path to the file
318
path = self._abspath(relpath)
319
f = self._sftp.file(path)
320
if self._do_prefetch and hasattr(f, 'prefetch'):
323
except (IOError, paramiko.SSHException), e:
324
self._translate_io_exception(e, path, ': error retrieving')
326
def get_partial(self, relpath, start, length=None):
328
Get just part of a file.
330
:param relpath: Path to the file, relative to base
331
:param start: The starting position to read from
332
:param length: The length to read. A length of None indicates
333
read to the end of the file.
334
:return: A file-like object containing at least the specified bytes.
335
Some implementations may return objects which can be read
336
past this length, but this is not guaranteed.
338
# TODO: implement get_partial_multi to help with knit support
339
f = self.get(relpath)
341
if self._do_prefetch and hasattr(f, 'prefetch'):
345
def put(self, relpath, f):
347
Copy the file-like or string object into the location.
349
:param relpath: Location to put the contents, relative to base.
350
:param f: File-like or string object.
352
final_path = self._abspath(relpath)
353
tmp_relpath = '%s.tmp.%.9f.%d.%d' % (relpath, time.time(),
354
os.getpid(), random.randint(0,0x7FFFFFFF))
355
tmp_abspath = self._abspath(tmp_relpath)
356
fout = self._sftp_open_exclusive(tmp_relpath)
361
except (paramiko.SSHException, IOError), e:
362
self._translate_io_exception(e, relpath, ': unable to write')
364
# If we fail, try to clean up the temporary file
365
# before we throw the exception
366
# but don't let another exception mess things up
369
self._sftp.remove(tmp_abspath)
374
# sftp rename doesn't allow overwriting, so play tricks:
375
tmp_safety = 'bzr.tmp.%.9f.%d.%d' % (time.time(), os.getpid(), random.randint(0, 0x7FFFFFFF))
376
tmp_safety = self._abspath(tmp_safety)
378
self._sftp.rename(final_path, tmp_safety)
385
self._sftp.rename(tmp_abspath, final_path)
386
except (paramiko.SSHException, IOError), e:
387
self._translate_io_exception(e, relpath, ': unable to rename')
393
self._sftp.unlink(tmp_safety)
395
self._sftp.rename(tmp_safety, final_path)
397
def iter_files_recursive(self):
398
"""Walk the relative paths of all files in this transport."""
399
queue = list(self.list_dir('.'))
401
relpath = urllib.quote(queue.pop(0))
402
st = self.stat(relpath)
403
if stat.S_ISDIR(st.st_mode):
404
for i, basename in enumerate(self.list_dir(relpath)):
405
queue.insert(i, relpath+'/'+basename)
409
def mkdir(self, relpath):
410
"""Create a directory at the given path."""
412
path = self._abspath(relpath)
413
self._sftp.mkdir(path)
414
except (paramiko.SSHException, IOError), e:
415
self._translate_io_exception(e, relpath, ': unable to mkdir',
416
failure_exc=FileExists)
418
def _translate_io_exception(self, e, path, more_info='', failure_exc=NoSuchFile):
419
"""Translate a paramiko or IOError into a friendlier exception.
421
:param e: The original exception
422
:param path: The path in question when the error is raised
423
:param more_info: Extra information that can be included,
424
such as what was going on
425
:param failure_exc: Paramiko has the super fun ability to raise completely
426
opaque errors that just set "e.args = ('Failure',)" with
428
This sometimes means FileExists, but it also sometimes
431
# paramiko seems to generate detailless errors.
432
self._translate_error(e, path, raise_generic=False)
433
if hasattr(e, 'args'):
434
if (e.args == ('No such file or directory',) or
435
e.args == ('No such file',)):
436
raise NoSuchFile(path, str(e) + more_info)
437
if (e.args == ('mkdir failed',)):
438
raise FileExists(path, str(e) + more_info)
439
# strange but true, for the paramiko server.
440
if (e.args == ('Failure',)):
441
raise failure_exc(path, str(e) + more_info)
444
def append(self, relpath, f):
446
Append the text in the file-like object into the final
450
path = self._abspath(relpath)
451
fout = self._sftp.file(path, 'ab')
453
except (IOError, paramiko.SSHException), e:
454
self._translate_io_exception(e, relpath, ': unable to append')
456
def copy(self, rel_from, rel_to):
457
"""Copy the item at rel_from to the location at rel_to"""
458
path_from = self._abspath(rel_from)
459
path_to = self._abspath(rel_to)
460
self._copy_abspaths(path_from, path_to)
462
def _copy_abspaths(self, path_from, path_to):
463
"""Copy files given an absolute path
465
:param path_from: Path on remote server to read
466
:param path_to: Path on remote server to write
469
TODO: Should the destination location be atomically created?
470
This has not been specified
471
TODO: This should use some sort of remote copy, rather than
472
pulling the data locally, and then writing it remotely
475
fin = self._sftp.file(path_from, 'rb')
477
fout = self._sftp.file(path_to, 'wb')
479
fout.set_pipelined(True)
480
self._pump(fin, fout)
485
except (IOError, paramiko.SSHException), e:
486
self._translate_io_exception(e, path_from, ': unable copy to: %r' % path_to)
488
def copy_to(self, relpaths, other, pb=None):
489
"""Copy a set of entries from self into another Transport.
491
:param relpaths: A list/generator of entries to be copied.
493
if isinstance(other, SFTPTransport) and other._sftp is self._sftp:
494
# Both from & to are on the same remote filesystem
495
# We can use a remote copy, instead of pulling locally, and pushing
497
total = self._get_total(relpaths)
499
for path in relpaths:
500
path_from = self._abspath(relpath)
501
path_to = other._abspath(relpath)
502
self._update_pb(pb, 'copy-to', count, total)
503
self._copy_abspaths(path_from, path_to)
507
return super(SFTPTransport, self).copy_to(relpaths, other, pb=pb)
509
# The dummy implementation just does a simple get + put
510
def copy_entry(path):
511
other.put(path, self.get(path))
513
return self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False)
515
def move(self, rel_from, rel_to):
516
"""Move the item at rel_from to the location at rel_to"""
517
path_from = self._abspath(rel_from)
518
path_to = self._abspath(rel_to)
520
self._sftp.rename(path_from, path_to)
521
except (IOError, paramiko.SSHException), e:
522
self._translate_io_exception(e, path_from, ': unable to move to: %r' % path_to)
524
def delete(self, relpath):
525
"""Delete the item at relpath"""
526
path = self._abspath(relpath)
528
self._sftp.remove(path)
529
except (IOError, paramiko.SSHException), e:
530
self._translate_io_exception(e, path, ': unable to delete')
533
"""Return True if this store supports listing."""
536
def list_dir(self, relpath):
538
Return a list of all files at the given location.
540
# does anything actually use this?
541
path = self._abspath(relpath)
543
return self._sftp.listdir(path)
544
except (IOError, paramiko.SSHException), e:
545
self._translate_io_exception(e, path, ': failed to list_dir')
547
def stat(self, relpath):
548
"""Return the stat information for a file."""
549
path = self._abspath(relpath)
551
return self._sftp.stat(path)
552
except (IOError, paramiko.SSHException), e:
553
self._translate_io_exception(e, path, ': unable to stat')
555
def lock_read(self, relpath):
557
Lock the given file for shared (read) access.
558
:return: A lock object, which has an unlock() member function
560
# FIXME: there should be something clever i can do here...
561
class BogusLock(object):
562
def __init__(self, path):
566
return BogusLock(relpath)
568
def lock_write(self, relpath):
570
Lock the given file for exclusive (write) access.
571
WARNING: many transports do not support this, so trying avoid using it
573
:return: A lock object, which has an unlock() member function
575
# This is a little bit bogus, but basically, we create a file
576
# which should not already exist, and if it does, we assume
577
# that there is a lock, and if it doesn't, the we assume
578
# that we have taken the lock.
579
return SFTPLock(relpath, self)
582
def _unparse_url(self, path=None):
585
path = urllib.quote(path)
586
if path.startswith('/'):
587
path = '/%2F' + path[1:]
590
netloc = urllib.quote(self._host)
591
if self._username is not None:
592
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
593
if self._port is not None:
594
netloc = '%s:%d' % (netloc, self._port)
596
return urlparse.urlunparse(('sftp', netloc, path, '', '', ''))
598
def _split_url(self, url):
599
if isinstance(url, unicode):
600
url = url.encode('utf-8')
601
(scheme, netloc, path, params,
602
query, fragment) = urlparse.urlparse(url, allow_fragments=False)
603
assert scheme == 'sftp'
604
username = password = host = port = None
606
username, host = netloc.split('@', 1)
608
username, password = username.split(':', 1)
609
password = urllib.unquote(password)
610
username = urllib.unquote(username)
615
host, port = host.rsplit(':', 1)
619
# TODO: Should this be ConnectionError?
620
raise TransportError('%s: invalid port number' % port)
621
host = urllib.unquote(host)
623
path = urllib.unquote(path)
625
# the initial slash should be removed from the path, and treated
626
# as a homedir relative path (the path begins with a double slash
627
# if it is absolute).
628
# see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt
629
if path.startswith('/'):
632
return (username, password, host, port, path)
634
def _parse_url(self, url):
635
(self._username, self._password,
636
self._host, self._port, self._path) = self._split_url(url)
638
def _sftp_connect(self):
639
"""Connect to the remote sftp server.
640
After this, self._sftp should have a valid connection (or
641
we raise an TransportError 'could not connect').
643
TODO: Raise a more reasonable ConnectionFailed exception
645
global _connected_hosts
647
idx = (self._host, self._port, self._username)
649
self._sftp = _connected_hosts[idx]
654
vendor = _get_ssh_vendor()
656
sock = SFTPSubprocess(self._host, self._port, self._username)
657
self._sftp = SFTPClient(sock)
659
self._paramiko_connect()
661
_connected_hosts[idx] = self._sftp
663
def _paramiko_connect(self):
664
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
669
t = paramiko.Transport((self._host, self._port or 22))
671
except paramiko.SSHException, e:
672
raise ConnectionError('Unable to reach SSH host %s:%d' %
673
(self._host, self._port), e)
675
server_key = t.get_remote_server_key()
676
server_key_hex = paramiko.util.hexify(server_key.get_fingerprint())
677
keytype = server_key.get_name()
678
if SYSTEM_HOSTKEYS.has_key(self._host) and SYSTEM_HOSTKEYS[self._host].has_key(keytype):
679
our_server_key = SYSTEM_HOSTKEYS[self._host][keytype]
680
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
681
elif BZR_HOSTKEYS.has_key(self._host) and BZR_HOSTKEYS[self._host].has_key(keytype):
682
our_server_key = BZR_HOSTKEYS[self._host][keytype]
683
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
685
warning('Adding %s host key for %s: %s' % (keytype, self._host, server_key_hex))
686
if not BZR_HOSTKEYS.has_key(self._host):
687
BZR_HOSTKEYS[self._host] = {}
688
BZR_HOSTKEYS[self._host][keytype] = server_key
689
our_server_key = server_key
690
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
692
if server_key != our_server_key:
693
filename1 = os.path.expanduser('~/.ssh/known_hosts')
694
filename2 = os.path.join(config_dir(), 'ssh_host_keys')
695
raise TransportError('Host keys for %s do not match! %s != %s' % \
696
(self._host, our_server_key_hex, server_key_hex),
697
['Try editing %s or %s' % (filename1, filename2)])
702
self._sftp = t.open_sftp_client()
703
except paramiko.SSHException, e:
704
raise ConnectionError('Unable to start sftp client %s:%d' %
705
(self._host, self._port), e)
707
def _sftp_auth(self, transport):
708
# paramiko requires a username, but it might be none if nothing was supplied
709
# use the local username, just in case.
710
# We don't override self._username, because if we aren't using paramiko,
711
# the username might be specified in ~/.ssh/config and we don't want to
712
# force it to something else
713
# Also, it would mess up the self.relpath() functionality
714
username = self._username or getpass.getuser()
716
# Paramiko tries to open a socket.AF_UNIX in order to connect
717
# to ssh-agent. That attribute doesn't exist on win32 (it does in cygwin)
718
# so we get an AttributeError exception. For now, just don't try to
719
# connect to an agent if we are on win32
720
if sys.platform != 'win32':
721
agent = paramiko.Agent()
722
for key in agent.get_keys():
723
mutter('Trying SSH agent key %s' % paramiko.util.hexify(key.get_fingerprint()))
725
transport.auth_publickey(username, key)
727
except paramiko.SSHException, e:
730
# okay, try finding id_rsa or id_dss? (posix only)
731
if self._try_pkey_auth(transport, paramiko.RSAKey, username, 'id_rsa'):
733
if self._try_pkey_auth(transport, paramiko.DSSKey, username, 'id_dsa'):
739
transport.auth_password(username, self._password)
741
except paramiko.SSHException, e:
744
# FIXME: Don't keep a password held in memory if you can help it
745
#self._password = None
747
# give up and ask for a password
748
password = bzrlib.ui.ui_factory.get_password(
749
prompt='SSH %(user)s@%(host)s password',
750
user=username, host=self._host)
752
transport.auth_password(username, password)
753
except paramiko.SSHException, e:
754
raise ConnectionError('Unable to authenticate to SSH host as %s@%s' %
755
(username, self._host), e)
757
def _try_pkey_auth(self, transport, pkey_class, username, filename):
758
filename = os.path.expanduser('~/.ssh/' + filename)
760
key = pkey_class.from_private_key_file(filename)
761
transport.auth_publickey(username, key)
763
except paramiko.PasswordRequiredException:
764
password = bzrlib.ui.ui_factory.get_password(
765
prompt='SSH %(filename)s password',
768
key = pkey_class.from_private_key_file(filename, password)
769
transport.auth_publickey(username, key)
771
except paramiko.SSHException:
772
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
773
except paramiko.SSHException:
774
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
779
def _sftp_open_exclusive(self, relpath):
780
"""Open a remote path exclusively.
782
SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if
783
the file already exists. However it does not expose this
784
at the higher level of SFTPClient.open(), so we have to
787
WARNING: This breaks the SFTPClient abstraction, so it
788
could easily break against an updated version of paramiko.
790
:param relpath: The relative path, where the file should be opened
792
path = self._sftp._adjust_cwd(self._abspath(relpath))
793
attr = SFTPAttributes()
794
mode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
795
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
797
t, msg = self._sftp._request(CMD_OPEN, path, mode, attr)
799
raise TransportError('Expected an SFTP handle')
800
handle = msg.get_string()
801
return SFTPFile(self._sftp, handle, 'w', -1)
802
except (paramiko.SSHException, IOError), e:
803
self._translate_io_exception(e, relpath, ': unable to open',
804
failure_exc=FileExists)
added
removed
bzrlib/transport/sftp.py
