1
# Copyright (C) 2005 Robey Pointer <robey@lag.net>, Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Implementation of Transport over SFTP, using paramiko."""
29
from bzrlib.errors import (FileExists,
30
TransportNotPossible, NoSuchFile, NonRelativePath,
33
from bzrlib.config import config_dir
34
from bzrlib.trace import mutter, warning, error
35
from bzrlib.transport import Transport, register_transport
40
error('The SFTP transport requires paramiko.')
43
from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE,
44
SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC,
46
from paramiko.sftp_attr import SFTPAttributes
47
from paramiko.sftp_file import SFTPFile
55
Load system host keys (probably doesn't work on windows) and any
56
"discovered" keys from previous sessions.
58
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
60
SYSTEM_HOSTKEYS = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
62
mutter('failed to load system host keys: ' + str(e))
63
bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')
65
BZR_HOSTKEYS = paramiko.util.load_host_keys(bzr_hostkey_path)
67
mutter('failed to load bzr host keys: ' + str(e))
72
Save "discovered" host keys in $(config)/ssh_host_keys/.
74
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
75
bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')
76
if not os.path.isdir(config_dir()):
77
os.mkdir(config_dir())
79
f = open(bzr_hostkey_path, 'w')
80
f.write('# SSH host keys collected by bzr\n')
81
for hostname, keys in BZR_HOSTKEYS.iteritems():
82
for keytype, key in keys.iteritems():
83
f.write('%s %s %s\n' % (hostname, keytype, key.get_base64()))
86
mutter('failed to save bzr host keys: ' + str(e))
90
class SFTPTransportError (TransportError):
93
class SFTPLock(object):
94
"""This fakes a lock in a remote location."""
95
__slots__ = ['path', 'lock_path', 'lock_file', 'transport']
96
def __init__(self, path, transport):
97
assert isinstance(transport, SFTPTransport)
101
self.lock_path = path + '.write-lock'
102
self.transport = transport
104
self.lock_file = transport._sftp_open_exclusive(self.lock_path)
106
raise LockError('File %r already locked' % (self.path,))
109
"""Should this warn, or actually try to cleanup?"""
111
warn("SFTPLock %r not explicitly unlocked" % (self.path,))
115
if not self.lock_file:
117
self.lock_file.close()
118
self.lock_file = None
120
self.transport.delete(self.lock_path)
121
except (NoSuchFile,):
122
# What specific errors should we catch here?
125
class SFTPTransport (Transport):
127
Transport implementation for SFTP access.
130
_url_matcher = re.compile(r'^sftp://([^:@]*(:[^@]*)?@)?(.*?)(:[^/]+)?(/.*)?$')
132
def __init__(self, base, clone_from=None):
133
assert base.startswith('sftp://')
134
super(SFTPTransport, self).__init__(base)
135
self._parse_url(base)
136
if clone_from is None:
139
# use the same ssh connection, etc
140
self._sftp = clone_from._sftp
141
# super saves 'self.base'
143
def should_cache(self):
145
Return True if the data pulled across should be cached locally.
149
def clone(self, offset=None):
151
Return a new SFTPTransport with root at self.base + offset.
152
We share the same SFTP session between such transports, because it's
153
fairly expensive to set them up.
156
return SFTPTransport(self.base, self)
158
return SFTPTransport(self.abspath(offset), self)
160
def abspath(self, relpath):
162
Return the full url to the given relative path.
164
@param relpath: the relative path or path components
165
@type relpath: str or list
167
return self._unparse_url(self._abspath(relpath))
169
def _abspath(self, relpath):
170
"""Return the absolute path segment without the SFTP URL."""
171
# FIXME: share the common code across transports
172
assert isinstance(relpath, basestring)
173
relpath = [urllib.unquote(relpath)]
174
basepath = self._path.split('/')
175
if len(basepath) > 0 and basepath[-1] == '':
176
basepath = basepath[:-1]
180
if len(basepath) == 0:
181
# In most filesystems, a request for the parent
182
# of root, just returns root.
190
path = '/'.join(basepath)
191
# could still be a "relative" path here, but relative on the sftp server
194
def relpath(self, abspath):
195
# FIXME: this is identical to HttpTransport -- share it
196
m = self._url_matcher.match(abspath)
198
if not path.startswith(self._path):
199
raise NonRelativePath('path %r is not under base URL %r'
200
% (abspath, self.base))
202
return abspath[pl:].lstrip('/')
204
def has(self, relpath):
206
Does the target location exist?
209
self._sftp.stat(self._abspath(relpath))
214
def get(self, relpath, decode=False):
216
Get the file at the given relative path.
218
:param relpath: The relative path to the file
221
path = self._abspath(relpath)
222
f = self._sftp.file(path)
225
except AttributeError:
226
# only works on paramiko 1.5.1 or greater
229
except (IOError, paramiko.SSHException), x:
230
raise NoSuchFile('Error retrieving %s: %s' % (path, str(x)), x)
232
def get_partial(self, relpath, start, length=None):
234
Get just part of a file.
236
:param relpath: Path to the file, relative to base
237
:param start: The starting position to read from
238
:param length: The length to read. A length of None indicates
239
read to the end of the file.
240
:return: A file-like object containing at least the specified bytes.
241
Some implementations may return objects which can be read
242
past this length, but this is not guaranteed.
244
# TODO: implement get_partial_multi to help with knit support
245
f = self.get(relpath)
249
except AttributeError:
250
# only works on paramiko 1.5.1 or greater
254
def put(self, relpath, f):
256
Copy the file-like or string object into the location.
258
:param relpath: Location to put the contents, relative to base.
259
:param f: File-like or string object.
261
final_path = self._abspath(relpath)
262
tmp_relpath = '%s.tmp.%.9f.%d.%d' % (relpath, time.time(),
263
os.getpid(), random.randint(0,0x7FFFFFFF))
264
tmp_abspath = self._abspath(tmp_relpath)
265
fout = self._sftp_open_exclusive(tmp_relpath)
271
self._translate_io_exception(e, relpath)
272
except paramiko.SSHException, x:
273
raise SFTPTransportError('Unable to write file %r' % (path,), x)
275
# If we fail, try to clean up the temporary file
276
# before we throw the exception
277
# but don't let another exception mess things up
280
self._sftp.remove(tmp_abspath)
285
# sftp rename doesn't allow overwriting, so play tricks:
286
tmp_safety = 'bzr.tmp.%.9f.%d.%d' % (time.time(), os.getpid(), random.randint(0, 0x7FFFFFFF))
287
tmp_safety = self._abspath(tmp_safety)
289
self._sftp.rename(final_path, tmp_safety)
294
self._sftp.rename(tmp_abspath, final_path)
296
self._translate_io_exception(e, relpath)
297
except paramiko.SSHException, x:
298
raise SFTPTransportError('Unable to rename into file %r'
301
self._sftp.unlink(tmp_safety)
303
def iter_files_recursive(self):
304
"""Walk the relative paths of all files in this transport."""
305
queue = list(self.list_dir('.'))
307
relpath = urllib.quote(queue.pop(0))
308
st = self.stat(relpath)
309
if stat.S_ISDIR(st.st_mode):
310
for i, basename in enumerate(self.list_dir(relpath)):
311
queue.insert(i, relpath+'/'+basename)
315
def mkdir(self, relpath):
316
"""Create a directory at the given path."""
318
path = self._abspath(relpath)
319
self._sftp.mkdir(path)
321
self._translate_io_exception(e, relpath)
322
except (IOError, paramiko.SSHException), x:
323
raise SFTPTransportError('Unable to mkdir %r' % (path,), x)
325
def _translate_io_exception(self, e, relpath):
326
# paramiko seems to generate detailless errors.
327
if (e.errno == errno.ENOENT or
328
e.args == ('No such file or directory',) or
329
e.args == ('No such file',)):
330
raise NoSuchFile(relpath)
331
if (e.args == ('mkdir failed',)):
332
raise FileExists(relpath)
333
# strange but true, for the paramiko server.
334
if (e.args == ('Failure',)):
335
raise FileExists(relpath)
338
def append(self, relpath, f):
340
Append the text in the file-like object into the final
344
path = self._abspath(relpath)
345
fout = self._sftp.file(path, 'ab')
347
except (IOError, paramiko.SSHException), x:
348
raise SFTPTransportError('Unable to append file %r' % (path,), x)
350
def copy(self, rel_from, rel_to):
351
"""Copy the item at rel_from to the location at rel_to"""
352
path_from = self._abspath(rel_from)
353
path_to = self._abspath(rel_to)
355
fin = self._sftp.file(path_from, 'rb')
357
fout = self._sftp.file(path_to, 'wb')
359
fout.set_pipelined(True)
360
self._pump(fin, fout)
365
except (IOError, paramiko.SSHException), x:
366
raise SFTPTransportError('Unable to copy %r to %r' % (path_from, path_to), x)
368
def move(self, rel_from, rel_to):
369
"""Move the item at rel_from to the location at rel_to"""
370
path_from = self._abspath(rel_from)
371
path_to = self._abspath(rel_to)
373
self._sftp.rename(path_from, path_to)
374
except (IOError, paramiko.SSHException), x:
375
raise SFTPTransportError('Unable to move %r to %r' % (path_from, path_to), x)
377
def delete(self, relpath):
378
"""Delete the item at relpath"""
379
path = self._abspath(relpath)
381
self._sftp.remove(path)
382
except (IOError, paramiko.SSHException), x:
383
raise SFTPTransportError('Unable to delete %r' % (path,), x)
386
"""Return True if this store supports listing."""
389
def list_dir(self, relpath):
391
Return a list of all files at the given location.
393
# does anything actually use this?
394
path = self._abspath(relpath)
396
return self._sftp.listdir(path)
397
except (IOError, paramiko.SSHException), x:
398
raise SFTPTransportError('Unable to list folder %r' % (path,), x)
400
def stat(self, relpath):
401
"""Return the stat information for a file."""
402
path = self._abspath(relpath)
404
return self._sftp.stat(path)
405
except (IOError, paramiko.SSHException), x:
406
raise SFTPTransportError('Unable to stat %r' % (path,), x)
408
def lock_read(self, relpath):
410
Lock the given file for shared (read) access.
411
:return: A lock object, which has an unlock() member function
413
# FIXME: there should be something clever i can do here...
414
class BogusLock(object):
415
def __init__(self, path):
419
return BogusLock(relpath)
421
def lock_write(self, relpath):
423
Lock the given file for exclusive (write) access.
424
WARNING: many transports do not support this, so trying avoid using it
426
:return: A lock object, which has an unlock() member function
428
# This is a little bit bogus, but basically, we create a file
429
# which should not already exist, and if it does, we assume
430
# that there is a lock, and if it doesn't, the we assume
431
# that we have taken the lock.
432
return SFTPLock(relpath, self)
435
def _unparse_url(self, path=None):
439
username = urllib.quote(self._username)
441
username += ':' + urllib.quote(self._password)
443
host += ':%d' % self._port
444
return 'sftp://%s@%s/%s' % (username, host, urllib.quote(path))
446
def _parse_url(self, url):
447
assert url[:7] == 'sftp://'
448
m = self._url_matcher.match(url)
450
raise SFTPTransportError('Unable to parse SFTP URL %r' % (url,))
451
self._username, self._password, self._host, self._port, self._path = m.groups()
452
if self._username is None:
453
self._username = getpass.getuser()
456
# username field is 'user:pass@' in this case, and password is ':pass'
457
username_len = len(self._username) - len(self._password) - 1
458
self._username = urllib.unquote(self._username[:username_len])
459
self._password = urllib.unquote(self._password[1:])
461
self._username = urllib.unquote(self._username[:-1])
462
if self._port is None:
465
self._port = int(self._port[1:])
466
if (self._path is None) or (self._path == ''):
470
self._path = urllib.unquote(self._path[1:])
472
def _sftp_connect(self):
473
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
478
t = paramiko.Transport((self._host, self._port))
480
except paramiko.SSHException:
481
raise SFTPTransportError('Unable to reach SSH host %s:%d' % (self._host, self._port))
483
server_key = t.get_remote_server_key()
484
server_key_hex = paramiko.util.hexify(server_key.get_fingerprint())
485
keytype = server_key.get_name()
486
if SYSTEM_HOSTKEYS.has_key(self._host) and SYSTEM_HOSTKEYS[self._host].has_key(keytype):
487
our_server_key = SYSTEM_HOSTKEYS[self._host][keytype]
488
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
489
elif BZR_HOSTKEYS.has_key(self._host) and BZR_HOSTKEYS[self._host].has_key(keytype):
490
our_server_key = BZR_HOSTKEYS[self._host][keytype]
491
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
493
warning('Adding %s host key for %s: %s' % (keytype, self._host, server_key_hex))
494
if not BZR_HOSTKEYS.has_key(self._host):
495
BZR_HOSTKEYS[self._host] = {}
496
BZR_HOSTKEYS[self._host][keytype] = server_key
497
our_server_key = server_key
498
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
500
if server_key != our_server_key:
501
filename1 = os.path.expanduser('~/.ssh/known_hosts')
502
filename2 = os.path.join(config_dir(), 'ssh_host_keys')
503
raise SFTPTransportError('Host keys for %s do not match! %s != %s' % \
504
(self._host, our_server_key_hex, server_key_hex),
505
['Try editing %s or %s' % (filename1, filename2)])
507
self._sftp_auth(t, self._username, self._host)
510
self._sftp = t.open_sftp_client()
511
except paramiko.SSHException:
512
raise BzrError('Unable to find path %s on SFTP server %s' % \
513
(self._path, self._host))
515
def _sftp_auth(self, transport, username, host):
516
agent = paramiko.Agent()
517
for key in agent.get_keys():
518
mutter('Trying SSH agent key %s' % paramiko.util.hexify(key.get_fingerprint()))
520
transport.auth_publickey(self._username, key)
522
except paramiko.SSHException, e:
525
# okay, try finding id_rsa or id_dss? (posix only)
526
if self._try_pkey_auth(transport, paramiko.RSAKey, 'id_rsa'):
528
if self._try_pkey_auth(transport, paramiko.DSSKey, 'id_dsa'):
533
transport.auth_password(self._username, self._password)
535
except paramiko.SSHException, e:
538
# give up and ask for a password
539
# FIXME: shouldn't be implementing UI this deep into bzrlib
540
enc = sys.stdout.encoding
541
password = getpass.getpass('SSH %s@%s password: ' %
542
(self._username.encode(enc, 'replace'), self._host.encode(enc, 'replace')))
544
transport.auth_password(self._username, password)
545
except paramiko.SSHException:
546
raise SFTPTransportError('Unable to authenticate to SSH host as %s@%s' % \
547
(self._username, self._host))
549
def _try_pkey_auth(self, transport, pkey_class, filename):
550
filename = os.path.expanduser('~/.ssh/' + filename)
552
key = pkey_class.from_private_key_file(filename)
553
transport.auth_publickey(self._username, key)
555
except paramiko.PasswordRequiredException:
556
# FIXME: shouldn't be implementing UI this deep into bzrlib
557
enc = sys.stdout.encoding
558
password = getpass.getpass('SSH %s password: ' %
559
(os.path.basename(filename).encode(enc, 'replace'),))
561
key = pkey_class.from_private_key_file(filename, password)
562
transport.auth_publickey(self._username, key)
564
except paramiko.SSHException:
565
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
566
except paramiko.SSHException:
567
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
572
def _sftp_open_exclusive(self, relpath):
573
"""Open a remote path exclusively.
575
SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if
576
the file already exists. However it does not expose this
577
at the higher level of SFTPClient.open(), so we have to
580
WARNING: This breaks the SFTPClient abstraction, so it
581
could easily break against an updated version of paramiko.
583
:param relpath: The relative path, where the file should be opened
585
path = self._abspath(relpath)
586
attr = SFTPAttributes()
587
mode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
588
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
590
t, msg = self._sftp._request(CMD_OPEN, path, mode, attr)
592
raise SFTPTransportError('Expected an SFTP handle')
593
handle = msg.get_string()
594
return SFTPFile(self._sftp, handle, 'w', -1)
596
self._translate_io_exception(e, relpath)
597
except paramiko.SSHException, x:
598
raise SFTPTransportError('Unable to open file %r' % (path,), x)
added
removed
bzrlib/transport/sftp.py
