« back to all changes in this revision
Viewing changes to bzrlib/transport/sftp.py
- browse files at revision 1518
- compare with another revision
- compare with revision 5114
- download diff
- download tarball
- view history from revision 1518
- Committer: Robert Collins
- Date: 2005-12-05 07:40:09 UTC
- mfrom: (1185.33.78 bzr.dev (Main development branch))
- Revision ID: robertc@robertcollins.net-20051205074009-ab11929b3456c4c9
Merge from mbp.
- files added:
- bzrlib/rio.py
- bzrlib/ui
- files removed:
- bzrlib/basicio.py
- files renamed:
- bzrlib/ui.py => bzrlib/ui/__init__.py
- files modified:
- Makefile
added
removed
bzrlib/transport/sftp.py
23
23
import stat
24
24
import sys
25
25
import urllib
26
import urlparse
26
27
import time
27
28
import random
29
import subprocess
30
import weakref
28
31
29
32
from bzrlib.errors import (FileExists,
30
33
TransportNotPossible, NoSuchFile, NonRelativePath,
33
36
from bzrlib.config import config_dir
34
37
from bzrlib.trace import mutter, warning, error
35
38
from bzrlib.transport import Transport, register_transport
39
from bzrlib.ui import ui_factory
36
40
37
41
try:
38
42
import paramiko
45
49
CMD_HANDLE, CMD_OPEN)
46
50
from paramiko.sftp_attr import SFTPAttributes
47
51
from paramiko.sftp_file import SFTPFile
52
from paramiko.sftp_client import SFTPClient
53
54
if 'sftp' not in urlparse.uses_netloc: urlparse.uses_netloc.append('sftp')
55
56
57
_close_fds = True
58
if sys.platform == 'win32':
59
# close_fds not supported on win32
60
_close_fds = False
61
62
_ssh_vendor = None
63
def _get_ssh_vendor():
64
"""Find out what version of SSH is on the system."""
65
global _ssh_vendor
66
if _ssh_vendor is not None:
67
return _ssh_vendor
68
69
_ssh_vendor = 'none'
70
71
try:
72
p = subprocess.Popen(['ssh', '-V'],
73
close_fds=_close_fds,
74
stdin=subprocess.PIPE,
75
stdout=subprocess.PIPE,
76
stderr=subprocess.PIPE)
77
returncode = p.returncode
78
stdout, stderr = p.communicate()
79
except OSError:
80
returncode = -1
81
stdout = stderr = ''
82
if 'OpenSSH' in stderr:
83
mutter('ssh implementation is OpenSSH')
84
_ssh_vendor = 'openssh'
85
elif 'SSH Secure Shell' in stderr:
86
mutter('ssh implementation is SSH Corp.')
87
_ssh_vendor = 'ssh'
88
89
if _ssh_vendor != 'none':
90
return _ssh_vendor
91
92
# XXX: 20051123 jamesh
93
# A check for putty's plink or lsh would go here.
94
95
mutter('falling back to paramiko implementation')
96
return _ssh_vendor
97
98
99
class SFTPSubprocess:
100
"""A socket-like object that talks to an ssh subprocess via pipes."""
101
def __init__(self, hostname, port=None, user=None):
102
vendor = _get_ssh_vendor()
103
assert vendor in ['openssh', 'ssh']
104
if vendor == 'openssh':
105
args = ['ssh',
106
'-oForwardX11=no', '-oForwardAgent=no',
107
'-oClearAllForwardings=yes', '-oProtocol=2',
108
'-oNoHostAuthenticationForLocalhost=yes']
109
if port is not None:
110
args.extend(['-p', str(port)])
111
if user is not None:
112
args.extend(['-l', user])
113
args.extend(['-s', hostname, 'sftp'])
114
elif vendor == 'ssh':
115
args = ['ssh', '-x']
116
if port is not None:
117
args.extend(['-p', str(port)])
118
if user is not None:
119
args.extend(['-l', user])
120
args.extend(['-s', 'sftp', hostname])
121
122
self.proc = subprocess.Popen(args, close_fds=_close_fds,
123
stdin=subprocess.PIPE,
124
stdout=subprocess.PIPE)
125
126
def send(self, data):
127
return os.write(self.proc.stdin.fileno(), data)
128
129
def recv(self, count):
130
return os.read(self.proc.stdout.fileno(), count)
131
132
def close(self):
133
self.proc.stdin.close()
134
self.proc.stdout.close()
135
self.proc.wait()
48
136
49
137
50
138
SYSTEM_HOSTKEYS = {}
51
139
BZR_HOSTKEYS = {}
52
140
141
# This is a weakref dictionary, so that we can reuse connections
142
# that are still active. Long term, it might be nice to have some
143
# sort of expiration policy, such as disconnect if inactive for
144
# X seconds. But that requires a lot more fanciness.
145
_connected_hosts = weakref.WeakValueDictionary()
146
53
147
def load_host_keys():
54
148
"""
55
149
Load system host keys (probably doesn't work on windows) and any
126
220
"""
127
221
Transport implementation for SFTP access.
128
222
"""
223
_do_prefetch = False # Right now Paramiko's prefetch support causes things to hang
129
224
130
_url_matcher = re.compile(r'^sftp://([^:@]*(:[^@]*)?@)?(.*?)(:[^/]+)?(/.*)?$')
131
132
225
def __init__(self, base, clone_from=None):
133
226
assert base.startswith('sftp://')
227
self._parse_url(base)
228
base = self._unparse_url()
134
229
super(SFTPTransport, self).__init__(base)
135
self._parse_url(base)
136
230
if clone_from is None:
137
231
self._sftp_connect()
138
232
else:
192
286
return path
193
287
194
288
def relpath(self, abspath):
195
# FIXME: this is identical to HttpTransport -- share it
196
m = self._url_matcher.match(abspath)
197
path = m.group(5)
198
if not path.startswith(self._path):
199
raise NonRelativePath('path %r is not under base URL %r'
200
% (abspath, self.base))
201
pl = len(self.base)
202
return abspath[pl:].lstrip('/')
289
username, password, host, port, path = self._split_url(abspath)
290
error = []
291
if (username != self._username):
292
error.append('username mismatch')
293
if (host != self._host):
294
error.append('host mismatch')
295
if (port != self._port):
296
error.append('port mismatch')
297
if (not path.startswith(self._path)):
298
error.append('path mismatch')
299
if error:
300
raise NonRelativePath('path %r is not under base URL %r: %s'
301
% (abspath, self.base, ', '.join(error)))
302
pl = len(self._path)
303
return path[pl:].lstrip('/')
203
304
204
305
def has(self, relpath):
205
306
"""
220
321
try:
221
322
path = self._abspath(relpath)
222
323
f = self._sftp.file(path)
223
try:
324
if self._do_prefetch and hasattr(f, 'prefetch'):
224
325
f.prefetch()
225
except AttributeError:
226
# only works on paramiko 1.5.1 or greater
227
pass
228
326
return f
229
327
except (IOError, paramiko.SSHException), x:
230
328
raise NoSuchFile('Error retrieving %s: %s' % (path, str(x)), x)
244
342
# TODO: implement get_partial_multi to help with knit support
245
343
f = self.get(relpath)
246
344
f.seek(start)
247
try:
345
if self._do_prefetch and hasattr(f, 'prefetch'):
248
346
f.prefetch()
249
except AttributeError:
250
# only works on paramiko 1.5.1 or greater
251
pass
252
347
return f
253
348
254
349
def put(self, relpath, f):
270
365
except IOError, e:
271
366
self._translate_io_exception(e, relpath)
272
367
except paramiko.SSHException, x:
273
raise SFTPTransportError('Unable to write file %r' % (path,), x)
368
raise SFTPTransportError('Unable to write file %r' % (relpath,), x)
274
369
except Exception, e:
275
370
# If we fail, try to clean up the temporary file
276
371
# before we throw the exception
290
385
file_existed = True
291
386
except:
292
387
file_existed = False
388
success = False
293
389
try:
294
self._sftp.rename(tmp_abspath, final_path)
295
except IOError, e:
296
self._translate_io_exception(e, relpath)
297
except paramiko.SSHException, x:
298
raise SFTPTransportError('Unable to rename into file %r'
299
% (path,), x)
300
if file_existed:
301
self._sftp.unlink(tmp_safety)
390
try:
391
self._sftp.rename(tmp_abspath, final_path)
392
except IOError, e:
393
self._translate_io_exception(e, relpath)
394
except paramiko.SSHException, x:
395
raise SFTPTransportError('Unable to rename into file %r' % (path,), x)
396
else:
397
success = True
398
finally:
399
if file_existed:
400
if success:
401
self._sftp.unlink(tmp_safety)
402
else:
403
self._sftp.rename(tmp_safety, final_path)
302
404
303
405
def iter_files_recursive(self):
304
406
"""Walk the relative paths of all files in this transport."""
351
453
"""Copy the item at rel_from to the location at rel_to"""
352
454
path_from = self._abspath(rel_from)
353
455
path_to = self._abspath(rel_to)
456
self._copy_abspaths(path_from, path_to)
457
458
def _copy_abspaths(self, path_from, path_to):
459
"""Copy files given an absolute path
460
461
:param path_from: Path on remote server to read
462
:param path_to: Path on remote server to write
463
:return: None
464
465
TODO: Should the destination location be atomically created?
466
This has not been specified
467
TODO: This should use some sort of remote copy, rather than
468
pulling the data locally, and then writing it remotely
469
"""
354
470
try:
355
471
fin = self._sftp.file(path_from, 'rb')
356
472
try:
365
481
except (IOError, paramiko.SSHException), x:
366
482
raise SFTPTransportError('Unable to copy %r to %r' % (path_from, path_to), x)
367
483
484
def copy_to(self, relpaths, other, pb=None):
485
"""Copy a set of entries from self into another Transport.
486
487
:param relpaths: A list/generator of entries to be copied.
488
"""
489
if isinstance(other, SFTPTransport) and other._sftp is self._sftp:
490
# Both from & to are on the same remote filesystem
491
# We can use a remote copy, instead of pulling locally, and pushing
492
493
total = self._get_total(relpaths)
494
count = 0
495
for path in relpaths:
496
path_from = self._abspath(relpath)
497
path_to = other._abspath(relpath)
498
self._update_pb(pb, 'copy-to', count, total)
499
self._copy_abspaths(path_from, path_to)
500
count += 1
501
return count
502
else:
503
return super(SFTPTransport, self).copy_to(relpaths, other, pb=pb)
504
505
# The dummy implementation just does a simple get + put
506
def copy_entry(path):
507
other.put(path, self.get(path))
508
509
return self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False)
510
368
511
def move(self, rel_from, rel_to):
369
512
"""Move the item at rel_from to the location at rel_to"""
370
513
path_from = self._abspath(rel_from)
435
578
def _unparse_url(self, path=None):
436
579
if path is None:
437
580
path = self._path
438
host = self._host
439
username = urllib.quote(self._username)
440
if self._password:
441
username += ':' + urllib.quote(self._password)
442
if self._port != 22:
443
host += ':%d' % self._port
444
return 'sftp://%s@%s/%s' % (username, host, urllib.quote(path))
581
path = urllib.quote(path)
582
if path.startswith('/'):
583
path = '/%2F' + path[1:]
584
else:
585
path = '/' + path
586
netloc = urllib.quote(self._host)
587
if self._username is not None:
588
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
589
if self._port is not None:
590
netloc = '%s:%d' % (netloc, self._port)
591
592
return urlparse.urlunparse(('sftp', netloc, path, '', '', ''))
593
594
def _split_url(self, url):
595
if isinstance(url, unicode):
596
url = url.encode('utf-8')
597
(scheme, netloc, path, params,
598
query, fragment) = urlparse.urlparse(url, allow_fragments=False)
599
assert scheme == 'sftp'
600
username = password = host = port = None
601
if '@' in netloc:
602
username, host = netloc.split('@', 1)
603
if ':' in username:
604
username, password = username.split(':', 1)
605
password = urllib.unquote(password)
606
username = urllib.unquote(username)
607
else:
608
host = netloc
609
610
if ':' in host:
611
host, port = host.rsplit(':', 1)
612
try:
613
port = int(port)
614
except ValueError:
615
raise SFTPTransportError('%s: invalid port number' % port)
616
host = urllib.unquote(host)
617
618
path = urllib.unquote(path)
619
620
# the initial slash should be removed from the path, and treated
621
# as a homedir relative path (the path begins with a double slash
622
# if it is absolute).
623
# see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt
624
if path.startswith('/'):
625
path = path[1:]
626
627
return (username, password, host, port, path)
445
628
446
629
def _parse_url(self, url):
447
assert url[:7] == 'sftp://'
448
m = self._url_matcher.match(url)
449
if m is None:
450
raise SFTPTransportError('Unable to parse SFTP URL %r' % (url,))
451
self._username, self._password, self._host, self._port, self._path = m.groups()
452
if self._username is None:
453
self._username = getpass.getuser()
454
else:
455
if self._password:
456
# username field is 'user:pass@' in this case, and password is ':pass'
457
username_len = len(self._username) - len(self._password) - 1
458
self._username = urllib.unquote(self._username[:username_len])
459
self._password = urllib.unquote(self._password[1:])
460
else:
461
self._username = urllib.unquote(self._username[:-1])
462
if self._port is None:
463
self._port = 22
464
else:
465
self._port = int(self._port[1:])
466
if (self._path is None) or (self._path == ''):
467
self._path = ''
468
else:
469
# remove leading '/'
470
self._path = urllib.unquote(self._path[1:])
630
(self._username, self._password,
631
self._host, self._port, self._path) = self._split_url(url)
471
632
472
633
def _sftp_connect(self):
634
"""Connect to the remote sftp server.
635
After this, self._sftp should have a valid connection (or
636
we raise an SFTPTransportError 'could not connect').
637
638
TODO: Raise a more reasonable ConnectionFailed exception
639
"""
640
global _connected_hosts
641
642
idx = (self._host, self._port, self._username)
643
try:
644
self._sftp = _connected_hosts[idx]
645
return
646
except KeyError:
647
pass
648
649
vendor = _get_ssh_vendor()
650
if vendor != 'none':
651
sock = SFTPSubprocess(self._host, self._port, self._username)
652
self._sftp = SFTPClient(sock)
653
else:
654
self._paramiko_connect()
655
656
_connected_hosts[idx] = self._sftp
657
658
def _paramiko_connect(self):
473
659
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
474
660
475
661
load_host_keys()
476
662
477
663
try:
478
664
t = paramiko.Transport((self._host, self._port))
479
665
t.start_client()
504
690
(self._host, our_server_key_hex, server_key_hex),
505
691
['Try editing %s or %s' % (filename1, filename2)])
506
692
507
self._sftp_auth(t, self._username, self._host)
693
self._sftp_auth(t)
508
694
509
695
try:
510
696
self._sftp = t.open_sftp_client()
512
698
raise BzrError('Unable to find path %s on SFTP server %s' % \
513
699
(self._path, self._host))
514
700
515
def _sftp_auth(self, transport, username, host):
701
def _sftp_auth(self, transport):
702
# paramiko requires a username, but it might be none if nothing was supplied
703
# use the local username, just in case.
704
# We don't override self._username, because if we aren't using paramiko,
705
# the username might be specified in ~/.ssh/config and we don't want to
706
# force it to something else
707
# Also, it would mess up the self.relpath() functionality
708
username = self._username or getpass.getuser()
709
516
710
agent = paramiko.Agent()
517
711
for key in agent.get_keys():
518
712
mutter('Trying SSH agent key %s' % paramiko.util.hexify(key.get_fingerprint()))
519
713
try:
520
transport.auth_publickey(self._username, key)
714
transport.auth_publickey(username, key)
521
715
return
522
716
except paramiko.SSHException, e:
523
717
pass
524
718
525
719
# okay, try finding id_rsa or id_dss? (posix only)
526
if self._try_pkey_auth(transport, paramiko.RSAKey, 'id_rsa'):
527
return
528
if self._try_pkey_auth(transport, paramiko.DSSKey, 'id_dsa'):
529
return
720
if self._try_pkey_auth(transport, paramiko.RSAKey, username, 'id_rsa'):
721
return
722
if self._try_pkey_auth(transport, paramiko.DSSKey, username, 'id_dsa'):
723
return
724
530
725
531
726
if self._password:
532
727
try:
533
transport.auth_password(self._username, self._password)
728
transport.auth_password(username, self._password)
534
729
return
535
730
except paramiko.SSHException, e:
536
731
pass
537
732
733
# FIXME: Don't keep a password held in memory if you can help it
734
#self._password = None
735
538
736
# give up and ask for a password
539
# FIXME: shouldn't be implementing UI this deep into bzrlib
540
enc = sys.stdout.encoding
541
password = getpass.getpass('SSH %s@%s password: ' %
542
(self._username.encode(enc, 'replace'), self._host.encode(enc, 'replace')))
737
password = ui_factory.get_password(prompt='SSH %(user)s@%(host)s password',
738
user=username, host=self._host)
543
739
try:
544
transport.auth_password(self._username, password)
740
transport.auth_password(username, password)
545
741
except paramiko.SSHException:
546
742
raise SFTPTransportError('Unable to authenticate to SSH host as %s@%s' % \
547
(self._username, self._host))
743
(username, self._host))
548
744
549
def _try_pkey_auth(self, transport, pkey_class, filename):
745
def _try_pkey_auth(self, transport, pkey_class, username, filename):
550
746
filename = os.path.expanduser('~/.ssh/' + filename)
551
747
try:
552
748
key = pkey_class.from_private_key_file(filename)
553
transport.auth_publickey(self._username, key)
749
transport.auth_publickey(username, key)
554
750
return True
555
751
except paramiko.PasswordRequiredException:
556
# FIXME: shouldn't be implementing UI this deep into bzrlib
557
enc = sys.stdout.encoding
558
password = getpass.getpass('SSH %s password: ' %
559
(os.path.basename(filename).encode(enc, 'replace'),))
752
password = ui_factory.get_password(prompt='SSH %(filename)s password',
753
filename=filename)
560
754
try:
561
755
key = pkey_class.from_private_key_file(filename, password)
562
transport.auth_publickey(self._username, key)
756
transport.auth_publickey(username, key)
563
757
return True
564
758
except paramiko.SSHException:
565
759
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
582
776
583
777
:param relpath: The relative path, where the file should be opened
584
778
"""
585
path = self._abspath(relpath)
779
path = self._sftp._adjust_cwd(self._abspath(relpath))
586
780
attr = SFTPAttributes()
587
781
mode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
588
782
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1