« back to all changes in this revision
Viewing changes to bzrlib/transport/sftp.py
- browse files at revision 1440
- compare with another revision
- download diff
- download tarball
- view history from revision 1440
- Committer: Robert Collins
- Date: 2005-10-11 03:19:29 UTC
- Revision ID: robertc@robertcollins.net-20051011031929-2d523107133c43be
further tuning of pull, do not do a local merge or fetch at all, if the remote branch is no newer than we are
- files added:
- bzrlib/mdiff.py
- files removed:
- BRANCH.TODO
- bzrlib/export
- bzrlib/tests/blackbox
- bzrlib/ui
- bzrlib/util/configobj
- bzrlib/util/configobj/docs
- files renamed:
- bzr_man.py => bzr-man.py
- bzrlib/conflicts.py => bzrlib/plugins/conflicts.py
- bzrlib/tests/ => bzrlib/selftest/
- bzrlib/tests/blackbox/test_too_much.py => bzrlib/selftest/blackbox.py
- bzrlib/tests/test_plugins.py => bzrlib/selftest/plugins.py
- bzrlib/tests/blackbox/test_revision_info.py => bzrlib/selftest/test_revision_info.py
- bzrlib/tests/test_branch_implementations.py => bzrlib/selftest/testbranch.py
- bzrlib/tests/test_diff.py => bzrlib/selftest/testdiff.py
- bzrlib/tests/test_fetch.py => bzrlib/selftest/testfetch.py
- bzrlib/tests/test_graph.py => bzrlib/selftest/testgraph.py
- bzrlib/tests/test_hashcache.py => bzrlib/selftest/testhashcache.py
- bzrlib/tests/test_identitymap.py => bzrlib/selftest/testidentitymap.py
- bzrlib/tests/test_inv.py => bzrlib/selftest/testinv.py
- bzrlib/tests/test_log.py => bzrlib/selftest/testlog.py
- bzrlib/tests/test_merge.py => bzrlib/selftest/testmerge.py
- bzrlib/tests/test_merge3.py => bzrlib/selftest/testmerge3.py
- bzrlib/tests/test_revision.py => bzrlib/selftest/testrevision.py
- bzrlib/tests/test_revisionnamespaces.py => bzrlib/selftest/testrevisionnamespaces.py
- bzrlib/tests/test_sampler.py => bzrlib/selftest/testsampler.py
- bzrlib/tests/test_status.py => bzrlib/selftest/teststatus.py
- bzrlib/tests/test_store.py => bzrlib/selftest/teststore.py
- bzrlib/tests/test_transactions.py => bzrlib/selftest/testtransactions.py
- bzrlib/tests/test_transport.py => bzrlib/selftest/testtransport.py
- bzrlib/tests/test_workingtree.py => bzrlib/selftest/testworkingtree.py
- bzrlib/tests/blackbox/test_versioning.py => bzrlib/selftest/versioning.py
- bzrlib/tests/test_whitebox.py => bzrlib/selftest/whitebox.py
- bzrlib/ui/__init__.py => bzrlib/ui.py
- files modified:
- .bzrignore
added
removed
bzrlib/transport/sftp.py
1
# Copyright (C) 2005 Robey Pointer <robey@lag.net>, Canonical Ltd
2
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
12
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
17
"""Implementation of Transport over SFTP, using paramiko."""
18
19
import errno
20
import getpass
21
import os
22
import re
23
import stat
24
import sys
25
import urllib
26
import urlparse
27
import time
28
import random
29
import subprocess
30
import weakref
31
32
from bzrlib.config import config_dir, ensure_config_dir_exists
33
from bzrlib.errors import (ConnectionError,
34
FileExists,
35
TransportNotPossible, NoSuchFile, PathNotChild,
36
TransportError,
37
LockError
38
)
39
from bzrlib.osutils import pathjoin, fancy_rename
40
from bzrlib.trace import mutter, warning, error
41
from bzrlib.transport import Transport, Server, urlescape
42
import bzrlib.ui
43
44
try:
45
import paramiko
46
except ImportError:
47
error('The SFTP transport requires paramiko.')
48
raise
49
else:
50
from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE,
51
SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC,
52
CMD_HANDLE, CMD_OPEN)
53
from paramiko.sftp_attr import SFTPAttributes
54
from paramiko.sftp_file import SFTPFile
55
from paramiko.sftp_client import SFTPClient
56
57
if 'sftp' not in urlparse.uses_netloc: urlparse.uses_netloc.append('sftp')
58
59
60
_close_fds = True
61
if sys.platform == 'win32':
62
# close_fds not supported on win32
63
_close_fds = False
64
65
_ssh_vendor = None
66
def _get_ssh_vendor():
67
"""Find out what version of SSH is on the system."""
68
global _ssh_vendor
69
if _ssh_vendor is not None:
70
return _ssh_vendor
71
72
_ssh_vendor = 'none'
73
74
try:
75
p = subprocess.Popen(['ssh', '-V'],
76
close_fds=_close_fds,
77
stdin=subprocess.PIPE,
78
stdout=subprocess.PIPE,
79
stderr=subprocess.PIPE)
80
returncode = p.returncode
81
stdout, stderr = p.communicate()
82
except OSError:
83
returncode = -1
84
stdout = stderr = ''
85
if 'OpenSSH' in stderr:
86
mutter('ssh implementation is OpenSSH')
87
_ssh_vendor = 'openssh'
88
elif 'SSH Secure Shell' in stderr:
89
mutter('ssh implementation is SSH Corp.')
90
_ssh_vendor = 'ssh'
91
92
if _ssh_vendor != 'none':
93
return _ssh_vendor
94
95
# XXX: 20051123 jamesh
96
# A check for putty's plink or lsh would go here.
97
98
mutter('falling back to paramiko implementation')
99
return _ssh_vendor
100
101
102
class SFTPSubprocess:
103
"""A socket-like object that talks to an ssh subprocess via pipes."""
104
def __init__(self, hostname, vendor, port=None, user=None):
105
assert vendor in ['openssh', 'ssh']
106
if vendor == 'openssh':
107
args = ['ssh',
108
'-oForwardX11=no', '-oForwardAgent=no',
109
'-oClearAllForwardings=yes', '-oProtocol=2',
110
'-oNoHostAuthenticationForLocalhost=yes']
111
if port is not None:
112
args.extend(['-p', str(port)])
113
if user is not None:
114
args.extend(['-l', user])
115
args.extend(['-s', hostname, 'sftp'])
116
elif vendor == 'ssh':
117
args = ['ssh', '-x']
118
if port is not None:
119
args.extend(['-p', str(port)])
120
if user is not None:
121
args.extend(['-l', user])
122
args.extend(['-s', 'sftp', hostname])
123
124
self.proc = subprocess.Popen(args, close_fds=_close_fds,
125
stdin=subprocess.PIPE,
126
stdout=subprocess.PIPE)
127
128
def send(self, data):
129
return os.write(self.proc.stdin.fileno(), data)
130
131
def recv_ready(self):
132
# TODO: jam 20051215 this function is necessary to support the
133
# pipelined() function. In reality, it probably should use
134
# poll() or select() to actually return if there is data
135
# available, otherwise we probably don't get any benefit
136
return True
137
138
def recv(self, count):
139
return os.read(self.proc.stdout.fileno(), count)
140
141
def close(self):
142
self.proc.stdin.close()
143
self.proc.stdout.close()
144
self.proc.wait()
145
146
147
SYSTEM_HOSTKEYS = {}
148
BZR_HOSTKEYS = {}
149
150
# This is a weakref dictionary, so that we can reuse connections
151
# that are still active. Long term, it might be nice to have some
152
# sort of expiration policy, such as disconnect if inactive for
153
# X seconds. But that requires a lot more fanciness.
154
_connected_hosts = weakref.WeakValueDictionary()
155
156
def load_host_keys():
157
"""
158
Load system host keys (probably doesn't work on windows) and any
159
"discovered" keys from previous sessions.
160
"""
161
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
162
try:
163
SYSTEM_HOSTKEYS = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
164
except Exception, e:
165
mutter('failed to load system host keys: ' + str(e))
166
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
167
try:
168
BZR_HOSTKEYS = paramiko.util.load_host_keys(bzr_hostkey_path)
169
except Exception, e:
170
mutter('failed to load bzr host keys: ' + str(e))
171
save_host_keys()
172
173
def save_host_keys():
174
"""
175
Save "discovered" host keys in $(config)/ssh_host_keys/.
176
"""
177
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
178
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
179
ensure_config_dir_exists()
180
181
try:
182
f = open(bzr_hostkey_path, 'w')
183
f.write('# SSH host keys collected by bzr\n')
184
for hostname, keys in BZR_HOSTKEYS.iteritems():
185
for keytype, key in keys.iteritems():
186
f.write('%s %s %s\n' % (hostname, keytype, key.get_base64()))
187
f.close()
188
except IOError, e:
189
mutter('failed to save bzr host keys: ' + str(e))
190
191
192
class SFTPLock(object):
193
"""This fakes a lock in a remote location."""
194
__slots__ = ['path', 'lock_path', 'lock_file', 'transport']
195
def __init__(self, path, transport):
196
assert isinstance(transport, SFTPTransport)
197
198
self.lock_file = None
199
self.path = path
200
self.lock_path = path + '.write-lock'
201
self.transport = transport
202
try:
203
# RBC 20060103 FIXME should we be using private methods here ?
204
abspath = transport._remote_path(self.lock_path)
205
self.lock_file = transport._sftp_open_exclusive(abspath)
206
except FileExists:
207
raise LockError('File %r already locked' % (self.path,))
208
209
def __del__(self):
210
"""Should this warn, or actually try to cleanup?"""
211
if self.lock_file:
212
warn("SFTPLock %r not explicitly unlocked" % (self.path,))
213
self.unlock()
214
215
def unlock(self):
216
if not self.lock_file:
217
return
218
self.lock_file.close()
219
self.lock_file = None
220
try:
221
self.transport.delete(self.lock_path)
222
except (NoSuchFile,):
223
# What specific errors should we catch here?
224
pass
225
226
227
228
class SFTPTransport (Transport):
229
"""
230
Transport implementation for SFTP access.
231
"""
232
_do_prefetch = False # Right now Paramiko's prefetch support causes things to hang
233
234
def __init__(self, base, clone_from=None):
235
assert base.startswith('sftp://')
236
self._parse_url(base)
237
base = self._unparse_url()
238
if base[-1] != '/':
239
base = base + '/'
240
super(SFTPTransport, self).__init__(base)
241
if clone_from is None:
242
self._sftp_connect()
243
else:
244
# use the same ssh connection, etc
245
self._sftp = clone_from._sftp
246
# super saves 'self.base'
247
248
def should_cache(self):
249
"""
250
Return True if the data pulled across should be cached locally.
251
"""
252
return True
253
254
def clone(self, offset=None):
255
"""
256
Return a new SFTPTransport with root at self.base + offset.
257
We share the same SFTP session between such transports, because it's
258
fairly expensive to set them up.
259
"""
260
if offset is None:
261
return SFTPTransport(self.base, self)
262
else:
263
return SFTPTransport(self.abspath(offset), self)
264
265
def abspath(self, relpath):
266
"""
267
Return the full url to the given relative path.
268
269
@param relpath: the relative path or path components
270
@type relpath: str or list
271
"""
272
return self._unparse_url(self._remote_path(relpath))
273
274
def _remote_path(self, relpath):
275
"""Return the path to be passed along the sftp protocol for relpath.
276
277
relpath is a urlencoded string.
278
"""
279
# FIXME: share the common code across transports
280
assert isinstance(relpath, basestring)
281
relpath = urllib.unquote(relpath).split('/')
282
basepath = self._path.split('/')
283
if len(basepath) > 0 and basepath[-1] == '':
284
basepath = basepath[:-1]
285
286
for p in relpath:
287
if p == '..':
288
if len(basepath) == 0:
289
# In most filesystems, a request for the parent
290
# of root, just returns root.
291
continue
292
basepath.pop()
293
elif p == '.':
294
continue # No-op
295
else:
296
basepath.append(p)
297
298
path = '/'.join(basepath)
299
return path
300
301
def relpath(self, abspath):
302
username, password, host, port, path = self._split_url(abspath)
303
error = []
304
if (username != self._username):
305
error.append('username mismatch')
306
if (host != self._host):
307
error.append('host mismatch')
308
if (port != self._port):
309
error.append('port mismatch')
310
if (not path.startswith(self._path)):
311
error.append('path mismatch')
312
if error:
313
extra = ': ' + ', '.join(error)
314
raise PathNotChild(abspath, self.base, extra=extra)
315
pl = len(self._path)
316
return path[pl:].strip('/')
317
318
def has(self, relpath):
319
"""
320
Does the target location exist?
321
"""
322
try:
323
self._sftp.stat(self._remote_path(relpath))
324
return True
325
except IOError:
326
return False
327
328
def get(self, relpath, decode=False):
329
"""
330
Get the file at the given relative path.
331
332
:param relpath: The relative path to the file
333
"""
334
try:
335
path = self._remote_path(relpath)
336
f = self._sftp.file(path, mode='rb')
337
if self._do_prefetch and hasattr(f, 'prefetch'):
338
f.prefetch()
339
return f
340
except (IOError, paramiko.SSHException), e:
341
self._translate_io_exception(e, path, ': error retrieving')
342
343
def get_partial(self, relpath, start, length=None):
344
"""
345
Get just part of a file.
346
347
:param relpath: Path to the file, relative to base
348
:param start: The starting position to read from
349
:param length: The length to read. A length of None indicates
350
read to the end of the file.
351
:return: A file-like object containing at least the specified bytes.
352
Some implementations may return objects which can be read
353
past this length, but this is not guaranteed.
354
"""
355
# TODO: implement get_partial_multi to help with knit support
356
f = self.get(relpath)
357
f.seek(start)
358
if self._do_prefetch and hasattr(f, 'prefetch'):
359
f.prefetch()
360
return f
361
362
def put(self, relpath, f, mode=None):
363
"""
364
Copy the file-like or string object into the location.
365
366
:param relpath: Location to put the contents, relative to base.
367
:param f: File-like or string object.
368
:param mode: The final mode for the file
369
"""
370
final_path = self._remote_path(relpath)
371
self._put(final_path, f, mode=mode)
372
373
def _put(self, abspath, f, mode=None):
374
"""Helper function so both put() and copy_abspaths can reuse the code"""
375
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
376
os.getpid(), random.randint(0,0x7FFFFFFF))
377
fout = self._sftp_open_exclusive(tmp_abspath, mode=mode)
378
closed = False
379
try:
380
try:
381
fout.set_pipelined(True)
382
self._pump(f, fout)
383
except (IOError, paramiko.SSHException), e:
384
self._translate_io_exception(e, tmp_abspath)
385
if mode is not None:
386
self._sftp.chmod(tmp_abspath, mode)
387
fout.close()
388
closed = True
389
self._rename(tmp_abspath, abspath)
390
except Exception, e:
391
# If we fail, try to clean up the temporary file
392
# before we throw the exception
393
# but don't let another exception mess things up
394
# Write out the traceback, because otherwise
395
# the catch and throw destroys it
396
import traceback
397
mutter(traceback.format_exc())
398
try:
399
if not closed:
400
fout.close()
401
self._sftp.remove(tmp_abspath)
402
except:
403
# raise the saved except
404
raise e
405
# raise the original with its traceback if we can.
406
raise
407
408
def iter_files_recursive(self):
409
"""Walk the relative paths of all files in this transport."""
410
queue = list(self.list_dir('.'))
411
while queue:
412
relpath = urllib.quote(queue.pop(0))
413
st = self.stat(relpath)
414
if stat.S_ISDIR(st.st_mode):
415
for i, basename in enumerate(self.list_dir(relpath)):
416
queue.insert(i, relpath+'/'+basename)
417
else:
418
yield relpath
419
420
def mkdir(self, relpath, mode=None):
421
"""Create a directory at the given path."""
422
try:
423
path = self._remote_path(relpath)
424
# In the paramiko documentation, it says that passing a mode flag
425
# will filtered against the server umask.
426
# StubSFTPServer does not do this, which would be nice, because it is
427
# what we really want :)
428
# However, real servers do use umask, so we really should do it that way
429
self._sftp.mkdir(path)
430
if mode is not None:
431
self._sftp.chmod(path, mode=mode)
432
except (paramiko.SSHException, IOError), e:
433
self._translate_io_exception(e, path, ': unable to mkdir',
434
failure_exc=FileExists)
435
436
def _translate_io_exception(self, e, path, more_info='', failure_exc=NoSuchFile):
437
"""Translate a paramiko or IOError into a friendlier exception.
438
439
:param e: The original exception
440
:param path: The path in question when the error is raised
441
:param more_info: Extra information that can be included,
442
such as what was going on
443
:param failure_exc: Paramiko has the super fun ability to raise completely
444
opaque errors that just set "e.args = ('Failure',)" with
445
no more information.
446
This sometimes means FileExists, but it also sometimes
447
means NoSuchFile
448
"""
449
# paramiko seems to generate detailless errors.
450
self._translate_error(e, path, raise_generic=False)
451
if hasattr(e, 'args'):
452
if (e.args == ('No such file or directory',) or
453
e.args == ('No such file',)):
454
raise NoSuchFile(path, str(e) + more_info)
455
if (e.args == ('mkdir failed',)):
456
raise FileExists(path, str(e) + more_info)
457
# strange but true, for the paramiko server.
458
if (e.args == ('Failure',)):
459
raise failure_exc(path, str(e) + more_info)
460
mutter('Raising exception with args %s', e.args)
461
if hasattr(e, 'errno'):
462
mutter('Raising exception with errno %s', e.errno)
463
raise e
464
465
def append(self, relpath, f):
466
"""
467
Append the text in the file-like object into the final
468
location.
469
"""
470
try:
471
path = self._remote_path(relpath)
472
fout = self._sftp.file(path, 'ab')
473
self._pump(f, fout)
474
except (IOError, paramiko.SSHException), e:
475
self._translate_io_exception(e, relpath, ': unable to append')
476
477
def copy(self, rel_from, rel_to):
478
"""Copy the item at rel_from to the location at rel_to"""
479
path_from = self._remote_path(rel_from)
480
path_to = self._remote_path(rel_to)
481
self._copy_abspaths(path_from, path_to)
482
483
def _copy_abspaths(self, path_from, path_to, mode=None):
484
"""Copy files given an absolute path
485
486
:param path_from: Path on remote server to read
487
:param path_to: Path on remote server to write
488
:return: None
489
490
TODO: Should the destination location be atomically created?
491
This has not been specified
492
TODO: This should use some sort of remote copy, rather than
493
pulling the data locally, and then writing it remotely
494
"""
495
try:
496
fin = self._sftp.file(path_from, 'rb')
497
try:
498
self._put(path_to, fin, mode=mode)
499
finally:
500
fin.close()
501
except (IOError, paramiko.SSHException), e:
502
self._translate_io_exception(e, path_from, ': unable copy to: %r' % path_to)
503
504
def copy_to(self, relpaths, other, mode=None, pb=None):
505
"""Copy a set of entries from self into another Transport.
506
507
:param relpaths: A list/generator of entries to be copied.
508
"""
509
if isinstance(other, SFTPTransport) and other._sftp is self._sftp:
510
# Both from & to are on the same remote filesystem
511
# We can use a remote copy, instead of pulling locally, and pushing
512
513
total = self._get_total(relpaths)
514
count = 0
515
for path in relpaths:
516
path_from = self._remote_path(relpath)
517
path_to = other._remote_path(relpath)
518
self._update_pb(pb, 'copy-to', count, total)
519
self._copy_abspaths(path_from, path_to, mode=mode)
520
count += 1
521
return count
522
else:
523
return super(SFTPTransport, self).copy_to(relpaths, other, mode=mode, pb=pb)
524
525
def _rename(self, abs_from, abs_to):
526
"""Do a fancy rename on the remote server.
527
528
Using the implementation provided by osutils.
529
"""
530
try:
531
fancy_rename(abs_from, abs_to,
532
rename_func=self._sftp.rename,
533
unlink_func=self._sftp.remove)
534
except (IOError, paramiko.SSHException), e:
535
self._translate_io_exception(e, abs_from, ': unable to rename to %r' % (abs_to))
536
537
def move(self, rel_from, rel_to):
538
"""Move the item at rel_from to the location at rel_to"""
539
path_from = self._remote_path(rel_from)
540
path_to = self._remote_path(rel_to)
541
self._rename(path_from, path_to)
542
543
def delete(self, relpath):
544
"""Delete the item at relpath"""
545
path = self._remote_path(relpath)
546
try:
547
self._sftp.remove(path)
548
except (IOError, paramiko.SSHException), e:
549
self._translate_io_exception(e, path, ': unable to delete')
550
551
def listable(self):
552
"""Return True if this store supports listing."""
553
return True
554
555
def list_dir(self, relpath):
556
"""
557
Return a list of all files at the given location.
558
"""
559
# does anything actually use this?
560
path = self._remote_path(relpath)
561
try:
562
return self._sftp.listdir(path)
563
except (IOError, paramiko.SSHException), e:
564
self._translate_io_exception(e, path, ': failed to list_dir')
565
566
def rmdir(self, relpath):
567
"""See Transport.rmdir."""
568
path = self._remote_path(relpath)
569
try:
570
return self._sftp.rmdir(path)
571
except (IOError, paramiko.SSHException), e:
572
self._translate_io_exception(e, path, ': failed to rmdir')
573
574
def stat(self, relpath):
575
"""Return the stat information for a file."""
576
path = self._remote_path(relpath)
577
try:
578
return self._sftp.stat(path)
579
except (IOError, paramiko.SSHException), e:
580
self._translate_io_exception(e, path, ': unable to stat')
581
582
def lock_read(self, relpath):
583
"""
584
Lock the given file for shared (read) access.
585
:return: A lock object, which has an unlock() member function
586
"""
587
# FIXME: there should be something clever i can do here...
588
class BogusLock(object):
589
def __init__(self, path):
590
self.path = path
591
def unlock(self):
592
pass
593
return BogusLock(relpath)
594
595
def lock_write(self, relpath):
596
"""
597
Lock the given file for exclusive (write) access.
598
WARNING: many transports do not support this, so trying avoid using it
599
600
:return: A lock object, which has an unlock() member function
601
"""
602
# This is a little bit bogus, but basically, we create a file
603
# which should not already exist, and if it does, we assume
604
# that there is a lock, and if it doesn't, the we assume
605
# that we have taken the lock.
606
return SFTPLock(relpath, self)
607
608
def _unparse_url(self, path=None):
609
if path is None:
610
path = self._path
611
path = urllib.quote(path)
612
# handle homedir paths
613
if not path.startswith('/'):
614
path = "/~/" + path
615
netloc = urllib.quote(self._host)
616
if self._username is not None:
617
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
618
if self._port is not None:
619
netloc = '%s:%d' % (netloc, self._port)
620
621
return urlparse.urlunparse(('sftp', netloc, path, '', '', ''))
622
623
def _split_url(self, url):
624
if isinstance(url, unicode):
625
url = url.encode('utf-8')
626
(scheme, netloc, path, params,
627
query, fragment) = urlparse.urlparse(url, allow_fragments=False)
628
assert scheme == 'sftp'
629
username = password = host = port = None
630
if '@' in netloc:
631
username, host = netloc.split('@', 1)
632
if ':' in username:
633
username, password = username.split(':', 1)
634
password = urllib.unquote(password)
635
username = urllib.unquote(username)
636
else:
637
host = netloc
638
639
if ':' in host:
640
host, port = host.rsplit(':', 1)
641
try:
642
port = int(port)
643
except ValueError:
644
# TODO: Should this be ConnectionError?
645
raise TransportError('%s: invalid port number' % port)
646
host = urllib.unquote(host)
647
648
path = urllib.unquote(path)
649
650
# the initial slash should be removed from the path, and treated
651
# as a homedir relative path (the path begins with a double slash
652
# if it is absolute).
653
# see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt
654
# RBC 20060118 we are not using this as its too user hostile. instead
655
# we are following lftp and using /~/foo to mean '~/foo'.
656
# handle homedir paths
657
if path.startswith('/~/'):
658
path = path[3:]
659
elif path == '/~':
660
path = ''
661
return (username, password, host, port, path)
662
663
def _parse_url(self, url):
664
(self._username, self._password,
665
self._host, self._port, self._path) = self._split_url(url)
666
667
def _sftp_connect(self):
668
"""Connect to the remote sftp server.
669
After this, self._sftp should have a valid connection (or
670
we raise an TransportError 'could not connect').
671
672
TODO: Raise a more reasonable ConnectionFailed exception
673
"""
674
global _connected_hosts
675
676
idx = (self._host, self._port, self._username)
677
try:
678
self._sftp = _connected_hosts[idx]
679
return
680
except KeyError:
681
pass
682
683
vendor = _get_ssh_vendor()
684
if vendor != 'none':
685
sock = SFTPSubprocess(self._host, vendor, self._port,
686
self._username)
687
self._sftp = SFTPClient(sock)
688
else:
689
self._paramiko_connect()
690
691
_connected_hosts[idx] = self._sftp
692
693
def _paramiko_connect(self):
694
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
695
696
load_host_keys()
697
698
try:
699
t = paramiko.Transport((self._host, self._port or 22))
700
t.start_client()
701
except paramiko.SSHException, e:
702
raise ConnectionError('Unable to reach SSH host %s:%d' %
703
(self._host, self._port), e)
704
705
server_key = t.get_remote_server_key()
706
server_key_hex = paramiko.util.hexify(server_key.get_fingerprint())
707
keytype = server_key.get_name()
708
if SYSTEM_HOSTKEYS.has_key(self._host) and SYSTEM_HOSTKEYS[self._host].has_key(keytype):
709
our_server_key = SYSTEM_HOSTKEYS[self._host][keytype]
710
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
711
elif BZR_HOSTKEYS.has_key(self._host) and BZR_HOSTKEYS[self._host].has_key(keytype):
712
our_server_key = BZR_HOSTKEYS[self._host][keytype]
713
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
714
else:
715
warning('Adding %s host key for %s: %s' % (keytype, self._host, server_key_hex))
716
if not BZR_HOSTKEYS.has_key(self._host):
717
BZR_HOSTKEYS[self._host] = {}
718
BZR_HOSTKEYS[self._host][keytype] = server_key
719
our_server_key = server_key
720
our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
721
save_host_keys()
722
if server_key != our_server_key:
723
filename1 = os.path.expanduser('~/.ssh/known_hosts')
724
filename2 = pathjoin(config_dir(), 'ssh_host_keys')
725
raise TransportError('Host keys for %s do not match! %s != %s' % \
726
(self._host, our_server_key_hex, server_key_hex),
727
['Try editing %s or %s' % (filename1, filename2)])
728
729
self._sftp_auth(t)
730
731
try:
732
self._sftp = t.open_sftp_client()
733
except paramiko.SSHException, e:
734
raise ConnectionError('Unable to start sftp client %s:%d' %
735
(self._host, self._port), e)
736
737
def _sftp_auth(self, transport):
738
# paramiko requires a username, but it might be none if nothing was supplied
739
# use the local username, just in case.
740
# We don't override self._username, because if we aren't using paramiko,
741
# the username might be specified in ~/.ssh/config and we don't want to
742
# force it to something else
743
# Also, it would mess up the self.relpath() functionality
744
username = self._username or getpass.getuser()
745
746
# Paramiko tries to open a socket.AF_UNIX in order to connect
747
# to ssh-agent. That attribute doesn't exist on win32 (it does in cygwin)
748
# so we get an AttributeError exception. For now, just don't try to
749
# connect to an agent if we are on win32
750
if sys.platform != 'win32':
751
agent = paramiko.Agent()
752
for key in agent.get_keys():
753
mutter('Trying SSH agent key %s' % paramiko.util.hexify(key.get_fingerprint()))
754
try:
755
transport.auth_publickey(username, key)
756
return
757
except paramiko.SSHException, e:
758
pass
759
760
# okay, try finding id_rsa or id_dss? (posix only)
761
if self._try_pkey_auth(transport, paramiko.RSAKey, username, 'id_rsa'):
762
return
763
if self._try_pkey_auth(transport, paramiko.DSSKey, username, 'id_dsa'):
764
return
765
766
if self._password:
767
try:
768
transport.auth_password(username, self._password)
769
return
770
except paramiko.SSHException, e:
771
pass
772
773
# FIXME: Don't keep a password held in memory if you can help it
774
#self._password = None
775
776
# give up and ask for a password
777
password = bzrlib.ui.ui_factory.get_password(
778
prompt='SSH %(user)s@%(host)s password',
779
user=username, host=self._host)
780
try:
781
transport.auth_password(username, password)
782
except paramiko.SSHException, e:
783
raise ConnectionError('Unable to authenticate to SSH host as %s@%s' %
784
(username, self._host), e)
785
786
def _try_pkey_auth(self, transport, pkey_class, username, filename):
787
filename = os.path.expanduser('~/.ssh/' + filename)
788
try:
789
key = pkey_class.from_private_key_file(filename)
790
transport.auth_publickey(username, key)
791
return True
792
except paramiko.PasswordRequiredException:
793
password = bzrlib.ui.ui_factory.get_password(
794
prompt='SSH %(filename)s password',
795
filename=filename)
796
try:
797
key = pkey_class.from_private_key_file(filename, password)
798
transport.auth_publickey(username, key)
799
return True
800
except paramiko.SSHException:
801
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
802
except paramiko.SSHException:
803
mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
804
except IOError:
805
pass
806
return False
807
808
def _sftp_open_exclusive(self, abspath, mode=None):
809
"""Open a remote path exclusively.
810
811
SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if
812
the file already exists. However it does not expose this
813
at the higher level of SFTPClient.open(), so we have to
814
sneak away with it.
815
816
WARNING: This breaks the SFTPClient abstraction, so it
817
could easily break against an updated version of paramiko.
818
819
:param abspath: The remote absolute path where the file should be opened
820
:param mode: The mode permissions bits for the new file
821
"""
822
path = self._sftp._adjust_cwd(abspath)
823
attr = SFTPAttributes()
824
if mode is not None:
825
attr.st_mode = mode
826
omode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
827
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
828
try:
829
t, msg = self._sftp._request(CMD_OPEN, path, omode, attr)
830
if t != CMD_HANDLE:
831
raise TransportError('Expected an SFTP handle')
832
handle = msg.get_string()
833
return SFTPFile(self._sftp, handle, 'wb', -1)
834
except (paramiko.SSHException, IOError), e:
835
self._translate_io_exception(e, abspath, ': unable to open',
836
failure_exc=FileExists)
837
838
839
# ------------- server test implementation --------------
840
import socket
841
import threading
842
843
from bzrlib.tests.stub_sftp import StubServer, StubSFTPServer
844
845
STUB_SERVER_KEY = """
846
-----BEGIN RSA PRIVATE KEY-----
847
MIICWgIBAAKBgQDTj1bqB4WmayWNPB+8jVSYpZYk80Ujvj680pOTh2bORBjbIAyz
848
oWGW+GUjzKxTiiPvVmxFgx5wdsFvF03v34lEVVhMpouqPAYQ15N37K/ir5XY+9m/
849
d8ufMCkjeXsQkKqFbAlQcnWMCRnOoPHS3I4vi6hmnDDeeYTSRvfLbW0fhwIBIwKB
850
gBIiOqZYaoqbeD9OS9z2K9KR2atlTxGxOJPXiP4ESqP3NVScWNwyZ3NXHpyrJLa0
851
EbVtzsQhLn6rF+TzXnOlcipFvjsem3iYzCpuChfGQ6SovTcOjHV9z+hnpXvQ/fon
852
soVRZY65wKnF7IAoUwTmJS9opqgrN6kRgCd3DASAMd1bAkEA96SBVWFt/fJBNJ9H
853
tYnBKZGw0VeHOYmVYbvMSstssn8un+pQpUm9vlG/bp7Oxd/m+b9KWEh2xPfv6zqU
854
avNwHwJBANqzGZa/EpzF4J8pGti7oIAPUIDGMtfIcmqNXVMckrmzQ2vTfqtkEZsA
855
4rE1IERRyiJQx6EJsz21wJmGV9WJQ5kCQQDwkS0uXqVdFzgHO6S++tjmjYcxwr3g
856
H0CoFYSgbddOT6miqRskOQF3DZVkJT3kyuBgU2zKygz52ukQZMqxCb1fAkASvuTv
857
qfpH87Qq5kQhNKdbbwbmd2NxlNabazPijWuphGTdW0VfJdWfklyS2Kr+iqrs/5wV
858
HhathJt636Eg7oIjAkA8ht3MQ+XSl9yIJIS8gVpbPxSw5OMfw0PjVE7tBdQruiSc
859
nvuQES5C9BMHjF39LZiGH1iLQy7FgdHyoP+eodI7
860
-----END RSA PRIVATE KEY-----
861
"""
862
863
864
class SingleListener(threading.Thread):
865
866
def __init__(self, callback):
867
threading.Thread.__init__(self)
868
self._callback = callback
869
self._socket = socket.socket()
870
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
871
self._socket.bind(('localhost', 0))
872
self._socket.listen(1)
873
self.port = self._socket.getsockname()[1]
874
self.stop_event = threading.Event()
875
876
def run(self):
877
s, _ = self._socket.accept()
878
# now close the listen socket
879
self._socket.close()
880
self._callback(s, self.stop_event)
881
882
def stop(self):
883
self.stop_event.set()
884
# We should consider waiting for the other thread
885
# to stop, because otherwise we get spurious
886
# bzr: ERROR: Socket exception: Connection reset by peer (54)
887
# because the test suite finishes before the thread has a chance
888
# to close. (Especially when only running a few tests)
889
890
891
class SFTPServer(Server):
892
"""Common code for SFTP server facilities."""
893
894
def _get_sftp_url(self, path):
895
"""Calculate a sftp url to this server for path."""
896
return 'sftp://foo:bar@localhost:%d/%s' % (self._listener.port, path)
897
898
def __init__(self):
899
self._original_vendor = None
900
self._homedir = None
901
self._server_homedir = None
902
self._listener = None
903
self._root = None
904
# sftp server logs
905
self.logs = []
906
907
def log(self, message):
908
"""What to do here? do we need this? Its for the StubServer.."""
909
self.logs.append(message)
910
911
def _run_server(self, s, stop_event):
912
ssh_server = paramiko.Transport(s)
913
key_file = os.path.join(self._homedir, 'test_rsa.key')
914
file(key_file, 'w').write(STUB_SERVER_KEY)
915
host_key = paramiko.RSAKey.from_private_key_file(key_file)
916
ssh_server.add_server_key(host_key)
917
server = StubServer(self)
918
ssh_server.set_subsystem_handler('sftp', paramiko.SFTPServer,
919
StubSFTPServer, root=self._root,
920
home=self._server_homedir)
921
event = threading.Event()
922
ssh_server.start_server(event, server)
923
event.wait(5.0)
924
stop_event.wait(30.0)
925
926
def setUp(self):
927
"""See bzrlib.transport.Server.setUp."""
928
# XXX: 20051124 jamesh
929
# The tests currently pop up a password prompt when an external ssh
930
# is used. This forces the use of the paramiko implementation.
931
global _ssh_vendor
932
self._original_vendor = _ssh_vendor
933
_ssh_vendor = 'none'
934
self._homedir = os.getcwdu()
935
if self._server_homedir is None:
936
self._server_homedir = self._homedir
937
self._root = '/'
938
# FIXME WINDOWS: _root should be _server_homedir[0]:/
939
self._listener = SingleListener(self._run_server)
940
self._listener.setDaemon(True)
941
self._listener.start()
942
943
def tearDown(self):
944
"""See bzrlib.transport.Server.tearDown."""
945
global _ssh_vendor
946
self._listener.stop()
947
_ssh_vendor = self._original_vendor
948
949
950
class SFTPAbsoluteServer(SFTPServer):
951
"""A test server for sftp transports, using absolute urls."""
952
953
def get_url(self):
954
"""See bzrlib.transport.Server.get_url."""
955
return self._get_sftp_url(urlescape(self._homedir[1:]))
956
957
958
class SFTPHomeDirServer(SFTPServer):
959
"""A test server for sftp transports, using homedir relative urls."""
960
961
def get_url(self):
962
"""See bzrlib.transport.Server.get_url."""
963
return self._get_sftp_url("~/")
964
965
966
class SFTPSiblingAbsoluteServer(SFTPAbsoluteServer):
967
"""A test servere for sftp transports, using absolute urls to non-home."""
968
969
def setUp(self):
970
self._server_homedir = '/dev/noone/runs/tests/here'
971
super(SFTPSiblingAbsoluteServer, self).setUp()
972
973
974
def get_test_permutations():
975
"""Return the permutations to be used in testing."""
976
return [(SFTPTransport, SFTPAbsoluteServer),
977
(SFTPTransport, SFTPHomeDirServer),
978
(SFTPTransport, SFTPSiblingAbsoluteServer),
979
]
Loggerhead is a web-based interface for Breezy
Version: 2.0.1