« back to all changes in this revision
Viewing changes to bzrlib/transport/sftp.py
- browse files at revision 1530.2.2
- compare with another revision
- compare with revision 1185.33.48
- download diff
- download tarball
- view history from revision 1530.2.2
- Committer: Robert Collins
- Date: 2006-01-12 02:59:28 UTC
- mfrom: (1534.1.6 integration)
- mto: (1185.50.57 bzr-jam-integration)
- mto: This revision was merged to the branch mainline in revision 1550.
- Revision ID: robertc@robertcollins.net-20060112025928-7ae30672ae84bc0e
Merge from integration.
- files added:
- BRANCH.TODO
- files renamed:
- bzrlib/tests/test_revision_info.py => bzrlib/tests/blackbox/test_revision_info.py
- bzrlib/tests/test_sftp.py => bzrlib/tests/test_sftp_transport.py
- files modified:
- HACKING
added
removed
bzrlib/transport/sftp.py
29
29
import subprocess
30
30
import weakref
31
31
32
from bzrlib.errors import (FileExists,
32
from bzrlib.config import config_dir, ensure_config_dir_exists
33
from bzrlib.errors import (ConnectionError,
34
FileExists,
33
35
TransportNotPossible, NoSuchFile, PathNotChild,
34
36
TransportError,
35
LockError)
36
from bzrlib.config import config_dir
37
LockError
38
)
39
from bzrlib.osutils import pathjoin, fancy_rename
37
40
from bzrlib.trace import mutter, warning, error
38
from bzrlib.transport import Transport, register_transport
41
from bzrlib.transport import Transport, Server, urlescape
39
42
import bzrlib.ui
40
43
41
44
try:
98
101
99
102
class SFTPSubprocess:
100
103
"""A socket-like object that talks to an ssh subprocess via pipes."""
101
def __init__(self, hostname, port=None, user=None):
102
vendor = _get_ssh_vendor()
104
def __init__(self, hostname, vendor, port=None, user=None):
103
105
assert vendor in ['openssh', 'ssh']
104
106
if vendor == 'openssh':
105
107
args = ['ssh',
126
128
def send(self, data):
127
129
return os.write(self.proc.stdin.fileno(), data)
128
130
131
def recv_ready(self):
132
# TODO: jam 20051215 this function is necessary to support the
133
# pipelined() function. In reality, it probably should use
134
# poll() or select() to actually return if there is data
135
# available, otherwise we probably don't get any benefit
136
return True
137
129
138
def recv(self, count):
130
139
return os.read(self.proc.stdout.fileno(), count)
131
140
154
163
SYSTEM_HOSTKEYS = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
155
164
except Exception, e:
156
165
mutter('failed to load system host keys: ' + str(e))
157
bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')
166
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
158
167
try:
159
168
BZR_HOSTKEYS = paramiko.util.load_host_keys(bzr_hostkey_path)
160
169
except Exception, e:
166
175
Save "discovered" host keys in $(config)/ssh_host_keys/.
167
176
"""
168
177
global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
169
bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')
170
if not os.path.isdir(config_dir()):
171
os.mkdir(config_dir())
178
bzr_hostkey_path = pathjoin(config_dir(), 'ssh_host_keys')
179
ensure_config_dir_exists()
180
172
181
try:
173
182
f = open(bzr_hostkey_path, 'w')
174
183
f.write('# SSH host keys collected by bzr\n')
191
200
self.lock_path = path + '.write-lock'
192
201
self.transport = transport
193
202
try:
194
self.lock_file = transport._sftp_open_exclusive(self.lock_path)
203
# RBC 20060103 FIXME should we be using private methods here ?
204
abspath = transport._remote_path(self.lock_path)
205
self.lock_file = transport._sftp_open_exclusive(abspath)
195
206
except FileExists:
196
207
raise LockError('File %r already locked' % (self.path,))
197
208
212
223
# What specific errors should we catch here?
213
224
pass
214
225
226
227
215
228
class SFTPTransport (Transport):
216
229
"""
217
230
Transport implementation for SFTP access.
222
235
assert base.startswith('sftp://')
223
236
self._parse_url(base)
224
237
base = self._unparse_url()
238
if base[-1] != '/':
239
base = base + '/'
225
240
super(SFTPTransport, self).__init__(base)
226
241
if clone_from is None:
227
242
self._sftp_connect()
254
269
@param relpath: the relative path or path components
255
270
@type relpath: str or list
256
271
"""
257
return self._unparse_url(self._abspath(relpath))
272
return self._unparse_url(self._remote_path(relpath))
258
273
259
def _abspath(self, relpath):
260
"""Return the absolute path segment without the SFTP URL."""
274
def _remote_path(self, relpath):
275
"""Return the path to be passed along the sftp protocol for relpath.
276
277
relpath is a urlencoded string.
278
"""
261
279
# FIXME: share the common code across transports
262
280
assert isinstance(relpath, basestring)
263
relpath = [urllib.unquote(relpath)]
281
relpath = urllib.unquote(relpath).split('/')
264
282
basepath = self._path.split('/')
265
283
if len(basepath) > 0 and basepath[-1] == '':
266
284
basepath = basepath[:-1]
278
296
basepath.append(p)
279
297
280
298
path = '/'.join(basepath)
281
# could still be a "relative" path here, but relative on the sftp server
282
299
return path
283
300
284
301
def relpath(self, abspath):
296
313
extra = ': ' + ', '.join(error)
297
314
raise PathNotChild(abspath, self.base, extra=extra)
298
315
pl = len(self._path)
299
return path[pl:].lstrip('/')
316
return path[pl:].strip('/')
300
317
301
318
def has(self, relpath):
302
319
"""
303
320
Does the target location exist?
304
321
"""
305
322
try:
306
self._sftp.stat(self._abspath(relpath))
323
self._sftp.stat(self._remote_path(relpath))
307
324
return True
308
325
except IOError:
309
326
return False
315
332
:param relpath: The relative path to the file
316
333
"""
317
334
try:
318
path = self._abspath(relpath)
319
f = self._sftp.file(path)
335
path = self._remote_path(relpath)
336
f = self._sftp.file(path, mode='rb')
320
337
if self._do_prefetch and hasattr(f, 'prefetch'):
321
338
f.prefetch()
322
339
return f
342
359
f.prefetch()
343
360
return f
344
361
345
def put(self, relpath, f):
362
def put(self, relpath, f, mode=None):
346
363
"""
347
364
Copy the file-like or string object into the location.
348
365
349
366
:param relpath: Location to put the contents, relative to base.
350
367
:param f: File-like or string object.
368
:param mode: The final mode for the file
351
369
"""
352
final_path = self._abspath(relpath)
353
tmp_relpath = '%s.tmp.%.9f.%d.%d' % (relpath, time.time(),
370
final_path = self._remote_path(relpath)
371
self._put(final_path, f, mode=mode)
372
373
def _put(self, abspath, f, mode=None):
374
"""Helper function so both put() and copy_abspaths can reuse the code"""
375
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
354
376
os.getpid(), random.randint(0,0x7FFFFFFF))
355
tmp_abspath = self._abspath(tmp_relpath)
356
fout = self._sftp_open_exclusive(tmp_relpath)
357
377
fout = self._sftp_open_exclusive(tmp_abspath, mode=mode)
378
closed = False
358
379
try:
359
380
try:
381
fout.set_pipelined(True)
360
382
self._pump(f, fout)
361
except (paramiko.SSHException, IOError), e:
362
self._translate_io_exception(e, relpath, ': unable to write')
383
except (IOError, paramiko.SSHException), e:
384
self._translate_io_exception(e, tmp_abspath)
385
if mode is not None:
386
self._sftp.chmod(tmp_abspath, mode)
387
fout.close()
388
closed = True
389
self._rename(tmp_abspath, abspath)
363
390
except Exception, e:
364
391
# If we fail, try to clean up the temporary file
365
392
# before we throw the exception
366
393
# but don't let another exception mess things up
394
# Write out the traceback, because otherwise
395
# the catch and throw destroys it
396
import traceback
397
mutter(traceback.format_exc())
367
398
try:
368
fout.close()
399
if not closed:
400
fout.close()
369
401
self._sftp.remove(tmp_abspath)
370
402
except:
371
pass
372
raise e
373
else:
374
# sftp rename doesn't allow overwriting, so play tricks:
375
tmp_safety = 'bzr.tmp.%.9f.%d.%d' % (time.time(), os.getpid(), random.randint(0, 0x7FFFFFFF))
376
tmp_safety = self._abspath(tmp_safety)
377
try:
378
self._sftp.rename(final_path, tmp_safety)
379
file_existed = True
380
except:
381
file_existed = False
382
success = False
383
try:
384
try:
385
self._sftp.rename(tmp_abspath, final_path)
386
except (paramiko.SSHException, IOError), e:
387
self._translate_io_exception(e, relpath, ': unable to rename')
388
else:
389
success = True
390
finally:
391
if file_existed:
392
if success:
393
self._sftp.unlink(tmp_safety)
394
else:
395
self._sftp.rename(tmp_safety, final_path)
403
# raise the saved except
404
raise e
405
# raise the original with its traceback if we can.
406
raise
396
407
397
408
def iter_files_recursive(self):
398
409
"""Walk the relative paths of all files in this transport."""
406
417
else:
407
418
yield relpath
408
419
409
def mkdir(self, relpath):
420
def mkdir(self, relpath, mode=None):
410
421
"""Create a directory at the given path."""
411
422
try:
412
path = self._abspath(relpath)
423
path = self._remote_path(relpath)
424
# In the paramiko documentation, it says that passing a mode flag
425
# will filtered against the server umask.
426
# StubSFTPServer does not do this, which would be nice, because it is
427
# what we really want :)
428
# However, real servers do use umask, so we really should do it that way
413
429
self._sftp.mkdir(path)
430
if mode is not None:
431
self._sftp.chmod(path, mode=mode)
414
432
except (paramiko.SSHException, IOError), e:
415
self._translate_io_exception(e, relpath, ': unable to mkdir',
433
self._translate_io_exception(e, path, ': unable to mkdir',
416
434
failure_exc=FileExists)
417
435
418
436
def _translate_io_exception(self, e, path, more_info='', failure_exc=NoSuchFile):
439
457
# strange but true, for the paramiko server.
440
458
if (e.args == ('Failure',)):
441
459
raise failure_exc(path, str(e) + more_info)
460
mutter('Raising exception with args %s', e.args)
461
if hasattr(e, 'errno'):
462
mutter('Raising exception with errno %s', e.errno)
442
463
raise e
443
464
444
465
def append(self, relpath, f):
447
468
location.
448
469
"""
449
470
try:
450
path = self._abspath(relpath)
471
path = self._remote_path(relpath)
451
472
fout = self._sftp.file(path, 'ab')
452
473
self._pump(f, fout)
453
474
except (IOError, paramiko.SSHException), e:
455
476
456
477
def copy(self, rel_from, rel_to):
457
478
"""Copy the item at rel_from to the location at rel_to"""
458
path_from = self._abspath(rel_from)
459
path_to = self._abspath(rel_to)
479
path_from = self._remote_path(rel_from)
480
path_to = self._remote_path(rel_to)
460
481
self._copy_abspaths(path_from, path_to)
461
482
462
def _copy_abspaths(self, path_from, path_to):
483
def _copy_abspaths(self, path_from, path_to, mode=None):
463
484
"""Copy files given an absolute path
464
485
465
486
:param path_from: Path on remote server to read
474
495
try:
475
496
fin = self._sftp.file(path_from, 'rb')
476
497
try:
477
fout = self._sftp.file(path_to, 'wb')
478
try:
479
fout.set_pipelined(True)
480
self._pump(fin, fout)
481
finally:
482
fout.close()
498
self._put(path_to, fin, mode=mode)
483
499
finally:
484
500
fin.close()
485
501
except (IOError, paramiko.SSHException), e:
486
502
self._translate_io_exception(e, path_from, ': unable copy to: %r' % path_to)
487
503
488
def copy_to(self, relpaths, other, pb=None):
504
def copy_to(self, relpaths, other, mode=None, pb=None):
489
505
"""Copy a set of entries from self into another Transport.
490
506
491
507
:param relpaths: A list/generator of entries to be copied.
497
513
total = self._get_total(relpaths)
498
514
count = 0
499
515
for path in relpaths:
500
path_from = self._abspath(relpath)
501
path_to = other._abspath(relpath)
516
path_from = self._remote_path(relpath)
517
path_to = other._remote_path(relpath)
502
518
self._update_pb(pb, 'copy-to', count, total)
503
self._copy_abspaths(path_from, path_to)
519
self._copy_abspaths(path_from, path_to, mode=mode)
504
520
count += 1
505
521
return count
506
522
else:
507
return super(SFTPTransport, self).copy_to(relpaths, other, pb=pb)
508
509
# The dummy implementation just does a simple get + put
510
def copy_entry(path):
511
other.put(path, self.get(path))
512
513
return self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False)
523
return super(SFTPTransport, self).copy_to(relpaths, other, mode=mode, pb=pb)
524
525
def _rename(self, abs_from, abs_to):
526
"""Do a fancy rename on the remote server.
527
528
Using the implementation provided by osutils.
529
"""
530
try:
531
fancy_rename(abs_from, abs_to,
532
rename_func=self._sftp.rename,
533
unlink_func=self._sftp.remove)
534
except (IOError, paramiko.SSHException), e:
535
self._translate_io_exception(e, abs_from, ': unable to rename to %r' % (abs_to))
514
536
515
537
def move(self, rel_from, rel_to):
516
538
"""Move the item at rel_from to the location at rel_to"""
517
path_from = self._abspath(rel_from)
518
path_to = self._abspath(rel_to)
519
try:
520
self._sftp.rename(path_from, path_to)
521
except (IOError, paramiko.SSHException), e:
522
self._translate_io_exception(e, path_from, ': unable to move to: %r' % path_to)
539
path_from = self._remote_path(rel_from)
540
path_to = self._remote_path(rel_to)
541
self._rename(path_from, path_to)
523
542
524
543
def delete(self, relpath):
525
544
"""Delete the item at relpath"""
526
path = self._abspath(relpath)
545
path = self._remote_path(relpath)
527
546
try:
528
547
self._sftp.remove(path)
529
548
except (IOError, paramiko.SSHException), e:
538
557
Return a list of all files at the given location.
539
558
"""
540
559
# does anything actually use this?
541
path = self._abspath(relpath)
560
path = self._remote_path(relpath)
542
561
try:
543
562
return self._sftp.listdir(path)
544
563
except (IOError, paramiko.SSHException), e:
546
565
547
566
def stat(self, relpath):
548
567
"""Return the stat information for a file."""
549
path = self._abspath(relpath)
568
path = self._remote_path(relpath)
550
569
try:
551
570
return self._sftp.stat(path)
552
571
except (IOError, paramiko.SSHException), e:
653
672
654
673
vendor = _get_ssh_vendor()
655
674
if vendor != 'none':
656
sock = SFTPSubprocess(self._host, self._port, self._username)
675
sock = SFTPSubprocess(self._host, vendor, self._port,
676
self._username)
657
677
self._sftp = SFTPClient(sock)
658
678
else:
659
679
self._paramiko_connect()
691
711
save_host_keys()
692
712
if server_key != our_server_key:
693
713
filename1 = os.path.expanduser('~/.ssh/known_hosts')
694
filename2 = os.path.join(config_dir(), 'ssh_host_keys')
714
filename2 = pathjoin(config_dir(), 'ssh_host_keys')
695
715
raise TransportError('Host keys for %s do not match! %s != %s' % \
696
716
(self._host, our_server_key_hex, server_key_hex),
697
717
['Try editing %s or %s' % (filename1, filename2)])
733
753
if self._try_pkey_auth(transport, paramiko.DSSKey, username, 'id_dsa'):
734
754
return
735
755
736
737
756
if self._password:
738
757
try:
739
758
transport.auth_password(username, self._password)
776
795
pass
777
796
return False
778
797
779
def _sftp_open_exclusive(self, relpath):
798
def _sftp_open_exclusive(self, abspath, mode=None):
780
799
"""Open a remote path exclusively.
781
800
782
801
SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if
787
806
WARNING: This breaks the SFTPClient abstraction, so it
788
807
could easily break against an updated version of paramiko.
789
808
790
:param relpath: The relative path, where the file should be opened
809
:param abspath: The remote absolute path where the file should be opened
810
:param mode: The mode permissions bits for the new file
791
811
"""
792
path = self._sftp._adjust_cwd(self._abspath(relpath))
812
path = self._sftp._adjust_cwd(abspath)
793
813
attr = SFTPAttributes()
794
mode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
814
if mode is not None:
815
attr.st_mode = mode
816
omode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
795
817
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
796
818
try:
797
t, msg = self._sftp._request(CMD_OPEN, path, mode, attr)
819
t, msg = self._sftp._request(CMD_OPEN, path, omode, attr)
798
820
if t != CMD_HANDLE:
799
821
raise TransportError('Expected an SFTP handle')
800
822
handle = msg.get_string()
801
return SFTPFile(self._sftp, handle, 'w', -1)
823
return SFTPFile(self._sftp, handle, 'wb', -1)
802
824
except (paramiko.SSHException, IOError), e:
803
self._translate_io_exception(e, relpath, ': unable to open',
825
self._translate_io_exception(e, abspath, ': unable to open',
804
826
failure_exc=FileExists)
805
827
828
829
# ------------- server test implementation --------------
830
import socket
831
import threading
832
833
from bzrlib.tests.stub_sftp import StubServer, StubSFTPServer
834
835
STUB_SERVER_KEY = """
836
-----BEGIN RSA PRIVATE KEY-----
837
MIICWgIBAAKBgQDTj1bqB4WmayWNPB+8jVSYpZYk80Ujvj680pOTh2bORBjbIAyz
838
oWGW+GUjzKxTiiPvVmxFgx5wdsFvF03v34lEVVhMpouqPAYQ15N37K/ir5XY+9m/
839
d8ufMCkjeXsQkKqFbAlQcnWMCRnOoPHS3I4vi6hmnDDeeYTSRvfLbW0fhwIBIwKB
840
gBIiOqZYaoqbeD9OS9z2K9KR2atlTxGxOJPXiP4ESqP3NVScWNwyZ3NXHpyrJLa0
841
EbVtzsQhLn6rF+TzXnOlcipFvjsem3iYzCpuChfGQ6SovTcOjHV9z+hnpXvQ/fon
842
soVRZY65wKnF7IAoUwTmJS9opqgrN6kRgCd3DASAMd1bAkEA96SBVWFt/fJBNJ9H
843
tYnBKZGw0VeHOYmVYbvMSstssn8un+pQpUm9vlG/bp7Oxd/m+b9KWEh2xPfv6zqU
844
avNwHwJBANqzGZa/EpzF4J8pGti7oIAPUIDGMtfIcmqNXVMckrmzQ2vTfqtkEZsA
845
4rE1IERRyiJQx6EJsz21wJmGV9WJQ5kCQQDwkS0uXqVdFzgHO6S++tjmjYcxwr3g
846
H0CoFYSgbddOT6miqRskOQF3DZVkJT3kyuBgU2zKygz52ukQZMqxCb1fAkASvuTv
847
qfpH87Qq5kQhNKdbbwbmd2NxlNabazPijWuphGTdW0VfJdWfklyS2Kr+iqrs/5wV
848
HhathJt636Eg7oIjAkA8ht3MQ+XSl9yIJIS8gVpbPxSw5OMfw0PjVE7tBdQruiSc
849
nvuQES5C9BMHjF39LZiGH1iLQy7FgdHyoP+eodI7
850
-----END RSA PRIVATE KEY-----
851
"""
852
853
854
class SingleListener(threading.Thread):
855
856
def __init__(self, callback):
857
threading.Thread.__init__(self)
858
self._callback = callback
859
self._socket = socket.socket()
860
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
861
self._socket.bind(('localhost', 0))
862
self._socket.listen(1)
863
self.port = self._socket.getsockname()[1]
864
self.stop_event = threading.Event()
865
866
def run(self):
867
s, _ = self._socket.accept()
868
# now close the listen socket
869
self._socket.close()
870
self._callback(s, self.stop_event)
871
872
def stop(self):
873
self.stop_event.set()
874
# We should consider waiting for the other thread
875
# to stop, because otherwise we get spurious
876
# bzr: ERROR: Socket exception: Connection reset by peer (54)
877
# because the test suite finishes before the thread has a chance
878
# to close. (Especially when only running a few tests)
879
880
881
class SFTPServer(Server):
882
"""Common code for SFTP server facilities."""
883
884
def _get_sftp_url(self, path):
885
"""Calculate a sftp url to this server for path."""
886
return 'sftp://foo:bar@localhost:%d/%s' % (self._listener.port, path)
887
888
def __init__(self):
889
self._original_vendor = None
890
self._homedir = None
891
self._server_homedir = None
892
self._listener = None
893
self._root = None
894
# sftp server logs
895
self.logs = []
896
897
def log(self, message):
898
"""What to do here? do we need this? Its for the StubServer.."""
899
self.logs.append(message)
900
901
def _run_server(self, s, stop_event):
902
ssh_server = paramiko.Transport(s)
903
key_file = os.path.join(self._homedir, 'test_rsa.key')
904
file(key_file, 'w').write(STUB_SERVER_KEY)
905
host_key = paramiko.RSAKey.from_private_key_file(key_file)
906
ssh_server.add_server_key(host_key)
907
server = StubServer(self)
908
ssh_server.set_subsystem_handler('sftp', paramiko.SFTPServer,
909
StubSFTPServer, root=self._root,
910
home=self._server_homedir)
911
event = threading.Event()
912
ssh_server.start_server(event, server)
913
event.wait(5.0)
914
stop_event.wait(30.0)
915
916
def setUp(self):
917
"""See bzrlib.transport.Server.setUp."""
918
# XXX: 20051124 jamesh
919
# The tests currently pop up a password prompt when an external ssh
920
# is used. This forces the use of the paramiko implementation.
921
global _ssh_vendor
922
self._original_vendor = _ssh_vendor
923
_ssh_vendor = 'none'
924
self._homedir = os.getcwdu()
925
if self._server_homedir is None:
926
self._server_homedir = self._homedir
927
self._root = '/'
928
# FIXME WINDOWS: _root should be _server_homedir[0]:/
929
self._listener = SingleListener(self._run_server)
930
self._listener.setDaemon(True)
931
self._listener.start()
932
933
def tearDown(self):
934
"""See bzrlib.transport.Server.tearDown."""
935
global _ssh_vendor
936
self._listener.stop()
937
_ssh_vendor = self._original_vendor
938
939
940
class SFTPAbsoluteServer(SFTPServer):
941
"""A test server for sftp transports, using absolute urls."""
942
943
def get_url(self):
944
"""See bzrlib.transport.Server.get_url."""
945
return self._get_sftp_url("%%2f%s" %
946
urlescape(self._homedir[1:]))
947
948
949
class SFTPHomeDirServer(SFTPServer):
950
"""A test server for sftp transports, using homedir relative urls."""
951
952
def get_url(self):
953
"""See bzrlib.transport.Server.get_url."""
954
return self._get_sftp_url("")
955
956
957
class SFTPSiblingAbsoluteServer(SFTPAbsoluteServer):
958
"""A test servere for sftp transports, using absolute urls to non-home."""
959
960
def setUp(self):
961
self._server_homedir = '/dev/noone/runs/tests/here'
962
super(SFTPSiblingAbsoluteServer, self).setUp()
963
964
965
def get_test_permutations():
966
"""Return the permutations to be used in testing."""
967
return [(SFTPTransport, SFTPAbsoluteServer),
968
(SFTPTransport, SFTPHomeDirServer),
969
(SFTPTransport, SFTPSiblingAbsoluteServer),
970
]
Loggerhead is a web-based interface for Breezy
Version: 2.0.1