1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
|
# Copyright (C) 2005 Robey Pointer <robey@lag.net>
# Copyright (C) 2005, 2006 Canonical Ltd
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""Implementation of Transport over SFTP, using paramiko."""
import errno
import os
import random
import select
import socket
import stat
import subprocess
import sys
import time
import urllib
import urlparse
import weakref
from bzrlib.errors import (FileExists,
NoSuchFile, PathNotChild,
TransportError,
LockError,
PathError,
ParamikoNotPresent,
UnknownSSH,
)
from bzrlib.osutils import pathjoin, fancy_rename, getcwd
from bzrlib.trace import mutter, warning
from bzrlib.transport import (
register_urlparse_netloc_protocol,
Server,
split_url,
ssh,
Transport,
)
import bzrlib.urlutils as urlutils
try:
import paramiko
except ImportError, e:
raise ParamikoNotPresent(e)
else:
from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE,
SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC,
CMD_HANDLE, CMD_OPEN)
from paramiko.sftp_attr import SFTPAttributes
from paramiko.sftp_file import SFTPFile
register_urlparse_netloc_protocol('sftp')
# This is a weakref dictionary, so that we can reuse connections
# that are still active. Long term, it might be nice to have some
# sort of expiration policy, such as disconnect if inactive for
# X seconds. But that requires a lot more fanciness.
_connected_hosts = weakref.WeakValueDictionary()
_paramiko_version = getattr(paramiko, '__version_info__', (0, 0, 0))
# don't use prefetch unless paramiko version >= 1.5.5 (there were bugs earlier)
_default_do_prefetch = (_paramiko_version >= (1, 5, 5))
def clear_connection_cache():
"""Remove all hosts from the SFTP connection cache.
Primarily useful for test cases wanting to force garbage collection.
"""
_connected_hosts.clear()
class SFTPLock(object):
"""This fakes a lock in a remote location."""
__slots__ = ['path', 'lock_path', 'lock_file', 'transport']
def __init__(self, path, transport):
assert isinstance(transport, SFTPTransport)
self.lock_file = None
self.path = path
self.lock_path = path + '.write-lock'
self.transport = transport
try:
# RBC 20060103 FIXME should we be using private methods here ?
abspath = transport._remote_path(self.lock_path)
self.lock_file = transport._sftp_open_exclusive(abspath)
except FileExists:
raise LockError('File %r already locked' % (self.path,))
def __del__(self):
"""Should this warn, or actually try to cleanup?"""
if self.lock_file:
warning("SFTPLock %r not explicitly unlocked" % (self.path,))
self.unlock()
def unlock(self):
if not self.lock_file:
return
self.lock_file.close()
self.lock_file = None
try:
self.transport.delete(self.lock_path)
except (NoSuchFile,):
# What specific errors should we catch here?
pass
class SFTPTransport(Transport):
"""Transport implementation for SFTP access."""
_do_prefetch = _default_do_prefetch
# TODO: jam 20060717 Conceivably these could be configurable, either
# by auto-tuning at run-time, or by a configuration (per host??)
# but the performance curve is pretty flat, so just going with
# reasonable defaults.
_max_readv_combine = 200
# Having to round trip to the server means waiting for a response,
# so it is better to download extra bytes.
# 8KiB had good performance for both local and remote network operations
_bytes_to_read_before_seek = 8192
# The sftp spec says that implementations SHOULD allow reads
# to be at least 32K. paramiko.readv() does an async request
# for the chunks. So we need to keep it within a single request
# size for paramiko <= 1.6.1. paramiko 1.6.2 will probably chop
# up the request itself, rather than us having to worry about it
_max_request_size = 32768
def __init__(self, base, clone_from=None):
assert base.startswith('sftp://')
self._parse_url(base)
base = self._unparse_url()
if base[-1] != '/':
base += '/'
super(SFTPTransport, self).__init__(base)
if clone_from is None:
self._sftp_connect()
else:
# use the same ssh connection, etc
self._sftp = clone_from._sftp
# super saves 'self.base'
def should_cache(self):
"""
Return True if the data pulled across should be cached locally.
"""
return True
def clone(self, offset=None):
"""
Return a new SFTPTransport with root at self.base + offset.
We share the same SFTP session between such transports, because it's
fairly expensive to set them up.
"""
if offset is None:
return SFTPTransport(self.base, self)
else:
return SFTPTransport(self.abspath(offset), self)
def abspath(self, relpath):
"""
Return the full url to the given relative path.
@param relpath: the relative path or path components
@type relpath: str or list
"""
return self._unparse_url(self._remote_path(relpath))
def _remote_path(self, relpath):
"""Return the path to be passed along the sftp protocol for relpath.
relpath is a urlencoded string.
"""
# FIXME: share the common code across transports
assert isinstance(relpath, basestring)
relpath = urlutils.unescape(relpath).split('/')
basepath = self._path.split('/')
if len(basepath) > 0 and basepath[-1] == '':
basepath = basepath[:-1]
for p in relpath:
if p == '..':
if len(basepath) == 0:
# In most filesystems, a request for the parent
# of root, just returns root.
continue
basepath.pop()
elif p == '.':
continue # No-op
else:
basepath.append(p)
path = '/'.join(basepath)
# mutter('relpath => remotepath %s => %s', relpath, path)
return path
def relpath(self, abspath):
username, password, host, port, path = self._split_url(abspath)
error = []
if (username != self._username):
error.append('username mismatch')
if (host != self._host):
error.append('host mismatch')
if (port != self._port):
error.append('port mismatch')
if (not path.startswith(self._path)):
error.append('path mismatch')
if error:
extra = ': ' + ', '.join(error)
raise PathNotChild(abspath, self.base, extra=extra)
pl = len(self._path)
return path[pl:].strip('/')
def has(self, relpath):
"""
Does the target location exist?
"""
try:
self._sftp.stat(self._remote_path(relpath))
return True
except IOError:
return False
def get(self, relpath):
"""
Get the file at the given relative path.
:param relpath: The relative path to the file
"""
try:
path = self._remote_path(relpath)
f = self._sftp.file(path, mode='rb')
if self._do_prefetch and (getattr(f, 'prefetch', None) is not None):
f.prefetch()
return f
except (IOError, paramiko.SSHException), e:
self._translate_io_exception(e, path, ': error retrieving')
def readv(self, relpath, offsets):
"""See Transport.readv()"""
# We overload the default readv() because we want to use a file
# that does not have prefetch enabled.
# Also, if we have a new paramiko, it implements an async readv()
if not offsets:
return
try:
path = self._remote_path(relpath)
fp = self._sftp.file(path, mode='rb')
readv = getattr(fp, 'readv', None)
if readv:
return self._sftp_readv(fp, offsets)
mutter('seek and read %s offsets', len(offsets))
return self._seek_and_read(fp, offsets)
except (IOError, paramiko.SSHException), e:
self._translate_io_exception(e, path, ': error retrieving')
def _sftp_readv(self, fp, offsets):
"""Use the readv() member of fp to do async readv.
And then read them using paramiko.readv(). paramiko.readv()
does not support ranges > 64K, so it caps the request size, and
just reads until it gets all the stuff it wants
"""
offsets = list(offsets)
sorted_offsets = sorted(offsets)
# The algorithm works as follows:
# 1) Coalesce nearby reads into a single chunk
# This generates a list of combined regions, the total size
# and the size of the sub regions. This coalescing step is limited
# in the number of nearby chunks to combine, and is allowed to
# skip small breaks in the requests. Limiting it makes sure that
# we can start yielding some data earlier, and skipping means we
# make fewer requests. (Beneficial even when using async)
# 2) Break up this combined regions into chunks that are smaller
# than 64KiB. Technically the limit is 65536, but we are a
# little bit conservative. This is because sftp has a maximum
# return chunk size of 64KiB (max size of an unsigned short)
# 3) Issue a readv() to paramiko to create an async request for
# all of this data
# 4) Read in the data as it comes back, until we've read one
# continuous section as determined in step 1
# 5) Break up the full sections into hunks for the original requested
# offsets. And put them in a cache
# 6) Check if the next request is in the cache, and if it is, remove
# it from the cache, and yield its data. Continue until no more
# entries are in the cache.
# 7) loop back to step 4 until all data has been read
#
# TODO: jam 20060725 This could be optimized one step further, by
# attempting to yield whatever data we have read, even before
# the first coallesced section has been fully processed.
# When coalescing for use with readv(), we don't really need to
# use any fudge factor, because the requests are made asynchronously
coalesced = list(self._coalesce_offsets(sorted_offsets,
limit=self._max_readv_combine,
fudge_factor=0,
))
requests = []
for c_offset in coalesced:
start = c_offset.start
size = c_offset.length
# We need to break this up into multiple requests
while size > 0:
next_size = min(size, self._max_request_size)
requests.append((start, next_size))
size -= next_size
start += next_size
mutter('SFTP.readv() %s offsets => %s coalesced => %s requests',
len(offsets), len(coalesced), len(requests))
# Queue the current read until we have read the full coalesced section
cur_data = []
cur_data_len = 0
cur_coalesced_stack = iter(coalesced)
cur_coalesced = cur_coalesced_stack.next()
# Cache the results, but only until they have been fulfilled
data_map = {}
# turn the list of offsets into a stack
offset_stack = iter(offsets)
cur_offset_and_size = offset_stack.next()
for data in fp.readv(requests):
cur_data += data
cur_data_len += len(data)
if cur_data_len < cur_coalesced.length:
continue
assert cur_data_len == cur_coalesced.length, \
"Somehow we read too much: %s != %s" % (cur_data_len,
cur_coalesced.length)
all_data = ''.join(cur_data)
cur_data = []
cur_data_len = 0
for suboffset, subsize in cur_coalesced.ranges:
key = (cur_coalesced.start+suboffset, subsize)
data_map[key] = all_data[suboffset:suboffset+subsize]
# Now that we've read some data, see if we can yield anything back
while cur_offset_and_size in data_map:
this_data = data_map.pop(cur_offset_and_size)
yield cur_offset_and_size[0], this_data
cur_offset_and_size = offset_stack.next()
# Now that we've read all of the data for this coalesced section
# on to the next
cur_coalesced = cur_coalesced_stack.next()
def put_file(self, relpath, f, mode=None):
"""
Copy the file-like object into the location.
:param relpath: Location to put the contents, relative to base.
:param f: File-like object.
:param mode: The final mode for the file
"""
final_path = self._remote_path(relpath)
self._put(final_path, f, mode=mode)
def _put(self, abspath, f, mode=None):
"""Helper function so both put() and copy_abspaths can reuse the code"""
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
os.getpid(), random.randint(0,0x7FFFFFFF))
fout = self._sftp_open_exclusive(tmp_abspath, mode=mode)
closed = False
try:
try:
fout.set_pipelined(True)
self._pump(f, fout)
except (IOError, paramiko.SSHException), e:
self._translate_io_exception(e, tmp_abspath)
# XXX: This doesn't truly help like we would like it to.
# The problem is that openssh strips sticky bits. So while we
# can properly set group write permission, we lose the group
# sticky bit. So it is probably best to stop chmodding, and
# just tell users that they need to set the umask correctly.
# The attr.st_mode = mode, in _sftp_open_exclusive
# will handle when the user wants the final mode to be more
# restrictive. And then we avoid a round trip. Unless
# paramiko decides to expose an async chmod()
# This is designed to chmod() right before we close.
# Because we set_pipelined() earlier, theoretically we might
# avoid the round trip for fout.close()
if mode is not None:
self._sftp.chmod(tmp_abspath, mode)
fout.close()
closed = True
self._rename_and_overwrite(tmp_abspath, abspath)
except Exception, e:
# If we fail, try to clean up the temporary file
# before we throw the exception
# but don't let another exception mess things up
# Write out the traceback, because otherwise
# the catch and throw destroys it
import traceback
mutter(traceback.format_exc())
try:
if not closed:
fout.close()
self._sftp.remove(tmp_abspath)
except:
# raise the saved except
raise e
# raise the original with its traceback if we can.
raise
def _non_atomic_put_helper(self, relpath, writer, mode=None,
create_parent_dir=False):
abspath = self._remote_path(relpath)
# TODO: jam 20060816 paramiko doesn't publicly expose a way to
# set the file mode at create time. If it does, use it.
# But for now, we just chmod later anyway.
def _open_and_write_file():
"""Try to open the target file, raise error on failure"""
fout = None
try:
try:
fout = self._sftp.file(abspath, mode='wb')
fout.set_pipelined(True)
writer(fout)
except (paramiko.SSHException, IOError), e:
self._translate_io_exception(e, abspath,
': unable to open')
# This is designed to chmod() right before we close.
# Because we set_pipelined() earlier, theoretically we might
# avoid the round trip for fout.close()
if mode is not None:
self._sftp.chmod(abspath, mode)
finally:
if fout is not None:
fout.close()
if not create_parent_dir:
_open_and_write_file()
return
# Try error handling to create the parent directory if we need to
try:
_open_and_write_file()
except NoSuchFile:
# Try to create the parent directory, and then go back to
# writing the file
parent_dir = os.path.dirname(abspath)
try:
self._sftp.mkdir(parent_dir)
except (paramiko.SSHException, IOError), e:
self._translate_io_exception(e, abspath, ': unable to open')
_open_and_write_file()
def non_atomic_put_file(self, relpath, f, mode=None,
create_parent_dir=False):
"""Copy the file-like object into the target location.
This function is not strictly safe to use. It is only meant to
be used when you already know that the target does not exist.
It is not safe, because it will open and truncate the remote
file. So there may be a time when the file has invalid contents.
:param relpath: The remote location to put the contents.
:param f: File-like object.
:param mode: Possible access permissions for new file.
None means do not set remote permissions.
:param create_parent_dir: If we cannot create the target file because
the parent directory does not exist, go ahead and
create it, and then try again.
"""
def writer(fout):
self._pump(f, fout)
self._non_atomic_put_helper(relpath, writer, mode=mode,
create_parent_dir=create_parent_dir)
def non_atomic_put_bytes(self, relpath, bytes, mode=None,
create_parent_dir=False):
def writer(fout):
fout.write(bytes)
self._non_atomic_put_helper(relpath, writer, mode=mode,
create_parent_dir=create_parent_dir)
def iter_files_recursive(self):
"""Walk the relative paths of all files in this transport."""
queue = list(self.list_dir('.'))
while queue:
relpath = queue.pop(0)
st = self.stat(relpath)
if stat.S_ISDIR(st.st_mode):
for i, basename in enumerate(self.list_dir(relpath)):
queue.insert(i, relpath+'/'+basename)
else:
yield relpath
def mkdir(self, relpath, mode=None):
"""Create a directory at the given path."""
path = self._remote_path(relpath)
try:
self._sftp.mkdir(path)
if mode is not None:
self._sftp.chmod(path, mode=mode)
except (paramiko.SSHException, IOError), e:
self._translate_io_exception(e, path, ': unable to mkdir',
failure_exc=FileExists)
def _translate_io_exception(self, e, path, more_info='',
failure_exc=PathError):
"""Translate a paramiko or IOError into a friendlier exception.
:param e: The original exception
:param path: The path in question when the error is raised
:param more_info: Extra information that can be included,
such as what was going on
:param failure_exc: Paramiko has the super fun ability to raise completely
opaque errors that just set "e.args = ('Failure',)" with
no more information.
If this parameter is set, it defines the exception
to raise in these cases.
"""
# paramiko seems to generate detailless errors.
self._translate_error(e, path, raise_generic=False)
if hasattr(e, 'args'):
if (e.args == ('No such file or directory',) or
e.args == ('No such file',)):
raise NoSuchFile(path, str(e) + more_info)
if (e.args == ('mkdir failed',)):
raise FileExists(path, str(e) + more_info)
# strange but true, for the paramiko server.
if (e.args == ('Failure',)):
raise failure_exc(path, str(e) + more_info)
mutter('Raising exception with args %s', e.args)
if hasattr(e, 'errno'):
mutter('Raising exception with errno %s', e.errno)
raise e
def append_file(self, relpath, f, mode=None):
"""
Append the text in the file-like object into the final
location.
"""
try:
path = self._remote_path(relpath)
fout = self._sftp.file(path, 'ab')
if mode is not None:
self._sftp.chmod(path, mode)
result = fout.tell()
self._pump(f, fout)
return result
except (IOError, paramiko.SSHException), e:
self._translate_io_exception(e, relpath, ': unable to append')
def rename(self, rel_from, rel_to):
"""Rename without special overwriting"""
try:
self._sftp.rename(self._remote_path(rel_from),
self._remote_path(rel_to))
except (IOError, paramiko.SSHException), e:
self._translate_io_exception(e, rel_from,
': unable to rename to %r' % (rel_to))
def _rename_and_overwrite(self, abs_from, abs_to):
"""Do a fancy rename on the remote server.
Using the implementation provided by osutils.
"""
try:
fancy_rename(abs_from, abs_to,
rename_func=self._sftp.rename,
unlink_func=self._sftp.remove)
except (IOError, paramiko.SSHException), e:
self._translate_io_exception(e, abs_from, ': unable to rename to %r' % (abs_to))
def move(self, rel_from, rel_to):
"""Move the item at rel_from to the location at rel_to"""
path_from = self._remote_path(rel_from)
path_to = self._remote_path(rel_to)
self._rename_and_overwrite(path_from, path_to)
def delete(self, relpath):
"""Delete the item at relpath"""
path = self._remote_path(relpath)
try:
self._sftp.remove(path)
except (IOError, paramiko.SSHException), e:
self._translate_io_exception(e, path, ': unable to delete')
def listable(self):
"""Return True if this store supports listing."""
return True
def list_dir(self, relpath):
"""
Return a list of all files at the given location.
"""
# does anything actually use this?
# -- Unknown
# This is at least used by copy_tree for remote upgrades.
# -- David Allouche 2006-08-11
path = self._remote_path(relpath)
try:
entries = self._sftp.listdir(path)
except (IOError, paramiko.SSHException), e:
self._translate_io_exception(e, path, ': failed to list_dir')
return [urlutils.escape(entry) for entry in entries]
def rmdir(self, relpath):
"""See Transport.rmdir."""
path = self._remote_path(relpath)
try:
return self._sftp.rmdir(path)
except (IOError, paramiko.SSHException), e:
self._translate_io_exception(e, path, ': failed to rmdir')
def stat(self, relpath):
"""Return the stat information for a file."""
path = self._remote_path(relpath)
try:
return self._sftp.stat(path)
except (IOError, paramiko.SSHException), e:
self._translate_io_exception(e, path, ': unable to stat')
def lock_read(self, relpath):
"""
Lock the given file for shared (read) access.
:return: A lock object, which has an unlock() member function
"""
# FIXME: there should be something clever i can do here...
class BogusLock(object):
def __init__(self, path):
self.path = path
def unlock(self):
pass
return BogusLock(relpath)
def lock_write(self, relpath):
"""
Lock the given file for exclusive (write) access.
WARNING: many transports do not support this, so trying avoid using it
:return: A lock object, which has an unlock() member function
"""
# This is a little bit bogus, but basically, we create a file
# which should not already exist, and if it does, we assume
# that there is a lock, and if it doesn't, the we assume
# that we have taken the lock.
return SFTPLock(relpath, self)
def _unparse_url(self, path=None):
if path is None:
path = self._path
path = urllib.quote(path)
# handle homedir paths
if not path.startswith('/'):
path = "/~/" + path
netloc = urllib.quote(self._host)
if self._username is not None:
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
if self._port is not None:
netloc = '%s:%d' % (netloc, self._port)
return urlparse.urlunparse(('sftp', netloc, path, '', '', ''))
def _split_url(self, url):
(scheme, username, password, host, port, path) = split_url(url)
assert scheme == 'sftp'
# the initial slash should be removed from the path, and treated
# as a homedir relative path (the path begins with a double slash
# if it is absolute).
# see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt
# RBC 20060118 we are not using this as its too user hostile. instead
# we are following lftp and using /~/foo to mean '~/foo'.
# handle homedir paths
if path.startswith('/~/'):
path = path[3:]
elif path == '/~':
path = ''
return (username, password, host, port, path)
def _parse_url(self, url):
(self._username, self._password,
self._host, self._port, self._path) = self._split_url(url)
def _sftp_connect(self):
"""Connect to the remote sftp server.
After this, self._sftp should have a valid connection (or
we raise an TransportError 'could not connect').
TODO: Raise a more reasonable ConnectionFailed exception
"""
self._sftp = _sftp_connect(self._host, self._port, self._username,
self._password)
def _sftp_open_exclusive(self, abspath, mode=None):
"""Open a remote path exclusively.
SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if
the file already exists. However it does not expose this
at the higher level of SFTPClient.open(), so we have to
sneak away with it.
WARNING: This breaks the SFTPClient abstraction, so it
could easily break against an updated version of paramiko.
:param abspath: The remote absolute path where the file should be opened
:param mode: The mode permissions bits for the new file
"""
# TODO: jam 20060816 Paramiko >= 1.6.2 (probably earlier) supports
# using the 'x' flag to indicate SFTP_FLAG_EXCL.
# However, there is no way to set the permission mode at open
# time using the sftp_client.file() functionality.
path = self._sftp._adjust_cwd(abspath)
# mutter('sftp abspath %s => %s', abspath, path)
attr = SFTPAttributes()
if mode is not None:
attr.st_mode = mode
omode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
| SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
try:
t, msg = self._sftp._request(CMD_OPEN, path, omode, attr)
if t != CMD_HANDLE:
raise TransportError('Expected an SFTP handle')
handle = msg.get_string()
return SFTPFile(self._sftp, handle, 'wb', -1)
except (paramiko.SSHException, IOError), e:
self._translate_io_exception(e, abspath, ': unable to open',
failure_exc=FileExists)
# ------------- server test implementation --------------
import threading
from bzrlib.tests.stub_sftp import StubServer, StubSFTPServer
STUB_SERVER_KEY = """
-----BEGIN RSA PRIVATE KEY-----
MIICWgIBAAKBgQDTj1bqB4WmayWNPB+8jVSYpZYk80Ujvj680pOTh2bORBjbIAyz
oWGW+GUjzKxTiiPvVmxFgx5wdsFvF03v34lEVVhMpouqPAYQ15N37K/ir5XY+9m/
d8ufMCkjeXsQkKqFbAlQcnWMCRnOoPHS3I4vi6hmnDDeeYTSRvfLbW0fhwIBIwKB
gBIiOqZYaoqbeD9OS9z2K9KR2atlTxGxOJPXiP4ESqP3NVScWNwyZ3NXHpyrJLa0
EbVtzsQhLn6rF+TzXnOlcipFvjsem3iYzCpuChfGQ6SovTcOjHV9z+hnpXvQ/fon
soVRZY65wKnF7IAoUwTmJS9opqgrN6kRgCd3DASAMd1bAkEA96SBVWFt/fJBNJ9H
tYnBKZGw0VeHOYmVYbvMSstssn8un+pQpUm9vlG/bp7Oxd/m+b9KWEh2xPfv6zqU
avNwHwJBANqzGZa/EpzF4J8pGti7oIAPUIDGMtfIcmqNXVMckrmzQ2vTfqtkEZsA
4rE1IERRyiJQx6EJsz21wJmGV9WJQ5kCQQDwkS0uXqVdFzgHO6S++tjmjYcxwr3g
H0CoFYSgbddOT6miqRskOQF3DZVkJT3kyuBgU2zKygz52ukQZMqxCb1fAkASvuTv
qfpH87Qq5kQhNKdbbwbmd2NxlNabazPijWuphGTdW0VfJdWfklyS2Kr+iqrs/5wV
HhathJt636Eg7oIjAkA8ht3MQ+XSl9yIJIS8gVpbPxSw5OMfw0PjVE7tBdQruiSc
nvuQES5C9BMHjF39LZiGH1iLQy7FgdHyoP+eodI7
-----END RSA PRIVATE KEY-----
"""
class SocketListener(threading.Thread):
def __init__(self, callback):
threading.Thread.__init__(self)
self._callback = callback
self._socket = socket.socket()
self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._socket.bind(('localhost', 0))
self._socket.listen(1)
self.port = self._socket.getsockname()[1]
self._stop_event = threading.Event()
def stop(self):
# called from outside this thread
self._stop_event.set()
# use a timeout here, because if the test fails, the server thread may
# never notice the stop_event.
self.join(5.0)
self._socket.close()
def run(self):
while True:
readable, writable_unused, exception_unused = \
select.select([self._socket], [], [], 0.1)
if self._stop_event.isSet():
return
if len(readable) == 0:
continue
try:
s, addr_unused = self._socket.accept()
# because the loopback socket is inline, and transports are
# never explicitly closed, best to launch a new thread.
threading.Thread(target=self._callback, args=(s,)).start()
except socket.error, x:
sys.excepthook(*sys.exc_info())
warning('Socket error during accept() within unit test server'
' thread: %r' % x)
except Exception, x:
# probably a failed test; unit test thread will log the
# failure/error
sys.excepthook(*sys.exc_info())
warning('Exception from within unit test server thread: %r' %
x)
class SocketDelay(object):
"""A socket decorator to make TCP appear slower.
This changes recv, send, and sendall to add a fixed latency to each python
call if a new roundtrip is detected. That is, when a recv is called and the
flag new_roundtrip is set, latency is charged. Every send and send_all
sets this flag.
In addition every send, sendall and recv sleeps a bit per character send to
simulate bandwidth.
Not all methods are implemented, this is deliberate as this class is not a
replacement for the builtin sockets layer. fileno is not implemented to
prevent the proxy being bypassed.
"""
simulated_time = 0
_proxied_arguments = dict.fromkeys([
"close", "getpeername", "getsockname", "getsockopt", "gettimeout",
"setblocking", "setsockopt", "settimeout", "shutdown"])
def __init__(self, sock, latency, bandwidth=1.0,
really_sleep=True):
"""
:param bandwith: simulated bandwith (MegaBit)
:param really_sleep: If set to false, the SocketDelay will just
increase a counter, instead of calling time.sleep. This is useful for
unittesting the SocketDelay.
"""
self.sock = sock
self.latency = latency
self.really_sleep = really_sleep
self.time_per_byte = 1 / (bandwidth / 8.0 * 1024 * 1024)
self.new_roundtrip = False
def sleep(self, s):
if self.really_sleep:
time.sleep(s)
else:
SocketDelay.simulated_time += s
def __getattr__(self, attr):
if attr in SocketDelay._proxied_arguments:
return getattr(self.sock, attr)
raise AttributeError("'SocketDelay' object has no attribute %r" %
attr)
def dup(self):
return SocketDelay(self.sock.dup(), self.latency, self.time_per_byte,
self._sleep)
def recv(self, *args):
data = self.sock.recv(*args)
if data and self.new_roundtrip:
self.new_roundtrip = False
self.sleep(self.latency)
self.sleep(len(data) * self.time_per_byte)
return data
def sendall(self, data, flags=0):
if not self.new_roundtrip:
self.new_roundtrip = True
self.sleep(self.latency)
self.sleep(len(data) * self.time_per_byte)
return self.sock.sendall(data, flags)
def send(self, data, flags=0):
if not self.new_roundtrip:
self.new_roundtrip = True
self.sleep(self.latency)
bytes_sent = self.sock.send(data, flags)
self.sleep(bytes_sent * self.time_per_byte)
return bytes_sent
class SFTPServer(Server):
"""Common code for SFTP server facilities."""
def __init__(self):
self._original_vendor = None
self._homedir = None
self._server_homedir = None
self._listener = None
self._root = None
self._vendor = ssh.ParamikoVendor()
# sftp server logs
self.logs = []
self.add_latency = 0
def _get_sftp_url(self, path):
"""Calculate an sftp url to this server for path."""
return 'sftp://foo:bar@localhost:%d/%s' % (self._listener.port, path)
def log(self, message):
"""StubServer uses this to log when a new server is created."""
self.logs.append(message)
def _run_server_entry(self, sock):
"""Entry point for all implementations of _run_server.
If self.add_latency is > 0.000001 then sock is given a latency adding
decorator.
"""
if self.add_latency > 0.000001:
sock = SocketDelay(sock, self.add_latency)
return self._run_server(sock)
def _run_server(self, s):
ssh_server = paramiko.Transport(s)
key_file = pathjoin(self._homedir, 'test_rsa.key')
f = open(key_file, 'w')
f.write(STUB_SERVER_KEY)
f.close()
host_key = paramiko.RSAKey.from_private_key_file(key_file)
ssh_server.add_server_key(host_key)
server = StubServer(self)
ssh_server.set_subsystem_handler('sftp', paramiko.SFTPServer,
StubSFTPServer, root=self._root,
home=self._server_homedir)
event = threading.Event()
ssh_server.start_server(event, server)
event.wait(5.0)
def setUp(self):
self._original_vendor = ssh._ssh_vendor
ssh._ssh_vendor = self._vendor
if sys.platform == 'win32':
# Win32 needs to use the UNICODE api
self._homedir = getcwd()
else:
# But Linux SFTP servers should just deal in bytestreams
self._homedir = os.getcwd()
if self._server_homedir is None:
self._server_homedir = self._homedir
self._root = '/'
if sys.platform == 'win32':
self._root = ''
self._listener = SocketListener(self._run_server_entry)
self._listener.setDaemon(True)
self._listener.start()
def tearDown(self):
"""See bzrlib.transport.Server.tearDown."""
self._listener.stop()
ssh._ssh_vendor = self._original_vendor
def get_bogus_url(self):
"""See bzrlib.transport.Server.get_bogus_url."""
# this is chosen to try to prevent trouble with proxies, wierd dns, etc
# we bind a random socket, so that we get a guaranteed unused port
# we just never listen on that port
s = socket.socket()
s.bind(('localhost', 0))
return 'sftp://%s:%s/' % s.getsockname()
class SFTPFullAbsoluteServer(SFTPServer):
"""A test server for sftp transports, using absolute urls and ssh."""
def get_url(self):
"""See bzrlib.transport.Server.get_url."""
return self._get_sftp_url(urlutils.escape(self._homedir[1:]))
class SFTPServerWithoutSSH(SFTPServer):
"""An SFTP server that uses a simple TCP socket pair rather than SSH."""
def __init__(self):
super(SFTPServerWithoutSSH, self).__init__()
self._vendor = ssh.LoopbackVendor()
def _run_server(self, sock):
# Re-import these as locals, so that they're still accessible during
# interpreter shutdown (when all module globals get set to None, leading
# to confusing errors like "'NoneType' object has no attribute 'error'".
import socket, errno
class FakeChannel(object):
def get_transport(self):
return self
def get_log_channel(self):
return 'paramiko'
def get_name(self):
return '1'
def get_hexdump(self):
return False
def close(self):
pass
server = paramiko.SFTPServer(FakeChannel(), 'sftp', StubServer(self), StubSFTPServer,
root=self._root, home=self._server_homedir)
try:
server.start_subsystem('sftp', None, sock)
except socket.error, e:
if (len(e.args) > 0) and (e.args[0] == errno.EPIPE):
# it's okay for the client to disconnect abruptly
# (bug in paramiko 1.6: it should absorb this exception)
pass
else:
raise
except Exception, e:
import sys; sys.stderr.write('\nEXCEPTION %r\n\n' % e.__class__)
server.finish_subsystem()
class SFTPAbsoluteServer(SFTPServerWithoutSSH):
"""A test server for sftp transports, using absolute urls."""
def get_url(self):
"""See bzrlib.transport.Server.get_url."""
if sys.platform == 'win32':
return self._get_sftp_url(urlutils.escape(self._homedir))
else:
return self._get_sftp_url(urlutils.escape(self._homedir[1:]))
class SFTPHomeDirServer(SFTPServerWithoutSSH):
"""A test server for sftp transports, using homedir relative urls."""
def get_url(self):
"""See bzrlib.transport.Server.get_url."""
return self._get_sftp_url("~/")
class SFTPSiblingAbsoluteServer(SFTPAbsoluteServer):
"""A test servere for sftp transports, using absolute urls to non-home."""
def setUp(self):
self._server_homedir = '/dev/noone/runs/tests/here'
super(SFTPSiblingAbsoluteServer, self).setUp()
def _sftp_connect(host, port, username, password):
"""Connect to the remote sftp server.
:raises: a TransportError 'could not connect'.
:returns: an paramiko.sftp_client.SFTPClient
TODO: Raise a more reasonable ConnectionFailed exception
"""
idx = (host, port, username)
try:
return _connected_hosts[idx]
except KeyError:
pass
sftp = _sftp_connect_uncached(host, port, username, password)
_connected_hosts[idx] = sftp
return sftp
def _sftp_connect_uncached(host, port, username, password):
vendor = ssh._get_ssh_vendor()
sftp = vendor.connect_sftp(username, password, host, port)
return sftp
def get_test_permutations():
"""Return the permutations to be used in testing."""
return [(SFTPTransport, SFTPAbsoluteServer),
(SFTPTransport, SFTPHomeDirServer),
(SFTPTransport, SFTPSiblingAbsoluteServer),
]
|