~bzr-pqm/bzr/bzr.dev

6609.2.1 by Vincent Ladeuil
Make all transport put_bytes() raises TypeError when given unicode strings rather than bytes.
1
# Copyright (C) 2005-2011, 2016 Canonical Ltd
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
2
#
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
7
#
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
12
#
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
16
6379.6.7 by Jelmer Vernooij
Move importing from future until after doc string, otherwise the doc string will disappear.
17
"""Implementation of Transport over SFTP, using paramiko."""
18
6379.6.3 by Jelmer Vernooij
Use absolute_import.
19
from __future__ import absolute_import
20
1910.7.17 by Andrew Bennetts
Various cosmetic changes.
21
# TODO: Remove the transport-based lock_read and lock_write methods.  They'll
22
# then raise TransportNotPossible, which will break remote access to any
23
# formats which rely on OS-level locks.  That should be fine as those formats
24
# are pretty old, but these combinations may have to be removed from the test
1910.7.19 by Andrew Bennetts
Expand comment about lock_read/lock_write.
25
# suite.  Those formats all date back to 0.7; so we should be able to remove
26
# these methods when we officially drop support for those formats.
1910.7.17 by Andrew Bennetts
Various cosmetic changes.
27
3686.1.3 by John Arbash Meinel
Finish up handling out-of-order requests.
28
import bisect
1489 by Robert Collins
Make the paramiko tests pass. Nice huh.
29
import errno
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
30
import itertools
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
31
import os
1540.2.2 by Röbey Pointer
allow forcing the use of paramiko via environ var; use prefetch on paramiko >= 1.5.2
32
import random
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
33
import stat
34
import sys
1540.2.2 by Röbey Pointer
allow forcing the use of paramiko via environ var; use prefetch on paramiko >= 1.5.2
35
import time
2711.1.1 by Martin Pool
Suppress warning about Paramiko use of struct.pack
36
import warnings
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
37
2001.3.2 by John Arbash Meinel
Force all transports to raise ShortReadvError if they can
38
from bzrlib import (
3777.1.1 by Aaron Bentley
Use auth.conf for sftp
39
    config,
3959.1.3 by Martin Pool
Guard common sftp mutter statements by debug flags
40
    debug,
2001.3.2 by John Arbash Meinel
Force all transports to raise ShortReadvError if they can
41
    errors,
42
    urlutils,
43
    )
1996.2.1 by Andrew Bennetts
Fix NameError reported by Alexander Belchenko.
44
from bzrlib.errors import (FileExists,
6055.2.1 by Jelmer Vernooij
Add UnparsedUrl.
45
                           NoSuchFile,
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
46
                           TransportError,
1996.2.1 by Andrew Bennetts
Fix NameError reported by Alexander Belchenko.
47
                           LockError,
1662.1.12 by Martin Pool
Translate unknown sftp errors to PathError, no NoSuchFile
48
                           PathError,
49
                           ParamikoNotPresent,
1530.1.3 by Robert Collins
transport implementations now tested consistently.
50
                           )
6055.2.1 by Jelmer Vernooij
Add UnparsedUrl.
51
from bzrlib.osutils import fancy_rename
1951.1.6 by Andrew Bennetts
Tidy up some unused imports.
52
from bzrlib.trace import mutter, warning
1636.1.2 by Robert Collins
More review fixen to the relpath at '/' fixes.
53
from bzrlib.transport import (
2671.3.6 by Robert Collins
Review feedback.
54
    FileFileStream,
2671.3.2 by Robert Collins
Start open_file_stream logic.
55
    _file_streams,
1951.1.4 by Andrew Bennetts
Start moving SSH connection code into bzrlib/transport/ssh.py
56
    ssh,
2485.8.16 by Vincent Ladeuil
Create a new, empty, ConnectedTransport class.
57
    ConnectedTransport,
1636.1.2 by Robert Collins
More review fixen to the relpath at '/' fixes.
58
    )
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
59
2711.1.1 by Martin Pool
Suppress warning about Paramiko use of struct.pack
60
# Disable one particular warning that comes from paramiko in Python2.5; if
61
# this is emitted at the wrong time it tends to cause spurious test failures
62
# or at least noise in the test case::
63
#
64
# [1770/7639 in 86s, 1 known failures, 50 skipped, 2 missing features]
65
# test_permissions.TestSftpPermissions.test_new_files
66
# /var/lib/python-support/python2.5/paramiko/message.py:226: DeprecationWarning: integer argument expected, got float
67
#  self.packet.write(struct.pack('>I', n))
68
warnings.filterwarnings('ignore',
69
        'integer argument expected, got float',
70
        category=DeprecationWarning,
71
        module='paramiko.message')
72
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
73
try:
74
    import paramiko
1185.62.24 by John Arbash Meinel
Changing the exception that sftp.py throws when it can't find paramiko, so that the test suite can handle it.
75
except ImportError, e:
76
    raise ParamikoNotPresent(e)
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
77
else:
78
    from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE,
79
                               SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC,
5056.1.1 by Neil Santos
Added default link() and symlink() methods to Transport.
80
                               SFTP_OK, CMD_HANDLE, CMD_OPEN)
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
81
    from paramiko.sftp_attr import SFTPAttributes
82
    from paramiko.sftp_file import SFTPFile
1636.1.1 by Robert Collins
Fix calling relpath() and abspath() on transports at their root.
83
1883.1.3 by Stefan (metze) Metzmacher
- fix typo
84
1711.6.2 by John Arbash Meinel
Clean up paramiko agent support.
85
_paramiko_version = getattr(paramiko, '__version_info__', (0, 0, 0))
86
# don't use prefetch unless paramiko version >= 1.5.5 (there were bugs earlier)
87
_default_do_prefetch = (_paramiko_version >= (1, 5, 5))
1540.2.3 by Robey Pointer
move the paramiko version check up to the top and use getattr instead of hasattr. use BZR_SSH instead of BZR_USE_PARAMIKO, to allow overriding the ssh vendor check in a general way.
88
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
89
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
90
class SFTPLock(object):
1910.7.17 by Andrew Bennetts
Various cosmetic changes.
91
    """This fakes a lock in a remote location.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
92
1910.7.17 by Andrew Bennetts
Various cosmetic changes.
93
    A present lock is indicated just by the existence of a file.  This
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
94
    doesn't work well on all transports and they are only used in
1910.7.17 by Andrew Bennetts
Various cosmetic changes.
95
    deprecated storage formats.
96
    """
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
97
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
98
    __slots__ = ['path', 'lock_path', 'lock_file', 'transport']
1910.7.17 by Andrew Bennetts
Various cosmetic changes.
99
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
100
    def __init__(self, path, transport):
101
        self.lock_file = None
102
        self.path = path
103
        self.lock_path = path + '.write-lock'
104
        self.transport = transport
105
        try:
1530.1.7 by Robert Collins
merge integration.
106
            # RBC 20060103 FIXME should we be using private methods here ?
107
            abspath = transport._remote_path(self.lock_path)
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
108
            self.lock_file = transport._sftp_open_exclusive(abspath)
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
109
        except FileExists:
110
            raise LockError('File %r already locked' % (self.path,))
111
112
    def unlock(self):
113
        if not self.lock_file:
114
            return
115
        self.lock_file.close()
116
        self.lock_file = None
117
        try:
118
            self.transport.delete(self.lock_path)
119
        except (NoSuchFile,):
120
            # What specific errors should we catch here?
121
            pass
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
122
1551.6.39 by Aaron Bentley
Apply Robey Pointer's listener patch
123
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
124
class _SFTPReadvHelper(object):
125
    """A class to help with managing the state of a readv request."""
126
127
    # See _get_requests for an explanation.
128
    _max_request_size = 32768
129
3882.7.2 by Martin Pool
Start reporting activity from sftp read access
130
    def __init__(self, original_offsets, relpath, _report_activity):
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
131
        """Create a new readv helper.
132
133
        :param original_offsets: The original requests given by the caller of
134
            readv()
135
        :param relpath: The name of the file (if known)
3882.7.2 by Martin Pool
Start reporting activity from sftp read access
136
        :param _report_activity: A Transport._report_activity bound method,
137
            to be called as data arrives.
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
138
        """
139
        self.original_offsets = list(original_offsets)
140
        self.relpath = relpath
3882.7.2 by Martin Pool
Start reporting activity from sftp read access
141
        self._report_activity = _report_activity
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
142
143
    def _get_requests(self):
144
        """Break up the offsets into individual requests over sftp.
145
146
        The SFTP spec only requires implementers to support 32kB requests. We
147
        could try something larger (openssh supports 64kB), but then we have to
148
        handle requests that fail.
149
        So instead, we just break up our maximum chunks into 32kB chunks, and
150
        asyncronously requests them.
151
        Newer versions of paramiko would do the chunking for us, but we want to
152
        start processing results right away, so we do it ourselves.
153
        """
154
        # TODO: Because we issue async requests, we don't 'fudge' any extra
155
        #       data.  I'm not 100% sure that is the best choice.
156
157
        # The first thing we do, is to collapse the individual requests as much
158
        # as possible, so we don't issues requests <32kB
159
        sorted_offsets = sorted(self.original_offsets)
160
        coalesced = list(ConnectedTransport._coalesce_offsets(sorted_offsets,
161
                                                        limit=0, fudge_factor=0))
162
        requests = []
163
        for c_offset in coalesced:
164
            start = c_offset.start
165
            size = c_offset.length
166
167
            # Break this up into 32kB requests
168
            while size > 0:
169
                next_size = min(size, self._max_request_size)
170
                requests.append((start, next_size))
171
                size -= next_size
172
                start += next_size
3959.1.3 by Martin Pool
Guard common sftp mutter statements by debug flags
173
        if 'sftp' in debug.debug_flags:
174
            mutter('SFTP.readv(%s) %s offsets => %s coalesced => %s requests',
175
                self.relpath, len(sorted_offsets), len(coalesced),
176
                len(requests))
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
177
        return requests
178
179
    def request_and_yield_offsets(self, fp):
180
        """Request the data from the remote machine, yielding the results.
181
182
        :param fp: A Paramiko SFTPFile object that supports readv.
183
        :return: Yield the data requested by the original readv caller, one by
184
            one.
185
        """
186
        requests = self._get_requests()
187
        offset_iter = iter(self.original_offsets)
188
        cur_offset, cur_size = offset_iter.next()
189
        # paramiko .readv() yields strings that are in the order of the requests
190
        # So we track the current request to know where the next data is
191
        # being returned from.
3686.1.4 by John Arbash Meinel
rename cur_start => input_start for clarity.
192
        input_start = None
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
193
        last_end = None
194
        buffered_data = []
195
        buffered_len = 0
196
197
        # This is used to buffer chunks which we couldn't process yet
198
        # It is (start, end, data) tuples.
199
        data_chunks = []
200
        # Create an 'unlimited' data stream, so we stop based on requests,
201
        # rather than just because the data stream ended. This lets us detect
202
        # short readv.
203
        data_stream = itertools.chain(fp.readv(requests),
204
                                      itertools.repeat(None))
3686.1.4 by John Arbash Meinel
rename cur_start => input_start for clarity.
205
        for (start, length), data in itertools.izip(requests, data_stream):
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
206
            if data is None:
207
                if cur_coalesced is not None:
208
                    raise errors.ShortReadvError(self.relpath,
3686.1.3 by John Arbash Meinel
Finish up handling out-of-order requests.
209
                        start, length, len(data))
210
            if len(data) != length:
211
                raise errors.ShortReadvError(self.relpath,
212
                    start, length, len(data))
3882.7.9 by Martin Pool
Move the point at which sftp issues transport activity (thanks vila)
213
            self._report_activity(length, 'read')
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
214
            if last_end is None:
215
                # This is the first request, just buffer it
3686.1.3 by John Arbash Meinel
Finish up handling out-of-order requests.
216
                buffered_data = [data]
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
217
                buffered_len = length
3686.1.4 by John Arbash Meinel
rename cur_start => input_start for clarity.
218
                input_start = start
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
219
            elif start == last_end:
220
                # The data we are reading fits neatly on the previous
221
                # buffer, so this is all part of a larger coalesced range.
222
                buffered_data.append(data)
223
                buffered_len += length
224
            else:
225
                # We have an 'interrupt' in the data stream. So we know we are
226
                # at a request boundary.
227
                if buffered_len > 0:
228
                    # We haven't consumed the buffer so far, so put it into
229
                    # data_chunks, and continue.
230
                    buffered = ''.join(buffered_data)
3686.1.4 by John Arbash Meinel
rename cur_start => input_start for clarity.
231
                    data_chunks.append((input_start, buffered))
232
                input_start = start
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
233
                buffered_data = [data]
3686.1.3 by John Arbash Meinel
Finish up handling out-of-order requests.
234
                buffered_len = length
235
            last_end = start + length
3686.1.4 by John Arbash Meinel
rename cur_start => input_start for clarity.
236
            if input_start == cur_offset and cur_size <= buffered_len:
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
237
                # Simplify the next steps a bit by transforming buffered_data
3686.1.4 by John Arbash Meinel
rename cur_start => input_start for clarity.
238
                # into a single string. We also have the nice property that
239
                # when there is only one string ''.join([x]) == x, so there is
240
                # no data copying.
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
241
                buffered = ''.join(buffered_data)
3686.1.7 by John Arbash Meinel
Purge the list buffer when we have a simple string buffer.
242
                # Clean out buffered data so that we keep memory
243
                # consumption low
244
                del buffered_data[:]
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
245
                buffered_offset = 0
3686.1.4 by John Arbash Meinel
rename cur_start => input_start for clarity.
246
                # TODO: We *could* also consider the case where cur_offset is in
247
                #       in the buffered range, even though it doesn't *start*
248
                #       the buffered range. But for packs we pretty much always
249
                #       read in order, so you won't get any extra data in the
250
                #       middle.
251
                while (input_start == cur_offset
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
252
                       and (buffered_offset + cur_size) <= buffered_len):
253
                    # We've buffered enough data to process this request, spit it
254
                    # out
255
                    cur_data = buffered[buffered_offset:buffered_offset + cur_size]
256
                    # move the direct pointer into our buffered data
257
                    buffered_offset += cur_size
258
                    # Move the start-of-buffer pointer
3686.1.4 by John Arbash Meinel
rename cur_start => input_start for clarity.
259
                    input_start += cur_size
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
260
                    # Yield the requested data
261
                    yield cur_offset, cur_data
262
                    cur_offset, cur_size = offset_iter.next()
263
                # at this point, we've consumed as much of buffered as we can,
264
                # so break off the portion that we consumed
265
                if buffered_offset == len(buffered_data):
266
                    # No tail to leave behind
267
                    buffered_data = []
268
                    buffered_len = 0
269
                else:
270
                    buffered = buffered[buffered_offset:]
271
                    buffered_data = [buffered]
272
                    buffered_len = len(buffered)
5807.5.1 by John Arbash Meinel
Fix bug #767177. Be more agressive with file.close() calls.
273
        # now that the data stream is done, close the handle
274
        fp.close()
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
275
        if buffered_len:
276
            buffered = ''.join(buffered_data)
3686.1.7 by John Arbash Meinel
Purge the list buffer when we have a simple string buffer.
277
            del buffered_data[:]
3686.1.4 by John Arbash Meinel
rename cur_start => input_start for clarity.
278
            data_chunks.append((input_start, buffered))
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
279
        if data_chunks:
3959.1.3 by Martin Pool
Guard common sftp mutter statements by debug flags
280
            if 'sftp' in debug.debug_flags:
281
                mutter('SFTP readv left with %d out-of-order bytes',
282
                    sum(map(lambda x: len(x[1]), data_chunks)))
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
283
            # We've processed all the readv data, at this point, anything we
284
            # couldn't process is in data_chunks. This doesn't happen often, so
285
            # this code path isn't optimized
3686.1.3 by John Arbash Meinel
Finish up handling out-of-order requests.
286
            # We use an interesting process for data_chunks
287
            # Specifically if we have "bisect_left([(start, len, entries)],
288
            #                                       (qstart,)])
289
            # If start == qstart, then we get the specific node. Otherwise we
290
            # get the previous node
291
            while True:
292
                idx = bisect.bisect_left(data_chunks, (cur_offset,))
3815.2.4 by Martin Pool
merge fix for out-of-order SFTP readv
293
                if idx < len(data_chunks) and data_chunks[idx][0] == cur_offset:
294
                    # The data starts here
3686.1.3 by John Arbash Meinel
Finish up handling out-of-order requests.
295
                    data = data_chunks[idx][1][:cur_size]
296
                elif idx > 0:
297
                    # The data is in a portion of a previous page
298
                    idx -= 1
299
                    sub_offset = cur_offset - data_chunks[idx][0]
300
                    data = data_chunks[idx][1]
301
                    data = data[sub_offset:sub_offset + cur_size]
302
                else:
303
                    # We are missing the page where the data should be found,
304
                    # something is wrong
305
                    data = ''
306
                if len(data) != cur_size:
307
                    raise AssertionError('We must have miscalulated.'
308
                        ' We expected %d bytes, but only found %d'
309
                        % (cur_size, len(data)))
310
                yield cur_offset, data
311
                cur_offset, cur_size = offset_iter.next()
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
312
313
2485.8.27 by Vincent Ladeuil
Hearing jam saying "vila, you're trying too hard", I simplified again.
314
class SFTPTransport(ConnectedTransport):
1752.2.40 by Andrew Bennetts
Merge from bzr.dev.
315
    """Transport implementation for SFTP access."""
1864.5.2 by John Arbash Meinel
always read in sorted order, and return in requested order, but only cache what is currently out of order
316
1540.2.3 by Robey Pointer
move the paramiko version check up to the top and use getattr instead of hasattr. use BZR_SSH instead of BZR_USE_PARAMIKO, to allow overriding the ssh vendor check in a general way.
317
    _do_prefetch = _default_do_prefetch
1864.5.6 by John Arbash Meinel
Setting defaults based on benchmarking data.
318
    # TODO: jam 20060717 Conceivably these could be configurable, either
319
    #       by auto-tuning at run-time, or by a configuration (per host??)
320
    #       but the performance curve is pretty flat, so just going with
321
    #       reasonable defaults.
1864.5.18 by John Arbash Meinel
Remove benchmarking stipple.
322
    _max_readv_combine = 200
1864.5.6 by John Arbash Meinel
Setting defaults based on benchmarking data.
323
    # Having to round trip to the server means waiting for a response,
324
    # so it is better to download extra bytes.
325
    # 8KiB had good performance for both local and remote network operations
1864.5.18 by John Arbash Meinel
Remove benchmarking stipple.
326
    _bytes_to_read_before_seek = 8192
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
327
1864.5.20 by John Arbash Meinel
Some small cleanups.
328
    # The sftp spec says that implementations SHOULD allow reads
329
    # to be at least 32K. paramiko.readv() does an async request
330
    # for the chunks. So we need to keep it within a single request
331
    # size for paramiko <= 1.6.1. paramiko 1.6.2 will probably chop
332
    # up the request itself, rather than us having to worry about it
333
    _max_request_size = 32768
1864.5.17 by John Arbash Meinel
A little cleanup, and restrict to 32K to conform better to sftp spec
334
2485.8.27 by Vincent Ladeuil
Hearing jam saying "vila, you're trying too hard", I simplified again.
335
    def _remote_path(self, relpath):
336
        """Return the path to be passed along the sftp protocol for relpath.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
337
2485.8.27 by Vincent Ladeuil
Hearing jam saying "vila, you're trying too hard", I simplified again.
338
        :param relpath: is a urlencoded string.
339
        """
6055.2.19 by Jelmer Vernooij
Use clone
340
        remote_path = self._parsed_url.clone(relpath).path
2485.8.27 by Vincent Ladeuil
Hearing jam saying "vila, you're trying too hard", I simplified again.
341
        # the initial slash should be removed from the path, and treated as a
342
        # homedir relative path (the path begins with a double slash if it is
343
        # absolute).  see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt
344
        # RBC 20060118 we are not using this as its too user hostile. instead
345
        # we are following lftp and using /~/foo to mean '~/foo'
346
        # vila--20070602 and leave absolute paths begin with a single slash.
347
        if remote_path.startswith('/~/'):
348
            remote_path = remote_path[3:]
349
        elif remote_path == '/~':
350
            remote_path = ''
351
        return remote_path
352
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
353
    def _create_connection(self, credentials=None):
354
        """Create a new connection with the provided credentials.
355
356
        :param credentials: The credentials needed to establish the connection.
357
358
        :return: The created connection and its associated credentials.
359
360
        The credentials are only the password as it may have been entered
361
        interactively by the user and may be different from the one provided
362
        in base url at transport creation time.
363
        """
364
        if credentials is None:
6055.2.1 by Jelmer Vernooij
Add UnparsedUrl.
365
            password = self._parsed_url.password
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
366
        else:
367
            password = credentials
368
369
        vendor = ssh._get_ssh_vendor()
6055.2.1 by Jelmer Vernooij
Add UnparsedUrl.
370
        user = self._parsed_url.user
3777.1.1 by Aaron Bentley
Use auth.conf for sftp
371
        if user is None:
372
            auth = config.AuthenticationConfig()
6055.2.1 by Jelmer Vernooij
Add UnparsedUrl.
373
            user = auth.get_user('ssh', self._parsed_url.host,
374
                self._parsed_url.port)
375
        connection = vendor.connect_sftp(self._parsed_url.user, password,
376
            self._parsed_url.host, self._parsed_url.port)
3777.1.1 by Aaron Bentley
Use auth.conf for sftp
377
        return connection, (user, password)
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
378
5247.2.12 by Vincent Ladeuil
Ensure that all transports close their underlying connection.
379
    def disconnect(self):
380
        connection = self._get_connection()
381
        if connection is not None:
382
            connection.close()
383
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
384
    def _get_sftp(self):
385
        """Ensures that a connection is established"""
386
        connection = self._get_connection()
387
        if connection is None:
388
            # First connection ever
389
            connection, credentials = self._create_connection()
390
            self._set_connection(connection, credentials)
391
        return connection
392
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
393
    def has(self, relpath):
394
        """
395
        Does the target location exist?
396
        """
397
        try:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
398
            self._get_sftp().stat(self._remote_path(relpath))
4103.3.1 by Martin Pool
More sftp progress hooks
399
            # stat result is about 20 bytes, let's say
400
            self._report_activity(20, 'read')
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
401
            return True
402
        except IOError:
403
            return False
404
2164.2.15 by Vincent Ladeuil
Http redirections are not followed by default. Do not use hints
405
    def get(self, relpath):
3882.7.2 by Martin Pool
Start reporting activity from sftp read access
406
        """Get the file at the given relative path.
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
407
408
        :param relpath: The relative path to the file
409
        """
410
        try:
1524.1.1 by Robert Collins
Test sftp with relative, absolute-in-homedir and absolute-not-in-homedir
411
            path = self._remote_path(relpath)
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
412
            f = self._get_sftp().file(path, mode='rb')
1540.2.3 by Robey Pointer
move the paramiko version check up to the top and use getattr instead of hasattr. use BZR_SSH instead of BZR_USE_PARAMIKO, to allow overriding the ssh vendor check in a general way.
413
            if self._do_prefetch and (getattr(f, 'prefetch', None) is not None):
1185.40.1 by Robey Pointer
prefetch files under paramiko 1.5.1 for improved speed
414
                f.prefetch()
415
            return f
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
416
        except (IOError, paramiko.SSHException), e:
2052.6.1 by Robert Collins
``Transport.get`` has had its interface made more clear for ease of use.
417
            self._translate_io_exception(e, path, ': error retrieving',
2052.6.4 by Robert Collins
Tidier SFTP code (feedback from Aaron).
418
                failure_exc=errors.ReadError)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
419
3882.7.4 by Martin Pool
SFTPTransport also sends activity from get()
420
    def get_bytes(self, relpath):
421
        # reimplement this here so that we can report how many bytes came back
422
        f = self.get(relpath)
423
        try:
424
            bytes = f.read()
425
            self._report_activity(len(bytes), 'read')
426
            return bytes
427
        finally:
428
            f.close()
429
2745.5.1 by Robert Collins
* New parameter on ``bzrlib.transport.Transport.readv``
430
    def _readv(self, relpath, offsets):
1864.5.11 by John Arbash Meinel
Factor out the workhorse of Transport.readv() into a helper function, and re-use that function in sftp but with a non-prefetch file.
431
        """See Transport.readv()"""
432
        # We overload the default readv() because we want to use a file
433
        # that does not have prefetch enabled.
1864.5.15 by John Arbash Meinel
Cleanup and document the new sftp readv implementation
434
        # Also, if we have a new paramiko, it implements an async readv()
1864.5.11 by John Arbash Meinel
Factor out the workhorse of Transport.readv() into a helper function, and re-use that function in sftp but with a non-prefetch file.
435
        if not offsets:
436
            return
437
438
        try:
439
            path = self._remote_path(relpath)
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
440
            fp = self._get_sftp().file(path, mode='rb')
1864.5.15 by John Arbash Meinel
Cleanup and document the new sftp readv implementation
441
            readv = getattr(fp, 'readv', None)
1864.5.18 by John Arbash Meinel
Remove benchmarking stipple.
442
            if readv:
2001.3.2 by John Arbash Meinel
Force all transports to raise ShortReadvError if they can
443
                return self._sftp_readv(fp, offsets, relpath)
3959.1.3 by Martin Pool
Guard common sftp mutter statements by debug flags
444
            if 'sftp' in debug.debug_flags:
445
                mutter('seek and read %s offsets', len(offsets))
2001.3.2 by John Arbash Meinel
Force all transports to raise ShortReadvError if they can
446
            return self._seek_and_read(fp, offsets, relpath)
1864.5.11 by John Arbash Meinel
Factor out the workhorse of Transport.readv() into a helper function, and re-use that function in sftp but with a non-prefetch file.
447
        except (IOError, paramiko.SSHException), e:
448
            self._translate_io_exception(e, path, ': error retrieving')
449
2671.3.1 by Robert Collins
* New method ``bzrlib.transport.Transport.get_recommended_page_size``.
450
    def recommended_page_size(self):
451
        """See Transport.recommended_page_size().
452
453
        For SFTP we suggest a large page size to reduce the overhead
454
        introduced by latency.
455
        """
456
        return 64 * 1024
457
3882.9.1 by Martin Pool
Fix strange default parameter to _sftp_readv
458
    def _sftp_readv(self, fp, offsets, relpath):
1864.5.15 by John Arbash Meinel
Cleanup and document the new sftp readv implementation
459
        """Use the readv() member of fp to do async readv.
1864.5.13 by John Arbash Meinel
Try using paramiko.readv()
460
3882.9.1 by Martin Pool
Fix strange default parameter to _sftp_readv
461
        Then read them using paramiko.readv(). paramiko.readv()
1864.5.13 by John Arbash Meinel
Try using paramiko.readv()
462
        does not support ranges > 64K, so it caps the request size, and
3882.9.1 by Martin Pool
Fix strange default parameter to _sftp_readv
463
        just reads until it gets all the stuff it wants.
1864.5.13 by John Arbash Meinel
Try using paramiko.readv()
464
        """
3882.7.2 by Martin Pool
Start reporting activity from sftp read access
465
        helper = _SFTPReadvHelper(offsets, relpath, self._report_activity)
3686.1.2 by John Arbash Meinel
Start moving the readv code into a helper.
466
        return helper.request_and_yield_offsets(fp)
2001.3.2 by John Arbash Meinel
Force all transports to raise ShortReadvError if they can
467
1955.3.6 by John Arbash Meinel
Lots of deprecation warnings, but no errors
468
    def put_file(self, relpath, f, mode=None):
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
469
        """
1955.3.6 by John Arbash Meinel
Lots of deprecation warnings, but no errors
470
        Copy the file-like object into the location.
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
471
472
        :param relpath: Location to put the contents, relative to base.
1955.3.6 by John Arbash Meinel
Lots of deprecation warnings, but no errors
473
        :param f:       File-like object.
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
474
        :param mode: The final mode for the file
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
475
        """
1524.1.1 by Robert Collins
Test sftp with relative, absolute-in-homedir and absolute-not-in-homedir
476
        final_path = self._remote_path(relpath)
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
477
        return self._put(final_path, f, mode=mode)
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
478
479
    def _put(self, abspath, f, mode=None):
480
        """Helper function so both put() and copy_abspaths can reuse the code"""
481
        tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
482
                        os.getpid(), random.randint(0,0x7FFFFFFF))
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
483
        fout = self._sftp_open_exclusive(tmp_abspath, mode=mode)
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
484
        closed = False
1185.41.6 by Robey Pointer
modified version of john's patch to add atomic put and locking to the sftp transport
485
        try:
1185.49.1 by John Arbash Meinel
Updating SftpTransport.put() so that it is atomic
486
            try:
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
487
                fout.set_pipelined(True)
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
488
                length = self._pump(f, fout)
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
489
            except (IOError, paramiko.SSHException), e:
490
                self._translate_io_exception(e, tmp_abspath)
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
491
            # XXX: This doesn't truly help like we would like it to.
492
            #      The problem is that openssh strips sticky bits. So while we
493
            #      can properly set group write permission, we lose the group
494
            #      sticky bit. So it is probably best to stop chmodding, and
495
            #      just tell users that they need to set the umask correctly.
496
            #      The attr.st_mode = mode, in _sftp_open_exclusive
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
497
            #      will handle when the user wants the final mode to be more
498
            #      restrictive. And then we avoid a round trip. Unless
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
499
            #      paramiko decides to expose an async chmod()
500
501
            # This is designed to chmod() right before we close.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
502
            # Because we set_pipelined() earlier, theoretically we might
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
503
            # avoid the round trip for fout.close()
1185.50.20 by John Arbash Meinel
merge permissions branch, also fixup tests so they are lined up with bzr.dev to help prevent conflicts.
504
            if mode is not None:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
505
                self._get_sftp().chmod(tmp_abspath, mode)
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
506
            fout.close()
507
            closed = True
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
508
            self._rename_and_overwrite(tmp_abspath, abspath)
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
509
            return length
1185.49.1 by John Arbash Meinel
Updating SftpTransport.put() so that it is atomic
510
        except Exception, e:
511
            # If we fail, try to clean up the temporary file
512
            # before we throw the exception
513
            # but don't let another exception mess things up
1185.50.20 by John Arbash Meinel
merge permissions branch, also fixup tests so they are lined up with bzr.dev to help prevent conflicts.
514
            # Write out the traceback, because otherwise
515
            # the catch and throw destroys it
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
516
            import traceback
517
            mutter(traceback.format_exc())
1185.49.1 by John Arbash Meinel
Updating SftpTransport.put() so that it is atomic
518
            try:
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
519
                if not closed:
520
                    fout.close()
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
521
                self._get_sftp().remove(tmp_abspath)
1185.49.1 by John Arbash Meinel
Updating SftpTransport.put() so that it is atomic
522
            except:
1532 by Robert Collins
Merge in John Meinels integration branch.
523
                # raise the saved except
524
                raise e
525
            # raise the original with its traceback if we can.
526
            raise
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
527
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
528
    def _put_non_atomic_helper(self, relpath, writer, mode=None,
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
529
                               create_parent_dir=False,
530
                               dir_mode=None):
1946.1.6 by John Arbash Meinel
Basic implementation of non_atomic_put for sftp
531
        abspath = self._remote_path(relpath)
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
532
533
        # TODO: jam 20060816 paramiko doesn't publicly expose a way to
534
        #       set the file mode at create time. If it does, use it.
535
        #       But for now, we just chmod later anyway.
1946.1.6 by John Arbash Meinel
Basic implementation of non_atomic_put for sftp
536
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
537
        def _open_and_write_file():
538
            """Try to open the target file, raise error on failure"""
1946.1.10 by John Arbash Meinel
make sure to always close the file object.
539
            fout = None
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
540
            try:
1946.1.10 by John Arbash Meinel
make sure to always close the file object.
541
                try:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
542
                    fout = self._get_sftp().file(abspath, mode='wb')
1946.1.10 by John Arbash Meinel
make sure to always close the file object.
543
                    fout.set_pipelined(True)
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
544
                    writer(fout)
1946.1.10 by John Arbash Meinel
make sure to always close the file object.
545
                except (paramiko.SSHException, IOError), e:
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
546
                    self._translate_io_exception(e, abspath,
547
                                                 ': unable to open')
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
548
1946.1.10 by John Arbash Meinel
make sure to always close the file object.
549
                # This is designed to chmod() right before we close.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
550
                # Because we set_pipelined() earlier, theoretically we might
1946.1.10 by John Arbash Meinel
make sure to always close the file object.
551
                # avoid the round trip for fout.close()
552
                if mode is not None:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
553
                    self._get_sftp().chmod(abspath, mode)
1946.1.10 by John Arbash Meinel
make sure to always close the file object.
554
            finally:
555
                if fout is not None:
556
                    fout.close()
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
557
558
        if not create_parent_dir:
559
            _open_and_write_file()
560
            return
561
562
        # Try error handling to create the parent directory if we need to
563
        try:
564
            _open_and_write_file()
565
        except NoSuchFile:
566
            # Try to create the parent directory, and then go back to
567
            # writing the file
568
            parent_dir = os.path.dirname(abspath)
1946.2.15 by John Arbash Meinel
Fix the sftp mkdir mode code
569
            self._mkdir(parent_dir, dir_mode)
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
570
            _open_and_write_file()
1946.1.6 by John Arbash Meinel
Basic implementation of non_atomic_put for sftp
571
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
572
    def put_file_non_atomic(self, relpath, f, mode=None,
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
573
                            create_parent_dir=False,
574
                            dir_mode=None):
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
575
        """Copy the file-like object into the target location.
576
577
        This function is not strictly safe to use. It is only meant to
578
        be used when you already know that the target does not exist.
579
        It is not safe, because it will open and truncate the remote
580
        file. So there may be a time when the file has invalid contents.
581
582
        :param relpath: The remote location to put the contents.
583
        :param f:       File-like object.
584
        :param mode:    Possible access permissions for new file.
585
                        None means do not set remote permissions.
586
        :param create_parent_dir: If we cannot create the target file because
587
                        the parent directory does not exist, go ahead and
588
                        create it, and then try again.
589
        """
590
        def writer(fout):
591
            self._pump(f, fout)
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
592
        self._put_non_atomic_helper(relpath, writer, mode=mode,
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
593
                                    create_parent_dir=create_parent_dir,
594
                                    dir_mode=dir_mode)
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
595
6609.2.1 by Vincent Ladeuil
Make all transport put_bytes() raises TypeError when given unicode strings rather than bytes.
596
    def put_bytes_non_atomic(self, relpath, raw_bytes, mode=None,
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
597
                             create_parent_dir=False,
598
                             dir_mode=None):
6609.2.1 by Vincent Ladeuil
Make all transport put_bytes() raises TypeError when given unicode strings rather than bytes.
599
        if not isinstance(raw_bytes, str):
600
            raise TypeError(
601
                'raw_bytes must be a plain string, not %s' % type(raw_bytes))
602
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
603
        def writer(fout):
6609.2.1 by Vincent Ladeuil
Make all transport put_bytes() raises TypeError when given unicode strings rather than bytes.
604
            fout.write(raw_bytes)
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
605
        self._put_non_atomic_helper(relpath, writer, mode=mode,
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
606
                                    create_parent_dir=create_parent_dir,
607
                                    dir_mode=dir_mode)
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
608
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
609
    def iter_files_recursive(self):
610
        """Walk the relative paths of all files in this transport."""
4103.3.1 by Martin Pool
More sftp progress hooks
611
        # progress is handled by list_dir
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
612
        queue = list(self.list_dir('.'))
613
        while queue:
1959.2.1 by John Arbash Meinel
David Allouche: Make transports return escaped paths
614
            relpath = queue.pop(0)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
615
            st = self.stat(relpath)
616
            if stat.S_ISDIR(st.st_mode):
617
                for i, basename in enumerate(self.list_dir(relpath)):
618
                    queue.insert(i, relpath+'/'+basename)
619
            else:
620
                yield relpath
621
1988.1.1 by John Arbash Meinel
Restore mode bit tests for sftp, and track down bugs
622
    def _mkdir(self, abspath, mode=None):
623
        if mode is None:
624
            local_mode = 0777
625
        else:
626
            local_mode = mode
627
        try:
4103.3.1 by Martin Pool
More sftp progress hooks
628
            self._report_activity(len(abspath), 'write')
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
629
            self._get_sftp().mkdir(abspath, local_mode)
4103.3.1 by Martin Pool
More sftp progress hooks
630
            self._report_activity(1, 'read')
1988.1.1 by John Arbash Meinel
Restore mode bit tests for sftp, and track down bugs
631
            if mode is not None:
3526.4.1 by Christophe Troestler
sftp transport: do not chmod a dir when unecessary (fix suid and sgid problems).
632
                # chmod a dir through sftp will erase any sgid bit set
633
                # on the server side.  So, if the bit mode are already
634
                # set, avoid the chmod.  If the mode is not fine but
635
                # the sgid bit is set, report a warning to the user
636
                # with the umask fix.
637
                stat = self._get_sftp().lstat(abspath)
638
                mode = mode & 0777 # can't set special bits anyway
639
                if mode != stat.st_mode & 0777:
3526.4.2 by Christophe Troestler
sftp: More informative warning message for chmod and suid/sgid bits problems.
640
                    if stat.st_mode & 06000:
641
                        warning('About to chmod %s over sftp, which will result'
642
                                ' in its suid or sgid bits being cleared.  If'
643
                                ' you want to preserve those bits, change your '
644
                                ' environment on the server to use umask 0%03o.'
3526.4.1 by Christophe Troestler
sftp transport: do not chmod a dir when unecessary (fix suid and sgid problems).
645
                                % (abspath, 0777 - mode))
646
                    self._get_sftp().chmod(abspath, mode=mode)
1988.1.1 by John Arbash Meinel
Restore mode bit tests for sftp, and track down bugs
647
        except (paramiko.SSHException, IOError), e:
648
            self._translate_io_exception(e, abspath, ': unable to mkdir',
649
                failure_exc=FileExists)
650
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
651
    def mkdir(self, relpath, mode=None):
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
652
        """Create a directory at the given path."""
1988.1.1 by John Arbash Meinel
Restore mode bit tests for sftp, and track down bugs
653
        self._mkdir(self._remote_path(relpath), mode=mode)
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
654
2671.3.9 by Robert Collins
Review feedback and fix VFat emulated transports to not claim to have unix permissions.
655
    def open_write_stream(self, relpath, mode=None):
656
        """See Transport.open_write_stream."""
2671.3.2 by Robert Collins
Start open_file_stream logic.
657
        # initialise the file to zero-length
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
658
        # this is three round trips, but we don't use this
659
        # api more than once per write_group at the moment so
2671.3.2 by Robert Collins
Start open_file_stream logic.
660
        # it is a tolerable overhead. Better would be to truncate
661
        # the file after opening. RBC 20070805
2671.3.3 by Robert Collins
Add mode parameter to Transport.open_file_stream.
662
        self.put_bytes_non_atomic(relpath, "", mode)
2671.3.2 by Robert Collins
Start open_file_stream logic.
663
        abspath = self._remote_path(relpath)
664
        # TODO: jam 20060816 paramiko doesn't publicly expose a way to
665
        #       set the file mode at create time. If it does, use it.
666
        #       But for now, we just chmod later anyway.
667
        handle = None
668
        try:
669
            handle = self._get_sftp().file(abspath, mode='wb')
670
            handle.set_pipelined(True)
671
        except (paramiko.SSHException, IOError), e:
672
            self._translate_io_exception(e, abspath,
673
                                         ': unable to open')
674
        _file_streams[self.abspath(relpath)] = handle
2671.3.6 by Robert Collins
Review feedback.
675
        return FileFileStream(self, relpath, handle)
2671.3.2 by Robert Collins
Start open_file_stream logic.
676
2052.6.4 by Robert Collins
Tidier SFTP code (feedback from Aaron).
677
    def _translate_io_exception(self, e, path, more_info='',
678
                                failure_exc=PathError):
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
679
        """Translate a paramiko or IOError into a friendlier exception.
680
681
        :param e: The original exception
682
        :param path: The path in question when the error is raised
683
        :param more_info: Extra information that can be included,
684
                          such as what was going on
685
        :param failure_exc: Paramiko has the super fun ability to raise completely
686
                           opaque errors that just set "e.args = ('Failure',)" with
687
                           no more information.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
688
                           If this parameter is set, it defines the exception
1662.1.12 by Martin Pool
Translate unknown sftp errors to PathError, no NoSuchFile
689
                           to raise in these cases.
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
690
        """
1489 by Robert Collins
Make the paramiko tests pass. Nice huh.
691
        # paramiko seems to generate detailless errors.
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
692
        self._translate_error(e, path, raise_generic=False)
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
693
        if getattr(e, 'args', None) is not None:
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
694
            if (e.args == ('No such file or directory',) or
695
                e.args == ('No such file',)):
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
696
                raise NoSuchFile(path, str(e) + more_info)
3565.4.1 by Christophe Troestler
[Bug #123475] SFTPTransport._translate_io_exception: better detection
697
            if (e.args == ('mkdir failed',) or
698
                e.args[0].startswith('syserr: File exists')):
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
699
                raise FileExists(path, str(e) + more_info)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
700
            # strange but true, for the paramiko server.
701
            if (e.args == ('Failure',)):
2052.6.4 by Robert Collins
Tidier SFTP code (feedback from Aaron).
702
                raise failure_exc(path, str(e) + more_info)
4105.2.2 by Martin Pool
Clarify comment
703
            # Can be something like args = ('Directory not empty:
704
            # '/srv/bazaar.launchpad.net/blah...: '
705
            # [Errno 39] Directory not empty',)
4105.2.1 by Martin Pool
Correctly translate Paramiko ENOTEMPTY to DirectoryNotEmpty
706
            if (e.args[0].startswith('Directory not empty: ')
707
                or getattr(e, 'errno', None) == errno.ENOTEMPTY):
708
                raise errors.DirectoryNotEmpty(path, str(e))
5349.2.2 by Martin Pool
Better exception when sftp server doesn't support symlinks
709
            if e.args == ('Operation unsupported',):
710
                raise errors.TransportNotPossible()
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
711
            mutter('Raising exception with args %s', e.args)
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
712
        if getattr(e, 'errno', None) is not None:
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
713
            mutter('Raising exception with errno %s', e.errno)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
714
        raise e
1489 by Robert Collins
Make the paramiko tests pass. Nice huh.
715
1955.3.15 by John Arbash Meinel
Deprecate 'Transport.append' in favor of Transport.append_file or Transport.append_bytes
716
    def append_file(self, relpath, f, mode=None):
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
717
        """
718
        Append the text in the file-like object into the final
719
        location.
720
        """
721
        try:
1524.1.1 by Robert Collins
Test sftp with relative, absolute-in-homedir and absolute-not-in-homedir
722
            path = self._remote_path(relpath)
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
723
            fout = self._get_sftp().file(path, 'ab')
1666.1.6 by Robert Collins
Make knit the default format.
724
            if mode is not None:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
725
                self._get_sftp().chmod(path, mode)
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
726
            result = fout.tell()
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
727
            self._pump(f, fout)
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
728
            return result
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
729
        except (IOError, paramiko.SSHException), e:
730
            self._translate_io_exception(e, relpath, ': unable to append')
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
731
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
732
    def rename(self, rel_from, rel_to):
733
        """Rename without special overwriting"""
734
        try:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
735
            self._get_sftp().rename(self._remote_path(rel_from),
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
736
                              self._remote_path(rel_to))
737
        except (IOError, paramiko.SSHException), e:
738
            self._translate_io_exception(e, rel_from,
739
                    ': unable to rename to %r' % (rel_to))
740
741
    def _rename_and_overwrite(self, abs_from, abs_to):
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
742
        """Do a fancy rename on the remote server.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
743
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
744
        Using the implementation provided by osutils.
745
        """
746
        try:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
747
            sftp = self._get_sftp()
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
748
            fancy_rename(abs_from, abs_to,
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
749
                         rename_func=sftp.rename,
750
                         unlink_func=sftp.remove)
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
751
        except (IOError, paramiko.SSHException), e:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
752
            self._translate_io_exception(e, abs_from,
753
                                         ': unable to rename to %r' % (abs_to))
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
754
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
755
    def move(self, rel_from, rel_to):
756
        """Move the item at rel_from to the location at rel_to"""
1524.1.1 by Robert Collins
Test sftp with relative, absolute-in-homedir and absolute-not-in-homedir
757
        path_from = self._remote_path(rel_from)
758
        path_to = self._remote_path(rel_to)
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
759
        self._rename_and_overwrite(path_from, path_to)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
760
761
    def delete(self, relpath):
762
        """Delete the item at relpath"""
1524.1.1 by Robert Collins
Test sftp with relative, absolute-in-homedir and absolute-not-in-homedir
763
        path = self._remote_path(relpath)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
764
        try:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
765
            self._get_sftp().remove(path)
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
766
        except (IOError, paramiko.SSHException), e:
767
            self._translate_io_exception(e, path, ': unable to delete')
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
768
2634.1.1 by Robert Collins
(robertc) Reinstate the accidentally backed out external_url patch.
769
    def external_url(self):
770
        """See bzrlib.transport.Transport.external_url."""
771
        # the external path for SFTP is the base
772
        return self.base
773
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
774
    def listable(self):
775
        """Return True if this store supports listing."""
776
        return True
777
778
    def list_dir(self, relpath):
779
        """
780
        Return a list of all files at the given location.
781
        """
782
        # does anything actually use this?
1959.2.1 by John Arbash Meinel
David Allouche: Make transports return escaped paths
783
        # -- Unknown
784
        # This is at least used by copy_tree for remote upgrades.
785
        # -- David Allouche 2006-08-11
1524.1.1 by Robert Collins
Test sftp with relative, absolute-in-homedir and absolute-not-in-homedir
786
        path = self._remote_path(relpath)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
787
        try:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
788
            entries = self._get_sftp().listdir(path)
4103.3.1 by Martin Pool
More sftp progress hooks
789
            self._report_activity(sum(map(len, entries)), 'read')
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
790
        except (IOError, paramiko.SSHException), e:
791
            self._translate_io_exception(e, path, ': failed to list_dir')
1959.2.2 by John Arbash Meinel
Apply Andrew Bennetts list_dir fixes
792
        return [urlutils.escape(entry) for entry in entries]
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
793
1534.4.15 by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports.
794
    def rmdir(self, relpath):
795
        """See Transport.rmdir."""
796
        path = self._remote_path(relpath)
797
        try:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
798
            return self._get_sftp().rmdir(path)
1534.4.15 by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports.
799
        except (IOError, paramiko.SSHException), e:
800
            self._translate_io_exception(e, path, ': failed to rmdir')
801
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
802
    def stat(self, relpath):
803
        """Return the stat information for a file."""
1524.1.1 by Robert Collins
Test sftp with relative, absolute-in-homedir and absolute-not-in-homedir
804
        path = self._remote_path(relpath)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
805
        try:
5056.1.5 by Neil Santos
Removed separate lstat() and reverted LocalTransport and SFTPTransport's stat() methods to using lstat() internally.
806
            return self._get_sftp().lstat(path)
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
807
        except (IOError, paramiko.SSHException), e:
808
            self._translate_io_exception(e, path, ': unable to stat')
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
809
5056.1.4 by Neil Santos
Changed stat() in SFTPTransport and LocalTransport back to calling plain stat(), instead of lstat().
810
    def readlink(self, relpath):
811
        """See Transport.readlink."""
812
        path = self._remote_path(relpath)
813
        try:
814
            return self._get_sftp().readlink(path)
815
        except (IOError, paramiko.SSHException), e:
816
            self._translate_io_exception(e, path, ': unable to readlink')
817
5056.1.1 by Neil Santos
Added default link() and symlink() methods to Transport.
818
    def symlink(self, source, link_name):
819
        """See Transport.symlink."""
820
        try:
821
            conn = self._get_sftp()
5056.1.5 by Neil Santos
Removed separate lstat() and reverted LocalTransport and SFTPTransport's stat() methods to using lstat() internally.
822
            sftp_retval = conn.symlink(source, link_name)
823
            if SFTP_OK != sftp_retval:
824
                raise TransportError(
825
                    '%r: unable to create symlink to %r' % (link_name, source),
826
                    sftp_retval
827
                )
5056.1.1 by Neil Santos
Added default link() and symlink() methods to Transport.
828
        except (IOError, paramiko.SSHException), e:
829
            self._translate_io_exception(e, link_name,
830
                                         ': unable to create symlink to %r' % (source))
831
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
832
    def lock_read(self, relpath):
833
        """
834
        Lock the given file for shared (read) access.
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
835
        :return: A lock object, which has an unlock() member function
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
836
        """
837
        # FIXME: there should be something clever i can do here...
838
        class BogusLock(object):
839
            def __init__(self, path):
840
                self.path = path
841
            def unlock(self):
842
                pass
843
        return BogusLock(relpath)
844
845
    def lock_write(self, relpath):
846
        """
847
        Lock the given file for exclusive (write) access.
848
        WARNING: many transports do not support this, so trying avoid using it
849
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
850
        :return: A lock object, which has an unlock() member function
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
851
        """
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
852
        # This is a little bit bogus, but basically, we create a file
853
        # which should not already exist, and if it does, we assume
854
        # that there is a lock, and if it doesn't, the we assume
855
        # that we have taken the lock.
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
856
        return SFTPLock(relpath, self)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
857
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
858
    def _sftp_open_exclusive(self, abspath, mode=None):
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
859
        """Open a remote path exclusively.
860
861
        SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if
862
        the file already exists. However it does not expose this
863
        at the higher level of SFTPClient.open(), so we have to
864
        sneak away with it.
865
866
        WARNING: This breaks the SFTPClient abstraction, so it
867
        could easily break against an updated version of paramiko.
868
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
869
        :param abspath: The remote absolute path where the file should be opened
870
        :param mode: The mode permissions bits for the new file
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
871
        """
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
872
        # TODO: jam 20060816 Paramiko >= 1.6.2 (probably earlier) supports
873
        #       using the 'x' flag to indicate SFTP_FLAG_EXCL.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
874
        #       However, there is no way to set the permission mode at open
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
875
        #       time using the sftp_client.file() functionality.
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
876
        path = self._get_sftp()._adjust_cwd(abspath)
1711.5.1 by John Arbash Meinel
Get most SFTP tests to pass. StubSFTPServer now talks the same path protocol that SFTPTransport talks. on win32
877
        # mutter('sftp abspath %s => %s', abspath, path)
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
878
        attr = SFTPAttributes()
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
879
        if mode is not None:
880
            attr.st_mode = mode
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
881
        omode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
882
                | SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
883
        try:
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
884
            t, msg = self._get_sftp()._request(CMD_OPEN, path, omode, attr)
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
885
            if t != CMD_HANDLE:
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
886
                raise TransportError('Expected an SFTP handle')
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
887
            handle = msg.get_string()
2485.8.38 by Vincent Ladeuil
Finish sftp refactoring. Test suite passing.
888
            return SFTPFile(self._get_sftp(), handle, 'wb', -1)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
889
        except (paramiko.SSHException, IOError), e:
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
890
            self._translate_io_exception(e, abspath, ': unable to open',
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
891
                failure_exc=FileExists)
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
892
1988.1.1 by John Arbash Meinel
Restore mode bit tests for sftp, and track down bugs
893
    def _can_roundtrip_unix_modebits(self):
894
        if sys.platform == 'win32':
895
            # anyone else?
896
            return False
897
        else:
898
            return True
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
899
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
900
901
def get_test_permutations():
902
    """Return the permutations to be used in testing."""
4797.11.2 by Vincent Ladeuil
Stop requiring testtools for sftp use.
903
    from bzrlib.tests import stub_sftp
904
    return [(SFTPTransport, stub_sftp.SFTPAbsoluteServer),
905
            (SFTPTransport, stub_sftp.SFTPHomeDirServer),
906
            (SFTPTransport, stub_sftp.SFTPSiblingAbsoluteServer),
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
907
            ]