~bzr-pqm/bzr/bzr.dev

1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
1
# Copyright (C) 2005 Robey Pointer <robey@lag.net>, Canonical Ltd
2
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
"""Implementation of Transport over SFTP, using paramiko."""
18
1489 by Robert Collins
Make the paramiko tests pass. Nice huh.
19
import errno
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
20
import getpass
21
import os
22
import re
23
import stat
24
import sys
25
import urllib
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
26
import urlparse
1185.49.1 by John Arbash Meinel
Updating SftpTransport.put() so that it is atomic
27
import time
28
import random
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
29
import subprocess
1185.49.10 by John Arbash Meinel
Use a weakref dictionary to enable re-use of a connection (for sftp).
30
import weakref
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
31
1489 by Robert Collins
Make the paramiko tests pass. Nice huh.
32
from bzrlib.errors import (FileExists, 
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
33
                           TransportNotPossible, NoSuchFile, PathNotChild,
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
34
                           TransportError,
35
                           LockError)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
36
from bzrlib.config import config_dir
37
from bzrlib.trace import mutter, warning, error
38
from bzrlib.transport import Transport, register_transport
1185.50.10 by John Arbash Meinel
Don't import ui_factory directly, in case it gets changed later.
39
import bzrlib.ui
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
40
41
try:
42
    import paramiko
43
except ImportError:
44
    error('The SFTP transport requires paramiko.')
45
    raise
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
46
else:
47
    from paramiko.sftp import (SFTP_FLAG_WRITE, SFTP_FLAG_CREATE,
48
                               SFTP_FLAG_EXCL, SFTP_FLAG_TRUNC,
49
                               CMD_HANDLE, CMD_OPEN)
50
    from paramiko.sftp_attr import SFTPAttributes
51
    from paramiko.sftp_file import SFTPFile
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
52
    from paramiko.sftp_client import SFTPClient
53
54
if 'sftp' not in urlparse.uses_netloc: urlparse.uses_netloc.append('sftp')
55
56
1185.50.2 by John Arbash Meinel
From Matt Lavin: close_fds is not supported on Windows platforms
57
_close_fds = True
58
if sys.platform == 'win32':
59
    # close_fds not supported on win32
60
    _close_fds = False
61
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
62
_ssh_vendor = None
63
def _get_ssh_vendor():
64
    """Find out what version of SSH is on the system."""
1185.48.2 by James Henstridge
Fix some bugs so it actually works
65
    global _ssh_vendor
66
    if _ssh_vendor is not None:
67
        return _ssh_vendor
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
68
1185.48.2 by James Henstridge
Fix some bugs so it actually works
69
    _ssh_vendor = 'none'
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
70
71
    try:
72
        p = subprocess.Popen(['ssh', '-V'],
1185.50.2 by John Arbash Meinel
From Matt Lavin: close_fds is not supported on Windows platforms
73
                             close_fds=_close_fds,
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
74
                             stdin=subprocess.PIPE,
75
                             stdout=subprocess.PIPE,
76
                             stderr=subprocess.PIPE)
77
        returncode = p.returncode
78
        stdout, stderr = p.communicate()
79
    except OSError:
80
        returncode = -1
81
        stdout = stderr = ''
82
    if 'OpenSSH' in stderr:
1185.48.2 by James Henstridge
Fix some bugs so it actually works
83
        mutter('ssh implementation is OpenSSH')
84
        _ssh_vendor = 'openssh'
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
85
    elif 'SSH Secure Shell' in stderr:
1185.48.2 by James Henstridge
Fix some bugs so it actually works
86
        mutter('ssh implementation is SSH Corp.')
87
        _ssh_vendor = 'ssh'
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
88
1185.48.2 by James Henstridge
Fix some bugs so it actually works
89
    if _ssh_vendor != 'none':
90
        return _ssh_vendor
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
91
92
    # XXX: 20051123 jamesh
93
    # A check for putty's plink or lsh would go here.
94
1185.48.2 by James Henstridge
Fix some bugs so it actually works
95
    mutter('falling back to paramiko implementation')
96
    return _ssh_vendor
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
97
98
99
class SFTPSubprocess:
100
    """A socket-like object that talks to an ssh subprocess via pipes."""
101
    def __init__(self, hostname, port=None, user=None):
102
        vendor = _get_ssh_vendor()
103
        assert vendor in ['openssh', 'ssh']
104
        if vendor == 'openssh':
105
            args = ['ssh',
106
                    '-oForwardX11=no', '-oForwardAgent=no',
107
                    '-oClearAllForwardings=yes', '-oProtocol=2',
108
                    '-oNoHostAuthenticationForLocalhost=yes']
109
            if port is not None:
110
                args.extend(['-p', str(port)])
111
            if user is not None:
112
                args.extend(['-l', user])
113
            args.extend(['-s', hostname, 'sftp'])
114
        elif vendor == 'ssh':
115
            args = ['ssh', '-x']
116
            if port is not None:
117
                args.extend(['-p', str(port)])
118
            if user is not None:
119
                args.extend(['-l', user])
120
            args.extend(['-s', 'sftp', hostname])
121
1185.50.2 by John Arbash Meinel
From Matt Lavin: close_fds is not supported on Windows platforms
122
        self.proc = subprocess.Popen(args, close_fds=_close_fds,
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
123
                                     stdin=subprocess.PIPE,
124
                                     stdout=subprocess.PIPE)
125
126
    def send(self, data):
127
        return os.write(self.proc.stdin.fileno(), data)
128
129
    def recv(self, count):
130
        return os.read(self.proc.stdout.fileno(), count)
131
132
    def close(self):
133
        self.proc.stdin.close()
134
        self.proc.stdout.close()
135
        self.proc.wait()
136
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
137
138
SYSTEM_HOSTKEYS = {}
139
BZR_HOSTKEYS = {}
140
1185.49.10 by John Arbash Meinel
Use a weakref dictionary to enable re-use of a connection (for sftp).
141
# This is a weakref dictionary, so that we can reuse connections
142
# that are still active. Long term, it might be nice to have some
143
# sort of expiration policy, such as disconnect if inactive for
144
# X seconds. But that requires a lot more fanciness.
145
_connected_hosts = weakref.WeakValueDictionary()
146
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
147
def load_host_keys():
148
    """
149
    Load system host keys (probably doesn't work on windows) and any
150
    "discovered" keys from previous sessions.
151
    """
152
    global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
153
    try:
154
        SYSTEM_HOSTKEYS = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
155
    except Exception, e:
156
        mutter('failed to load system host keys: ' + str(e))
157
    bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')
158
    try:
159
        BZR_HOSTKEYS = paramiko.util.load_host_keys(bzr_hostkey_path)
160
    except Exception, e:
161
        mutter('failed to load bzr host keys: ' + str(e))
162
        save_host_keys()
163
164
def save_host_keys():
165
    """
166
    Save "discovered" host keys in $(config)/ssh_host_keys/.
167
    """
168
    global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
169
    bzr_hostkey_path = os.path.join(config_dir(), 'ssh_host_keys')
170
    if not os.path.isdir(config_dir()):
171
        os.mkdir(config_dir())
172
    try:
173
        f = open(bzr_hostkey_path, 'w')
174
        f.write('# SSH host keys collected by bzr\n')
175
        for hostname, keys in BZR_HOSTKEYS.iteritems():
176
            for keytype, key in keys.iteritems():
177
                f.write('%s %s %s\n' % (hostname, keytype, key.get_base64()))
178
        f.close()
179
    except IOError, e:
180
        mutter('failed to save bzr host keys: ' + str(e))
181
182
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
183
class SFTPLock(object):
184
    """This fakes a lock in a remote location."""
185
    __slots__ = ['path', 'lock_path', 'lock_file', 'transport']
186
    def __init__(self, path, transport):
187
        assert isinstance(transport, SFTPTransport)
188
189
        self.lock_file = None
190
        self.path = path
191
        self.lock_path = path + '.write-lock'
192
        self.transport = transport
193
        try:
194
            self.lock_file = transport._sftp_open_exclusive(self.lock_path)
195
        except FileExists:
196
            raise LockError('File %r already locked' % (self.path,))
197
198
    def __del__(self):
199
        """Should this warn, or actually try to cleanup?"""
200
        if self.lock_file:
201
            warn("SFTPLock %r not explicitly unlocked" % (self.path,))
202
            self.unlock()
203
204
    def unlock(self):
205
        if not self.lock_file:
206
            return
207
        self.lock_file.close()
208
        self.lock_file = None
209
        try:
210
            self.transport.delete(self.lock_path)
211
        except (NoSuchFile,):
212
            # What specific errors should we catch here?
213
            pass
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
214
215
class SFTPTransport (Transport):
216
    """
217
    Transport implementation for SFTP access.
218
    """
1185.49.16 by John Arbash Meinel
Disabling prefetch
219
    _do_prefetch = False # Right now Paramiko's prefetch support causes things to hang
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
220
221
    def __init__(self, base, clone_from=None):
222
        assert base.startswith('sftp://')
1185.49.6 by John Arbash Meinel
Fixed the double rename, to rename the safety in case of problem.
223
        self._parse_url(base)
224
        base = self._unparse_url()
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
225
        super(SFTPTransport, self).__init__(base)
226
        if clone_from is None:
227
            self._sftp_connect()
228
        else:
229
            # use the same ssh connection, etc
230
            self._sftp = clone_from._sftp
231
        # super saves 'self.base'
232
    
233
    def should_cache(self):
234
        """
235
        Return True if the data pulled across should be cached locally.
236
        """
237
        return True
238
239
    def clone(self, offset=None):
240
        """
241
        Return a new SFTPTransport with root at self.base + offset.
242
        We share the same SFTP session between such transports, because it's
243
        fairly expensive to set them up.
244
        """
245
        if offset is None:
246
            return SFTPTransport(self.base, self)
247
        else:
248
            return SFTPTransport(self.abspath(offset), self)
249
250
    def abspath(self, relpath):
251
        """
252
        Return the full url to the given relative path.
253
        
254
        @param relpath: the relative path or path components
255
        @type relpath: str or list
256
        """
257
        return self._unparse_url(self._abspath(relpath))
258
    
259
    def _abspath(self, relpath):
260
        """Return the absolute path segment without the SFTP URL."""
261
        # FIXME: share the common code across transports
262
        assert isinstance(relpath, basestring)
263
        relpath = [urllib.unquote(relpath)]
264
        basepath = self._path.split('/')
265
        if len(basepath) > 0 and basepath[-1] == '':
266
            basepath = basepath[:-1]
267
268
        for p in relpath:
269
            if p == '..':
270
                if len(basepath) == 0:
271
                    # In most filesystems, a request for the parent
272
                    # of root, just returns root.
273
                    continue
274
                basepath.pop()
275
            elif p == '.':
276
                continue # No-op
277
            else:
278
                basepath.append(p)
279
280
        path = '/'.join(basepath)
1185.40.4 by Robey Pointer
fix sftp urls to support the ietf draft url spec wrt relative vs absolute sftp urls (this will break existing branch urls); fix username/password parsing in sftp urls; add unit tests to make sure sftp url parsing is working
281
        # could still be a "relative" path here, but relative on the sftp server
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
282
        return path
283
284
    def relpath(self, abspath):
1185.48.8 by James Henstridge
More URL handling fixes
285
        username, password, host, port, path = self._split_url(abspath)
1185.49.23 by John Arbash Meinel
bugreport from Matthieu Moy: relpath was failing, but throwing an unhelpful exception.
286
        error = []
287
        if (username != self._username):
288
            error.append('username mismatch')
289
        if (host != self._host):
290
            error.append('host mismatch')
291
        if (port != self._port):
292
            error.append('port mismatch')
293
        if (not path.startswith(self._path)):
294
            error.append('path mismatch')
295
        if error:
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
296
            extra = ': ' + ', '.join(error)
297
            raise PathNotChild(abspath, self.base, extra=extra)
1185.48.8 by James Henstridge
More URL handling fixes
298
        pl = len(self._path)
299
        return path[pl:].lstrip('/')
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
300
301
    def has(self, relpath):
302
        """
303
        Does the target location exist?
304
        """
305
        try:
306
            self._sftp.stat(self._abspath(relpath))
307
            return True
308
        except IOError:
309
            return False
310
311
    def get(self, relpath, decode=False):
312
        """
313
        Get the file at the given relative path.
314
315
        :param relpath: The relative path to the file
316
        """
317
        try:
318
            path = self._abspath(relpath)
1185.40.1 by Robey Pointer
prefetch files under paramiko 1.5.1 for improved speed
319
            f = self._sftp.file(path)
1185.49.16 by John Arbash Meinel
Disabling prefetch
320
            if self._do_prefetch and hasattr(f, 'prefetch'):
1185.40.1 by Robey Pointer
prefetch files under paramiko 1.5.1 for improved speed
321
                f.prefetch()
322
            return f
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
323
        except (IOError, paramiko.SSHException), e:
324
            self._translate_io_exception(e, path, ': error retrieving')
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
325
326
    def get_partial(self, relpath, start, length=None):
327
        """
328
        Get just part of a file.
329
330
        :param relpath: Path to the file, relative to base
331
        :param start: The starting position to read from
332
        :param length: The length to read. A length of None indicates
333
                       read to the end of the file.
334
        :return: A file-like object containing at least the specified bytes.
335
                 Some implementations may return objects which can be read
336
                 past this length, but this is not guaranteed.
337
        """
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
338
        # TODO: implement get_partial_multi to help with knit support
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
339
        f = self.get(relpath)
340
        f.seek(start)
1185.49.16 by John Arbash Meinel
Disabling prefetch
341
        if self._do_prefetch and hasattr(f, 'prefetch'):
1185.40.1 by Robey Pointer
prefetch files under paramiko 1.5.1 for improved speed
342
            f.prefetch()
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
343
        return f
344
345
    def put(self, relpath, f):
346
        """
347
        Copy the file-like or string object into the location.
348
349
        :param relpath: Location to put the contents, relative to base.
350
        :param f:       File-like or string object.
351
        """
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
352
        final_path = self._abspath(relpath)
353
        tmp_relpath = '%s.tmp.%.9f.%d.%d' % (relpath, time.time(),
354
                        os.getpid(), random.randint(0,0x7FFFFFFF))
355
        tmp_abspath = self._abspath(tmp_relpath)
356
        fout = self._sftp_open_exclusive(tmp_relpath)
1185.49.1 by John Arbash Meinel
Updating SftpTransport.put() so that it is atomic
357
1185.41.6 by Robey Pointer
modified version of john's patch to add atomic put and locking to the sftp transport
358
        try:
1185.49.1 by John Arbash Meinel
Updating SftpTransport.put() so that it is atomic
359
            try:
360
                self._pump(f, fout)
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
361
            except (paramiko.SSHException, IOError), e:
362
                self._translate_io_exception(e, relpath, ': unable to write')
1185.49.1 by John Arbash Meinel
Updating SftpTransport.put() so that it is atomic
363
        except Exception, e:
364
            # If we fail, try to clean up the temporary file
365
            # before we throw the exception
366
            # but don't let another exception mess things up
367
            try:
368
                fout.close()
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
369
                self._sftp.remove(tmp_abspath)
1185.49.1 by John Arbash Meinel
Updating SftpTransport.put() so that it is atomic
370
            except:
371
                pass
372
            raise e
373
        else:
1185.40.15 by Robey Pointer
sftp doesn't allow rename of A to B if B already exists, so play some tricks (worked out on irc) where we move any existing file out of the way before renaming, and blow it away later
374
            # sftp rename doesn't allow overwriting, so play tricks:
375
            tmp_safety = 'bzr.tmp.%.9f.%d.%d' % (time.time(), os.getpid(), random.randint(0, 0x7FFFFFFF))
376
            tmp_safety = self._abspath(tmp_safety)
377
            try:
378
                self._sftp.rename(final_path, tmp_safety)
379
                file_existed = True
380
            except:
381
                file_existed = False
1185.49.6 by John Arbash Meinel
Fixed the double rename, to rename the safety in case of problem.
382
            success = False
1185.49.1 by John Arbash Meinel
Updating SftpTransport.put() so that it is atomic
383
            try:
1185.49.6 by John Arbash Meinel
Fixed the double rename, to rename the safety in case of problem.
384
                try:
385
                    self._sftp.rename(tmp_abspath, final_path)
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
386
                except (paramiko.SSHException, IOError), e:
387
                    self._translate_io_exception(e, relpath, ': unable to rename')
1185.49.6 by John Arbash Meinel
Fixed the double rename, to rename the safety in case of problem.
388
                else:
389
                    success = True
390
            finally:
391
                if file_existed:
392
                    if success:
393
                        self._sftp.unlink(tmp_safety)
394
                    else:
395
                        self._sftp.rename(tmp_safety, final_path)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
396
397
    def iter_files_recursive(self):
398
        """Walk the relative paths of all files in this transport."""
399
        queue = list(self.list_dir('.'))
400
        while queue:
1185.12.96 by Aaron Bentley
Merge from mpool
401
            relpath = urllib.quote(queue.pop(0))
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
402
            st = self.stat(relpath)
403
            if stat.S_ISDIR(st.st_mode):
404
                for i, basename in enumerate(self.list_dir(relpath)):
405
                    queue.insert(i, relpath+'/'+basename)
406
            else:
407
                yield relpath
408
409
    def mkdir(self, relpath):
410
        """Create a directory at the given path."""
411
        try:
412
            path = self._abspath(relpath)
413
            self._sftp.mkdir(path)
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
414
        except (paramiko.SSHException, IOError), e:
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
415
            self._translate_io_exception(e, relpath, ': unable to mkdir',
416
                failure_exc=FileExists)
417
418
    def _translate_io_exception(self, e, path, more_info='', failure_exc=NoSuchFile):
419
        """Translate a paramiko or IOError into a friendlier exception.
420
421
        :param e: The original exception
422
        :param path: The path in question when the error is raised
423
        :param more_info: Extra information that can be included,
424
                          such as what was going on
425
        :param failure_exc: Paramiko has the super fun ability to raise completely
426
                           opaque errors that just set "e.args = ('Failure',)" with
427
                           no more information.
428
                           This sometimes means FileExists, but it also sometimes
429
                           means NoSuchFile
430
        """
1489 by Robert Collins
Make the paramiko tests pass. Nice huh.
431
        # paramiko seems to generate detailless errors.
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
432
        self._translate_error(e, path, raise_generic=False)
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
433
        if hasattr(e, 'args'):
434
            if (e.args == ('No such file or directory',) or
435
                e.args == ('No such file',)):
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
436
                raise NoSuchFile(path, str(e) + more_info)
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
437
            if (e.args == ('mkdir failed',)):
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
438
                raise FileExists(path, str(e) + more_info)
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
439
            # strange but true, for the paramiko server.
440
            if (e.args == ('Failure',)):
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
441
                raise failure_exc(path, str(e) + more_info)
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
442
        raise e
1489 by Robert Collins
Make the paramiko tests pass. Nice huh.
443
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
444
    def append(self, relpath, f):
445
        """
446
        Append the text in the file-like object into the final
447
        location.
448
        """
449
        try:
450
            path = self._abspath(relpath)
451
            fout = self._sftp.file(path, 'ab')
452
            self._pump(f, fout)
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
453
        except (IOError, paramiko.SSHException), e:
454
            self._translate_io_exception(e, relpath, ': unable to append')
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
455
456
    def copy(self, rel_from, rel_to):
457
        """Copy the item at rel_from to the location at rel_to"""
458
        path_from = self._abspath(rel_from)
459
        path_to = self._abspath(rel_to)
1185.49.11 by John Arbash Meinel
Setting up framework for making sftp remote copy faster
460
        self._copy_abspaths(path_from, path_to)
461
462
    def _copy_abspaths(self, path_from, path_to):
463
        """Copy files given an absolute path
464
465
        :param path_from: Path on remote server to read
466
        :param path_to: Path on remote server to write
467
        :return: None
468
469
        TODO: Should the destination location be atomically created?
470
              This has not been specified
471
        TODO: This should use some sort of remote copy, rather than
472
              pulling the data locally, and then writing it remotely
473
        """
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
474
        try:
475
            fin = self._sftp.file(path_from, 'rb')
476
            try:
477
                fout = self._sftp.file(path_to, 'wb')
478
                try:
479
                    fout.set_pipelined(True)
480
                    self._pump(fin, fout)
481
                finally:
482
                    fout.close()
483
            finally:
484
                fin.close()
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
485
        except (IOError, paramiko.SSHException), e:
486
            self._translate_io_exception(e, path_from, ': unable copy to: %r' % path_to)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
487
1185.49.11 by John Arbash Meinel
Setting up framework for making sftp remote copy faster
488
    def copy_to(self, relpaths, other, pb=None):
489
        """Copy a set of entries from self into another Transport.
490
491
        :param relpaths: A list/generator of entries to be copied.
492
        """
493
        if isinstance(other, SFTPTransport) and other._sftp is self._sftp:
494
            # Both from & to are on the same remote filesystem
495
            # We can use a remote copy, instead of pulling locally, and pushing
496
497
            total = self._get_total(relpaths)
498
            count = 0
499
            for path in relpaths:
500
                path_from = self._abspath(relpath)
501
                path_to = other._abspath(relpath)
502
                self._update_pb(pb, 'copy-to', count, total)
503
                self._copy_abspaths(path_from, path_to)
504
                count += 1
505
            return count
506
        else:
1185.49.13 by John Arbash Meinel
Removed delayed setup, since it broke some tests. Fixed other small bugs. All tests pass.
507
            return super(SFTPTransport, self).copy_to(relpaths, other, pb=pb)
1185.49.11 by John Arbash Meinel
Setting up framework for making sftp remote copy faster
508
509
        # The dummy implementation just does a simple get + put
510
        def copy_entry(path):
511
            other.put(path, self.get(path))
512
513
        return self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False)
514
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
515
    def move(self, rel_from, rel_to):
516
        """Move the item at rel_from to the location at rel_to"""
517
        path_from = self._abspath(rel_from)
518
        path_to = self._abspath(rel_to)
519
        try:
520
            self._sftp.rename(path_from, path_to)
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
521
        except (IOError, paramiko.SSHException), e:
522
            self._translate_io_exception(e, path_from, ': unable to move to: %r' % path_to)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
523
524
    def delete(self, relpath):
525
        """Delete the item at relpath"""
526
        path = self._abspath(relpath)
527
        try:
528
            self._sftp.remove(path)
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
529
        except (IOError, paramiko.SSHException), e:
530
            self._translate_io_exception(e, path, ': unable to delete')
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
531
            
532
    def listable(self):
533
        """Return True if this store supports listing."""
534
        return True
535
536
    def list_dir(self, relpath):
537
        """
538
        Return a list of all files at the given location.
539
        """
540
        # does anything actually use this?
541
        path = self._abspath(relpath)
542
        try:
543
            return self._sftp.listdir(path)
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
544
        except (IOError, paramiko.SSHException), e:
545
            self._translate_io_exception(e, path, ': failed to list_dir')
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
546
547
    def stat(self, relpath):
548
        """Return the stat information for a file."""
549
        path = self._abspath(relpath)
550
        try:
551
            return self._sftp.stat(path)
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
552
        except (IOError, paramiko.SSHException), e:
553
            self._translate_io_exception(e, path, ': unable to stat')
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
554
555
    def lock_read(self, relpath):
556
        """
557
        Lock the given file for shared (read) access.
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
558
        :return: A lock object, which has an unlock() member function
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
559
        """
560
        # FIXME: there should be something clever i can do here...
561
        class BogusLock(object):
562
            def __init__(self, path):
563
                self.path = path
564
            def unlock(self):
565
                pass
566
        return BogusLock(relpath)
567
568
    def lock_write(self, relpath):
569
        """
570
        Lock the given file for exclusive (write) access.
571
        WARNING: many transports do not support this, so trying avoid using it
572
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
573
        :return: A lock object, which has an unlock() member function
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
574
        """
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
575
        # This is a little bit bogus, but basically, we create a file
576
        # which should not already exist, and if it does, we assume
577
        # that there is a lock, and if it doesn't, the we assume
578
        # that we have taken the lock.
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
579
        return SFTPLock(relpath, self)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
580
581
582
    def _unparse_url(self, path=None):
583
        if path is None:
1185.48.5 by James Henstridge
Change SFTP url parsing back to treat the path in sftp://host/path as
584
            path = self._path
585
        path = urllib.quote(path)
586
        if path.startswith('/'):
587
            path = '/%2F' + path[1:]
588
        else:
589
            path = '/' + path
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
590
        netloc = urllib.quote(self._host)
591
        if self._username is not None:
592
            netloc = '%s@%s' % (urllib.quote(self._username), netloc)
1185.49.23 by John Arbash Meinel
bugreport from Matthieu Moy: relpath was failing, but throwing an unhelpful exception.
593
        if self._port is not None:
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
594
            netloc = '%s:%d' % (netloc, self._port)
595
596
        return urlparse.urlunparse(('sftp', netloc, path, '', '', ''))
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
597
1185.48.8 by James Henstridge
More URL handling fixes
598
    def _split_url(self, url):
1185.48.3 by James Henstridge
More fixes
599
        if isinstance(url, unicode):
600
            url = url.encode('utf-8')
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
601
        (scheme, netloc, path, params,
602
         query, fragment) = urlparse.urlparse(url, allow_fragments=False)
603
        assert scheme == 'sftp'
1185.48.8 by James Henstridge
More URL handling fixes
604
        username = password = host = port = None
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
605
        if '@' in netloc:
1185.48.8 by James Henstridge
More URL handling fixes
606
            username, host = netloc.split('@', 1)
607
            if ':' in username:
608
                username, password = username.split(':', 1)
609
                password = urllib.unquote(password)
610
            username = urllib.unquote(username)
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
611
        else:
1185.48.8 by James Henstridge
More URL handling fixes
612
            host = netloc
613
614
        if ':' in host:
615
            host, port = host.rsplit(':', 1)
1185.33.67 by Martin Pool
[merge] use /usr/bin/ssh if we can (jamesh)
616
            try:
617
                port = int(port)
618
            except ValueError:
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
619
                # TODO: Should this be ConnectionError?
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
620
                raise TransportError('%s: invalid port number' % port)
1185.48.8 by James Henstridge
More URL handling fixes
621
        host = urllib.unquote(host)
622
623
        path = urllib.unquote(path)
1185.48.5 by James Henstridge
Change SFTP url parsing back to treat the path in sftp://host/path as
624
625
        # the initial slash should be removed from the path, and treated
626
        # as a homedir relative path (the path begins with a double slash
627
        # if it is absolute).
628
        # see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt
1185.48.8 by James Henstridge
More URL handling fixes
629
        if path.startswith('/'):
630
            path = path[1:]
631
632
        return (username, password, host, port, path)
633
634
    def _parse_url(self, url):
635
        (self._username, self._password,
636
         self._host, self._port, self._path) = self._split_url(url)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
637
638
    def _sftp_connect(self):
1185.49.14 by John Arbash Meinel
[merge] bzr.dev
639
        """Connect to the remote sftp server.
640
        After this, self._sftp should have a valid connection (or
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
641
        we raise an TransportError 'could not connect').
1185.49.14 by John Arbash Meinel
[merge] bzr.dev
642
643
        TODO: Raise a more reasonable ConnectionFailed exception
644
        """
645
        global _connected_hosts
1185.49.10 by John Arbash Meinel
Use a weakref dictionary to enable re-use of a connection (for sftp).
646
647
        idx = (self._host, self._port, self._username)
648
        try:
649
            self._sftp = _connected_hosts[idx]
650
            return
651
        except KeyError:
652
            pass
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
653
        
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
654
        vendor = _get_ssh_vendor()
655
        if vendor != 'none':
656
            sock = SFTPSubprocess(self._host, self._port, self._username)
657
            self._sftp = SFTPClient(sock)
658
        else:
659
            self._paramiko_connect()
660
1185.49.14 by John Arbash Meinel
[merge] bzr.dev
661
        _connected_hosts[idx] = self._sftp
662
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
663
    def _paramiko_connect(self):
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
664
        global SYSTEM_HOSTKEYS, BZR_HOSTKEYS
665
        
666
        load_host_keys()
1185.48.1 by James Henstridge
Use /usr/bin/ssh if we can.
667
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
668
        try:
1185.49.27 by John Arbash Meinel
James Henstridge confirmed that sftp.py needs self._port or 22
669
            t = paramiko.Transport((self._host, self._port or 22))
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
670
            t.start_client()
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
671
        except paramiko.SSHException, e:
672
            raise ConnectionError('Unable to reach SSH host %s:%d' %
673
                                  (self._host, self._port), e)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
674
            
675
        server_key = t.get_remote_server_key()
676
        server_key_hex = paramiko.util.hexify(server_key.get_fingerprint())
677
        keytype = server_key.get_name()
678
        if SYSTEM_HOSTKEYS.has_key(self._host) and SYSTEM_HOSTKEYS[self._host].has_key(keytype):
679
            our_server_key = SYSTEM_HOSTKEYS[self._host][keytype]
680
            our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
681
        elif BZR_HOSTKEYS.has_key(self._host) and BZR_HOSTKEYS[self._host].has_key(keytype):
682
            our_server_key = BZR_HOSTKEYS[self._host][keytype]
683
            our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
684
        else:
685
            warning('Adding %s host key for %s: %s' % (keytype, self._host, server_key_hex))
686
            if not BZR_HOSTKEYS.has_key(self._host):
687
                BZR_HOSTKEYS[self._host] = {}
688
            BZR_HOSTKEYS[self._host][keytype] = server_key
689
            our_server_key = server_key
690
            our_server_key_hex = paramiko.util.hexify(our_server_key.get_fingerprint())
691
            save_host_keys()
692
        if server_key != our_server_key:
693
            filename1 = os.path.expanduser('~/.ssh/known_hosts')
694
            filename2 = os.path.join(config_dir(), 'ssh_host_keys')
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
695
            raise TransportError('Host keys for %s do not match!  %s != %s' % \
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
696
                (self._host, our_server_key_hex, server_key_hex),
697
                ['Try editing %s or %s' % (filename1, filename2)])
698
1185.49.22 by John Arbash Meinel
Added get_password to the UIFactory, using it inside of sftp.py
699
        self._sftp_auth(t)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
700
        
701
        try:
702
            self._sftp = t.open_sftp_client()
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
703
        except paramiko.SSHException, e:
704
            raise ConnectionError('Unable to start sftp client %s:%d' %
705
                                  (self._host, self._port), e)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
706
1185.49.22 by John Arbash Meinel
Added get_password to the UIFactory, using it inside of sftp.py
707
    def _sftp_auth(self, transport):
708
        # paramiko requires a username, but it might be none if nothing was supplied
709
        # use the local username, just in case.
710
        # We don't override self._username, because if we aren't using paramiko,
711
        # the username might be specified in ~/.ssh/config and we don't want to
712
        # force it to something else
713
        # Also, it would mess up the self.relpath() functionality
714
        username = self._username or getpass.getuser()
715
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
716
        # Paramiko tries to open a socket.AF_UNIX in order to connect
717
        # to ssh-agent. That attribute doesn't exist on win32 (it does in cygwin)
718
        # so we get an AttributeError exception. For now, just don't try to
719
        # connect to an agent if we are on win32
720
        if sys.platform != 'win32':
721
            agent = paramiko.Agent()
722
            for key in agent.get_keys():
723
                mutter('Trying SSH agent key %s' % paramiko.util.hexify(key.get_fingerprint()))
724
                try:
725
                    transport.auth_publickey(username, key)
726
                    return
727
                except paramiko.SSHException, e:
728
                    pass
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
729
        
730
        # okay, try finding id_rsa or id_dss?  (posix only)
1185.49.22 by John Arbash Meinel
Added get_password to the UIFactory, using it inside of sftp.py
731
        if self._try_pkey_auth(transport, paramiko.RSAKey, username, 'id_rsa'):
732
            return
733
        if self._try_pkey_auth(transport, paramiko.DSSKey, username, 'id_dsa'):
734
            return
735
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
736
1185.27.1 by Jelmer Vernooij
Parse passwords in sftp URLs (sftp://foo:bar@localhost/).
737
        if self._password:
738
            try:
1185.49.22 by John Arbash Meinel
Added get_password to the UIFactory, using it inside of sftp.py
739
                transport.auth_password(username, self._password)
1185.27.1 by Jelmer Vernooij
Parse passwords in sftp URLs (sftp://foo:bar@localhost/).
740
                return
741
            except paramiko.SSHException, e:
742
                pass
743
1185.49.22 by John Arbash Meinel
Added get_password to the UIFactory, using it inside of sftp.py
744
            # FIXME: Don't keep a password held in memory if you can help it
745
            #self._password = None
746
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
747
        # give up and ask for a password
1185.50.10 by John Arbash Meinel
Don't import ui_factory directly, in case it gets changed later.
748
        password = bzrlib.ui.ui_factory.get_password(
749
                prompt='SSH %(user)s@%(host)s password',
750
                user=username, host=self._host)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
751
        try:
1185.49.22 by John Arbash Meinel
Added get_password to the UIFactory, using it inside of sftp.py
752
            transport.auth_password(username, password)
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
753
        except paramiko.SSHException, e:
754
            raise ConnectionError('Unable to authenticate to SSH host as %s@%s' %
755
                                  (username, self._host), e)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
756
1185.49.22 by John Arbash Meinel
Added get_password to the UIFactory, using it inside of sftp.py
757
    def _try_pkey_auth(self, transport, pkey_class, username, filename):
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
758
        filename = os.path.expanduser('~/.ssh/' + filename)
759
        try:
760
            key = pkey_class.from_private_key_file(filename)
1185.49.22 by John Arbash Meinel
Added get_password to the UIFactory, using it inside of sftp.py
761
            transport.auth_publickey(username, key)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
762
            return True
763
        except paramiko.PasswordRequiredException:
1185.50.10 by John Arbash Meinel
Don't import ui_factory directly, in case it gets changed later.
764
            password = bzrlib.ui.ui_factory.get_password(
765
                    prompt='SSH %(filename)s password',
766
                    filename=filename)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
767
            try:
768
                key = pkey_class.from_private_key_file(filename, password)
1185.49.22 by John Arbash Meinel
Added get_password to the UIFactory, using it inside of sftp.py
769
                transport.auth_publickey(username, key)
1185.16.87 by mbp at sourcefrog
- fix line endings in sftp transport
770
                return True
771
            except paramiko.SSHException:
772
                mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
773
        except paramiko.SSHException:
774
            mutter('SSH authentication via %s key failed.' % (os.path.basename(filename),))
775
        except IOError:
776
            pass
777
        return False
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
778
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
779
    def _sftp_open_exclusive(self, relpath):
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
780
        """Open a remote path exclusively.
781
782
        SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if
783
        the file already exists. However it does not expose this
784
        at the higher level of SFTPClient.open(), so we have to
785
        sneak away with it.
786
787
        WARNING: This breaks the SFTPClient abstraction, so it
788
        could easily break against an updated version of paramiko.
789
1185.49.3 by John Arbash Meinel
Added a form of locking to sftp branches. Refactored _sftp_open_exclusive to take a relative path
790
        :param relpath: The relative path, where the file should be opened
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
791
        """
1185.49.15 by John Arbash Meinel
[patch] Robey Pointer - Need to adjust_cwd in _sftp_openexclusive
792
        path = self._sftp._adjust_cwd(self._abspath(relpath))
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
793
        attr = SFTPAttributes()
794
        mode = (SFTP_FLAG_WRITE | SFTP_FLAG_CREATE 
795
                | SFTP_FLAG_TRUNC | SFTP_FLAG_EXCL)
796
        try:
797
            t, msg = self._sftp._request(CMD_OPEN, path, mode, attr)
798
            if t != CMD_HANDLE:
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
799
                raise TransportError('Expected an SFTP handle')
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
800
            handle = msg.get_string()
801
            return SFTPFile(self._sftp, handle, 'w', -1)
1185.50.11 by John Arbash Meinel
[merge] Refactor NoSuchFile style exceptions.
802
        except (paramiko.SSHException, IOError), e:
1185.50.13 by John Arbash Meinel
Expanded the Transport test suite. Including delete, copy, move, etc. Updated SftpTransport to conform.
803
            self._translate_io_exception(e, relpath, ': unable to open',
804
                failure_exc=FileExists)
1185.49.2 by John Arbash Meinel
Adding a open_exclusive function since paramiko supports it, but doesn't expose it
805