~bzr-pqm/bzr/bzr.dev

3508.1.7 by Vincent Ladeuil
Prepare test framework for pyftpdlib injection.
1
# Copyright (C) 2005, 2006, 2007, 2008, 2009 Canonical Ltd
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
2
#
1185.36.4 by Daniel Silverstone
Add FTP transport
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
7
#
1185.36.4 by Daniel Silverstone
Add FTP transport
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
12
#
1185.36.4 by Daniel Silverstone
Add FTP transport
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
1185.36.4 by Daniel Silverstone
Add FTP transport
16
"""Implementation of Transport over ftp.
17
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
20
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
25
"""
26
27
from cStringIO import StringIO
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
28
import errno
1185.36.4 by Daniel Silverstone
Add FTP transport
29
import ftplib
3021.1.2 by Vincent Ladeuil
Fix bug #164567 by catching connection errors.
30
import getpass
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
31
import os
1551.15.2 by Aaron Bentley
FTP does not require atomic rename. Fixes #89436
32
import os.path
1185.36.4 by Daniel Silverstone
Add FTP transport
33
import urlparse
3508.1.15 by Vincent Ladeuil
Tweak chmod bits output for easier debug.
34
import random
3021.1.2 by Vincent Ladeuil
Fix bug #164567 by catching connection errors.
35
import socket
1185.36.4 by Daniel Silverstone
Add FTP transport
36
import stat
1185.72.14 by Matthieu Moy
Sleep between retries when the connection closes
37
import time
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
38
from warnings import warn
39
1948.1.7 by John Arbash Meinel
Fix FtpTransport so that it raises UnicodeError and skips the unicode path tests
40
from bzrlib import (
2900.2.5 by Vincent Ladeuil
ake ftp aware of authentication config.
41
    config,
1948.1.7 by John Arbash Meinel
Fix FtpTransport so that it raises UnicodeError and skips the unicode path tests
42
    errors,
1551.15.2 by Aaron Bentley
FTP does not require atomic rename. Fixes #89436
43
    osutils,
1948.1.7 by John Arbash Meinel
Fix FtpTransport so that it raises UnicodeError and skips the unicode path tests
44
    urlutils,
45
    )
46
from bzrlib.trace import mutter, warning
1707.3.4 by John Arbash Meinel
Moved most of sftp.split_url into a Transport function.
47
from bzrlib.transport import (
2671.3.6 by Robert Collins
Review feedback.
48
    AppendBasedFileStream,
2900.2.16 by Vincent Ladeuil
Make hhtp proxy aware of AuthenticationConfig (for password).
49
    ConnectedTransport,
2671.3.2 by Robert Collins
Start open_file_stream logic.
50
    _file_streams,
2900.2.16 by Vincent Ladeuil
Make hhtp proxy aware of AuthenticationConfig (for password).
51
    register_urlparse_netloc_protocol,
1707.3.4 by John Arbash Meinel
Moved most of sftp.split_url into a Transport function.
52
    Server,
53
    )
2413.1.1 by John Arbash Meinel
Implement TestCaseWithFTPServer using the new shiny Feature mechanism.
54
from bzrlib.transport.local import LocalURLServer
1707.3.24 by John Arbash Meinel
Add prompt for password for ftp.
55
import bzrlib.ui
1551.3.11 by Aaron Bentley
Merge from Robert
56
2900.2.16 by Vincent Ladeuil
Make hhtp proxy aware of AuthenticationConfig (for password).
57
58
register_urlparse_netloc_protocol('aftp')
1707.3.30 by John Arbash Meinel
Only load medusa if you are running the test suite.
59
1551.3.11 by Aaron Bentley
Merge from Robert
60
1707.3.25 by John Arbash Meinel
Setting some default exception classes, which can help a lot.
61
class FtpPathError(errors.PathError):
62
    """FTP failed for path: %(path)s%(extra)s"""
63
64
1185.36.4 by Daniel Silverstone
Add FTP transport
65
class FtpStatResult(object):
3508.1.4 by Vincent Ladeuil
Ensure that ftp transport are always in binary mode.
66
67
    def __init__(self, f, abspath):
1185.36.4 by Daniel Silverstone
Add FTP transport
68
        try:
3508.1.4 by Vincent Ladeuil
Ensure that ftp transport are always in binary mode.
69
            self.st_size = f.size(abspath)
1185.36.4 by Daniel Silverstone
Add FTP transport
70
            self.st_mode = stat.S_IFREG
71
        except ftplib.error_perm:
72
            pwd = f.pwd()
73
            try:
3508.1.4 by Vincent Ladeuil
Ensure that ftp transport are always in binary mode.
74
                f.cwd(abspath)
1185.36.4 by Daniel Silverstone
Add FTP transport
75
                self.st_mode = stat.S_IFDIR
76
            finally:
77
                f.cwd(pwd)
78
79
1185.72.15 by Matthieu Moy
better error messages on ftp failures
80
_number_of_retries = 2
1185.72.14 by Matthieu Moy
Sleep between retries when the connection closes
81
_sleep_between_retries = 5
1185.72.12 by Matthieu Moy
made __number_of_retries global
82
2485.8.46 by Vincent Ladeuil
Add some remarks about current limitations in connection sharing.
83
# FIXME: there are inconsistencies in the way temporary errors are
84
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
85
# be taken to analyze the implications for write operations (read operations
2485.8.59 by Vincent Ladeuil
Update from review comments.
86
# are safe to retry). Overall even some read operations are never
87
# retried. --vila 20070720 (Bug #127164)
2485.8.16 by Vincent Ladeuil
Create a new, empty, ConnectedTransport class.
88
class FtpTransport(ConnectedTransport):
1185.36.4 by Daniel Silverstone
Add FTP transport
89
    """This is the transport agent for ftp:// access."""
90
2485.8.59 by Vincent Ladeuil
Update from review comments.
91
    def __init__(self, base, _from_transport=None):
1185.36.4 by Daniel Silverstone
Add FTP transport
92
        """Set the base path where files will be stored."""
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
93
        if not (base.startswith('ftp://') or base.startswith('aftp://')):
94
            raise ValueError(base)
2485.8.59 by Vincent Ladeuil
Update from review comments.
95
        super(FtpTransport, self).__init__(base,
96
                                           _from_transport=_from_transport)
2485.8.25 by Vincent Ladeuil
Separate abspath from _remote_path, the intents are different.
97
        self._unqualified_scheme = 'ftp'
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
98
        if self._scheme == 'aftp':
99
            self.is_active = True
100
        else:
101
            self.is_active = False
1707.3.4 by John Arbash Meinel
Moved most of sftp.split_url into a Transport function.
102
1185.36.4 by Daniel Silverstone
Add FTP transport
103
    def _get_FTP(self):
104
        """Return the ftplib.FTP instance for this object."""
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
105
        # Ensures that a connection is established
106
        connection = self._get_connection()
107
        if connection is None:
108
            # First connection ever
109
            connection, credentials = self._create_connection()
110
            self._set_connection(connection, credentials)
111
        return connection
112
113
    def _create_connection(self, credentials=None):
114
        """Create a new connection with the provided credentials.
115
116
        :param credentials: The credentials needed to establish the connection.
117
118
        :return: The created connection and its associated credentials.
119
3777.1.1 by Aaron Bentley
Use auth.conf for sftp
120
        The input credentials are only the password as it may have been
121
        entered interactively by the user and may be different from the one
122
        provided in base url at transport creation time.  The returned
123
        credentials are username, password.
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
124
        """
125
        if credentials is None:
2900.2.15 by Vincent Ladeuil
AuthenticationConfig can be queried for logins too (first step).
126
            user, password = self._user, self._password
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
127
        else:
2900.2.15 by Vincent Ladeuil
AuthenticationConfig can be queried for logins too (first step).
128
            user, password = credentials
129
130
        auth = config.AuthenticationConfig()
131
        if user is None:
4304.2.1 by Vincent Ladeuil
Fix bug #367726 by reverting some default user handling introduced
132
            user = auth.get_user('ftp', self._host, port=self._port,
133
                                 default=getpass.getuser())
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
134
        mutter("Constructing FTP instance against %r" %
2900.2.15 by Vincent Ladeuil
AuthenticationConfig can be queried for logins too (first step).
135
               ((self._host, self._port, user, '********',
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
136
                self.is_active),))
137
        try:
3331.2.4 by Jelmer Vernooij
Move GSSAPI support to a separate file.
138
            connection = ftplib.FTP()
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
139
            connection.connect(host=self._host, port=self._port)
3331.2.4 by Jelmer Vernooij
Move GSSAPI support to a separate file.
140
            if user and user != 'anonymous' and \
141
                    password is None: # '' is a valid password
142
                password = auth.get_password('ftp', self._host, user,
143
                                             port=self._port)
144
            connection.login(user=user, passwd=password)
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
145
            connection.set_pasv(not self.is_active)
3508.1.4 by Vincent Ladeuil
Ensure that ftp transport are always in binary mode.
146
            # binary mode is the default
147
            connection.voidcmd('TYPE I')
3021.1.2 by Vincent Ladeuil
Fix bug #164567 by catching connection errors.
148
        except socket.error, e:
3021.1.4 by Vincent Ladeuil
Use the right execption.
149
            raise errors.SocketConnectionError(self._host, self._port,
150
                                               msg='Unable to connect to',
151
                                               orig_error= e)
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
152
        except ftplib.error_perm, e:
153
            raise errors.TransportError(msg="Error setting up connection:"
154
                                        " %s" % str(e), orig_error=e)
2900.2.15 by Vincent Ladeuil
AuthenticationConfig can be queried for logins too (first step).
155
        return connection, (user, password)
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
156
157
    def _reconnect(self):
158
        """Create a new connection with the previously used credentials"""
2917.1.2 by Vincent Ladeuil
Fix typo in ftp.py making the reconnection fail on temporary errors (154259).
159
        credentials = self._get_credentials()
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
160
        connection, credentials = self._create_connection(credentials)
161
        self._set_connection(connection, credentials)
162
163
    def _translate_perm_error(self, err, path, extra=None,
164
                              unknown_exc=FtpPathError):
1707.3.19 by John Arbash Meinel
For some reason after RNTO we get a failure for has()
165
        """Try to translate an ftplib.error_perm exception.
166
167
        :param err: The error to translate into a bzr error
168
        :param path: The path which had problems
169
        :param extra: Extra information which can be included
170
        :param unknown_exc: If None, we will just raise the original exception
171
                    otherwise we raise unknown_exc(path, extra=extra)
172
        """
1707.3.10 by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works.
173
        s = str(err).lower()
174
        if not extra:
175
            extra = str(err)
1707.3.25 by John Arbash Meinel
Setting some default exception classes, which can help a lot.
176
        else:
177
            extra += ': ' + str(err)
1707.3.10 by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works.
178
        if ('no such file' in s
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
179
            or 'could not open' in s
180
            or 'no such dir' in s
2338.1.1 by John Arbash Meinel
(Arnaud Fontaine) Compatibility with vsftpd error messages.
181
            or 'could not create file' in s # vsftpd
2570.1.1 by John Arbash Meinel
(John Arbash Meinel) A couple small updates for pushing over ftp.
182
            or 'file doesn\'t exist' in s
3315.1.1 by Christian Tschabuschnig
support vsftp (#129786)
183
            or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
2885.3.1 by Gary van der Merwe
Correctly detect a NoSuchFile when using a filezilla server.
184
            or 'file/directory not found' in s # filezilla server
3382.1.1 by Ian Clatworthy
commit now works with Microsoft FTP (Andreas Deininger)
185
            # Microsoft FTP-Service RNFR reply if file not found
186
            or (s.startswith('550 ') and 'unable to rename to' in extra)
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
187
            ):
188
            raise errors.NoSuchFile(path, extra=extra)
1707.3.10 by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works.
189
        if ('file exists' in s):
1707.3.15 by John Arbash Meinel
Handle trying to list a file instead of a dir
190
            raise errors.FileExists(path, extra=extra)
191
        if ('not a directory' in s):
192
            raise errors.PathError(path, extra=extra)
193
1707.3.18 by John Arbash Meinel
Fix error handling for ftp.put()
194
        mutter('unable to understand error for path: %s: %s', path, err)
1707.3.19 by John Arbash Meinel
For some reason after RNTO we get a failure for has()
195
196
        if unknown_exc:
197
            raise unknown_exc(path, extra=extra)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
198
        # TODO: jam 20060516 Consider re-raising the error wrapped in
1707.3.10 by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works.
199
        #       something like TransportError, but this loses the traceback
1707.3.15 by John Arbash Meinel
Handle trying to list a file instead of a dir
200
        #       Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
201
        #       to handle. Consider doing something like that here.
1707.3.10 by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works.
202
        #raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
1707.3.15 by John Arbash Meinel
Handle trying to list a file instead of a dir
203
        raise
1707.3.10 by John Arbash Meinel
Getting the append() tests to pass, and adding better error translation. These are probably dependent on the ftp test server, but for now it works.
204
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
205
    def _remote_path(self, relpath):
1948.1.7 by John Arbash Meinel
Fix FtpTransport so that it raises UnicodeError and skips the unicode path tests
206
        # XXX: It seems that ftplib does not handle Unicode paths
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
207
        # at the same time, medusa won't handle utf8 paths So if
208
        # we .encode(utf8) here (see ConnectedTransport
209
        # implementation), then we get a Server failure.  while
210
        # if we use str(), we get a UnicodeError, and the test
211
        # suite just skips testing UnicodePaths.
212
        relative = str(urlutils.unescape(relpath))
213
        remote_path = self._combine_paths(self._path, relative)
214
        return remote_path
1185.36.4 by Daniel Silverstone
Add FTP transport
215
216
    def has(self, relpath):
1707.3.16 by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories.
217
        """Does the target location exist?"""
218
        # FIXME jam 20060516 We *do* ask about directories in the test suite
219
        #       We don't seem to in the actual codebase
220
        # XXX: I assume we're never asked has(dirname) and thus I use
221
        # the FTP size command and assume that if it doesn't raise,
222
        # all is good.
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
223
        abspath = self._remote_path(relpath)
1185.36.4 by Daniel Silverstone
Add FTP transport
224
        try:
225
            f = self._get_FTP()
1707.3.19 by John Arbash Meinel
For some reason after RNTO we get a failure for has()
226
            mutter('FTP has check: %s => %s', relpath, abspath)
1707.3.16 by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories.
227
            s = f.size(abspath)
228
            mutter("FTP has: %s", abspath)
1185.36.4 by Daniel Silverstone
Add FTP transport
229
            return True
1707.3.16 by John Arbash Meinel
Overloading cmd_size and cmd_mkd so that the test suite can handle directories.
230
        except ftplib.error_perm, e:
231
            if ('is a directory' in str(e).lower()):
232
                mutter("FTP has dir: %s: %s", abspath, e)
233
                return True
234
            mutter("FTP has not: %s: %s", abspath, e)
1185.36.4 by Daniel Silverstone
Add FTP transport
235
            return False
236
2164.2.15 by Vincent Ladeuil
Http redirections are not followed by default. Do not use hints
237
    def get(self, relpath, decode=False, retries=0):
1185.36.4 by Daniel Silverstone
Add FTP transport
238
        """Get the file at the given relative path.
239
240
        :param relpath: The relative path to the file
1185.72.9 by Matthieu Moy
suggestions by jam and robertc
241
        :param retries: Number of retries after temporary failures so far
242
                        for this operation.
1185.36.4 by Daniel Silverstone
Add FTP transport
243
244
        We're meant to return a file-like object which bzr will
245
        then read from. For now we do this via the magic of StringIO
246
        """
1540.3.6 by Martin Pool
[merge] update from bzr.dev
247
        # TODO: decode should be deprecated
1185.36.4 by Daniel Silverstone
Add FTP transport
248
        try:
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
249
            mutter("FTP get: %s", self._remote_path(relpath))
1185.36.4 by Daniel Silverstone
Add FTP transport
250
            f = self._get_FTP()
251
            ret = StringIO()
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
252
            f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
1185.36.4 by Daniel Silverstone
Add FTP transport
253
            ret.seek(0)
254
            return ret
255
        except ftplib.error_perm, e:
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
256
            raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
1185.72.8 by Matthieu Moy
try to solve the problem of ftp servers closing the connection
257
        except ftplib.error_temp, e:
1185.72.15 by Matthieu Moy
better error messages on ftp failures
258
            if retries > _number_of_retries:
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
259
                raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
1185.72.15 by Matthieu Moy
better error messages on ftp failures
260
                                     % self.abspath(relpath),
261
                                     orig_error=e)
1185.72.8 by Matthieu Moy
try to solve the problem of ftp servers closing the connection
262
            else:
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
263
                warning("FTP temporary error: %s. Retrying.", str(e))
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
264
                self._reconnect()
1185.72.8 by Matthieu Moy
try to solve the problem of ftp servers closing the connection
265
                return self.get(relpath, decode, retries+1)
1185.72.15 by Matthieu Moy
better error messages on ftp failures
266
        except EOFError, e:
1185.72.9 by Matthieu Moy
suggestions by jam and robertc
267
            if retries > _number_of_retries:
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
268
                raise errors.TransportError("FTP control connection closed during GET %s."
1185.72.15 by Matthieu Moy
better error messages on ftp failures
269
                                     % self.abspath(relpath),
270
                                     orig_error=e)
1185.72.8 by Matthieu Moy
try to solve the problem of ftp servers closing the connection
271
            else:
1185.72.9 by Matthieu Moy
suggestions by jam and robertc
272
                warning("FTP control connection closed. Trying to reopen.")
1185.72.14 by Matthieu Moy
Sleep between retries when the connection closes
273
                time.sleep(_sleep_between_retries)
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
274
                self._reconnect()
1185.72.8 by Matthieu Moy
try to solve the problem of ftp servers closing the connection
275
                return self.get(relpath, decode, retries+1)
1185.36.4 by Daniel Silverstone
Add FTP transport
276
1955.3.6 by John Arbash Meinel
Lots of deprecation warnings, but no errors
277
    def put_file(self, relpath, fp, mode=None, retries=0):
1185.36.4 by Daniel Silverstone
Add FTP transport
278
        """Copy the file-like or string object into the location.
279
280
        :param relpath: Location to put the contents, relative to base.
1185.72.13 by Matthieu Moy
Make ftp put atomic
281
        :param fp:       File-like or string object.
1185.72.9 by Matthieu Moy
suggestions by jam and robertc
282
        :param retries: Number of retries after temporary failures so far
283
                        for this operation.
284
1551.15.2 by Aaron Bentley
FTP does not require atomic rename. Fixes #89436
285
        TODO: jam 20051215 ftp as a protocol seems to support chmod, but
286
        ftplib does not
1185.36.4 by Daniel Silverstone
Add FTP transport
287
        """
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
288
        abspath = self._remote_path(relpath)
1707.3.18 by John Arbash Meinel
Fix error handling for ftp.put()
289
        tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
1185.72.13 by Matthieu Moy
Make ftp put atomic
290
                        os.getpid(), random.randint(0,0x7FFFFFFF))
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
291
        bytes = None
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
292
        if getattr(fp, 'read', None) is None:
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
293
            # hand in a string IO
294
            bytes = fp
295
            fp = StringIO(bytes)
296
        else:
297
            # capture the byte count; .read() may be read only so
298
            # decorate it.
299
            class byte_counter(object):
300
                def __init__(self, fp):
301
                    self.fp = fp
302
                    self.counted_bytes = 0
303
                def read(self, count):
304
                    result = self.fp.read(count)
305
                    self.counted_bytes += len(result)
306
                    return result
307
            fp = byte_counter(fp)
1185.36.4 by Daniel Silverstone
Add FTP transport
308
        try:
1707.3.18 by John Arbash Meinel
Fix error handling for ftp.put()
309
            mutter("FTP put: %s", abspath)
1185.36.4 by Daniel Silverstone
Add FTP transport
310
            f = self._get_FTP()
1185.72.13 by Matthieu Moy
Make ftp put atomic
311
            try:
312
                f.storbinary('STOR '+tmp_abspath, fp)
1551.15.2 by Aaron Bentley
FTP does not require atomic rename. Fixes #89436
313
                self._rename_and_overwrite(tmp_abspath, abspath, f)
3508.1.1 by Vincent Ladeuil
Fix ftp transport so that it handles the 'mode' parameter when provided.
314
                self._setmode(relpath, mode)
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
315
                if bytes is not None:
316
                    return len(bytes)
317
                else:
318
                    return fp.counted_bytes
1185.72.13 by Matthieu Moy
Make ftp put atomic
319
            except (ftplib.error_temp,EOFError), e:
1185.72.15 by Matthieu Moy
better error messages on ftp failures
320
                warning("Failure during ftp PUT. Deleting temporary file.")
1185.72.13 by Matthieu Moy
Make ftp put atomic
321
                try:
322
                    f.delete(tmp_abspath)
323
                except:
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
324
                    warning("Failed to delete temporary file on the"
325
                            " server.\nFile: %s", tmp_abspath)
1185.72.13 by Matthieu Moy
Make ftp put atomic
326
                    raise e
327
                raise
1185.36.4 by Daniel Silverstone
Add FTP transport
328
        except ftplib.error_perm, e:
2570.1.1 by John Arbash Meinel
(John Arbash Meinel) A couple small updates for pushing over ftp.
329
            self._translate_perm_error(e, abspath, extra='could not store',
330
                                       unknown_exc=errors.NoSuchFile)
1185.72.8 by Matthieu Moy
try to solve the problem of ftp servers closing the connection
331
        except ftplib.error_temp, e:
1185.72.9 by Matthieu Moy
suggestions by jam and robertc
332
            if retries > _number_of_retries:
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
333
                raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
1185.72.15 by Matthieu Moy
better error messages on ftp failures
334
                                     % self.abspath(relpath), orig_error=e)
1185.72.8 by Matthieu Moy
try to solve the problem of ftp servers closing the connection
335
            else:
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
336
                warning("FTP temporary error: %s. Retrying.", str(e))
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
337
                self._reconnect()
1955.3.6 by John Arbash Meinel
Lots of deprecation warnings, but no errors
338
                self.put_file(relpath, fp, mode, retries+1)
1185.72.9 by Matthieu Moy
suggestions by jam and robertc
339
        except EOFError:
1185.72.10 by Matthieu Moy
One 1 -> _number_of_retries was missing
340
            if retries > _number_of_retries:
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
341
                raise errors.TransportError("FTP control connection closed during PUT %s."
1185.72.15 by Matthieu Moy
better error messages on ftp failures
342
                                     % self.abspath(relpath), orig_error=e)
1185.72.8 by Matthieu Moy
try to solve the problem of ftp servers closing the connection
343
            else:
1185.72.14 by Matthieu Moy
Sleep between retries when the connection closes
344
                warning("FTP control connection closed. Trying to reopen.")
345
                time.sleep(_sleep_between_retries)
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
346
                self._reconnect()
1955.3.6 by John Arbash Meinel
Lots of deprecation warnings, but no errors
347
                self.put_file(relpath, fp, mode, retries+1)
1185.72.8 by Matthieu Moy
try to solve the problem of ftp servers closing the connection
348
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
349
    def mkdir(self, relpath, mode=None):
1185.36.4 by Daniel Silverstone
Add FTP transport
350
        """Create a directory at the given path."""
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
351
        abspath = self._remote_path(relpath)
1185.36.4 by Daniel Silverstone
Add FTP transport
352
        try:
1707.3.14 by John Arbash Meinel
Cleanup mkdir and rmdir error handling.
353
            mutter("FTP mkd: %s", abspath)
1185.36.4 by Daniel Silverstone
Add FTP transport
354
            f = self._get_FTP()
1707.3.14 by John Arbash Meinel
Cleanup mkdir and rmdir error handling.
355
            f.mkd(abspath)
3508.1.1 by Vincent Ladeuil
Fix ftp transport so that it handles the 'mode' parameter when provided.
356
            self._setmode(relpath, mode)
1185.36.4 by Daniel Silverstone
Add FTP transport
357
        except ftplib.error_perm, e:
1707.3.25 by John Arbash Meinel
Setting some default exception classes, which can help a lot.
358
            self._translate_perm_error(e, abspath,
359
                unknown_exc=errors.FileExists)
1185.36.4 by Daniel Silverstone
Add FTP transport
360
2671.3.9 by Robert Collins
Review feedback and fix VFat emulated transports to not claim to have unix permissions.
361
    def open_write_stream(self, relpath, mode=None):
362
        """See Transport.open_write_stream."""
2671.3.6 by Robert Collins
Review feedback.
363
        self.put_bytes(relpath, "", mode)
364
        result = AppendBasedFileStream(self, relpath)
365
        _file_streams[self.abspath(relpath)] = result
366
        return result
2671.3.2 by Robert Collins
Start open_file_stream logic.
367
2671.3.1 by Robert Collins
* New method ``bzrlib.transport.Transport.get_recommended_page_size``.
368
    def recommended_page_size(self):
369
        """See Transport.recommended_page_size().
370
371
        For FTP we suggest a large page size to reduce the overhead
372
        introduced by latency.
373
        """
374
        return 64 * 1024
375
1658.1.1 by Martin Pool
Fix FTP push with metadir format (Alexandre Saint)
376
    def rmdir(self, rel_path):
377
        """Delete the directory at rel_path"""
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
378
        abspath = self._remote_path(rel_path)
1658.1.1 by Martin Pool
Fix FTP push with metadir format (Alexandre Saint)
379
        try:
1707.3.14 by John Arbash Meinel
Cleanup mkdir and rmdir error handling.
380
            mutter("FTP rmd: %s", abspath)
1551.15.4 by Aaron Bentley
Revert now-unnecessary changes
381
            f = self._get_FTP()
1707.3.14 by John Arbash Meinel
Cleanup mkdir and rmdir error handling.
382
            f.rmd(abspath)
1658.1.1 by Martin Pool
Fix FTP push with metadir format (Alexandre Saint)
383
        except ftplib.error_perm, e:
1551.15.4 by Aaron Bentley
Revert now-unnecessary changes
384
            self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
1658.1.1 by Martin Pool
Fix FTP push with metadir format (Alexandre Saint)
385
1955.3.15 by John Arbash Meinel
Deprecate 'Transport.append' in favor of Transport.append_file or Transport.append_bytes
386
    def append_file(self, relpath, f, mode=None):
1185.36.4 by Daniel Silverstone
Add FTP transport
387
        """Append the text in the file-like object into the final
388
        location.
389
        """
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
390
        abspath = self._remote_path(relpath)
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
391
        if self.has(relpath):
392
            ftp = self._get_FTP()
1707.3.19 by John Arbash Meinel
For some reason after RNTO we get a failure for has()
393
            result = ftp.size(abspath)
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
394
        else:
395
            result = 0
396
1707.3.19 by John Arbash Meinel
For some reason after RNTO we get a failure for has()
397
        mutter("FTP appe to %s", abspath)
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
398
        self._try_append(relpath, f.read(), mode)
399
400
        return result
401
402
    def _try_append(self, relpath, text, mode=None, retries=0):
403
        """Try repeatedly to append the given text to the file at relpath.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
404
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
405
        This is a recursive function. On errors, it will be called until the
406
        number of retries is exceeded.
407
        """
408
        try:
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
409
            abspath = self._remote_path(relpath)
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
410
            mutter("FTP appe (try %d) to %s", retries, abspath)
411
            ftp = self._get_FTP()
412
            cmd = "APPE %s" % abspath
413
            conn = ftp.transfercmd(cmd)
414
            conn.sendall(text)
415
            conn.close()
3508.1.1 by Vincent Ladeuil
Fix ftp transport so that it handles the 'mode' parameter when provided.
416
            self._setmode(relpath, mode)
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
417
            ftp.getresp()
418
        except ftplib.error_perm, e:
1707.3.26 by John Arbash Meinel
Added a default exception for append
419
            self._translate_perm_error(e, abspath, extra='error appending',
420
                unknown_exc=errors.NoSuchFile)
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
421
        except ftplib.error_temp, e:
422
            if retries > _number_of_retries:
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
423
                raise errors.TransportError("FTP temporary error during APPEND %s." \
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
424
                        "Aborting." % abspath, orig_error=e)
425
            else:
426
                warning("FTP temporary error: %s. Retrying.", str(e))
2485.8.33 by Vincent Ladeuil
ftp connection sharing refactored. Tests passing.
427
                self._reconnect()
1707.3.8 by John Arbash Meinel
[patch] Alexandre Saint: enable append support for ftp
428
                self._try_append(relpath, text, mode, retries+1)
429
430
    def _setmode(self, relpath, mode):
431
        """Set permissions on a path.
432
433
        Only set permissions if the FTP server supports the 'SITE CHMOD'
434
        extension.
435
        """
3508.1.1 by Vincent Ladeuil
Fix ftp transport so that it handles the 'mode' parameter when provided.
436
        if mode:
437
            try:
438
                mutter("FTP site chmod: setting permissions to %s on %s",
3508.1.10 by Vincent Ladeuil
Start supporting pyftpdlib as an ftp test server.
439
                       oct(mode), self._remote_path(relpath))
3508.1.1 by Vincent Ladeuil
Fix ftp transport so that it handles the 'mode' parameter when provided.
440
                ftp = self._get_FTP()
441
                cmd = "SITE CHMOD %s %s" % (oct(mode),
442
                                            self._remote_path(relpath))
443
                ftp.sendcmd(cmd)
444
            except ftplib.error_perm, e:
445
                # Command probably not available on this server
446
                warning("FTP Could not set permissions to %s on %s. %s",
3508.1.10 by Vincent Ladeuil
Start supporting pyftpdlib as an ftp test server.
447
                        oct(mode), self._remote_path(relpath), str(e))
1185.36.4 by Daniel Silverstone
Add FTP transport
448
1707.3.11 by John Arbash Meinel
fixing more tests.
449
    # TODO: jam 20060516 I believe ftp allows you to tell an ftp server
450
    #       to copy something to another machine. And you may be able
451
    #       to give it its own address as the 'to' location.
452
    #       So implement a fancier 'copy()'
1185.36.4 by Daniel Silverstone
Add FTP transport
453
1551.15.3 by Aaron Bentley
Separate FtpTransport.rename from FtpTransport.move
454
    def rename(self, rel_from, rel_to):
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
455
        abs_from = self._remote_path(rel_from)
456
        abs_to = self._remote_path(rel_to)
1551.15.3 by Aaron Bentley
Separate FtpTransport.rename from FtpTransport.move
457
        mutter("FTP rename: %s => %s", abs_from, abs_to)
458
        f = self._get_FTP()
459
        return self._rename(abs_from, abs_to, f)
460
461
    def _rename(self, abs_from, abs_to, f):
462
        try:
463
            f.rename(abs_from, abs_to)
464
        except ftplib.error_perm, e:
465
            self._translate_perm_error(e, abs_from,
466
                ': unable to rename to %r' % (abs_to))
467
1185.36.4 by Daniel Silverstone
Add FTP transport
468
    def move(self, rel_from, rel_to):
469
        """Move the item at rel_from to the location at rel_to"""
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
470
        abs_from = self._remote_path(rel_from)
471
        abs_to = self._remote_path(rel_to)
1185.36.4 by Daniel Silverstone
Add FTP transport
472
        try:
1707.3.19 by John Arbash Meinel
For some reason after RNTO we get a failure for has()
473
            mutter("FTP mv: %s => %s", abs_from, abs_to)
1185.36.4 by Daniel Silverstone
Add FTP transport
474
            f = self._get_FTP()
1551.15.2 by Aaron Bentley
FTP does not require atomic rename. Fixes #89436
475
            self._rename_and_overwrite(abs_from, abs_to, f)
1185.36.4 by Daniel Silverstone
Add FTP transport
476
        except ftplib.error_perm, e:
1707.3.19 by John Arbash Meinel
For some reason after RNTO we get a failure for has()
477
            self._translate_perm_error(e, abs_from,
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
478
                extra='unable to rename to %r' % (rel_to,),
1707.3.19 by John Arbash Meinel
For some reason after RNTO we get a failure for has()
479
                unknown_exc=errors.PathError)
1185.36.4 by Daniel Silverstone
Add FTP transport
480
1551.15.2 by Aaron Bentley
FTP does not require atomic rename. Fixes #89436
481
    def _rename_and_overwrite(self, abs_from, abs_to, f):
482
        """Do a fancy rename on the remote server.
483
484
        Using the implementation provided by osutils.
485
        """
1551.15.3 by Aaron Bentley
Separate FtpTransport.rename from FtpTransport.move
486
        osutils.fancy_rename(abs_from, abs_to,
1551.15.4 by Aaron Bentley
Revert now-unnecessary changes
487
            rename_func=lambda p1, p2: self._rename(p1, p2, f),
488
            unlink_func=lambda p: self._delete(p, f))
1551.15.2 by Aaron Bentley
FTP does not require atomic rename. Fixes #89436
489
1185.36.4 by Daniel Silverstone
Add FTP transport
490
    def delete(self, relpath):
491
        """Delete the item at relpath"""
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
492
        abspath = self._remote_path(relpath)
1551.15.2 by Aaron Bentley
FTP does not require atomic rename. Fixes #89436
493
        f = self._get_FTP()
494
        self._delete(abspath, f)
495
496
    def _delete(self, abspath, f):
1185.36.4 by Daniel Silverstone
Add FTP transport
497
        try:
1707.3.11 by John Arbash Meinel
fixing more tests.
498
            mutter("FTP rm: %s", abspath)
499
            f.delete(abspath)
1185.36.4 by Daniel Silverstone
Add FTP transport
500
        except ftplib.error_perm, e:
1707.3.25 by John Arbash Meinel
Setting some default exception classes, which can help a lot.
501
            self._translate_perm_error(e, abspath, 'error deleting',
502
                unknown_exc=errors.NoSuchFile)
1185.36.4 by Daniel Silverstone
Add FTP transport
503
2634.1.1 by Robert Collins
(robertc) Reinstate the accidentally backed out external_url patch.
504
    def external_url(self):
505
        """See bzrlib.transport.Transport.external_url."""
506
        # FTP URL's are externally usable.
507
        return self.base
508
1185.36.4 by Daniel Silverstone
Add FTP transport
509
    def listable(self):
510
        """See Transport.listable."""
511
        return True
512
513
    def list_dir(self, relpath):
514
        """See Transport.list_dir."""
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
515
        basepath = self._remote_path(relpath)
1959.2.5 by John Arbash Meinel
Small fixes in ftp.list_dir()
516
        mutter("FTP nlst: %s", basepath)
1959.2.1 by John Arbash Meinel
David Allouche: Make transports return escaped paths
517
        f = self._get_FTP()
1185.36.4 by Daniel Silverstone
Add FTP transport
518
        try:
3508.1.6 by Vincent Ladeuil
Fix python-2.4 compatibility.
519
            try:
520
                paths = f.nlst(basepath)
521
            except ftplib.error_perm, e:
522
                self._translate_perm_error(e, relpath,
523
                                           extra='error with list_dir')
524
            except ftplib.error_temp, e:
525
                # xs4all's ftp server raises a 450 temp error when listing an
526
                # empty directory. Check for that and just return an empty list
527
                # in that case. See bug #215522
528
                if str(e).lower().startswith('450 no files found'):
529
                    mutter('FTP Server returned "%s" for nlst.'
530
                           ' Assuming it means empty directory',
531
                           str(e))
532
                    return []
533
                raise
3508.1.4 by Vincent Ladeuil
Ensure that ftp transport are always in binary mode.
534
        finally:
535
            # Restore binary mode as nlst switch to ascii mode to retrieve file
536
            # list
537
            f.voidcmd('TYPE I')
538
1959.2.1 by John Arbash Meinel
David Allouche: Make transports return escaped paths
539
        # If FTP.nlst returns paths prefixed by relpath, strip 'em
540
        if paths and paths[0].startswith(basepath):
541
            entries = [path[len(basepath)+1:] for path in paths]
542
        else:
543
            entries = paths
544
        # Remove . and .. if present
1959.2.5 by John Arbash Meinel
Small fixes in ftp.list_dir()
545
        return [urlutils.escape(entry) for entry in entries
546
                if entry not in ('.', '..')]
1185.36.4 by Daniel Silverstone
Add FTP transport
547
548
    def iter_files_recursive(self):
549
        """See Transport.iter_files_recursive.
550
551
        This is cargo-culted from the SFTP transport"""
552
        mutter("FTP iter_files_recursive")
553
        queue = list(self.list_dir("."))
554
        while queue:
1959.2.1 by John Arbash Meinel
David Allouche: Make transports return escaped paths
555
            relpath = queue.pop(0)
1185.36.4 by Daniel Silverstone
Add FTP transport
556
            st = self.stat(relpath)
557
            if stat.S_ISDIR(st.st_mode):
558
                for i, basename in enumerate(self.list_dir(relpath)):
559
                    queue.insert(i, relpath+"/"+basename)
560
            else:
561
                yield relpath
562
563
    def stat(self, relpath):
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
564
        """Return the stat information for a file."""
2485.8.22 by Vincent Ladeuil
Finish ftp refactoring. Test suite passing.
565
        abspath = self._remote_path(relpath)
1185.36.4 by Daniel Silverstone
Add FTP transport
566
        try:
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
567
            mutter("FTP stat: %s", abspath)
1185.36.4 by Daniel Silverstone
Add FTP transport
568
            f = self._get_FTP()
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
569
            return FtpStatResult(f, abspath)
1185.36.4 by Daniel Silverstone
Add FTP transport
570
        except ftplib.error_perm, e:
1707.3.13 by John Arbash Meinel
Changing raise Error to raise errors.Error
571
            self._translate_perm_error(e, abspath, extra='error w/ stat')
1185.36.4 by Daniel Silverstone
Add FTP transport
572
573
    def lock_read(self, relpath):
574
        """Lock the given file for shared (read) access.
575
        :return: A lock object, which should be passed to Transport.unlock()
576
        """
577
        # The old RemoteBranch ignore lock for reading, so we will
578
        # continue that tradition and return a bogus lock object.
579
        class BogusLock(object):
580
            def __init__(self, path):
581
                self.path = path
582
            def unlock(self):
583
                pass
584
        return BogusLock(relpath)
585
586
    def lock_write(self, relpath):
587
        """Lock the given file for exclusive (write) access.
588
        WARNING: many transports do not support this, so trying avoid using it
589
590
        :return: A lock object, which should be passed to Transport.unlock()
591
        """
592
        return self.lock_read(relpath)
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
593
594
595
def get_test_permutations():
596
    """Return the permutations to be used in testing."""
3508.1.7 by Vincent Ladeuil
Prepare test framework for pyftpdlib injection.
597
    from bzrlib.tests import ftp_server
3508.1.23 by Vincent Ladeuil
Fix as per Martin's review.
598
    return [(FtpTransport, ftp_server.FTPTestServer)]