1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
36
from warnings import warn
44
from bzrlib.trace import mutter, warning
45
from bzrlib.transport import (
46
AppendBasedFileStream,
49
register_urlparse_netloc_protocol,
52
from bzrlib.transport.local import LocalURLServer
56
register_urlparse_netloc_protocol('aftp')
59
class FtpPathError(errors.PathError):
60
"""FTP failed for path: %(path)s%(extra)s"""
63
class FtpStatResult(object):
64
def __init__(self, f, relpath):
66
self.st_size = f.size(relpath)
67
self.st_mode = stat.S_IFREG
68
except ftplib.error_perm:
72
self.st_mode = stat.S_IFDIR
77
_number_of_retries = 2
78
_sleep_between_retries = 5
80
# FIXME: there are inconsistencies in the way temporary errors are
81
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
82
# be taken to analyze the implications for write operations (read operations
83
# are safe to retry). Overall even some read operations are never
84
# retried. --vila 20070720 (Bug #127164)
85
class FtpTransport(ConnectedTransport):
86
"""This is the transport agent for ftp:// access."""
88
def __init__(self, base, _from_transport=None):
89
"""Set the base path where files will be stored."""
90
assert base.startswith('ftp://') or base.startswith('aftp://')
91
super(FtpTransport, self).__init__(base,
92
_from_transport=_from_transport)
93
self._unqualified_scheme = 'ftp'
94
if self._scheme == 'aftp':
97
self.is_active = False
100
"""Return the ftplib.FTP instance for this object."""
101
# Ensures that a connection is established
102
connection = self._get_connection()
103
if connection is None:
104
# First connection ever
105
connection, credentials = self._create_connection()
106
self._set_connection(connection, credentials)
109
def _create_connection(self, credentials=None):
110
"""Create a new connection with the provided credentials.
112
:param credentials: The credentials needed to establish the connection.
114
:return: The created connection and its associated credentials.
116
The credentials are only the password as it may have been entered
117
interactively by the user and may be different from the one provided
118
in base url at transport creation time.
120
if credentials is None:
121
user, password = self._user, self._password
123
user, password = credentials
125
auth = config.AuthenticationConfig()
127
user = auth.get_user('ftp', self._host, port=self._port)
129
# Default to local user
130
user = getpass.getuser()
132
mutter("Constructing FTP instance against %r" %
133
((self._host, self._port, user, '********',
136
connection = ftplib.FTP()
137
connection.connect(host=self._host, port=self._port)
138
if user and user != 'anonymous' and \
139
password is None: # '' is a valid password
140
password = auth.get_password('ftp', self._host, user,
142
connection.login(user=user, passwd=password)
143
connection.set_pasv(not self.is_active)
144
except ftplib.error_perm, e:
145
raise errors.TransportError(msg="Error setting up connection:"
146
" %s" % str(e), orig_error=e)
147
return connection, (user, password)
149
def _reconnect(self):
150
"""Create a new connection with the previously used credentials"""
151
credentials = self._get_credentials()
152
connection, credentials = self._create_connection(credentials)
153
self._set_connection(connection, credentials)
155
def _translate_perm_error(self, err, path, extra=None,
156
unknown_exc=FtpPathError):
157
"""Try to translate an ftplib.error_perm exception.
159
:param err: The error to translate into a bzr error
160
:param path: The path which had problems
161
:param extra: Extra information which can be included
162
:param unknown_exc: If None, we will just raise the original exception
163
otherwise we raise unknown_exc(path, extra=extra)
169
extra += ': ' + str(err)
170
if ('no such file' in s
171
or 'could not open' in s
172
or 'no such dir' in s
173
or 'could not create file' in s # vsftpd
174
or 'file doesn\'t exist' in s
175
or 'file/directory not found' in s # filezilla server
177
raise errors.NoSuchFile(path, extra=extra)
178
if ('file exists' in s):
179
raise errors.FileExists(path, extra=extra)
180
if ('not a directory' in s):
181
raise errors.PathError(path, extra=extra)
183
mutter('unable to understand error for path: %s: %s', path, err)
186
raise unknown_exc(path, extra=extra)
187
# TODO: jam 20060516 Consider re-raising the error wrapped in
188
# something like TransportError, but this loses the traceback
189
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
190
# to handle. Consider doing something like that here.
191
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
194
def _remote_path(self, relpath):
195
# XXX: It seems that ftplib does not handle Unicode paths
196
# at the same time, medusa won't handle utf8 paths So if
197
# we .encode(utf8) here (see ConnectedTransport
198
# implementation), then we get a Server failure. while
199
# if we use str(), we get a UnicodeError, and the test
200
# suite just skips testing UnicodePaths.
201
relative = str(urlutils.unescape(relpath))
202
remote_path = self._combine_paths(self._path, relative)
205
def has(self, relpath):
206
"""Does the target location exist?"""
207
# FIXME jam 20060516 We *do* ask about directories in the test suite
208
# We don't seem to in the actual codebase
209
# XXX: I assume we're never asked has(dirname) and thus I use
210
# the FTP size command and assume that if it doesn't raise,
212
abspath = self._remote_path(relpath)
215
mutter('FTP has check: %s => %s', relpath, abspath)
217
mutter("FTP has: %s", abspath)
219
except ftplib.error_perm, e:
220
if ('is a directory' in str(e).lower()):
221
mutter("FTP has dir: %s: %s", abspath, e)
223
mutter("FTP has not: %s: %s", abspath, e)
226
def get(self, relpath, decode=False, retries=0):
227
"""Get the file at the given relative path.
229
:param relpath: The relative path to the file
230
:param retries: Number of retries after temporary failures so far
233
We're meant to return a file-like object which bzr will
234
then read from. For now we do this via the magic of StringIO
236
# TODO: decode should be deprecated
238
mutter("FTP get: %s", self._remote_path(relpath))
241
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
244
except ftplib.error_perm, e:
245
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
246
except ftplib.error_temp, e:
247
if retries > _number_of_retries:
248
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
249
% self.abspath(relpath),
252
warning("FTP temporary error: %s. Retrying.", str(e))
254
return self.get(relpath, decode, retries+1)
256
if retries > _number_of_retries:
257
raise errors.TransportError("FTP control connection closed during GET %s."
258
% self.abspath(relpath),
261
warning("FTP control connection closed. Trying to reopen.")
262
time.sleep(_sleep_between_retries)
264
return self.get(relpath, decode, retries+1)
266
def put_file(self, relpath, fp, mode=None, retries=0):
267
"""Copy the file-like or string object into the location.
269
:param relpath: Location to put the contents, relative to base.
270
:param fp: File-like or string object.
271
:param retries: Number of retries after temporary failures so far
274
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
277
abspath = self._remote_path(relpath)
278
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
279
os.getpid(), random.randint(0,0x7FFFFFFF))
281
if getattr(fp, 'read', None) is None:
282
# hand in a string IO
286
# capture the byte count; .read() may be read only so
288
class byte_counter(object):
289
def __init__(self, fp):
291
self.counted_bytes = 0
292
def read(self, count):
293
result = self.fp.read(count)
294
self.counted_bytes += len(result)
296
fp = byte_counter(fp)
298
mutter("FTP put: %s", abspath)
301
f.storbinary('STOR '+tmp_abspath, fp)
302
self._rename_and_overwrite(tmp_abspath, abspath, f)
303
if bytes is not None:
306
return fp.counted_bytes
307
except (ftplib.error_temp,EOFError), e:
308
warning("Failure during ftp PUT. Deleting temporary file.")
310
f.delete(tmp_abspath)
312
warning("Failed to delete temporary file on the"
313
" server.\nFile: %s", tmp_abspath)
316
except ftplib.error_perm, e:
317
self._translate_perm_error(e, abspath, extra='could not store',
318
unknown_exc=errors.NoSuchFile)
319
except ftplib.error_temp, e:
320
if retries > _number_of_retries:
321
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
322
% self.abspath(relpath), orig_error=e)
324
warning("FTP temporary error: %s. Retrying.", str(e))
326
self.put_file(relpath, fp, mode, retries+1)
328
if retries > _number_of_retries:
329
raise errors.TransportError("FTP control connection closed during PUT %s."
330
% self.abspath(relpath), orig_error=e)
332
warning("FTP control connection closed. Trying to reopen.")
333
time.sleep(_sleep_between_retries)
335
self.put_file(relpath, fp, mode, retries+1)
337
def mkdir(self, relpath, mode=None):
338
"""Create a directory at the given path."""
339
abspath = self._remote_path(relpath)
341
mutter("FTP mkd: %s", abspath)
344
except ftplib.error_perm, e:
345
self._translate_perm_error(e, abspath,
346
unknown_exc=errors.FileExists)
348
def open_write_stream(self, relpath, mode=None):
349
"""See Transport.open_write_stream."""
350
self.put_bytes(relpath, "", mode)
351
result = AppendBasedFileStream(self, relpath)
352
_file_streams[self.abspath(relpath)] = result
355
def recommended_page_size(self):
356
"""See Transport.recommended_page_size().
358
For FTP we suggest a large page size to reduce the overhead
359
introduced by latency.
363
def rmdir(self, rel_path):
364
"""Delete the directory at rel_path"""
365
abspath = self._remote_path(rel_path)
367
mutter("FTP rmd: %s", abspath)
370
except ftplib.error_perm, e:
371
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
373
def append_file(self, relpath, f, mode=None):
374
"""Append the text in the file-like object into the final
377
abspath = self._remote_path(relpath)
378
if self.has(relpath):
379
ftp = self._get_FTP()
380
result = ftp.size(abspath)
384
mutter("FTP appe to %s", abspath)
385
self._try_append(relpath, f.read(), mode)
389
def _try_append(self, relpath, text, mode=None, retries=0):
390
"""Try repeatedly to append the given text to the file at relpath.
392
This is a recursive function. On errors, it will be called until the
393
number of retries is exceeded.
396
abspath = self._remote_path(relpath)
397
mutter("FTP appe (try %d) to %s", retries, abspath)
398
ftp = self._get_FTP()
399
ftp.voidcmd("TYPE I")
400
cmd = "APPE %s" % abspath
401
conn = ftp.transfercmd(cmd)
405
self._setmode(relpath, mode)
407
except ftplib.error_perm, e:
408
self._translate_perm_error(e, abspath, extra='error appending',
409
unknown_exc=errors.NoSuchFile)
410
except ftplib.error_temp, e:
411
if retries > _number_of_retries:
412
raise errors.TransportError("FTP temporary error during APPEND %s." \
413
"Aborting." % abspath, orig_error=e)
415
warning("FTP temporary error: %s. Retrying.", str(e))
417
self._try_append(relpath, text, mode, retries+1)
419
def _setmode(self, relpath, mode):
420
"""Set permissions on a path.
422
Only set permissions if the FTP server supports the 'SITE CHMOD'
426
mutter("FTP site chmod: setting permissions to %s on %s",
427
str(mode), self._remote_path(relpath))
428
ftp = self._get_FTP()
429
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
431
except ftplib.error_perm, e:
432
# Command probably not available on this server
433
warning("FTP Could not set permissions to %s on %s. %s",
434
str(mode), self._remote_path(relpath), str(e))
436
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
437
# to copy something to another machine. And you may be able
438
# to give it its own address as the 'to' location.
439
# So implement a fancier 'copy()'
441
def rename(self, rel_from, rel_to):
442
abs_from = self._remote_path(rel_from)
443
abs_to = self._remote_path(rel_to)
444
mutter("FTP rename: %s => %s", abs_from, abs_to)
446
return self._rename(abs_from, abs_to, f)
448
def _rename(self, abs_from, abs_to, f):
450
f.rename(abs_from, abs_to)
451
except ftplib.error_perm, e:
452
self._translate_perm_error(e, abs_from,
453
': unable to rename to %r' % (abs_to))
455
def move(self, rel_from, rel_to):
456
"""Move the item at rel_from to the location at rel_to"""
457
abs_from = self._remote_path(rel_from)
458
abs_to = self._remote_path(rel_to)
460
mutter("FTP mv: %s => %s", abs_from, abs_to)
462
self._rename_and_overwrite(abs_from, abs_to, f)
463
except ftplib.error_perm, e:
464
self._translate_perm_error(e, abs_from,
465
extra='unable to rename to %r' % (rel_to,),
466
unknown_exc=errors.PathError)
468
def _rename_and_overwrite(self, abs_from, abs_to, f):
469
"""Do a fancy rename on the remote server.
471
Using the implementation provided by osutils.
473
osutils.fancy_rename(abs_from, abs_to,
474
rename_func=lambda p1, p2: self._rename(p1, p2, f),
475
unlink_func=lambda p: self._delete(p, f))
477
def delete(self, relpath):
478
"""Delete the item at relpath"""
479
abspath = self._remote_path(relpath)
481
self._delete(abspath, f)
483
def _delete(self, abspath, f):
485
mutter("FTP rm: %s", abspath)
487
except ftplib.error_perm, e:
488
self._translate_perm_error(e, abspath, 'error deleting',
489
unknown_exc=errors.NoSuchFile)
491
def external_url(self):
492
"""See bzrlib.transport.Transport.external_url."""
493
# FTP URL's are externally usable.
497
"""See Transport.listable."""
500
def list_dir(self, relpath):
501
"""See Transport.list_dir."""
502
basepath = self._remote_path(relpath)
503
mutter("FTP nlst: %s", basepath)
506
paths = f.nlst(basepath)
507
except ftplib.error_perm, e:
508
self._translate_perm_error(e, relpath, extra='error with list_dir')
509
# If FTP.nlst returns paths prefixed by relpath, strip 'em
510
if paths and paths[0].startswith(basepath):
511
entries = [path[len(basepath)+1:] for path in paths]
514
# Remove . and .. if present
515
return [urlutils.escape(entry) for entry in entries
516
if entry not in ('.', '..')]
518
def iter_files_recursive(self):
519
"""See Transport.iter_files_recursive.
521
This is cargo-culted from the SFTP transport"""
522
mutter("FTP iter_files_recursive")
523
queue = list(self.list_dir("."))
525
relpath = queue.pop(0)
526
st = self.stat(relpath)
527
if stat.S_ISDIR(st.st_mode):
528
for i, basename in enumerate(self.list_dir(relpath)):
529
queue.insert(i, relpath+"/"+basename)
533
def stat(self, relpath):
534
"""Return the stat information for a file."""
535
abspath = self._remote_path(relpath)
537
mutter("FTP stat: %s", abspath)
539
return FtpStatResult(f, abspath)
540
except ftplib.error_perm, e:
541
self._translate_perm_error(e, abspath, extra='error w/ stat')
543
def lock_read(self, relpath):
544
"""Lock the given file for shared (read) access.
545
:return: A lock object, which should be passed to Transport.unlock()
547
# The old RemoteBranch ignore lock for reading, so we will
548
# continue that tradition and return a bogus lock object.
549
class BogusLock(object):
550
def __init__(self, path):
554
return BogusLock(relpath)
556
def lock_write(self, relpath):
557
"""Lock the given file for exclusive (write) access.
558
WARNING: many transports do not support this, so trying avoid using it
560
:return: A lock object, which should be passed to Transport.unlock()
562
return self.lock_read(relpath)
565
def get_test_permutations():
566
"""Return the permutations to be used in testing."""
567
from bzrlib import tests
568
if tests.FTPServerFeature.available():
569
from bzrlib.tests import ftp_server
570
return [(FtpTransport, ftp_server.FTPServer)]
572
# Dummy server to have the test suite report the number of tests
573
# needing that feature.
574
class UnavailableFTPServer(object):
576
raise tests.UnavailableFeature(tests.FTPServerFeature)
578
return [(FtpTransport, UnavailableFTPServer)]
added
removed
bzrlib/transport/ftp.py
