1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
38
from warnings import warn
46
from bzrlib.trace import mutter, warning
47
from bzrlib.transport import (
48
AppendBasedFileStream,
51
register_urlparse_netloc_protocol,
54
from bzrlib.transport.local import LocalURLServer
58
register_urlparse_netloc_protocol('aftp')
61
class FtpPathError(errors.PathError):
62
"""FTP failed for path: %(path)s%(extra)s"""
65
class FtpStatResult(object):
66
def __init__(self, f, relpath):
68
self.st_size = f.size(relpath)
69
self.st_mode = stat.S_IFREG
70
except ftplib.error_perm:
74
self.st_mode = stat.S_IFDIR
79
_number_of_retries = 2
80
_sleep_between_retries = 5
82
# FIXME: there are inconsistencies in the way temporary errors are
83
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
84
# be taken to analyze the implications for write operations (read operations
85
# are safe to retry). Overall even some read operations are never
86
# retried. --vila 20070720 (Bug #127164)
87
class FtpTransport(ConnectedTransport):
88
"""This is the transport agent for ftp:// access."""
90
def __init__(self, base, _from_transport=None):
91
"""Set the base path where files will be stored."""
92
assert base.startswith('ftp://') or base.startswith('aftp://')
93
super(FtpTransport, self).__init__(base,
94
_from_transport=_from_transport)
95
self._unqualified_scheme = 'ftp'
96
if self._scheme == 'aftp':
99
self.is_active = False
102
"""Return the ftplib.FTP instance for this object."""
103
# Ensures that a connection is established
104
connection = self._get_connection()
105
if connection is None:
106
# First connection ever
107
connection, credentials = self._create_connection()
108
self._set_connection(connection, credentials)
111
def _create_connection(self, credentials=None):
112
"""Create a new connection with the provided credentials.
114
:param credentials: The credentials needed to establish the connection.
116
:return: The created connection and its associated credentials.
118
The credentials are only the password as it may have been entered
119
interactively by the user and may be different from the one provided
120
in base url at transport creation time.
122
if credentials is None:
123
user, password = self._user, self._password
125
user, password = credentials
127
auth = config.AuthenticationConfig()
129
user = auth.get_user('ftp', self._host, port=self._port)
131
# Default to local user
132
user = getpass.getuser()
134
mutter("Constructing FTP instance against %r" %
135
((self._host, self._port, user, '********',
138
connection = ftplib.FTP()
139
connection.connect(host=self._host, port=self._port)
140
if user and user != 'anonymous' and \
141
password is None: # '' is a valid password
142
password = auth.get_password('ftp', self._host, user,
144
connection.login(user=user, passwd=password)
145
connection.set_pasv(not self.is_active)
146
except socket.error, e:
147
raise errors.SocketConnectionError(self._host, self._port,
148
msg='Unable to connect to',
150
except ftplib.error_perm, e:
151
raise errors.TransportError(msg="Error setting up connection:"
152
" %s" % str(e), orig_error=e)
153
return connection, (user, password)
155
def _reconnect(self):
156
"""Create a new connection with the previously used credentials"""
157
credentials = self._get_credentials()
158
connection, credentials = self._create_connection(credentials)
159
self._set_connection(connection, credentials)
161
def _translate_perm_error(self, err, path, extra=None,
162
unknown_exc=FtpPathError):
163
"""Try to translate an ftplib.error_perm exception.
165
:param err: The error to translate into a bzr error
166
:param path: The path which had problems
167
:param extra: Extra information which can be included
168
:param unknown_exc: If None, we will just raise the original exception
169
otherwise we raise unknown_exc(path, extra=extra)
175
extra += ': ' + str(err)
176
if ('no such file' in s
177
or 'could not open' in s
178
or 'no such dir' in s
179
or 'could not create file' in s # vsftpd
180
or 'file doesn\'t exist' in s
181
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
182
or 'file/directory not found' in s # filezilla server
184
raise errors.NoSuchFile(path, extra=extra)
185
if ('file exists' in s):
186
raise errors.FileExists(path, extra=extra)
187
if ('not a directory' in s):
188
raise errors.PathError(path, extra=extra)
190
mutter('unable to understand error for path: %s: %s', path, err)
193
raise unknown_exc(path, extra=extra)
194
# TODO: jam 20060516 Consider re-raising the error wrapped in
195
# something like TransportError, but this loses the traceback
196
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
197
# to handle. Consider doing something like that here.
198
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
201
def _remote_path(self, relpath):
202
# XXX: It seems that ftplib does not handle Unicode paths
203
# at the same time, medusa won't handle utf8 paths So if
204
# we .encode(utf8) here (see ConnectedTransport
205
# implementation), then we get a Server failure. while
206
# if we use str(), we get a UnicodeError, and the test
207
# suite just skips testing UnicodePaths.
208
relative = str(urlutils.unescape(relpath))
209
remote_path = self._combine_paths(self._path, relative)
212
def has(self, relpath):
213
"""Does the target location exist?"""
214
# FIXME jam 20060516 We *do* ask about directories in the test suite
215
# We don't seem to in the actual codebase
216
# XXX: I assume we're never asked has(dirname) and thus I use
217
# the FTP size command and assume that if it doesn't raise,
219
abspath = self._remote_path(relpath)
222
mutter('FTP has check: %s => %s', relpath, abspath)
224
mutter("FTP has: %s", abspath)
226
except ftplib.error_perm, e:
227
if ('is a directory' in str(e).lower()):
228
mutter("FTP has dir: %s: %s", abspath, e)
230
mutter("FTP has not: %s: %s", abspath, e)
233
def get(self, relpath, decode=False, retries=0):
234
"""Get the file at the given relative path.
236
:param relpath: The relative path to the file
237
:param retries: Number of retries after temporary failures so far
240
We're meant to return a file-like object which bzr will
241
then read from. For now we do this via the magic of StringIO
243
# TODO: decode should be deprecated
245
mutter("FTP get: %s", self._remote_path(relpath))
248
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
251
except ftplib.error_perm, e:
252
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
253
except ftplib.error_temp, e:
254
if retries > _number_of_retries:
255
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
256
% self.abspath(relpath),
259
warning("FTP temporary error: %s. Retrying.", str(e))
261
return self.get(relpath, decode, retries+1)
263
if retries > _number_of_retries:
264
raise errors.TransportError("FTP control connection closed during GET %s."
265
% self.abspath(relpath),
268
warning("FTP control connection closed. Trying to reopen.")
269
time.sleep(_sleep_between_retries)
271
return self.get(relpath, decode, retries+1)
273
def put_file(self, relpath, fp, mode=None, retries=0):
274
"""Copy the file-like or string object into the location.
276
:param relpath: Location to put the contents, relative to base.
277
:param fp: File-like or string object.
278
:param retries: Number of retries after temporary failures so far
281
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
284
abspath = self._remote_path(relpath)
285
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
286
os.getpid(), random.randint(0,0x7FFFFFFF))
288
if getattr(fp, 'read', None) is None:
289
# hand in a string IO
293
# capture the byte count; .read() may be read only so
295
class byte_counter(object):
296
def __init__(self, fp):
298
self.counted_bytes = 0
299
def read(self, count):
300
result = self.fp.read(count)
301
self.counted_bytes += len(result)
303
fp = byte_counter(fp)
305
mutter("FTP put: %s", abspath)
308
f.storbinary('STOR '+tmp_abspath, fp)
309
self._rename_and_overwrite(tmp_abspath, abspath, f)
310
if bytes is not None:
313
return fp.counted_bytes
314
except (ftplib.error_temp,EOFError), e:
315
warning("Failure during ftp PUT. Deleting temporary file.")
317
f.delete(tmp_abspath)
319
warning("Failed to delete temporary file on the"
320
" server.\nFile: %s", tmp_abspath)
323
except ftplib.error_perm, e:
324
self._translate_perm_error(e, abspath, extra='could not store',
325
unknown_exc=errors.NoSuchFile)
326
except ftplib.error_temp, e:
327
if retries > _number_of_retries:
328
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
329
% self.abspath(relpath), orig_error=e)
331
warning("FTP temporary error: %s. Retrying.", str(e))
333
self.put_file(relpath, fp, mode, retries+1)
335
if retries > _number_of_retries:
336
raise errors.TransportError("FTP control connection closed during PUT %s."
337
% self.abspath(relpath), orig_error=e)
339
warning("FTP control connection closed. Trying to reopen.")
340
time.sleep(_sleep_between_retries)
342
self.put_file(relpath, fp, mode, retries+1)
344
def mkdir(self, relpath, mode=None):
345
"""Create a directory at the given path."""
346
abspath = self._remote_path(relpath)
348
mutter("FTP mkd: %s", abspath)
351
except ftplib.error_perm, e:
352
self._translate_perm_error(e, abspath,
353
unknown_exc=errors.FileExists)
355
def open_write_stream(self, relpath, mode=None):
356
"""See Transport.open_write_stream."""
357
self.put_bytes(relpath, "", mode)
358
result = AppendBasedFileStream(self, relpath)
359
_file_streams[self.abspath(relpath)] = result
362
def recommended_page_size(self):
363
"""See Transport.recommended_page_size().
365
For FTP we suggest a large page size to reduce the overhead
366
introduced by latency.
370
def rmdir(self, rel_path):
371
"""Delete the directory at rel_path"""
372
abspath = self._remote_path(rel_path)
374
mutter("FTP rmd: %s", abspath)
377
except ftplib.error_perm, e:
378
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
380
def append_file(self, relpath, f, mode=None):
381
"""Append the text in the file-like object into the final
384
abspath = self._remote_path(relpath)
385
if self.has(relpath):
386
ftp = self._get_FTP()
387
result = ftp.size(abspath)
391
mutter("FTP appe to %s", abspath)
392
self._try_append(relpath, f.read(), mode)
396
def _try_append(self, relpath, text, mode=None, retries=0):
397
"""Try repeatedly to append the given text to the file at relpath.
399
This is a recursive function. On errors, it will be called until the
400
number of retries is exceeded.
403
abspath = self._remote_path(relpath)
404
mutter("FTP appe (try %d) to %s", retries, abspath)
405
ftp = self._get_FTP()
406
ftp.voidcmd("TYPE I")
407
cmd = "APPE %s" % abspath
408
conn = ftp.transfercmd(cmd)
412
self._setmode(relpath, mode)
414
except ftplib.error_perm, e:
415
self._translate_perm_error(e, abspath, extra='error appending',
416
unknown_exc=errors.NoSuchFile)
417
except ftplib.error_temp, e:
418
if retries > _number_of_retries:
419
raise errors.TransportError("FTP temporary error during APPEND %s." \
420
"Aborting." % abspath, orig_error=e)
422
warning("FTP temporary error: %s. Retrying.", str(e))
424
self._try_append(relpath, text, mode, retries+1)
426
def _setmode(self, relpath, mode):
427
"""Set permissions on a path.
429
Only set permissions if the FTP server supports the 'SITE CHMOD'
433
mutter("FTP site chmod: setting permissions to %s on %s",
434
str(mode), self._remote_path(relpath))
435
ftp = self._get_FTP()
436
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
438
except ftplib.error_perm, e:
439
# Command probably not available on this server
440
warning("FTP Could not set permissions to %s on %s. %s",
441
str(mode), self._remote_path(relpath), str(e))
443
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
444
# to copy something to another machine. And you may be able
445
# to give it its own address as the 'to' location.
446
# So implement a fancier 'copy()'
448
def rename(self, rel_from, rel_to):
449
abs_from = self._remote_path(rel_from)
450
abs_to = self._remote_path(rel_to)
451
mutter("FTP rename: %s => %s", abs_from, abs_to)
453
return self._rename(abs_from, abs_to, f)
455
def _rename(self, abs_from, abs_to, f):
457
f.rename(abs_from, abs_to)
458
except ftplib.error_perm, e:
459
self._translate_perm_error(e, abs_from,
460
': unable to rename to %r' % (abs_to))
462
def move(self, rel_from, rel_to):
463
"""Move the item at rel_from to the location at rel_to"""
464
abs_from = self._remote_path(rel_from)
465
abs_to = self._remote_path(rel_to)
467
mutter("FTP mv: %s => %s", abs_from, abs_to)
469
self._rename_and_overwrite(abs_from, abs_to, f)
470
except ftplib.error_perm, e:
471
self._translate_perm_error(e, abs_from,
472
extra='unable to rename to %r' % (rel_to,),
473
unknown_exc=errors.PathError)
475
def _rename_and_overwrite(self, abs_from, abs_to, f):
476
"""Do a fancy rename on the remote server.
478
Using the implementation provided by osutils.
480
osutils.fancy_rename(abs_from, abs_to,
481
rename_func=lambda p1, p2: self._rename(p1, p2, f),
482
unlink_func=lambda p: self._delete(p, f))
484
def delete(self, relpath):
485
"""Delete the item at relpath"""
486
abspath = self._remote_path(relpath)
488
self._delete(abspath, f)
490
def _delete(self, abspath, f):
492
mutter("FTP rm: %s", abspath)
494
except ftplib.error_perm, e:
495
self._translate_perm_error(e, abspath, 'error deleting',
496
unknown_exc=errors.NoSuchFile)
498
def external_url(self):
499
"""See bzrlib.transport.Transport.external_url."""
500
# FTP URL's are externally usable.
504
"""See Transport.listable."""
507
def list_dir(self, relpath):
508
"""See Transport.list_dir."""
509
basepath = self._remote_path(relpath)
510
mutter("FTP nlst: %s", basepath)
513
paths = f.nlst(basepath)
514
except ftplib.error_perm, e:
515
self._translate_perm_error(e, relpath, extra='error with list_dir')
516
# If FTP.nlst returns paths prefixed by relpath, strip 'em
517
if paths and paths[0].startswith(basepath):
518
entries = [path[len(basepath)+1:] for path in paths]
521
# Remove . and .. if present
522
return [urlutils.escape(entry) for entry in entries
523
if entry not in ('.', '..')]
525
def iter_files_recursive(self):
526
"""See Transport.iter_files_recursive.
528
This is cargo-culted from the SFTP transport"""
529
mutter("FTP iter_files_recursive")
530
queue = list(self.list_dir("."))
532
relpath = queue.pop(0)
533
st = self.stat(relpath)
534
if stat.S_ISDIR(st.st_mode):
535
for i, basename in enumerate(self.list_dir(relpath)):
536
queue.insert(i, relpath+"/"+basename)
540
def stat(self, relpath):
541
"""Return the stat information for a file."""
542
abspath = self._remote_path(relpath)
544
mutter("FTP stat: %s", abspath)
546
return FtpStatResult(f, abspath)
547
except ftplib.error_perm, e:
548
self._translate_perm_error(e, abspath, extra='error w/ stat')
550
def lock_read(self, relpath):
551
"""Lock the given file for shared (read) access.
552
:return: A lock object, which should be passed to Transport.unlock()
554
# The old RemoteBranch ignore lock for reading, so we will
555
# continue that tradition and return a bogus lock object.
556
class BogusLock(object):
557
def __init__(self, path):
561
return BogusLock(relpath)
563
def lock_write(self, relpath):
564
"""Lock the given file for exclusive (write) access.
565
WARNING: many transports do not support this, so trying avoid using it
567
:return: A lock object, which should be passed to Transport.unlock()
569
return self.lock_read(relpath)
572
def get_test_permutations():
573
"""Return the permutations to be used in testing."""
574
from bzrlib import tests
575
if tests.FTPServerFeature.available():
576
from bzrlib.tests import ftp_server
577
return [(FtpTransport, ftp_server.FTPServer)]
579
# Dummy server to have the test suite report the number of tests
580
# needing that feature. We raise UnavailableFeature from methods before
581
# the test server is being used. Doing so in the setUp method has bad
582
# side-effects (tearDown is never called).
583
class UnavailableFTPServer(object):
592
raise tests.UnavailableFeature(tests.FTPServerFeature)
594
def get_bogus_url(self):
595
raise tests.UnavailableFeature(tests.FTPServerFeature)
597
return [(FtpTransport, UnavailableFTPServer)]
added
removed
bzrlib/transport/ftp.py
