1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
38
from warnings import warn
46
from bzrlib.trace import mutter, warning
47
from bzrlib.transport import (
48
AppendBasedFileStream,
51
register_urlparse_netloc_protocol,
54
from bzrlib.transport.local import LocalURLServer
58
register_urlparse_netloc_protocol('aftp')
61
class FtpPathError(errors.PathError):
62
"""FTP failed for path: %(path)s%(extra)s"""
65
class FtpStatResult(object):
66
def __init__(self, f, relpath):
68
self.st_size = f.size(relpath)
69
self.st_mode = stat.S_IFREG
70
except ftplib.error_perm:
74
self.st_mode = stat.S_IFDIR
79
_number_of_retries = 2
80
_sleep_between_retries = 5
82
# FIXME: there are inconsistencies in the way temporary errors are
83
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
84
# be taken to analyze the implications for write operations (read operations
85
# are safe to retry). Overall even some read operations are never
86
# retried. --vila 20070720 (Bug #127164)
87
class FtpTransport(ConnectedTransport):
88
"""This is the transport agent for ftp:// access."""
90
def __init__(self, base, _from_transport=None):
91
"""Set the base path where files will be stored."""
92
assert base.startswith('ftp://') or base.startswith('aftp://')
93
super(FtpTransport, self).__init__(base,
94
_from_transport=_from_transport)
95
self._unqualified_scheme = 'ftp'
96
if self._scheme == 'aftp':
99
self.is_active = False
102
"""Return the ftplib.FTP instance for this object."""
103
# Ensures that a connection is established
104
connection = self._get_connection()
105
if connection is None:
106
# First connection ever
107
connection, credentials = self._create_connection()
108
self._set_connection(connection, credentials)
111
def _create_connection(self, credentials=None):
112
"""Create a new connection with the provided credentials.
114
:param credentials: The credentials needed to establish the connection.
116
:return: The created connection and its associated credentials.
118
The credentials are only the password as it may have been entered
119
interactively by the user and may be different from the one provided
120
in base url at transport creation time.
122
if credentials is None:
123
user, password = self._user, self._password
125
user, password = credentials
127
auth = config.AuthenticationConfig()
129
user = auth.get_user('ftp', self._host, port=self._port)
131
# Default to local user
132
user = getpass.getuser()
134
mutter("Constructing FTP instance against %r" %
135
((self._host, self._port, user, '********',
138
connection = ftplib.FTP()
139
connection.connect(host=self._host, port=self._port)
140
if user and user != 'anonymous' and \
141
password is None: # '' is a valid password
142
password = auth.get_password('ftp', self._host, user,
144
connection.login(user=user, passwd=password)
145
connection.set_pasv(not self.is_active)
146
except socket.error, e:
147
raise errors.SocketConnectionError(self._host, self._port,
148
msg='Unable to connect to',
150
except ftplib.error_perm, e:
151
raise errors.TransportError(msg="Error setting up connection:"
152
" %s" % str(e), orig_error=e)
153
return connection, (user, password)
155
def _reconnect(self):
156
"""Create a new connection with the previously used credentials"""
157
credentials = self._get_credentials()
158
connection, credentials = self._create_connection(credentials)
159
self._set_connection(connection, credentials)
161
def _translate_perm_error(self, err, path, extra=None,
162
unknown_exc=FtpPathError):
163
"""Try to translate an ftplib.error_perm exception.
165
:param err: The error to translate into a bzr error
166
:param path: The path which had problems
167
:param extra: Extra information which can be included
168
:param unknown_exc: If None, we will just raise the original exception
169
otherwise we raise unknown_exc(path, extra=extra)
175
extra += ': ' + str(err)
176
if ('no such file' in s
177
or 'could not open' in s
178
or 'no such dir' in s
179
or 'could not create file' in s # vsftpd
180
or 'file doesn\'t exist' in s
181
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
182
or 'file/directory not found' in s # filezilla server
183
# Microsoft FTP-Service RNFR reply if file not found
184
or (s.startswith('550 ') and 'unable to rename to' in extra)
186
raise errors.NoSuchFile(path, extra=extra)
187
if ('file exists' in s):
188
raise errors.FileExists(path, extra=extra)
189
if ('not a directory' in s):
190
raise errors.PathError(path, extra=extra)
192
mutter('unable to understand error for path: %s: %s', path, err)
195
raise unknown_exc(path, extra=extra)
196
# TODO: jam 20060516 Consider re-raising the error wrapped in
197
# something like TransportError, but this loses the traceback
198
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
199
# to handle. Consider doing something like that here.
200
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
203
def _remote_path(self, relpath):
204
# XXX: It seems that ftplib does not handle Unicode paths
205
# at the same time, medusa won't handle utf8 paths So if
206
# we .encode(utf8) here (see ConnectedTransport
207
# implementation), then we get a Server failure. while
208
# if we use str(), we get a UnicodeError, and the test
209
# suite just skips testing UnicodePaths.
210
relative = str(urlutils.unescape(relpath))
211
remote_path = self._combine_paths(self._path, relative)
214
def has(self, relpath):
215
"""Does the target location exist?"""
216
# FIXME jam 20060516 We *do* ask about directories in the test suite
217
# We don't seem to in the actual codebase
218
# XXX: I assume we're never asked has(dirname) and thus I use
219
# the FTP size command and assume that if it doesn't raise,
221
abspath = self._remote_path(relpath)
224
mutter('FTP has check: %s => %s', relpath, abspath)
226
mutter("FTP has: %s", abspath)
228
except ftplib.error_perm, e:
229
if ('is a directory' in str(e).lower()):
230
mutter("FTP has dir: %s: %s", abspath, e)
232
mutter("FTP has not: %s: %s", abspath, e)
235
def get(self, relpath, decode=False, retries=0):
236
"""Get the file at the given relative path.
238
:param relpath: The relative path to the file
239
:param retries: Number of retries after temporary failures so far
242
We're meant to return a file-like object which bzr will
243
then read from. For now we do this via the magic of StringIO
245
# TODO: decode should be deprecated
247
mutter("FTP get: %s", self._remote_path(relpath))
250
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
253
except ftplib.error_perm, e:
254
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
255
except ftplib.error_temp, e:
256
if retries > _number_of_retries:
257
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
258
% self.abspath(relpath),
261
warning("FTP temporary error: %s. Retrying.", str(e))
263
return self.get(relpath, decode, retries+1)
265
if retries > _number_of_retries:
266
raise errors.TransportError("FTP control connection closed during GET %s."
267
% self.abspath(relpath),
270
warning("FTP control connection closed. Trying to reopen.")
271
time.sleep(_sleep_between_retries)
273
return self.get(relpath, decode, retries+1)
275
def put_file(self, relpath, fp, mode=None, retries=0):
276
"""Copy the file-like or string object into the location.
278
:param relpath: Location to put the contents, relative to base.
279
:param fp: File-like or string object.
280
:param retries: Number of retries after temporary failures so far
283
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
286
abspath = self._remote_path(relpath)
287
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
288
os.getpid(), random.randint(0,0x7FFFFFFF))
290
if getattr(fp, 'read', None) is None:
291
# hand in a string IO
295
# capture the byte count; .read() may be read only so
297
class byte_counter(object):
298
def __init__(self, fp):
300
self.counted_bytes = 0
301
def read(self, count):
302
result = self.fp.read(count)
303
self.counted_bytes += len(result)
305
fp = byte_counter(fp)
307
mutter("FTP put: %s", abspath)
310
f.storbinary('STOR '+tmp_abspath, fp)
311
self._rename_and_overwrite(tmp_abspath, abspath, f)
312
if bytes is not None:
315
return fp.counted_bytes
316
except (ftplib.error_temp,EOFError), e:
317
warning("Failure during ftp PUT. Deleting temporary file.")
319
f.delete(tmp_abspath)
321
warning("Failed to delete temporary file on the"
322
" server.\nFile: %s", tmp_abspath)
325
except ftplib.error_perm, e:
326
self._translate_perm_error(e, abspath, extra='could not store',
327
unknown_exc=errors.NoSuchFile)
328
except ftplib.error_temp, e:
329
if retries > _number_of_retries:
330
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
331
% self.abspath(relpath), orig_error=e)
333
warning("FTP temporary error: %s. Retrying.", str(e))
335
self.put_file(relpath, fp, mode, retries+1)
337
if retries > _number_of_retries:
338
raise errors.TransportError("FTP control connection closed during PUT %s."
339
% self.abspath(relpath), orig_error=e)
341
warning("FTP control connection closed. Trying to reopen.")
342
time.sleep(_sleep_between_retries)
344
self.put_file(relpath, fp, mode, retries+1)
346
def mkdir(self, relpath, mode=None):
347
"""Create a directory at the given path."""
348
abspath = self._remote_path(relpath)
350
mutter("FTP mkd: %s", abspath)
353
except ftplib.error_perm, e:
354
self._translate_perm_error(e, abspath,
355
unknown_exc=errors.FileExists)
357
def open_write_stream(self, relpath, mode=None):
358
"""See Transport.open_write_stream."""
359
self.put_bytes(relpath, "", mode)
360
result = AppendBasedFileStream(self, relpath)
361
_file_streams[self.abspath(relpath)] = result
364
def recommended_page_size(self):
365
"""See Transport.recommended_page_size().
367
For FTP we suggest a large page size to reduce the overhead
368
introduced by latency.
372
def rmdir(self, rel_path):
373
"""Delete the directory at rel_path"""
374
abspath = self._remote_path(rel_path)
376
mutter("FTP rmd: %s", abspath)
379
except ftplib.error_perm, e:
380
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
382
def append_file(self, relpath, f, mode=None):
383
"""Append the text in the file-like object into the final
386
abspath = self._remote_path(relpath)
387
if self.has(relpath):
388
ftp = self._get_FTP()
389
result = ftp.size(abspath)
393
mutter("FTP appe to %s", abspath)
394
self._try_append(relpath, f.read(), mode)
398
def _try_append(self, relpath, text, mode=None, retries=0):
399
"""Try repeatedly to append the given text to the file at relpath.
401
This is a recursive function. On errors, it will be called until the
402
number of retries is exceeded.
405
abspath = self._remote_path(relpath)
406
mutter("FTP appe (try %d) to %s", retries, abspath)
407
ftp = self._get_FTP()
408
ftp.voidcmd("TYPE I")
409
cmd = "APPE %s" % abspath
410
conn = ftp.transfercmd(cmd)
414
self._setmode(relpath, mode)
416
except ftplib.error_perm, e:
417
self._translate_perm_error(e, abspath, extra='error appending',
418
unknown_exc=errors.NoSuchFile)
419
except ftplib.error_temp, e:
420
if retries > _number_of_retries:
421
raise errors.TransportError("FTP temporary error during APPEND %s." \
422
"Aborting." % abspath, orig_error=e)
424
warning("FTP temporary error: %s. Retrying.", str(e))
426
self._try_append(relpath, text, mode, retries+1)
428
def _setmode(self, relpath, mode):
429
"""Set permissions on a path.
431
Only set permissions if the FTP server supports the 'SITE CHMOD'
435
mutter("FTP site chmod: setting permissions to %s on %s",
436
str(mode), self._remote_path(relpath))
437
ftp = self._get_FTP()
438
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
440
except ftplib.error_perm, e:
441
# Command probably not available on this server
442
warning("FTP Could not set permissions to %s on %s. %s",
443
str(mode), self._remote_path(relpath), str(e))
445
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
446
# to copy something to another machine. And you may be able
447
# to give it its own address as the 'to' location.
448
# So implement a fancier 'copy()'
450
def rename(self, rel_from, rel_to):
451
abs_from = self._remote_path(rel_from)
452
abs_to = self._remote_path(rel_to)
453
mutter("FTP rename: %s => %s", abs_from, abs_to)
455
return self._rename(abs_from, abs_to, f)
457
def _rename(self, abs_from, abs_to, f):
459
f.rename(abs_from, abs_to)
460
except ftplib.error_perm, e:
461
self._translate_perm_error(e, abs_from,
462
': unable to rename to %r' % (abs_to))
464
def move(self, rel_from, rel_to):
465
"""Move the item at rel_from to the location at rel_to"""
466
abs_from = self._remote_path(rel_from)
467
abs_to = self._remote_path(rel_to)
469
mutter("FTP mv: %s => %s", abs_from, abs_to)
471
self._rename_and_overwrite(abs_from, abs_to, f)
472
except ftplib.error_perm, e:
473
self._translate_perm_error(e, abs_from,
474
extra='unable to rename to %r' % (rel_to,),
475
unknown_exc=errors.PathError)
477
def _rename_and_overwrite(self, abs_from, abs_to, f):
478
"""Do a fancy rename on the remote server.
480
Using the implementation provided by osutils.
482
osutils.fancy_rename(abs_from, abs_to,
483
rename_func=lambda p1, p2: self._rename(p1, p2, f),
484
unlink_func=lambda p: self._delete(p, f))
486
def delete(self, relpath):
487
"""Delete the item at relpath"""
488
abspath = self._remote_path(relpath)
490
self._delete(abspath, f)
492
def _delete(self, abspath, f):
494
mutter("FTP rm: %s", abspath)
496
except ftplib.error_perm, e:
497
self._translate_perm_error(e, abspath, 'error deleting',
498
unknown_exc=errors.NoSuchFile)
500
def external_url(self):
501
"""See bzrlib.transport.Transport.external_url."""
502
# FTP URL's are externally usable.
506
"""See Transport.listable."""
509
def list_dir(self, relpath):
510
"""See Transport.list_dir."""
511
basepath = self._remote_path(relpath)
512
mutter("FTP nlst: %s", basepath)
515
paths = f.nlst(basepath)
516
except ftplib.error_perm, e:
517
self._translate_perm_error(e, relpath, extra='error with list_dir')
518
# If FTP.nlst returns paths prefixed by relpath, strip 'em
519
if paths and paths[0].startswith(basepath):
520
entries = [path[len(basepath)+1:] for path in paths]
523
# Remove . and .. if present
524
return [urlutils.escape(entry) for entry in entries
525
if entry not in ('.', '..')]
527
def iter_files_recursive(self):
528
"""See Transport.iter_files_recursive.
530
This is cargo-culted from the SFTP transport"""
531
mutter("FTP iter_files_recursive")
532
queue = list(self.list_dir("."))
534
relpath = queue.pop(0)
535
st = self.stat(relpath)
536
if stat.S_ISDIR(st.st_mode):
537
for i, basename in enumerate(self.list_dir(relpath)):
538
queue.insert(i, relpath+"/"+basename)
542
def stat(self, relpath):
543
"""Return the stat information for a file."""
544
abspath = self._remote_path(relpath)
546
mutter("FTP stat: %s", abspath)
548
return FtpStatResult(f, abspath)
549
except ftplib.error_perm, e:
550
self._translate_perm_error(e, abspath, extra='error w/ stat')
552
def lock_read(self, relpath):
553
"""Lock the given file for shared (read) access.
554
:return: A lock object, which should be passed to Transport.unlock()
556
# The old RemoteBranch ignore lock for reading, so we will
557
# continue that tradition and return a bogus lock object.
558
class BogusLock(object):
559
def __init__(self, path):
563
return BogusLock(relpath)
565
def lock_write(self, relpath):
566
"""Lock the given file for exclusive (write) access.
567
WARNING: many transports do not support this, so trying avoid using it
569
:return: A lock object, which should be passed to Transport.unlock()
571
return self.lock_read(relpath)
574
def get_test_permutations():
575
"""Return the permutations to be used in testing."""
576
from bzrlib import tests
577
if tests.FTPServerFeature.available():
578
from bzrlib.tests import ftp_server
579
return [(FtpTransport, ftp_server.FTPServer)]
581
# Dummy server to have the test suite report the number of tests
582
# needing that feature. We raise UnavailableFeature from methods before
583
# the test server is being used. Doing so in the setUp method has bad
584
# side-effects (tearDown is never called).
585
class UnavailableFTPServer(object):
594
raise tests.UnavailableFeature(tests.FTPServerFeature)
596
def get_bogus_url(self):
597
raise tests.UnavailableFeature(tests.FTPServerFeature)
599
return [(FtpTransport, UnavailableFTPServer)]
added
removed
bzrlib/transport/ftp.py
