1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
40
from warnings import warn
47
from bzrlib.trace import mutter, warning
48
from bzrlib.transport import (
52
from bzrlib.transport.local import LocalURLServer
58
class FtpPathError(errors.PathError):
59
"""FTP failed for path: %(path)s%(extra)s"""
62
class FtpStatResult(object):
63
def __init__(self, f, relpath):
65
self.st_size = f.size(relpath)
66
self.st_mode = stat.S_IFREG
67
except ftplib.error_perm:
71
self.st_mode = stat.S_IFDIR
76
_number_of_retries = 2
77
_sleep_between_retries = 5
79
# FIXME: there are inconsistencies in the way temporary errors are
80
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
81
# be taken to analyze the implications for write operations (read operations
82
# are safe to retry). Overall even some read operations are never retried.
83
class FtpTransport(ConnectedTransport):
84
"""This is the transport agent for ftp:// access."""
86
def __init__(self, base, from_transport=None):
87
"""Set the base path where files will be stored."""
88
assert base.startswith('ftp://') or base.startswith('aftp://')
89
super(FtpTransport, self).__init__(base, from_transport)
90
self._unqualified_scheme = 'ftp'
91
if self._scheme == 'aftp':
94
self.is_active = False
97
"""Return the ftplib.FTP instance for this object."""
98
# Ensures that a connection is established
99
connection = self._get_connection()
100
if connection is None:
101
# First connection ever
102
connection, credentials = self._create_connection()
103
self._set_connection(connection, credentials)
106
def _create_connection(self, credentials=None):
107
"""Create a new connection with the provided credentials.
109
:param credentials: The credentials needed to establish the connection.
111
:return: The created connection and its associated credentials.
113
The credentials are only the password as it may have been entered
114
interactively by the user and may be different from the one provided
115
in base url at transport creation time.
117
if credentials is None:
118
password = self._password
120
password = credentials
122
mutter("Constructing FTP instance against %r" %
123
((self._host, self._port, self._user, '********',
126
connection = ftplib.FTP()
127
connection.connect(host=self._host, port=self._port)
128
if self._user and self._user != 'anonymous' and \
129
password is not None: # '' is a valid password
130
get_password = bzrlib.ui.ui_factory.get_password
131
password = get_password(prompt='FTP %(user)s@%(host)s password',
132
user=self._user, host=self._host)
133
connection.login(user=self._user, passwd=password)
134
connection.set_pasv(not self.is_active)
135
except ftplib.error_perm, e:
136
raise errors.TransportError(msg="Error setting up connection:"
137
" %s" % str(e), orig_error=e)
138
return connection, password
140
def _reconnect(self):
141
"""Create a new connection with the previously used credentials"""
142
credentials = self.get_credentials()
143
connection, credentials = self._create_connection(credentials)
144
self._set_connection(connection, credentials)
146
def _translate_perm_error(self, err, path, extra=None,
147
unknown_exc=FtpPathError):
148
"""Try to translate an ftplib.error_perm exception.
150
:param err: The error to translate into a bzr error
151
:param path: The path which had problems
152
:param extra: Extra information which can be included
153
:param unknown_exc: If None, we will just raise the original exception
154
otherwise we raise unknown_exc(path, extra=extra)
160
extra += ': ' + str(err)
161
if ('no such file' in s
162
or 'could not open' in s
163
or 'no such dir' in s
164
or 'could not create file' in s # vsftpd
165
or 'file doesn\'t exist' in s
167
raise errors.NoSuchFile(path, extra=extra)
168
if ('file exists' in s):
169
raise errors.FileExists(path, extra=extra)
170
if ('not a directory' in s):
171
raise errors.PathError(path, extra=extra)
173
mutter('unable to understand error for path: %s: %s', path, err)
176
raise unknown_exc(path, extra=extra)
177
# TODO: jam 20060516 Consider re-raising the error wrapped in
178
# something like TransportError, but this loses the traceback
179
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
180
# to handle. Consider doing something like that here.
181
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
184
def should_cache(self):
185
"""Return True if the data pulled across should be cached locally.
189
def _remote_path(self, relpath):
190
# XXX: It seems that ftplib does not handle Unicode paths
191
# at the same time, medusa won't handle utf8 paths So if
192
# we .encode(utf8) here (see ConnectedTransport
193
# implementation), then we get a Server failure. while
194
# if we use str(), we get a UnicodeError, and the test
195
# suite just skips testing UnicodePaths.
196
relative = str(urlutils.unescape(relpath))
197
remote_path = self._combine_paths(self._path, relative)
200
def has(self, relpath):
201
"""Does the target location exist?"""
202
# FIXME jam 20060516 We *do* ask about directories in the test suite
203
# We don't seem to in the actual codebase
204
# XXX: I assume we're never asked has(dirname) and thus I use
205
# the FTP size command and assume that if it doesn't raise,
207
abspath = self._remote_path(relpath)
210
mutter('FTP has check: %s => %s', relpath, abspath)
212
mutter("FTP has: %s", abspath)
214
except ftplib.error_perm, e:
215
if ('is a directory' in str(e).lower()):
216
mutter("FTP has dir: %s: %s", abspath, e)
218
mutter("FTP has not: %s: %s", abspath, e)
221
def get(self, relpath, decode=False, retries=0):
222
"""Get the file at the given relative path.
224
:param relpath: The relative path to the file
225
:param retries: Number of retries after temporary failures so far
228
We're meant to return a file-like object which bzr will
229
then read from. For now we do this via the magic of StringIO
231
# TODO: decode should be deprecated
233
mutter("FTP get: %s", self._remote_path(relpath))
236
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
239
except ftplib.error_perm, e:
240
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
241
except ftplib.error_temp, e:
242
if retries > _number_of_retries:
243
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
244
% self.abspath(relpath),
247
warning("FTP temporary error: %s. Retrying.", str(e))
249
return self.get(relpath, decode, retries+1)
251
if retries > _number_of_retries:
252
raise errors.TransportError("FTP control connection closed during GET %s."
253
% self.abspath(relpath),
256
warning("FTP control connection closed. Trying to reopen.")
257
time.sleep(_sleep_between_retries)
259
return self.get(relpath, decode, retries+1)
261
def put_file(self, relpath, fp, mode=None, retries=0):
262
"""Copy the file-like or string object into the location.
264
:param relpath: Location to put the contents, relative to base.
265
:param fp: File-like or string object.
266
:param retries: Number of retries after temporary failures so far
269
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
272
abspath = self._remote_path(relpath)
273
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
274
os.getpid(), random.randint(0,0x7FFFFFFF))
275
if getattr(fp, 'read', None) is None:
278
mutter("FTP put: %s", abspath)
281
f.storbinary('STOR '+tmp_abspath, fp)
282
self._rename_and_overwrite(tmp_abspath, abspath, f)
283
except (ftplib.error_temp,EOFError), e:
284
warning("Failure during ftp PUT. Deleting temporary file.")
286
f.delete(tmp_abspath)
288
warning("Failed to delete temporary file on the"
289
" server.\nFile: %s", tmp_abspath)
292
except ftplib.error_perm, e:
293
self._translate_perm_error(e, abspath, extra='could not store',
294
unknown_exc=errors.NoSuchFile)
295
except ftplib.error_temp, e:
296
if retries > _number_of_retries:
297
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
298
% self.abspath(relpath), orig_error=e)
300
warning("FTP temporary error: %s. Retrying.", str(e))
302
self.put_file(relpath, fp, mode, retries+1)
304
if retries > _number_of_retries:
305
raise errors.TransportError("FTP control connection closed during PUT %s."
306
% self.abspath(relpath), orig_error=e)
308
warning("FTP control connection closed. Trying to reopen.")
309
time.sleep(_sleep_between_retries)
311
self.put_file(relpath, fp, mode, retries+1)
313
def mkdir(self, relpath, mode=None):
314
"""Create a directory at the given path."""
315
abspath = self._remote_path(relpath)
317
mutter("FTP mkd: %s", abspath)
320
except ftplib.error_perm, e:
321
self._translate_perm_error(e, abspath,
322
unknown_exc=errors.FileExists)
324
def rmdir(self, rel_path):
325
"""Delete the directory at rel_path"""
326
abspath = self._remote_path(rel_path)
328
mutter("FTP rmd: %s", abspath)
331
except ftplib.error_perm, e:
332
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
334
def append_file(self, relpath, f, mode=None):
335
"""Append the text in the file-like object into the final
338
abspath = self._remote_path(relpath)
339
if self.has(relpath):
340
ftp = self._get_FTP()
341
result = ftp.size(abspath)
345
mutter("FTP appe to %s", abspath)
346
self._try_append(relpath, f.read(), mode)
350
def _try_append(self, relpath, text, mode=None, retries=0):
351
"""Try repeatedly to append the given text to the file at relpath.
353
This is a recursive function. On errors, it will be called until the
354
number of retries is exceeded.
357
abspath = self._remote_path(relpath)
358
mutter("FTP appe (try %d) to %s", retries, abspath)
359
ftp = self._get_FTP()
360
ftp.voidcmd("TYPE I")
361
cmd = "APPE %s" % abspath
362
conn = ftp.transfercmd(cmd)
366
self._setmode(relpath, mode)
368
except ftplib.error_perm, e:
369
self._translate_perm_error(e, abspath, extra='error appending',
370
unknown_exc=errors.NoSuchFile)
371
except ftplib.error_temp, e:
372
if retries > _number_of_retries:
373
raise errors.TransportError("FTP temporary error during APPEND %s." \
374
"Aborting." % abspath, orig_error=e)
376
warning("FTP temporary error: %s. Retrying.", str(e))
378
self._try_append(relpath, text, mode, retries+1)
380
def _setmode(self, relpath, mode):
381
"""Set permissions on a path.
383
Only set permissions if the FTP server supports the 'SITE CHMOD'
387
mutter("FTP site chmod: setting permissions to %s on %s",
388
str(mode), self._remote_path(relpath))
389
ftp = self._get_FTP()
390
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
392
except ftplib.error_perm, e:
393
# Command probably not available on this server
394
warning("FTP Could not set permissions to %s on %s. %s",
395
str(mode), self._remote_path(relpath), str(e))
397
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
398
# to copy something to another machine. And you may be able
399
# to give it its own address as the 'to' location.
400
# So implement a fancier 'copy()'
402
def rename(self, rel_from, rel_to):
403
abs_from = self._remote_path(rel_from)
404
abs_to = self._remote_path(rel_to)
405
mutter("FTP rename: %s => %s", abs_from, abs_to)
407
return self._rename(abs_from, abs_to, f)
409
def _rename(self, abs_from, abs_to, f):
411
f.rename(abs_from, abs_to)
412
except ftplib.error_perm, e:
413
self._translate_perm_error(e, abs_from,
414
': unable to rename to %r' % (abs_to))
416
def move(self, rel_from, rel_to):
417
"""Move the item at rel_from to the location at rel_to"""
418
abs_from = self._remote_path(rel_from)
419
abs_to = self._remote_path(rel_to)
421
mutter("FTP mv: %s => %s", abs_from, abs_to)
423
self._rename_and_overwrite(abs_from, abs_to, f)
424
except ftplib.error_perm, e:
425
self._translate_perm_error(e, abs_from,
426
extra='unable to rename to %r' % (rel_to,),
427
unknown_exc=errors.PathError)
429
def _rename_and_overwrite(self, abs_from, abs_to, f):
430
"""Do a fancy rename on the remote server.
432
Using the implementation provided by osutils.
434
osutils.fancy_rename(abs_from, abs_to,
435
rename_func=lambda p1, p2: self._rename(p1, p2, f),
436
unlink_func=lambda p: self._delete(p, f))
438
def delete(self, relpath):
439
"""Delete the item at relpath"""
440
abspath = self._remote_path(relpath)
442
self._delete(abspath, f)
444
def _delete(self, abspath, f):
446
mutter("FTP rm: %s", abspath)
448
except ftplib.error_perm, e:
449
self._translate_perm_error(e, abspath, 'error deleting',
450
unknown_exc=errors.NoSuchFile)
453
"""See Transport.listable."""
456
def list_dir(self, relpath):
457
"""See Transport.list_dir."""
458
basepath = self._remote_path(relpath)
459
mutter("FTP nlst: %s", basepath)
462
paths = f.nlst(basepath)
463
except ftplib.error_perm, e:
464
self._translate_perm_error(e, relpath, extra='error with list_dir')
465
# If FTP.nlst returns paths prefixed by relpath, strip 'em
466
if paths and paths[0].startswith(basepath):
467
entries = [path[len(basepath)+1:] for path in paths]
470
# Remove . and .. if present
471
return [urlutils.escape(entry) for entry in entries
472
if entry not in ('.', '..')]
474
def iter_files_recursive(self):
475
"""See Transport.iter_files_recursive.
477
This is cargo-culted from the SFTP transport"""
478
mutter("FTP iter_files_recursive")
479
queue = list(self.list_dir("."))
481
relpath = queue.pop(0)
482
st = self.stat(relpath)
483
if stat.S_ISDIR(st.st_mode):
484
for i, basename in enumerate(self.list_dir(relpath)):
485
queue.insert(i, relpath+"/"+basename)
489
def stat(self, relpath):
490
"""Return the stat information for a file."""
491
abspath = self._remote_path(relpath)
493
mutter("FTP stat: %s", abspath)
495
return FtpStatResult(f, abspath)
496
except ftplib.error_perm, e:
497
self._translate_perm_error(e, abspath, extra='error w/ stat')
499
def lock_read(self, relpath):
500
"""Lock the given file for shared (read) access.
501
:return: A lock object, which should be passed to Transport.unlock()
503
# The old RemoteBranch ignore lock for reading, so we will
504
# continue that tradition and return a bogus lock object.
505
class BogusLock(object):
506
def __init__(self, path):
510
return BogusLock(relpath)
512
def lock_write(self, relpath):
513
"""Lock the given file for exclusive (write) access.
514
WARNING: many transports do not support this, so trying avoid using it
516
:return: A lock object, which should be passed to Transport.unlock()
518
return self.lock_read(relpath)
521
class FtpServer(Server):
522
"""Common code for FTP server facilities."""
526
self._ftp_server = None
528
self._async_thread = None
533
"""Calculate an ftp url to this server."""
534
return 'ftp://foo:bar@localhost:%d/' % (self._port)
536
# def get_bogus_url(self):
537
# """Return a URL which cannot be connected to."""
538
# return 'ftp://127.0.0.1:1'
540
def log(self, message):
541
"""This is used by medusa.ftp_server to log connections, etc."""
542
self.logs.append(message)
544
def setUp(self, vfs_server=None):
546
raise RuntimeError('Must have medusa to run the FtpServer')
548
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
549
"FtpServer currently assumes local transport, got %s" % vfs_server
551
self._root = os.getcwdu()
552
self._ftp_server = _ftp_server(
553
authorizer=_test_authorizer(root=self._root),
555
port=0, # bind to a random port
557
logger_object=self # Use FtpServer.log() for messages
559
self._port = self._ftp_server.getsockname()[1]
560
# Don't let it loop forever, or handle an infinite number of requests.
561
# In this case it will run for 1000s, or 10000 requests
562
self._async_thread = threading.Thread(
563
target=FtpServer._asyncore_loop_ignore_EBADF,
564
kwargs={'timeout':0.1, 'count':10000})
565
self._async_thread.setDaemon(True)
566
self._async_thread.start()
569
"""See bzrlib.transport.Server.tearDown."""
570
# have asyncore release the channel
571
self._ftp_server.del_channel()
573
self._async_thread.join()
576
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
577
"""Ignore EBADF during server shutdown.
579
We close the socket to get the server to shutdown, but this causes
580
select.select() to raise EBADF.
583
asyncore.loop(*args, **kwargs)
584
# FIXME: If we reach that point, we should raise an exception
585
# explaining that the 'count' parameter in setUp is too low or
586
# testers may wonder why their test just sits there waiting for a
587
# server that is already dead. Note that if the tester waits too
588
# long under pdb the server will also die.
589
except select.error, e:
590
if e.args[0] != errno.EBADF:
596
_test_authorizer = None
600
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
603
import medusa.filesys
604
import medusa.ftp_server
610
class test_authorizer(object):
611
"""A custom Authorizer object for running the test suite.
613
The reason we cannot use dummy_authorizer, is because it sets the
614
channel to readonly, which we don't always want to do.
617
def __init__(self, root):
620
def authorize(self, channel, username, password):
621
"""Return (success, reply_string, filesystem)"""
623
return 0, 'No Medusa.', None
625
channel.persona = -1, -1
626
if username == 'anonymous':
627
channel.read_only = 1
629
channel.read_only = 0
631
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
634
class ftp_channel(medusa.ftp_server.ftp_channel):
635
"""Customized ftp channel"""
637
def log(self, message):
638
"""Redirect logging requests."""
639
mutter('_ftp_channel: %s', message)
641
def log_info(self, message, type='info'):
642
"""Redirect logging requests."""
643
mutter('_ftp_channel %s: %s', type, message)
645
def cmd_rnfr(self, line):
646
"""Prepare for renaming a file."""
647
self._renaming = line[1]
648
self.respond('350 Ready for RNTO')
649
# TODO: jam 20060516 in testing, the ftp server seems to
650
# check that the file already exists, or it sends
651
# 550 RNFR command failed
653
def cmd_rnto(self, line):
654
"""Rename a file based on the target given.
656
rnto must be called after calling rnfr.
658
if not self._renaming:
659
self.respond('503 RNFR required first.')
660
pfrom = self.filesystem.translate(self._renaming)
661
self._renaming = None
662
pto = self.filesystem.translate(line[1])
663
if os.path.exists(pto):
664
self.respond('550 RNTO failed: file exists')
667
os.rename(pfrom, pto)
668
except (IOError, OSError), e:
669
# TODO: jam 20060516 return custom responses based on
670
# why the command failed
671
# (bialix 20070418) str(e) on Python 2.5 @ Windows
672
# sometimes don't provide expected error message;
673
# so we obtain such message via os.strerror()
674
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
676
self.respond('550 RNTO failed')
677
# For a test server, we will go ahead and just die
680
self.respond('250 Rename successful.')
682
def cmd_size(self, line):
683
"""Return the size of a file
685
This is overloaded to help the test suite determine if the
686
target is a directory.
689
if not self.filesystem.isfile(filename):
690
if self.filesystem.isdir(filename):
691
self.respond('550 "%s" is a directory' % (filename,))
693
self.respond('550 "%s" is not a file' % (filename,))
695
self.respond('213 %d'
696
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
698
def cmd_mkd(self, line):
699
"""Create a directory.
701
Overloaded because default implementation does not distinguish
702
*why* it cannot make a directory.
705
self.command_not_understood(''.join(line))
709
self.filesystem.mkdir (path)
710
self.respond ('257 MKD command successful.')
711
except (IOError, OSError), e:
712
# (bialix 20070418) str(e) on Python 2.5 @ Windows
713
# sometimes don't provide expected error message;
714
# so we obtain such message via os.strerror()
715
self.respond ('550 error creating directory: %s' %
716
os.strerror(e.errno))
718
self.respond ('550 error creating directory.')
721
class ftp_server(medusa.ftp_server.ftp_server):
722
"""Customize the behavior of the Medusa ftp_server.
724
There are a few warts on the ftp_server, based on how it expects
728
ftp_channel_class = ftp_channel
730
def __init__(self, *args, **kwargs):
731
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
732
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
734
def log(self, message):
735
"""Redirect logging requests."""
736
mutter('_ftp_server: %s', message)
738
def log_info(self, message, type='info'):
739
"""Override the asyncore.log_info so we don't stipple the screen."""
740
mutter('_ftp_server %s: %s', type, message)
742
_test_authorizer = test_authorizer
743
_ftp_channel = ftp_channel
744
_ftp_server = ftp_server
749
def get_test_permutations():
750
"""Return the permutations to be used in testing."""
751
if not _setup_medusa():
752
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
755
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
