1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
40
from warnings import warn
47
from bzrlib.trace import mutter, warning
48
from bzrlib.transport import (
53
from bzrlib.transport.local import LocalURLServer
59
class FtpPathError(errors.PathError):
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
85
class FtpStatResult(object):
86
def __init__(self, f, relpath):
88
self.st_size = f.size(relpath)
89
self.st_mode = stat.S_IFREG
90
except ftplib.error_perm:
94
self.st_mode = stat.S_IFDIR
99
_number_of_retries = 2
100
_sleep_between_retries = 5
102
class FtpTransport(Transport):
103
"""This is the transport agent for ftp:// access."""
105
def __init__(self, base, _provided_instance=None):
106
"""Set the base path where files will be stored."""
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://
113
if not base.endswith('/'):
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
123
def _unparse_url(self, path=None):
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
138
"""Return the ftplib.FTP instance for this object."""
139
if self._FTP_instance is not None:
140
return self._FTP_instance
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
146
return self._FTP_instance
147
except ftplib.error_perm, e:
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
152
"""Try to translate an ftplib.error_perm exception.
154
:param err: The error to translate into a bzr error
155
:param path: The path which had problems
156
:param extra: Extra information which can be included
157
:param unknown_exc: If None, we will just raise the original exception
158
otherwise we raise unknown_exc(path, extra=extra)
164
extra += ': ' + str(err)
165
if ('no such file' in s
166
or 'could not open' in s
167
or 'no such dir' in s
168
or 'could not create file' in s # vsftpd
169
or 'file doesn\'t exist' in s
171
raise errors.NoSuchFile(path, extra=extra)
172
if ('file exists' in s):
173
raise errors.FileExists(path, extra=extra)
174
if ('not a directory' in s):
175
raise errors.PathError(path, extra=extra)
177
mutter('unable to understand error for path: %s: %s', path, err)
180
raise unknown_exc(path, extra=extra)
181
# TODO: jam 20060516 Consider re-raising the error wrapped in
182
# something like TransportError, but this loses the traceback
183
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
184
# to handle. Consider doing something like that here.
185
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
188
def should_cache(self):
189
"""Return True if the data pulled across should be cached locally.
193
def clone(self, offset=None):
194
"""Return a new FtpTransport with root at self.base + offset.
198
return FtpTransport(self.base, self._FTP_instance)
200
return FtpTransport(self.abspath(offset), self._FTP_instance)
202
def _abspath(self, relpath):
203
assert isinstance(relpath, basestring)
204
relpath = urlutils.unescape(relpath)
205
if relpath.startswith('/'):
208
basepath = self._path.split('/')
209
if len(basepath) > 0 and basepath[-1] == '':
210
basepath = basepath[:-1]
211
for p in relpath.split('/'):
213
if len(basepath) == 0:
214
# In most filesystems, a request for the parent
215
# of root, just returns root.
218
elif p == '.' or p == '':
222
# Possibly, we could use urlparse.urljoin() here, but
223
# I'm concerned about when it chooses to strip the last
224
# portion of the path, and when it doesn't.
226
# XXX: It seems that ftplib does not handle Unicode paths
227
# at the same time, medusa won't handle utf8 paths
228
# So if we .encode(utf8) here, then we get a Server failure.
229
# while if we use str(), we get a UnicodeError, and the test suite
230
# just skips testing UnicodePaths.
231
return str('/'.join(basepath) or '/')
233
def abspath(self, relpath):
234
"""Return the full url to the given relative path.
235
This can be supplied with a string or a list
237
path = self._abspath(relpath)
238
return self._unparse_url(path)
240
def has(self, relpath):
241
"""Does the target location exist?"""
242
# FIXME jam 20060516 We *do* ask about directories in the test suite
243
# We don't seem to in the actual codebase
244
# XXX: I assume we're never asked has(dirname) and thus I use
245
# the FTP size command and assume that if it doesn't raise,
247
abspath = self._abspath(relpath)
250
mutter('FTP has check: %s => %s', relpath, abspath)
252
mutter("FTP has: %s", abspath)
254
except ftplib.error_perm, e:
255
if ('is a directory' in str(e).lower()):
256
mutter("FTP has dir: %s: %s", abspath, e)
258
mutter("FTP has not: %s: %s", abspath, e)
261
def get(self, relpath, decode=False, retries=0):
262
"""Get the file at the given relative path.
264
:param relpath: The relative path to the file
265
:param retries: Number of retries after temporary failures so far
268
We're meant to return a file-like object which bzr will
269
then read from. For now we do this via the magic of StringIO
271
# TODO: decode should be deprecated
273
mutter("FTP get: %s", self._abspath(relpath))
276
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
279
except ftplib.error_perm, e:
280
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
281
except ftplib.error_temp, e:
282
if retries > _number_of_retries:
283
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
284
% self.abspath(relpath),
287
warning("FTP temporary error: %s. Retrying.", str(e))
288
self._FTP_instance = None
289
return self.get(relpath, decode, retries+1)
291
if retries > _number_of_retries:
292
raise errors.TransportError("FTP control connection closed during GET %s."
293
% self.abspath(relpath),
296
warning("FTP control connection closed. Trying to reopen.")
297
time.sleep(_sleep_between_retries)
298
self._FTP_instance = None
299
return self.get(relpath, decode, retries+1)
301
def put_file(self, relpath, fp, mode=None, retries=0):
302
"""Copy the file-like or string object into the location.
304
:param relpath: Location to put the contents, relative to base.
305
:param fp: File-like or string object.
306
:param retries: Number of retries after temporary failures so far
309
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
312
abspath = self._abspath(relpath)
313
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
314
os.getpid(), random.randint(0,0x7FFFFFFF))
315
if getattr(fp, 'read', None) is None:
318
mutter("FTP put: %s", abspath)
321
f.storbinary('STOR '+tmp_abspath, fp)
322
self._rename_and_overwrite(tmp_abspath, abspath, f)
323
except (ftplib.error_temp,EOFError), e:
324
warning("Failure during ftp PUT. Deleting temporary file.")
326
f.delete(tmp_abspath)
328
warning("Failed to delete temporary file on the"
329
" server.\nFile: %s", tmp_abspath)
332
except ftplib.error_perm, e:
333
self._translate_perm_error(e, abspath, extra='could not store',
334
unknown_exc=errors.NoSuchFile)
335
except ftplib.error_temp, e:
336
if retries > _number_of_retries:
337
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
338
% self.abspath(relpath), orig_error=e)
340
warning("FTP temporary error: %s. Retrying.", str(e))
341
self._FTP_instance = None
342
self.put_file(relpath, fp, mode, retries+1)
344
if retries > _number_of_retries:
345
raise errors.TransportError("FTP control connection closed during PUT %s."
346
% self.abspath(relpath), orig_error=e)
348
warning("FTP control connection closed. Trying to reopen.")
349
time.sleep(_sleep_between_retries)
350
self._FTP_instance = None
351
self.put_file(relpath, fp, mode, retries+1)
353
def mkdir(self, relpath, mode=None):
354
"""Create a directory at the given path."""
355
abspath = self._abspath(relpath)
357
mutter("FTP mkd: %s", abspath)
360
except ftplib.error_perm, e:
361
self._translate_perm_error(e, abspath,
362
unknown_exc=errors.FileExists)
364
def rmdir(self, rel_path):
365
"""Delete the directory at rel_path"""
366
abspath = self._abspath(rel_path)
368
mutter("FTP rmd: %s", abspath)
371
except ftplib.error_perm, e:
372
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
374
def append_file(self, relpath, f, mode=None):
375
"""Append the text in the file-like object into the final
378
abspath = self._abspath(relpath)
379
if self.has(relpath):
380
ftp = self._get_FTP()
381
result = ftp.size(abspath)
385
mutter("FTP appe to %s", abspath)
386
self._try_append(relpath, f.read(), mode)
390
def _try_append(self, relpath, text, mode=None, retries=0):
391
"""Try repeatedly to append the given text to the file at relpath.
393
This is a recursive function. On errors, it will be called until the
394
number of retries is exceeded.
397
abspath = self._abspath(relpath)
398
mutter("FTP appe (try %d) to %s", retries, abspath)
399
ftp = self._get_FTP()
400
ftp.voidcmd("TYPE I")
401
cmd = "APPE %s" % abspath
402
conn = ftp.transfercmd(cmd)
406
self._setmode(relpath, mode)
408
except ftplib.error_perm, e:
409
self._translate_perm_error(e, abspath, extra='error appending',
410
unknown_exc=errors.NoSuchFile)
411
except ftplib.error_temp, e:
412
if retries > _number_of_retries:
413
raise errors.TransportError("FTP temporary error during APPEND %s." \
414
"Aborting." % abspath, orig_error=e)
416
warning("FTP temporary error: %s. Retrying.", str(e))
417
self._FTP_instance = None
418
self._try_append(relpath, text, mode, retries+1)
420
def _setmode(self, relpath, mode):
421
"""Set permissions on a path.
423
Only set permissions if the FTP server supports the 'SITE CHMOD'
427
mutter("FTP site chmod: setting permissions to %s on %s",
428
str(mode), self._abspath(relpath))
429
ftp = self._get_FTP()
430
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
432
except ftplib.error_perm, e:
433
# Command probably not available on this server
434
warning("FTP Could not set permissions to %s on %s. %s",
435
str(mode), self._abspath(relpath), str(e))
437
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
438
# to copy something to another machine. And you may be able
439
# to give it its own address as the 'to' location.
440
# So implement a fancier 'copy()'
442
def rename(self, rel_from, rel_to):
443
abs_from = self._abspath(rel_from)
444
abs_to = self._abspath(rel_to)
445
mutter("FTP rename: %s => %s", abs_from, abs_to)
447
return self._rename(abs_from, abs_to, f)
449
def _rename(self, abs_from, abs_to, f):
451
f.rename(abs_from, abs_to)
452
except ftplib.error_perm, e:
453
self._translate_perm_error(e, abs_from,
454
': unable to rename to %r' % (abs_to))
456
def move(self, rel_from, rel_to):
457
"""Move the item at rel_from to the location at rel_to"""
458
abs_from = self._abspath(rel_from)
459
abs_to = self._abspath(rel_to)
461
mutter("FTP mv: %s => %s", abs_from, abs_to)
463
self._rename_and_overwrite(abs_from, abs_to, f)
464
except ftplib.error_perm, e:
465
self._translate_perm_error(e, abs_from,
466
extra='unable to rename to %r' % (rel_to,),
467
unknown_exc=errors.PathError)
469
def _rename_and_overwrite(self, abs_from, abs_to, f):
470
"""Do a fancy rename on the remote server.
472
Using the implementation provided by osutils.
474
osutils.fancy_rename(abs_from, abs_to,
475
rename_func=lambda p1, p2: self._rename(p1, p2, f),
476
unlink_func=lambda p: self._delete(p, f))
478
def delete(self, relpath):
479
"""Delete the item at relpath"""
480
abspath = self._abspath(relpath)
482
self._delete(abspath, f)
484
def _delete(self, abspath, f):
486
mutter("FTP rm: %s", abspath)
488
except ftplib.error_perm, e:
489
self._translate_perm_error(e, abspath, 'error deleting',
490
unknown_exc=errors.NoSuchFile)
492
def external_url(self):
493
"""See bzrlib.transport.Transport.external_url."""
494
# FTP URL's are externally usable.
498
"""See Transport.listable."""
501
def list_dir(self, relpath):
502
"""See Transport.list_dir."""
503
basepath = self._abspath(relpath)
504
mutter("FTP nlst: %s", basepath)
507
paths = f.nlst(basepath)
508
except ftplib.error_perm, e:
509
self._translate_perm_error(e, relpath, extra='error with list_dir')
510
# If FTP.nlst returns paths prefixed by relpath, strip 'em
511
if paths and paths[0].startswith(basepath):
512
entries = [path[len(basepath)+1:] for path in paths]
515
# Remove . and .. if present
516
return [urlutils.escape(entry) for entry in entries
517
if entry not in ('.', '..')]
519
def iter_files_recursive(self):
520
"""See Transport.iter_files_recursive.
522
This is cargo-culted from the SFTP transport"""
523
mutter("FTP iter_files_recursive")
524
queue = list(self.list_dir("."))
526
relpath = queue.pop(0)
527
st = self.stat(relpath)
528
if stat.S_ISDIR(st.st_mode):
529
for i, basename in enumerate(self.list_dir(relpath)):
530
queue.insert(i, relpath+"/"+basename)
534
def stat(self, relpath):
535
"""Return the stat information for a file."""
536
abspath = self._abspath(relpath)
538
mutter("FTP stat: %s", abspath)
540
return FtpStatResult(f, abspath)
541
except ftplib.error_perm, e:
542
self._translate_perm_error(e, abspath, extra='error w/ stat')
544
def lock_read(self, relpath):
545
"""Lock the given file for shared (read) access.
546
:return: A lock object, which should be passed to Transport.unlock()
548
# The old RemoteBranch ignore lock for reading, so we will
549
# continue that tradition and return a bogus lock object.
550
class BogusLock(object):
551
def __init__(self, path):
555
return BogusLock(relpath)
557
def lock_write(self, relpath):
558
"""Lock the given file for exclusive (write) access.
559
WARNING: many transports do not support this, so trying avoid using it
561
:return: A lock object, which should be passed to Transport.unlock()
563
return self.lock_read(relpath)
566
class FtpServer(Server):
567
"""Common code for FTP server facilities."""
571
self._ftp_server = None
573
self._async_thread = None
578
"""Calculate an ftp url to this server."""
579
return 'ftp://foo:bar@localhost:%d/' % (self._port)
581
# def get_bogus_url(self):
582
# """Return a URL which cannot be connected to."""
583
# return 'ftp://127.0.0.1:1'
585
def log(self, message):
586
"""This is used by medusa.ftp_server to log connections, etc."""
587
self.logs.append(message)
589
def setUp(self, vfs_server=None):
591
raise RuntimeError('Must have medusa to run the FtpServer')
593
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
594
"FtpServer currently assumes local transport, got %s" % vfs_server
596
self._root = os.getcwdu()
597
self._ftp_server = _ftp_server(
598
authorizer=_test_authorizer(root=self._root),
600
port=0, # bind to a random port
602
logger_object=self # Use FtpServer.log() for messages
604
self._port = self._ftp_server.getsockname()[1]
605
# Don't let it loop forever, or handle an infinite number of requests.
606
# In this case it will run for 100s, or 1000 requests
607
self._async_thread = threading.Thread(
608
target=FtpServer._asyncore_loop_ignore_EBADF,
609
kwargs={'timeout':0.1, 'count':1000})
610
self._async_thread.setDaemon(True)
611
self._async_thread.start()
614
"""See bzrlib.transport.Server.tearDown."""
615
# have asyncore release the channel
616
self._ftp_server.del_channel()
618
self._async_thread.join()
621
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
622
"""Ignore EBADF during server shutdown.
624
We close the socket to get the server to shutdown, but this causes
625
select.select() to raise EBADF.
628
asyncore.loop(*args, **kwargs)
629
except select.error, e:
630
if e.args[0] != errno.EBADF:
636
_test_authorizer = None
640
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
643
import medusa.filesys
644
import medusa.ftp_server
650
class test_authorizer(object):
651
"""A custom Authorizer object for running the test suite.
653
The reason we cannot use dummy_authorizer, is because it sets the
654
channel to readonly, which we don't always want to do.
657
def __init__(self, root):
660
def authorize(self, channel, username, password):
661
"""Return (success, reply_string, filesystem)"""
663
return 0, 'No Medusa.', None
665
channel.persona = -1, -1
666
if username == 'anonymous':
667
channel.read_only = 1
669
channel.read_only = 0
671
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
674
class ftp_channel(medusa.ftp_server.ftp_channel):
675
"""Customized ftp channel"""
677
def log(self, message):
678
"""Redirect logging requests."""
679
mutter('_ftp_channel: %s', message)
681
def log_info(self, message, type='info'):
682
"""Redirect logging requests."""
683
mutter('_ftp_channel %s: %s', type, message)
685
def cmd_rnfr(self, line):
686
"""Prepare for renaming a file."""
687
self._renaming = line[1]
688
self.respond('350 Ready for RNTO')
689
# TODO: jam 20060516 in testing, the ftp server seems to
690
# check that the file already exists, or it sends
691
# 550 RNFR command failed
693
def cmd_rnto(self, line):
694
"""Rename a file based on the target given.
696
rnto must be called after calling rnfr.
698
if not self._renaming:
699
self.respond('503 RNFR required first.')
700
pfrom = self.filesystem.translate(self._renaming)
701
self._renaming = None
702
pto = self.filesystem.translate(line[1])
703
if os.path.exists(pto):
704
self.respond('550 RNTO failed: file exists')
707
os.rename(pfrom, pto)
708
except (IOError, OSError), e:
709
# TODO: jam 20060516 return custom responses based on
710
# why the command failed
711
# (bialix 20070418) str(e) on Python 2.5 @ Windows
712
# sometimes don't provide expected error message;
713
# so we obtain such message via os.strerror()
714
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
716
self.respond('550 RNTO failed')
717
# For a test server, we will go ahead and just die
720
self.respond('250 Rename successful.')
722
def cmd_size(self, line):
723
"""Return the size of a file
725
This is overloaded to help the test suite determine if the
726
target is a directory.
729
if not self.filesystem.isfile(filename):
730
if self.filesystem.isdir(filename):
731
self.respond('550 "%s" is a directory' % (filename,))
733
self.respond('550 "%s" is not a file' % (filename,))
735
self.respond('213 %d'
736
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
738
def cmd_mkd(self, line):
739
"""Create a directory.
741
Overloaded because default implementation does not distinguish
742
*why* it cannot make a directory.
745
self.command_not_understood(''.join(line))
749
self.filesystem.mkdir (path)
750
self.respond ('257 MKD command successful.')
751
except (IOError, OSError), e:
752
# (bialix 20070418) str(e) on Python 2.5 @ Windows
753
# sometimes don't provide expected error message;
754
# so we obtain such message via os.strerror()
755
self.respond ('550 error creating directory: %s' %
756
os.strerror(e.errno))
758
self.respond ('550 error creating directory.')
761
class ftp_server(medusa.ftp_server.ftp_server):
762
"""Customize the behavior of the Medusa ftp_server.
764
There are a few warts on the ftp_server, based on how it expects
768
ftp_channel_class = ftp_channel
770
def __init__(self, *args, **kwargs):
771
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
772
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
774
def log(self, message):
775
"""Redirect logging requests."""
776
mutter('_ftp_server: %s', message)
778
def log_info(self, message, type='info'):
779
"""Override the asyncore.log_info so we don't stipple the screen."""
780
mutter('_ftp_server %s: %s', type, message)
782
_test_authorizer = test_authorizer
783
_ftp_channel = ftp_channel
784
_ftp_server = ftp_server
789
def get_test_permutations():
790
"""Return the permutations to be used in testing."""
791
if not _setup_medusa():
792
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
795
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
