1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
40
from warnings import warn
47
from bzrlib.trace import mutter, warning
48
from bzrlib.transport import (
53
from bzrlib.transport.local import LocalURLServer
59
class FtpPathError(errors.PathError):
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
85
class FtpStatResult(object):
86
def __init__(self, f, relpath):
88
self.st_size = f.size(relpath)
89
self.st_mode = stat.S_IFREG
90
except ftplib.error_perm:
94
self.st_mode = stat.S_IFDIR
99
_number_of_retries = 2
100
_sleep_between_retries = 5
102
class FtpTransport(Transport):
103
"""This is the transport agent for ftp:// access."""
105
def __init__(self, base, _provided_instance=None):
106
"""Set the base path where files will be stored."""
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://
113
if not base.endswith('/'):
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
123
def _unparse_url(self, path=None):
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
138
"""Return the ftplib.FTP instance for this object."""
139
if self._FTP_instance is not None:
140
return self._FTP_instance
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
146
return self._FTP_instance
147
except ftplib.error_perm, e:
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
152
"""Try to translate an ftplib.error_perm exception.
154
:param err: The error to translate into a bzr error
155
:param path: The path which had problems
156
:param extra: Extra information which can be included
157
:param unknown_exc: If None, we will just raise the original exception
158
otherwise we raise unknown_exc(path, extra=extra)
164
extra += ': ' + str(err)
165
if ('no such file' in s
166
or 'could not open' in s
167
or 'no such dir' in s
168
or 'could not create file' in s # vsftpd
169
or 'file doesn\'t exist' in s
171
raise errors.NoSuchFile(path, extra=extra)
172
if ('file exists' in s):
173
raise errors.FileExists(path, extra=extra)
174
if ('not a directory' in s):
175
raise errors.PathError(path, extra=extra)
177
mutter('unable to understand error for path: %s: %s', path, err)
180
raise unknown_exc(path, extra=extra)
181
# TODO: jam 20060516 Consider re-raising the error wrapped in
182
# something like TransportError, but this loses the traceback
183
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
184
# to handle. Consider doing something like that here.
185
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
188
def should_cache(self):
189
"""Return True if the data pulled across should be cached locally.
193
def clone(self, offset=None):
194
"""Return a new FtpTransport with root at self.base + offset.
198
return FtpTransport(self.base, self._FTP_instance)
200
return FtpTransport(self.abspath(offset), self._FTP_instance)
202
def _abspath(self, relpath):
203
assert isinstance(relpath, basestring)
204
relpath = urlutils.unescape(relpath)
205
if relpath.startswith('/'):
208
basepath = self._path.split('/')
209
if len(basepath) > 0 and basepath[-1] == '':
210
basepath = basepath[:-1]
211
for p in relpath.split('/'):
213
if len(basepath) == 0:
214
# In most filesystems, a request for the parent
215
# of root, just returns root.
218
elif p == '.' or p == '':
222
# Possibly, we could use urlparse.urljoin() here, but
223
# I'm concerned about when it chooses to strip the last
224
# portion of the path, and when it doesn't.
226
# XXX: It seems that ftplib does not handle Unicode paths
227
# at the same time, medusa won't handle utf8 paths
228
# So if we .encode(utf8) here, then we get a Server failure.
229
# while if we use str(), we get a UnicodeError, and the test suite
230
# just skips testing UnicodePaths.
231
return str('/'.join(basepath) or '/')
233
def abspath(self, relpath):
234
"""Return the full url to the given relative path.
235
This can be supplied with a string or a list
237
path = self._abspath(relpath)
238
return self._unparse_url(path)
240
def has(self, relpath):
241
"""Does the target location exist?"""
242
# FIXME jam 20060516 We *do* ask about directories in the test suite
243
# We don't seem to in the actual codebase
244
# XXX: I assume we're never asked has(dirname) and thus I use
245
# the FTP size command and assume that if it doesn't raise,
247
abspath = self._abspath(relpath)
250
mutter('FTP has check: %s => %s', relpath, abspath)
252
mutter("FTP has: %s", abspath)
254
except ftplib.error_perm, e:
255
if ('is a directory' in str(e).lower()):
256
mutter("FTP has dir: %s: %s", abspath, e)
258
mutter("FTP has not: %s: %s", abspath, e)
261
def get(self, relpath, decode=False, retries=0):
262
"""Get the file at the given relative path.
264
:param relpath: The relative path to the file
265
:param retries: Number of retries after temporary failures so far
268
We're meant to return a file-like object which bzr will
269
then read from. For now we do this via the magic of StringIO
271
# TODO: decode should be deprecated
273
mutter("FTP get: %s", self._abspath(relpath))
276
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
279
except ftplib.error_perm, e:
280
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
281
except ftplib.error_temp, e:
282
if retries > _number_of_retries:
283
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
284
% self.abspath(relpath),
287
warning("FTP temporary error: %s. Retrying.", str(e))
288
self._FTP_instance = None
289
return self.get(relpath, decode, retries+1)
291
if retries > _number_of_retries:
292
raise errors.TransportError("FTP control connection closed during GET %s."
293
% self.abspath(relpath),
296
warning("FTP control connection closed. Trying to reopen.")
297
time.sleep(_sleep_between_retries)
298
self._FTP_instance = None
299
return self.get(relpath, decode, retries+1)
301
def put_file(self, relpath, fp, mode=None, retries=0):
302
"""Copy the file-like or string object into the location.
304
:param relpath: Location to put the contents, relative to base.
305
:param fp: File-like or string object.
306
:param retries: Number of retries after temporary failures so far
309
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
312
abspath = self._abspath(relpath)
313
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
314
os.getpid(), random.randint(0,0x7FFFFFFF))
315
if getattr(fp, 'read', None) is None:
318
mutter("FTP put: %s", abspath)
321
f.storbinary('STOR '+tmp_abspath, fp)
322
self._rename_and_overwrite(tmp_abspath, abspath, f)
323
except (ftplib.error_temp,EOFError), e:
324
warning("Failure during ftp PUT. Deleting temporary file.")
326
f.delete(tmp_abspath)
328
warning("Failed to delete temporary file on the"
329
" server.\nFile: %s", tmp_abspath)
332
except ftplib.error_perm, e:
333
self._translate_perm_error(e, abspath, extra='could not store',
334
unknown_exc=errors.NoSuchFile)
335
except ftplib.error_temp, e:
336
if retries > _number_of_retries:
337
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
338
% self.abspath(relpath), orig_error=e)
340
warning("FTP temporary error: %s. Retrying.", str(e))
341
self._FTP_instance = None
342
self.put_file(relpath, fp, mode, retries+1)
344
if retries > _number_of_retries:
345
raise errors.TransportError("FTP control connection closed during PUT %s."
346
% self.abspath(relpath), orig_error=e)
348
warning("FTP control connection closed. Trying to reopen.")
349
time.sleep(_sleep_between_retries)
350
self._FTP_instance = None
351
self.put_file(relpath, fp, mode, retries+1)
353
def mkdir(self, relpath, mode=None):
354
"""Create a directory at the given path."""
355
abspath = self._abspath(relpath)
357
mutter("FTP mkd: %s", abspath)
360
except ftplib.error_perm, e:
361
self._translate_perm_error(e, abspath,
362
unknown_exc=errors.FileExists)
364
def rmdir(self, rel_path):
365
"""Delete the directory at rel_path"""
366
abspath = self._abspath(rel_path)
368
mutter("FTP rmd: %s", abspath)
371
except ftplib.error_perm, e:
372
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
374
def append_file(self, relpath, f, mode=None):
375
"""Append the text in the file-like object into the final
378
abspath = self._abspath(relpath)
379
if self.has(relpath):
380
ftp = self._get_FTP()
381
result = ftp.size(abspath)
385
mutter("FTP appe to %s", abspath)
386
self._try_append(relpath, f.read(), mode)
390
def _try_append(self, relpath, text, mode=None, retries=0):
391
"""Try repeatedly to append the given text to the file at relpath.
393
This is a recursive function. On errors, it will be called until the
394
number of retries is exceeded.
397
abspath = self._abspath(relpath)
398
mutter("FTP appe (try %d) to %s", retries, abspath)
399
ftp = self._get_FTP()
400
ftp.voidcmd("TYPE I")
401
cmd = "APPE %s" % abspath
402
conn = ftp.transfercmd(cmd)
406
self._setmode(relpath, mode)
408
except ftplib.error_perm, e:
409
self._translate_perm_error(e, abspath, extra='error appending',
410
unknown_exc=errors.NoSuchFile)
411
except ftplib.error_temp, e:
412
if retries > _number_of_retries:
413
raise errors.TransportError("FTP temporary error during APPEND %s." \
414
"Aborting." % abspath, orig_error=e)
416
warning("FTP temporary error: %s. Retrying.", str(e))
417
self._FTP_instance = None
418
self._try_append(relpath, text, mode, retries+1)
420
def _setmode(self, relpath, mode):
421
"""Set permissions on a path.
423
Only set permissions if the FTP server supports the 'SITE CHMOD'
427
mutter("FTP site chmod: setting permissions to %s on %s",
428
str(mode), self._abspath(relpath))
429
ftp = self._get_FTP()
430
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
432
except ftplib.error_perm, e:
433
# Command probably not available on this server
434
warning("FTP Could not set permissions to %s on %s. %s",
435
str(mode), self._abspath(relpath), str(e))
437
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
438
# to copy something to another machine. And you may be able
439
# to give it its own address as the 'to' location.
440
# So implement a fancier 'copy()'
442
def rename(self, rel_from, rel_to):
443
abs_from = self._abspath(rel_from)
444
abs_to = self._abspath(rel_to)
445
mutter("FTP rename: %s => %s", abs_from, abs_to)
447
return self._rename(abs_from, abs_to, f)
449
def _rename(self, abs_from, abs_to, f):
451
f.rename(abs_from, abs_to)
452
except ftplib.error_perm, e:
453
self._translate_perm_error(e, abs_from,
454
': unable to rename to %r' % (abs_to))
456
def move(self, rel_from, rel_to):
457
"""Move the item at rel_from to the location at rel_to"""
458
abs_from = self._abspath(rel_from)
459
abs_to = self._abspath(rel_to)
461
mutter("FTP mv: %s => %s", abs_from, abs_to)
463
self._rename_and_overwrite(abs_from, abs_to, f)
464
except ftplib.error_perm, e:
465
self._translate_perm_error(e, abs_from,
466
extra='unable to rename to %r' % (rel_to,),
467
unknown_exc=errors.PathError)
469
def _rename_and_overwrite(self, abs_from, abs_to, f):
470
"""Do a fancy rename on the remote server.
472
Using the implementation provided by osutils.
474
osutils.fancy_rename(abs_from, abs_to,
475
rename_func=lambda p1, p2: self._rename(p1, p2, f),
476
unlink_func=lambda p: self._delete(p, f))
478
def delete(self, relpath):
479
"""Delete the item at relpath"""
480
abspath = self._abspath(relpath)
482
self._delete(abspath, f)
484
def _delete(self, abspath, f):
486
mutter("FTP rm: %s", abspath)
488
except ftplib.error_perm, e:
489
self._translate_perm_error(e, abspath, 'error deleting',
490
unknown_exc=errors.NoSuchFile)
493
"""See Transport.listable."""
496
def list_dir(self, relpath):
497
"""See Transport.list_dir."""
498
basepath = self._abspath(relpath)
499
mutter("FTP nlst: %s", basepath)
502
paths = f.nlst(basepath)
503
except ftplib.error_perm, e:
504
self._translate_perm_error(e, relpath, extra='error with list_dir')
505
# If FTP.nlst returns paths prefixed by relpath, strip 'em
506
if paths and paths[0].startswith(basepath):
507
entries = [path[len(basepath)+1:] for path in paths]
510
# Remove . and .. if present
511
return [urlutils.escape(entry) for entry in entries
512
if entry not in ('.', '..')]
514
def iter_files_recursive(self):
515
"""See Transport.iter_files_recursive.
517
This is cargo-culted from the SFTP transport"""
518
mutter("FTP iter_files_recursive")
519
queue = list(self.list_dir("."))
521
relpath = queue.pop(0)
522
st = self.stat(relpath)
523
if stat.S_ISDIR(st.st_mode):
524
for i, basename in enumerate(self.list_dir(relpath)):
525
queue.insert(i, relpath+"/"+basename)
529
def stat(self, relpath):
530
"""Return the stat information for a file."""
531
abspath = self._abspath(relpath)
533
mutter("FTP stat: %s", abspath)
535
return FtpStatResult(f, abspath)
536
except ftplib.error_perm, e:
537
self._translate_perm_error(e, abspath, extra='error w/ stat')
539
def lock_read(self, relpath):
540
"""Lock the given file for shared (read) access.
541
:return: A lock object, which should be passed to Transport.unlock()
543
# The old RemoteBranch ignore lock for reading, so we will
544
# continue that tradition and return a bogus lock object.
545
class BogusLock(object):
546
def __init__(self, path):
550
return BogusLock(relpath)
552
def lock_write(self, relpath):
553
"""Lock the given file for exclusive (write) access.
554
WARNING: many transports do not support this, so trying avoid using it
556
:return: A lock object, which should be passed to Transport.unlock()
558
return self.lock_read(relpath)
561
class FtpServer(Server):
562
"""Common code for SFTP server facilities."""
566
self._ftp_server = None
568
self._async_thread = None
573
"""Calculate an ftp url to this server."""
574
return 'ftp://foo:bar@localhost:%d/' % (self._port)
576
# def get_bogus_url(self):
577
# """Return a URL which cannot be connected to."""
578
# return 'ftp://127.0.0.1:1'
580
def log(self, message):
581
"""This is used by medusa.ftp_server to log connections, etc."""
582
self.logs.append(message)
584
def setUp(self, vfs_server=None):
586
raise RuntimeError('Must have medusa to run the FtpServer')
588
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
589
"FtpServer currently assumes local transport, got %s" % vfs_server
591
self._root = os.getcwdu()
592
self._ftp_server = _ftp_server(
593
authorizer=_test_authorizer(root=self._root),
595
port=0, # bind to a random port
597
logger_object=self # Use FtpServer.log() for messages
599
self._port = self._ftp_server.getsockname()[1]
600
# Don't let it loop forever, or handle an infinite number of requests.
601
# In this case it will run for 100s, or 1000 requests
602
self._async_thread = threading.Thread(
603
target=FtpServer._asyncore_loop_ignore_EBADF,
604
kwargs={'timeout':0.1, 'count':1000})
605
self._async_thread.setDaemon(True)
606
self._async_thread.start()
609
"""See bzrlib.transport.Server.tearDown."""
610
# have asyncore release the channel
611
self._ftp_server.del_channel()
613
self._async_thread.join()
616
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
617
"""Ignore EBADF during server shutdown.
619
We close the socket to get the server to shutdown, but this causes
620
select.select() to raise EBADF.
623
asyncore.loop(*args, **kwargs)
624
except select.error, e:
625
if e.args[0] != errno.EBADF:
631
_test_authorizer = None
635
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
638
import medusa.filesys
639
import medusa.ftp_server
645
class test_authorizer(object):
646
"""A custom Authorizer object for running the test suite.
648
The reason we cannot use dummy_authorizer, is because it sets the
649
channel to readonly, which we don't always want to do.
652
def __init__(self, root):
655
def authorize(self, channel, username, password):
656
"""Return (success, reply_string, filesystem)"""
658
return 0, 'No Medusa.', None
660
channel.persona = -1, -1
661
if username == 'anonymous':
662
channel.read_only = 1
664
channel.read_only = 0
666
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
669
class ftp_channel(medusa.ftp_server.ftp_channel):
670
"""Customized ftp channel"""
672
def log(self, message):
673
"""Redirect logging requests."""
674
mutter('_ftp_channel: %s', message)
676
def log_info(self, message, type='info'):
677
"""Redirect logging requests."""
678
mutter('_ftp_channel %s: %s', type, message)
680
def cmd_rnfr(self, line):
681
"""Prepare for renaming a file."""
682
self._renaming = line[1]
683
self.respond('350 Ready for RNTO')
684
# TODO: jam 20060516 in testing, the ftp server seems to
685
# check that the file already exists, or it sends
686
# 550 RNFR command failed
688
def cmd_rnto(self, line):
689
"""Rename a file based on the target given.
691
rnto must be called after calling rnfr.
693
if not self._renaming:
694
self.respond('503 RNFR required first.')
695
pfrom = self.filesystem.translate(self._renaming)
696
self._renaming = None
697
pto = self.filesystem.translate(line[1])
698
if os.path.exists(pto):
699
self.respond('550 RNTO failed: file exists')
702
os.rename(pfrom, pto)
703
except (IOError, OSError), e:
704
# TODO: jam 20060516 return custom responses based on
705
# why the command failed
706
# (bialix 20070418) str(e) on Python 2.5 @ Windows
707
# sometimes don't provide expected error message;
708
# so we obtain such message via os.strerror()
709
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
711
self.respond('550 RNTO failed')
712
# For a test server, we will go ahead and just die
715
self.respond('250 Rename successful.')
717
def cmd_size(self, line):
718
"""Return the size of a file
720
This is overloaded to help the test suite determine if the
721
target is a directory.
724
if not self.filesystem.isfile(filename):
725
if self.filesystem.isdir(filename):
726
self.respond('550 "%s" is a directory' % (filename,))
728
self.respond('550 "%s" is not a file' % (filename,))
730
self.respond('213 %d'
731
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
733
def cmd_mkd(self, line):
734
"""Create a directory.
736
Overloaded because default implementation does not distinguish
737
*why* it cannot make a directory.
740
self.command_not_understood(''.join(line))
744
self.filesystem.mkdir (path)
745
self.respond ('257 MKD command successful.')
746
except (IOError, OSError), e:
747
# (bialix 20070418) str(e) on Python 2.5 @ Windows
748
# sometimes don't provide expected error message;
749
# so we obtain such message via os.strerror()
750
self.respond ('550 error creating directory: %s' %
751
os.strerror(e.errno))
753
self.respond ('550 error creating directory.')
756
class ftp_server(medusa.ftp_server.ftp_server):
757
"""Customize the behavior of the Medusa ftp_server.
759
There are a few warts on the ftp_server, based on how it expects
763
ftp_channel_class = ftp_channel
765
def __init__(self, *args, **kwargs):
766
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
767
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
769
def log(self, message):
770
"""Redirect logging requests."""
771
mutter('_ftp_server: %s', message)
773
def log_info(self, message, type='info'):
774
"""Override the asyncore.log_info so we don't stipple the screen."""
775
mutter('_ftp_server %s: %s', type, message)
777
_test_authorizer = test_authorizer
778
_ftp_channel = ftp_channel
779
_ftp_server = ftp_server
784
def get_test_permutations():
785
"""Return the permutations to be used in testing."""
786
if not _setup_medusa():
787
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
790
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
