1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
40
from warnings import warn
47
from bzrlib.trace import mutter, warning
48
from bzrlib.transport import (
53
from bzrlib.transport.local import LocalURLServer
59
class FtpPathError(errors.PathError):
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
85
class FtpStatResult(object):
86
def __init__(self, f, relpath):
88
self.st_size = f.size(relpath)
89
self.st_mode = stat.S_IFREG
90
except ftplib.error_perm:
94
self.st_mode = stat.S_IFDIR
99
_number_of_retries = 2
100
_sleep_between_retries = 5
102
class FtpTransport(Transport):
103
"""This is the transport agent for ftp:// access."""
105
def __init__(self, base, _provided_instance=None):
106
"""Set the base path where files will be stored."""
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://, delete the leading 'a'
112
# FIXME: This breaks even hopes of connection sharing
113
# (by reusing the url instead of true cloning) by
114
# modifying the the url coming from the user.
116
if not base.endswith('/'):
118
(self._proto, self._username,
119
self._password, self._host,
120
self._port, self._path) = split_url(base)
121
base = self._unparse_url()
123
super(FtpTransport, self).__init__(base)
124
self._FTP_instance = _provided_instance
126
def _unparse_url(self, path=None):
129
path = urllib.quote(path)
130
netloc = urllib.quote(self._host)
131
if self._username is not None:
132
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
133
if self._port is not None:
134
netloc = '%s:%d' % (netloc, self._port)
138
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
141
"""Return the ftplib.FTP instance for this object."""
142
if self._FTP_instance is not None:
143
return self._FTP_instance
146
self._FTP_instance = _find_FTP(self._host, self._port,
147
self._username, self._password,
149
return self._FTP_instance
150
except ftplib.error_perm, e:
151
raise errors.TransportError(msg="Error setting up connection: %s"
152
% str(e), orig_error=e)
154
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
155
"""Try to translate an ftplib.error_perm exception.
157
:param err: The error to translate into a bzr error
158
:param path: The path which had problems
159
:param extra: Extra information which can be included
160
:param unknown_exc: If None, we will just raise the original exception
161
otherwise we raise unknown_exc(path, extra=extra)
167
extra += ': ' + str(err)
168
if ('no such file' in s
169
or 'could not open' in s
170
or 'no such dir' in s
171
or 'could not create file' in s # vsftpd
173
raise errors.NoSuchFile(path, extra=extra)
174
if ('file exists' in s):
175
raise errors.FileExists(path, extra=extra)
176
if ('not a directory' in s):
177
raise errors.PathError(path, extra=extra)
179
mutter('unable to understand error for path: %s: %s', path, err)
182
raise unknown_exc(path, extra=extra)
183
# TODO: jam 20060516 Consider re-raising the error wrapped in
184
# something like TransportError, but this loses the traceback
185
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
186
# to handle. Consider doing something like that here.
187
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
190
def should_cache(self):
191
"""Return True if the data pulled across should be cached locally.
195
def clone(self, offset=None):
196
"""Return a new FtpTransport with root at self.base + offset.
200
return FtpTransport(self.base, self._FTP_instance)
202
return FtpTransport(self.abspath(offset), self._FTP_instance)
204
def _abspath(self, relpath):
205
assert isinstance(relpath, basestring)
206
relpath = urlutils.unescape(relpath)
207
if relpath.startswith('/'):
210
basepath = self._path.split('/')
211
if len(basepath) > 0 and basepath[-1] == '':
212
basepath = basepath[:-1]
213
for p in relpath.split('/'):
215
if len(basepath) == 0:
216
# In most filesystems, a request for the parent
217
# of root, just returns root.
220
elif p == '.' or p == '':
224
# Possibly, we could use urlparse.urljoin() here, but
225
# I'm concerned about when it chooses to strip the last
226
# portion of the path, and when it doesn't.
228
# XXX: It seems that ftplib does not handle Unicode paths
229
# at the same time, medusa won't handle utf8 paths
230
# So if we .encode(utf8) here, then we get a Server failure.
231
# while if we use str(), we get a UnicodeError, and the test suite
232
# just skips testing UnicodePaths.
233
return str('/'.join(basepath) or '/')
235
def abspath(self, relpath):
236
"""Return the full url to the given relative path.
237
This can be supplied with a string or a list
239
path = self._abspath(relpath)
240
return self._unparse_url(path)
242
def has(self, relpath):
243
"""Does the target location exist?"""
244
# FIXME jam 20060516 We *do* ask about directories in the test suite
245
# We don't seem to in the actual codebase
246
# XXX: I assume we're never asked has(dirname) and thus I use
247
# the FTP size command and assume that if it doesn't raise,
249
abspath = self._abspath(relpath)
252
mutter('FTP has check: %s => %s', relpath, abspath)
254
mutter("FTP has: %s", abspath)
256
except ftplib.error_perm, e:
257
if ('is a directory' in str(e).lower()):
258
mutter("FTP has dir: %s: %s", abspath, e)
260
mutter("FTP has not: %s: %s", abspath, e)
263
def get(self, relpath, decode=False, retries=0):
264
"""Get the file at the given relative path.
266
:param relpath: The relative path to the file
267
:param retries: Number of retries after temporary failures so far
270
We're meant to return a file-like object which bzr will
271
then read from. For now we do this via the magic of StringIO
273
# TODO: decode should be deprecated
275
mutter("FTP get: %s", self._abspath(relpath))
278
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
281
except ftplib.error_perm, e:
282
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
283
except ftplib.error_temp, e:
284
if retries > _number_of_retries:
285
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
286
% self.abspath(relpath),
289
warning("FTP temporary error: %s. Retrying.", str(e))
290
self._FTP_instance = None
291
return self.get(relpath, decode, retries+1)
293
if retries > _number_of_retries:
294
raise errors.TransportError("FTP control connection closed during GET %s."
295
% self.abspath(relpath),
298
warning("FTP control connection closed. Trying to reopen.")
299
time.sleep(_sleep_between_retries)
300
self._FTP_instance = None
301
return self.get(relpath, decode, retries+1)
303
def put_file(self, relpath, fp, mode=None, retries=0):
304
"""Copy the file-like or string object into the location.
306
:param relpath: Location to put the contents, relative to base.
307
:param fp: File-like or string object.
308
:param retries: Number of retries after temporary failures so far
311
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
314
abspath = self._abspath(relpath)
315
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
316
os.getpid(), random.randint(0,0x7FFFFFFF))
317
if getattr(fp, 'read', None) is None:
320
mutter("FTP put: %s", abspath)
323
f.storbinary('STOR '+tmp_abspath, fp)
324
self._rename_and_overwrite(tmp_abspath, abspath, f)
325
except (ftplib.error_temp,EOFError), e:
326
warning("Failure during ftp PUT. Deleting temporary file.")
328
f.delete(tmp_abspath)
330
warning("Failed to delete temporary file on the"
331
" server.\nFile: %s", tmp_abspath)
334
except ftplib.error_perm, e:
335
self._translate_perm_error(e, abspath, extra='could not store')
336
except ftplib.error_temp, e:
337
if retries > _number_of_retries:
338
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
339
% self.abspath(relpath), orig_error=e)
341
warning("FTP temporary error: %s. Retrying.", str(e))
342
self._FTP_instance = None
343
self.put_file(relpath, fp, mode, retries+1)
345
if retries > _number_of_retries:
346
raise errors.TransportError("FTP control connection closed during PUT %s."
347
% self.abspath(relpath), orig_error=e)
349
warning("FTP control connection closed. Trying to reopen.")
350
time.sleep(_sleep_between_retries)
351
self._FTP_instance = None
352
self.put_file(relpath, fp, mode, retries+1)
354
def mkdir(self, relpath, mode=None):
355
"""Create a directory at the given path."""
356
abspath = self._abspath(relpath)
358
mutter("FTP mkd: %s", abspath)
361
except ftplib.error_perm, e:
362
self._translate_perm_error(e, abspath,
363
unknown_exc=errors.FileExists)
365
def rmdir(self, rel_path):
366
"""Delete the directory at rel_path"""
367
abspath = self._abspath(rel_path)
369
mutter("FTP rmd: %s", abspath)
372
except ftplib.error_perm, e:
373
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
375
def append_file(self, relpath, f, mode=None):
376
"""Append the text in the file-like object into the final
379
abspath = self._abspath(relpath)
380
if self.has(relpath):
381
ftp = self._get_FTP()
382
result = ftp.size(abspath)
386
mutter("FTP appe to %s", abspath)
387
self._try_append(relpath, f.read(), mode)
391
def _try_append(self, relpath, text, mode=None, retries=0):
392
"""Try repeatedly to append the given text to the file at relpath.
394
This is a recursive function. On errors, it will be called until the
395
number of retries is exceeded.
398
abspath = self._abspath(relpath)
399
mutter("FTP appe (try %d) to %s", retries, abspath)
400
ftp = self._get_FTP()
401
ftp.voidcmd("TYPE I")
402
cmd = "APPE %s" % abspath
403
conn = ftp.transfercmd(cmd)
407
self._setmode(relpath, mode)
409
except ftplib.error_perm, e:
410
self._translate_perm_error(e, abspath, extra='error appending',
411
unknown_exc=errors.NoSuchFile)
412
except ftplib.error_temp, e:
413
if retries > _number_of_retries:
414
raise errors.TransportError("FTP temporary error during APPEND %s." \
415
"Aborting." % abspath, orig_error=e)
417
warning("FTP temporary error: %s. Retrying.", str(e))
418
self._FTP_instance = None
419
self._try_append(relpath, text, mode, retries+1)
421
def _setmode(self, relpath, mode):
422
"""Set permissions on a path.
424
Only set permissions if the FTP server supports the 'SITE CHMOD'
428
mutter("FTP site chmod: setting permissions to %s on %s",
429
str(mode), self._abspath(relpath))
430
ftp = self._get_FTP()
431
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
433
except ftplib.error_perm, e:
434
# Command probably not available on this server
435
warning("FTP Could not set permissions to %s on %s. %s",
436
str(mode), self._abspath(relpath), str(e))
438
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
439
# to copy something to another machine. And you may be able
440
# to give it its own address as the 'to' location.
441
# So implement a fancier 'copy()'
443
def rename(self, rel_from, rel_to):
444
abs_from = self._abspath(rel_from)
445
abs_to = self._abspath(rel_to)
446
mutter("FTP rename: %s => %s", abs_from, abs_to)
448
return self._rename(abs_from, abs_to, f)
450
def _rename(self, abs_from, abs_to, f):
452
f.rename(abs_from, abs_to)
453
except ftplib.error_perm, e:
454
self._translate_perm_error(e, abs_from,
455
': unable to rename to %r' % (abs_to))
457
def move(self, rel_from, rel_to):
458
"""Move the item at rel_from to the location at rel_to"""
459
abs_from = self._abspath(rel_from)
460
abs_to = self._abspath(rel_to)
462
mutter("FTP mv: %s => %s", abs_from, abs_to)
464
self._rename_and_overwrite(abs_from, abs_to, f)
465
except ftplib.error_perm, e:
466
self._translate_perm_error(e, abs_from,
467
extra='unable to rename to %r' % (rel_to,),
468
unknown_exc=errors.PathError)
470
def _rename_and_overwrite(self, abs_from, abs_to, f):
471
"""Do a fancy rename on the remote server.
473
Using the implementation provided by osutils.
475
osutils.fancy_rename(abs_from, abs_to,
476
rename_func=lambda p1, p2: self._rename(p1, p2, f),
477
unlink_func=lambda p: self._delete(p, f))
479
def delete(self, relpath):
480
"""Delete the item at relpath"""
481
abspath = self._abspath(relpath)
483
self._delete(abspath, f)
485
def _delete(self, abspath, f):
487
mutter("FTP rm: %s", abspath)
489
except ftplib.error_perm, e:
490
self._translate_perm_error(e, abspath, 'error deleting',
491
unknown_exc=errors.NoSuchFile)
494
"""See Transport.listable."""
497
def list_dir(self, relpath):
498
"""See Transport.list_dir."""
499
basepath = self._abspath(relpath)
500
mutter("FTP nlst: %s", basepath)
503
paths = f.nlst(basepath)
504
except ftplib.error_perm, e:
505
self._translate_perm_error(e, relpath, extra='error with list_dir')
506
# If FTP.nlst returns paths prefixed by relpath, strip 'em
507
if paths and paths[0].startswith(basepath):
508
entries = [path[len(basepath)+1:] for path in paths]
511
# Remove . and .. if present
512
return [urlutils.escape(entry) for entry in entries
513
if entry not in ('.', '..')]
515
def iter_files_recursive(self):
516
"""See Transport.iter_files_recursive.
518
This is cargo-culted from the SFTP transport"""
519
mutter("FTP iter_files_recursive")
520
queue = list(self.list_dir("."))
522
relpath = queue.pop(0)
523
st = self.stat(relpath)
524
if stat.S_ISDIR(st.st_mode):
525
for i, basename in enumerate(self.list_dir(relpath)):
526
queue.insert(i, relpath+"/"+basename)
530
def stat(self, relpath):
531
"""Return the stat information for a file."""
532
abspath = self._abspath(relpath)
534
mutter("FTP stat: %s", abspath)
536
return FtpStatResult(f, abspath)
537
except ftplib.error_perm, e:
538
self._translate_perm_error(e, abspath, extra='error w/ stat')
540
def lock_read(self, relpath):
541
"""Lock the given file for shared (read) access.
542
:return: A lock object, which should be passed to Transport.unlock()
544
# The old RemoteBranch ignore lock for reading, so we will
545
# continue that tradition and return a bogus lock object.
546
class BogusLock(object):
547
def __init__(self, path):
551
return BogusLock(relpath)
553
def lock_write(self, relpath):
554
"""Lock the given file for exclusive (write) access.
555
WARNING: many transports do not support this, so trying avoid using it
557
:return: A lock object, which should be passed to Transport.unlock()
559
return self.lock_read(relpath)
562
class FtpServer(Server):
563
"""Common code for SFTP server facilities."""
567
self._ftp_server = None
569
self._async_thread = None
574
"""Calculate an ftp url to this server."""
575
return 'ftp://foo:bar@localhost:%d/' % (self._port)
577
# def get_bogus_url(self):
578
# """Return a URL which cannot be connected to."""
579
# return 'ftp://127.0.0.1:1'
581
def log(self, message):
582
"""This is used by medusa.ftp_server to log connections, etc."""
583
self.logs.append(message)
585
def setUp(self, vfs_server=None):
587
raise RuntimeError('Must have medusa to run the FtpServer')
589
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
590
"FtpServer currently assumes local transport, got %s" % vfs_server
592
self._root = os.getcwdu()
593
self._ftp_server = _ftp_server(
594
authorizer=_test_authorizer(root=self._root),
596
port=0, # bind to a random port
598
logger_object=self # Use FtpServer.log() for messages
600
self._port = self._ftp_server.getsockname()[1]
601
# Don't let it loop forever, or handle an infinite number of requests.
602
# In this case it will run for 100s, or 1000 requests
603
self._async_thread = threading.Thread(
604
target=FtpServer._asyncore_loop_ignore_EBADF,
605
kwargs={'timeout':0.1, 'count':10000})
606
self._async_thread.setDaemon(True)
607
self._async_thread.start()
610
"""See bzrlib.transport.Server.tearDown."""
611
# have asyncore release the channel
612
self._ftp_server.del_channel()
614
self._async_thread.join()
617
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
618
"""Ignore EBADF during server shutdown.
620
We close the socket to get the server to shutdown, but this causes
621
select.select() to raise EBADF.
624
asyncore.loop(*args, **kwargs)
625
# FIXME: If we reach that point, we should raise an
626
# exception explaining that the 'count' parameter in
627
# setUp is too low or testers may wonder why their
628
# test just sits there waiting for a server that is
630
except select.error, e:
631
if e.args[0] != errno.EBADF:
637
_test_authorizer = None
641
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
644
import medusa.filesys
645
import medusa.ftp_server
651
class test_authorizer(object):
652
"""A custom Authorizer object for running the test suite.
654
The reason we cannot use dummy_authorizer, is because it sets the
655
channel to readonly, which we don't always want to do.
658
def __init__(self, root):
661
def authorize(self, channel, username, password):
662
"""Return (success, reply_string, filesystem)"""
664
return 0, 'No Medusa.', None
666
channel.persona = -1, -1
667
if username == 'anonymous':
668
channel.read_only = 1
670
channel.read_only = 0
672
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
675
class ftp_channel(medusa.ftp_server.ftp_channel):
676
"""Customized ftp channel"""
678
def log(self, message):
679
"""Redirect logging requests."""
680
mutter('_ftp_channel: %s', message)
682
def log_info(self, message, type='info'):
683
"""Redirect logging requests."""
684
mutter('_ftp_channel %s: %s', type, message)
686
def cmd_rnfr(self, line):
687
"""Prepare for renaming a file."""
688
self._renaming = line[1]
689
self.respond('350 Ready for RNTO')
690
# TODO: jam 20060516 in testing, the ftp server seems to
691
# check that the file already exists, or it sends
692
# 550 RNFR command failed
694
def cmd_rnto(self, line):
695
"""Rename a file based on the target given.
697
rnto must be called after calling rnfr.
699
if not self._renaming:
700
self.respond('503 RNFR required first.')
701
pfrom = self.filesystem.translate(self._renaming)
702
self._renaming = None
703
pto = self.filesystem.translate(line[1])
704
if os.path.exists(pto):
705
self.respond('550 RNTO failed: file exists')
708
os.rename(pfrom, pto)
709
except (IOError, OSError), e:
710
# TODO: jam 20060516 return custom responses based on
711
# why the command failed
712
# (bialix 20070418) str(e) on Python 2.5 @ Windows
713
# sometimes don't provide expected error message;
714
# so we obtain such message via os.strerror()
715
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
717
self.respond('550 RNTO failed')
718
# For a test server, we will go ahead and just die
721
self.respond('250 Rename successful.')
723
def cmd_size(self, line):
724
"""Return the size of a file
726
This is overloaded to help the test suite determine if the
727
target is a directory.
730
if not self.filesystem.isfile(filename):
731
if self.filesystem.isdir(filename):
732
self.respond('550 "%s" is a directory' % (filename,))
734
self.respond('550 "%s" is not a file' % (filename,))
736
self.respond('213 %d'
737
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
739
def cmd_mkd(self, line):
740
"""Create a directory.
742
Overloaded because default implementation does not distinguish
743
*why* it cannot make a directory.
746
self.command_not_understood(''.join(line))
750
self.filesystem.mkdir (path)
751
self.respond ('257 MKD command successful.')
752
except (IOError, OSError), e:
753
# (bialix 20070418) str(e) on Python 2.5 @ Windows
754
# sometimes don't provide expected error message;
755
# so we obtain such message via os.strerror()
756
self.respond ('550 error creating directory: %s' %
757
os.strerror(e.errno))
759
self.respond ('550 error creating directory.')
762
class ftp_server(medusa.ftp_server.ftp_server):
763
"""Customize the behavior of the Medusa ftp_server.
765
There are a few warts on the ftp_server, based on how it expects
769
ftp_channel_class = ftp_channel
771
def __init__(self, *args, **kwargs):
772
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
773
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
775
def log(self, message):
776
"""Redirect logging requests."""
777
mutter('_ftp_server: %s', message)
779
def log_info(self, message, type='info'):
780
"""Override the asyncore.log_info so we don't stipple the screen."""
781
mutter('_ftp_server %s: %s', type, message)
783
_test_authorizer = test_authorizer
784
_ftp_channel = ftp_channel
785
_ftp_server = ftp_server
790
def get_test_permutations():
791
"""Return the permutations to be used in testing."""
792
if not _setup_medusa():
793
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
796
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
