1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
38
from warnings import warn
44
from bzrlib.trace import mutter, warning
45
from bzrlib.transport import (
55
class FtpPathError(errors.PathError):
56
"""FTP failed for path: %(path)s%(extra)s"""
60
def _find_FTP(hostname, port, username, password, is_active):
61
"""Find an ftplib.FTP instance attached to this triplet."""
62
key = (hostname, port, username, password, is_active)
63
alt_key = (hostname, port, username, '********', is_active)
64
if key not in _FTP_cache:
65
mutter("Constructing FTP instance against %r" % (alt_key,))
68
conn.connect(host=hostname, port=port)
69
if username and username != 'anonymous' and not password:
70
password = bzrlib.ui.ui_factory.get_password(
71
prompt='FTP %(user)s@%(host)s password',
72
user=username, host=hostname)
73
conn.login(user=username, passwd=password)
74
conn.set_pasv(not is_active)
76
_FTP_cache[key] = conn
78
return _FTP_cache[key]
81
class FtpStatResult(object):
82
def __init__(self, f, relpath):
84
self.st_size = f.size(relpath)
85
self.st_mode = stat.S_IFREG
86
except ftplib.error_perm:
90
self.st_mode = stat.S_IFDIR
95
_number_of_retries = 2
96
_sleep_between_retries = 5
98
class FtpTransport(Transport):
99
"""This is the transport agent for ftp:// access."""
101
def __init__(self, base, _provided_instance=None):
102
"""Set the base path where files will be stored."""
103
assert base.startswith('ftp://') or base.startswith('aftp://')
105
self.is_active = base.startswith('aftp://')
107
# urlparse won't handle aftp://
109
if not base.endswith('/'):
111
(self._proto, self._username,
112
self._password, self._host,
113
self._port, self._path) = split_url(base)
114
base = self._unparse_url()
116
super(FtpTransport, self).__init__(base)
117
self._FTP_instance = _provided_instance
119
def _unparse_url(self, path=None):
122
path = urllib.quote(path)
123
netloc = urllib.quote(self._host)
124
if self._username is not None:
125
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
126
if self._port is not None:
127
netloc = '%s:%d' % (netloc, self._port)
131
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
134
"""Return the ftplib.FTP instance for this object."""
135
if self._FTP_instance is not None:
136
return self._FTP_instance
139
self._FTP_instance = _find_FTP(self._host, self._port,
140
self._username, self._password,
142
return self._FTP_instance
143
except ftplib.error_perm, e:
144
raise errors.TransportError(msg="Error setting up connection: %s"
145
% str(e), orig_error=e)
147
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
148
"""Try to translate an ftplib.error_perm exception.
150
:param err: The error to translate into a bzr error
151
:param path: The path which had problems
152
:param extra: Extra information which can be included
153
:param unknown_exc: If None, we will just raise the original exception
154
otherwise we raise unknown_exc(path, extra=extra)
160
extra += ': ' + str(err)
161
if ('no such file' in s
162
or 'could not open' in s
163
or 'no such dir' in s
164
or 'could not create file' in s # vsftpd
166
raise errors.NoSuchFile(path, extra=extra)
167
if ('file exists' in s):
168
raise errors.FileExists(path, extra=extra)
169
if ('not a directory' in s):
170
raise errors.PathError(path, extra=extra)
172
mutter('unable to understand error for path: %s: %s', path, err)
175
raise unknown_exc(path, extra=extra)
176
# TODO: jam 20060516 Consider re-raising the error wrapped in
177
# something like TransportError, but this loses the traceback
178
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
179
# to handle. Consider doing something like that here.
180
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
183
def should_cache(self):
184
"""Return True if the data pulled across should be cached locally.
188
def clone(self, offset=None):
189
"""Return a new FtpTransport with root at self.base + offset.
193
return FtpTransport(self.base, self._FTP_instance)
195
return FtpTransport(self.abspath(offset), self._FTP_instance)
197
def _abspath(self, relpath):
198
assert isinstance(relpath, basestring)
199
relpath = urlutils.unescape(relpath)
200
if relpath.startswith('/'):
203
basepath = self._path.split('/')
204
if len(basepath) > 0 and basepath[-1] == '':
205
basepath = basepath[:-1]
206
for p in relpath.split('/'):
208
if len(basepath) == 0:
209
# In most filesystems, a request for the parent
210
# of root, just returns root.
213
elif p == '.' or p == '':
217
# Possibly, we could use urlparse.urljoin() here, but
218
# I'm concerned about when it chooses to strip the last
219
# portion of the path, and when it doesn't.
221
# XXX: It seems that ftplib does not handle Unicode paths
222
# at the same time, medusa won't handle utf8 paths
223
# So if we .encode(utf8) here, then we get a Server failure.
224
# while if we use str(), we get a UnicodeError, and the test suite
225
# just skips testing UnicodePaths.
226
return str('/'.join(basepath) or '/')
228
def abspath(self, relpath):
229
"""Return the full url to the given relative path.
230
This can be supplied with a string or a list
232
path = self._abspath(relpath)
233
return self._unparse_url(path)
235
def has(self, relpath):
236
"""Does the target location exist?"""
237
# FIXME jam 20060516 We *do* ask about directories in the test suite
238
# We don't seem to in the actual codebase
239
# XXX: I assume we're never asked has(dirname) and thus I use
240
# the FTP size command and assume that if it doesn't raise,
242
abspath = self._abspath(relpath)
245
mutter('FTP has check: %s => %s', relpath, abspath)
247
mutter("FTP has: %s", abspath)
249
except ftplib.error_perm, e:
250
if ('is a directory' in str(e).lower()):
251
mutter("FTP has dir: %s: %s", abspath, e)
253
mutter("FTP has not: %s: %s", abspath, e)
256
def get(self, relpath, decode=False, retries=0):
257
"""Get the file at the given relative path.
259
:param relpath: The relative path to the file
260
:param retries: Number of retries after temporary failures so far
263
We're meant to return a file-like object which bzr will
264
then read from. For now we do this via the magic of StringIO
266
# TODO: decode should be deprecated
268
mutter("FTP get: %s", self._abspath(relpath))
271
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
274
except ftplib.error_perm, e:
275
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
276
except ftplib.error_temp, e:
277
if retries > _number_of_retries:
278
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
279
% self.abspath(relpath),
282
warning("FTP temporary error: %s. Retrying.", str(e))
283
self._FTP_instance = None
284
return self.get(relpath, decode, retries+1)
286
if retries > _number_of_retries:
287
raise errors.TransportError("FTP control connection closed during GET %s."
288
% self.abspath(relpath),
291
warning("FTP control connection closed. Trying to reopen.")
292
time.sleep(_sleep_between_retries)
293
self._FTP_instance = None
294
return self.get(relpath, decode, retries+1)
296
def put_file(self, relpath, fp, mode=None, retries=0):
297
"""Copy the file-like or string object into the location.
299
:param relpath: Location to put the contents, relative to base.
300
:param fp: File-like or string object.
301
:param retries: Number of retries after temporary failures so far
304
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
306
abspath = self._abspath(relpath)
307
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
308
os.getpid(), random.randint(0,0x7FFFFFFF))
309
if getattr(fp, 'read', None) is None:
312
mutter("FTP put: %s", abspath)
315
f.storbinary('STOR '+tmp_abspath, fp)
316
f.rename(tmp_abspath, abspath)
317
except (ftplib.error_temp,EOFError), e:
318
warning("Failure during ftp PUT. Deleting temporary file.")
320
f.delete(tmp_abspath)
322
warning("Failed to delete temporary file on the"
323
" server.\nFile: %s", tmp_abspath)
326
except ftplib.error_perm, e:
327
self._translate_perm_error(e, abspath, extra='could not store')
328
except ftplib.error_temp, e:
329
if retries > _number_of_retries:
330
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
331
% self.abspath(relpath), orig_error=e)
333
warning("FTP temporary error: %s. Retrying.", str(e))
334
self._FTP_instance = None
335
self.put_file(relpath, fp, mode, retries+1)
337
if retries > _number_of_retries:
338
raise errors.TransportError("FTP control connection closed during PUT %s."
339
% self.abspath(relpath), orig_error=e)
341
warning("FTP control connection closed. Trying to reopen.")
342
time.sleep(_sleep_between_retries)
343
self._FTP_instance = None
344
self.put_file(relpath, fp, mode, retries+1)
346
def mkdir(self, relpath, mode=None):
347
"""Create a directory at the given path."""
348
abspath = self._abspath(relpath)
350
mutter("FTP mkd: %s", abspath)
353
except ftplib.error_perm, e:
354
self._translate_perm_error(e, abspath,
355
unknown_exc=errors.FileExists)
357
def rmdir(self, rel_path):
358
"""Delete the directory at rel_path"""
359
abspath = self._abspath(rel_path)
361
mutter("FTP rmd: %s", abspath)
364
except ftplib.error_perm, e:
365
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
367
def append_file(self, relpath, f, mode=None):
368
"""Append the text in the file-like object into the final
371
abspath = self._abspath(relpath)
372
if self.has(relpath):
373
ftp = self._get_FTP()
374
result = ftp.size(abspath)
378
mutter("FTP appe to %s", abspath)
379
self._try_append(relpath, f.read(), mode)
383
def _try_append(self, relpath, text, mode=None, retries=0):
384
"""Try repeatedly to append the given text to the file at relpath.
386
This is a recursive function. On errors, it will be called until the
387
number of retries is exceeded.
390
abspath = self._abspath(relpath)
391
mutter("FTP appe (try %d) to %s", retries, abspath)
392
ftp = self._get_FTP()
393
ftp.voidcmd("TYPE I")
394
cmd = "APPE %s" % abspath
395
conn = ftp.transfercmd(cmd)
399
self._setmode(relpath, mode)
401
except ftplib.error_perm, e:
402
self._translate_perm_error(e, abspath, extra='error appending',
403
unknown_exc=errors.NoSuchFile)
404
except ftplib.error_temp, e:
405
if retries > _number_of_retries:
406
raise errors.TransportError("FTP temporary error during APPEND %s." \
407
"Aborting." % abspath, orig_error=e)
409
warning("FTP temporary error: %s. Retrying.", str(e))
410
self._FTP_instance = None
411
self._try_append(relpath, text, mode, retries+1)
413
def _setmode(self, relpath, mode):
414
"""Set permissions on a path.
416
Only set permissions if the FTP server supports the 'SITE CHMOD'
420
mutter("FTP site chmod: setting permissions to %s on %s",
421
str(mode), self._abspath(relpath))
422
ftp = self._get_FTP()
423
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
425
except ftplib.error_perm, e:
426
# Command probably not available on this server
427
warning("FTP Could not set permissions to %s on %s. %s",
428
str(mode), self._abspath(relpath), str(e))
430
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
431
# to copy something to another machine. And you may be able
432
# to give it its own address as the 'to' location.
433
# So implement a fancier 'copy()'
435
def move(self, rel_from, rel_to):
436
"""Move the item at rel_from to the location at rel_to"""
437
abs_from = self._abspath(rel_from)
438
abs_to = self._abspath(rel_to)
440
mutter("FTP mv: %s => %s", abs_from, abs_to)
442
f.rename(abs_from, abs_to)
443
except ftplib.error_perm, e:
444
self._translate_perm_error(e, abs_from,
445
extra='unable to rename to %r' % (rel_to,),
446
unknown_exc=errors.PathError)
450
def delete(self, relpath):
451
"""Delete the item at relpath"""
452
abspath = self._abspath(relpath)
454
mutter("FTP rm: %s", abspath)
457
except ftplib.error_perm, e:
458
self._translate_perm_error(e, abspath, 'error deleting',
459
unknown_exc=errors.NoSuchFile)
462
"""See Transport.listable."""
465
def list_dir(self, relpath):
466
"""See Transport.list_dir."""
467
basepath = self._abspath(relpath)
468
mutter("FTP nlst: %s", basepath)
471
paths = f.nlst(basepath)
472
except ftplib.error_perm, e:
473
self._translate_perm_error(e, relpath, extra='error with list_dir')
474
# If FTP.nlst returns paths prefixed by relpath, strip 'em
475
if paths and paths[0].startswith(basepath):
476
entries = [path[len(basepath)+1:] for path in paths]
479
# Remove . and .. if present
480
return [urlutils.escape(entry) for entry in entries
481
if entry not in ('.', '..')]
483
def iter_files_recursive(self):
484
"""See Transport.iter_files_recursive.
486
This is cargo-culted from the SFTP transport"""
487
mutter("FTP iter_files_recursive")
488
queue = list(self.list_dir("."))
490
relpath = queue.pop(0)
491
st = self.stat(relpath)
492
if stat.S_ISDIR(st.st_mode):
493
for i, basename in enumerate(self.list_dir(relpath)):
494
queue.insert(i, relpath+"/"+basename)
498
def stat(self, relpath):
499
"""Return the stat information for a file."""
500
abspath = self._abspath(relpath)
502
mutter("FTP stat: %s", abspath)
504
return FtpStatResult(f, abspath)
505
except ftplib.error_perm, e:
506
self._translate_perm_error(e, abspath, extra='error w/ stat')
508
def lock_read(self, relpath):
509
"""Lock the given file for shared (read) access.
510
:return: A lock object, which should be passed to Transport.unlock()
512
# The old RemoteBranch ignore lock for reading, so we will
513
# continue that tradition and return a bogus lock object.
514
class BogusLock(object):
515
def __init__(self, path):
519
return BogusLock(relpath)
521
def lock_write(self, relpath):
522
"""Lock the given file for exclusive (write) access.
523
WARNING: many transports do not support this, so trying avoid using it
525
:return: A lock object, which should be passed to Transport.unlock()
527
return self.lock_read(relpath)
530
class FtpServer(Server):
531
"""Common code for SFTP server facilities."""
535
self._ftp_server = None
537
self._async_thread = None
542
"""Calculate an ftp url to this server."""
543
return 'ftp://foo:bar@localhost:%d/' % (self._port)
545
# def get_bogus_url(self):
546
# """Return a URL which cannot be connected to."""
547
# return 'ftp://127.0.0.1:1'
549
def log(self, message):
550
"""This is used by medusa.ftp_server to log connections, etc."""
551
self.logs.append(message)
556
raise RuntimeError('Must have medusa to run the FtpServer')
558
self._root = os.getcwdu()
559
self._ftp_server = _ftp_server(
560
authorizer=_test_authorizer(root=self._root),
562
port=0, # bind to a random port
564
logger_object=self # Use FtpServer.log() for messages
566
self._port = self._ftp_server.getsockname()[1]
567
# Don't let it loop forever, or handle an infinite number of requests.
568
# In this case it will run for 100s, or 1000 requests
569
self._async_thread = threading.Thread(target=asyncore.loop,
570
kwargs={'timeout':0.1, 'count':1000})
571
self._async_thread.setDaemon(True)
572
self._async_thread.start()
575
"""See bzrlib.transport.Server.tearDown."""
576
# have asyncore release the channel
577
self._ftp_server.del_channel()
579
self._async_thread.join()
584
_test_authorizer = None
588
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
591
import medusa.filesys
592
import medusa.ftp_server
598
class test_authorizer(object):
599
"""A custom Authorizer object for running the test suite.
601
The reason we cannot use dummy_authorizer, is because it sets the
602
channel to readonly, which we don't always want to do.
605
def __init__(self, root):
608
def authorize(self, channel, username, password):
609
"""Return (success, reply_string, filesystem)"""
611
return 0, 'No Medusa.', None
613
channel.persona = -1, -1
614
if username == 'anonymous':
615
channel.read_only = 1
617
channel.read_only = 0
619
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
622
class ftp_channel(medusa.ftp_server.ftp_channel):
623
"""Customized ftp channel"""
625
def log(self, message):
626
"""Redirect logging requests."""
627
mutter('_ftp_channel: %s', message)
629
def log_info(self, message, type='info'):
630
"""Redirect logging requests."""
631
mutter('_ftp_channel %s: %s', type, message)
633
def cmd_rnfr(self, line):
634
"""Prepare for renaming a file."""
635
self._renaming = line[1]
636
self.respond('350 Ready for RNTO')
637
# TODO: jam 20060516 in testing, the ftp server seems to
638
# check that the file already exists, or it sends
639
# 550 RNFR command failed
641
def cmd_rnto(self, line):
642
"""Rename a file based on the target given.
644
rnto must be called after calling rnfr.
646
if not self._renaming:
647
self.respond('503 RNFR required first.')
648
pfrom = self.filesystem.translate(self._renaming)
649
self._renaming = None
650
pto = self.filesystem.translate(line[1])
652
os.rename(pfrom, pto)
653
except (IOError, OSError), e:
654
# TODO: jam 20060516 return custom responses based on
655
# why the command failed
656
self.respond('550 RNTO failed: %s' % (e,))
658
self.respond('550 RNTO failed')
659
# For a test server, we will go ahead and just die
662
self.respond('250 Rename successful.')
664
def cmd_size(self, line):
665
"""Return the size of a file
667
This is overloaded to help the test suite determine if the
668
target is a directory.
671
if not self.filesystem.isfile(filename):
672
if self.filesystem.isdir(filename):
673
self.respond('550 "%s" is a directory' % (filename,))
675
self.respond('550 "%s" is not a file' % (filename,))
677
self.respond('213 %d'
678
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
680
def cmd_mkd(self, line):
681
"""Create a directory.
683
Overloaded because default implementation does not distinguish
684
*why* it cannot make a directory.
687
self.command_not_understood(''.join(line))
691
self.filesystem.mkdir (path)
692
self.respond ('257 MKD command successful.')
693
except (IOError, OSError), e:
694
self.respond ('550 error creating directory: %s' % (e,))
696
self.respond ('550 error creating directory.')
699
class ftp_server(medusa.ftp_server.ftp_server):
700
"""Customize the behavior of the Medusa ftp_server.
702
There are a few warts on the ftp_server, based on how it expects
706
ftp_channel_class = ftp_channel
708
def __init__(self, *args, **kwargs):
709
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
710
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
712
def log(self, message):
713
"""Redirect logging requests."""
714
mutter('_ftp_server: %s', message)
716
def log_info(self, message, type='info'):
717
"""Override the asyncore.log_info so we don't stipple the screen."""
718
mutter('_ftp_server %s: %s', type, message)
720
_test_authorizer = test_authorizer
721
_ftp_channel = ftp_channel
722
_ftp_server = ftp_server
727
def get_test_permutations():
728
"""Return the permutations to be used in testing."""
729
if not _setup_medusa():
730
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
733
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
