1
# Copyright (C) 2005 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
38
from warnings import warn
44
from bzrlib.trace import mutter, warning
45
from bzrlib.transport import (
55
class FtpPathError(errors.PathError):
56
"""FTP failed for path: %(path)s%(extra)s"""
60
def _find_FTP(hostname, port, username, password, is_active):
61
"""Find an ftplib.FTP instance attached to this triplet."""
62
key = (hostname, port, username, password, is_active)
63
alt_key = (hostname, port, username, '********', is_active)
64
if key not in _FTP_cache:
65
mutter("Constructing FTP instance against %r" % (alt_key,))
68
conn.connect(host=hostname, port=port)
69
if username and username != 'anonymous' and not password:
70
password = bzrlib.ui.ui_factory.get_password(
71
prompt='FTP %(user)s@%(host)s password',
72
user=username, host=hostname)
73
conn.login(user=username, passwd=password)
74
conn.set_pasv(not is_active)
76
_FTP_cache[key] = conn
78
return _FTP_cache[key]
81
class FtpStatResult(object):
82
def __init__(self, f, relpath):
84
self.st_size = f.size(relpath)
85
self.st_mode = stat.S_IFREG
86
except ftplib.error_perm:
90
self.st_mode = stat.S_IFDIR
95
_number_of_retries = 2
96
_sleep_between_retries = 5
98
class FtpTransport(Transport):
99
"""This is the transport agent for ftp:// access."""
101
def __init__(self, base, _provided_instance=None):
102
"""Set the base path where files will be stored."""
103
assert base.startswith('ftp://') or base.startswith('aftp://')
105
self.is_active = base.startswith('aftp://')
107
# urlparse won't handle aftp://
109
if not base.endswith('/'):
111
(self._proto, self._username,
112
self._password, self._host,
113
self._port, self._path) = split_url(base)
114
base = self._unparse_url()
116
super(FtpTransport, self).__init__(base)
117
self._FTP_instance = _provided_instance
119
def _unparse_url(self, path=None):
122
path = urllib.quote(path)
123
netloc = urllib.quote(self._host)
124
if self._username is not None:
125
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
126
if self._port is not None:
127
netloc = '%s:%d' % (netloc, self._port)
131
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
134
"""Return the ftplib.FTP instance for this object."""
135
if self._FTP_instance is not None:
136
return self._FTP_instance
139
self._FTP_instance = _find_FTP(self._host, self._port,
140
self._username, self._password,
142
return self._FTP_instance
143
except ftplib.error_perm, e:
144
raise errors.TransportError(msg="Error setting up connection: %s"
145
% str(e), orig_error=e)
147
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
148
"""Try to translate an ftplib.error_perm exception.
150
:param err: The error to translate into a bzr error
151
:param path: The path which had problems
152
:param extra: Extra information which can be included
153
:param unknown_exc: If None, we will just raise the original exception
154
otherwise we raise unknown_exc(path, extra=extra)
160
extra += ': ' + str(err)
161
if ('no such file' in s
162
or 'could not open' in s
163
or 'no such dir' in s
165
raise errors.NoSuchFile(path, extra=extra)
166
if ('file exists' in s):
167
raise errors.FileExists(path, extra=extra)
168
if ('not a directory' in s):
169
raise errors.PathError(path, extra=extra)
171
mutter('unable to understand error for path: %s: %s', path, err)
174
raise unknown_exc(path, extra=extra)
175
# TODO: jam 20060516 Consider re-raising the error wrapped in
176
# something like TransportError, but this loses the traceback
177
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
178
# to handle. Consider doing something like that here.
179
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
182
def should_cache(self):
183
"""Return True if the data pulled across should be cached locally.
187
def clone(self, offset=None):
188
"""Return a new FtpTransport with root at self.base + offset.
192
return FtpTransport(self.base, self._FTP_instance)
194
return FtpTransport(self.abspath(offset), self._FTP_instance)
196
def _abspath(self, relpath):
197
assert isinstance(relpath, basestring)
198
relpath = urlutils.unescape(relpath)
199
if relpath.startswith('/'):
202
basepath = self._path.split('/')
203
if len(basepath) > 0 and basepath[-1] == '':
204
basepath = basepath[:-1]
205
for p in relpath.split('/'):
207
if len(basepath) == 0:
208
# In most filesystems, a request for the parent
209
# of root, just returns root.
212
elif p == '.' or p == '':
216
# Possibly, we could use urlparse.urljoin() here, but
217
# I'm concerned about when it chooses to strip the last
218
# portion of the path, and when it doesn't.
220
# XXX: It seems that ftplib does not handle Unicode paths
221
# at the same time, medusa won't handle utf8 paths
222
# So if we .encode(utf8) here, then we get a Server failure.
223
# while if we use str(), we get a UnicodeError, and the test suite
224
# just skips testing UnicodePaths.
225
return str('/'.join(basepath) or '/')
227
def abspath(self, relpath):
228
"""Return the full url to the given relative path.
229
This can be supplied with a string or a list
231
path = self._abspath(relpath)
232
return self._unparse_url(path)
234
def has(self, relpath):
235
"""Does the target location exist?"""
236
# FIXME jam 20060516 We *do* ask about directories in the test suite
237
# We don't seem to in the actual codebase
238
# XXX: I assume we're never asked has(dirname) and thus I use
239
# the FTP size command and assume that if it doesn't raise,
241
abspath = self._abspath(relpath)
244
mutter('FTP has check: %s => %s', relpath, abspath)
246
mutter("FTP has: %s", abspath)
248
except ftplib.error_perm, e:
249
if ('is a directory' in str(e).lower()):
250
mutter("FTP has dir: %s: %s", abspath, e)
252
mutter("FTP has not: %s: %s", abspath, e)
255
def get(self, relpath, decode=False, retries=0):
256
"""Get the file at the given relative path.
258
:param relpath: The relative path to the file
259
:param retries: Number of retries after temporary failures so far
262
We're meant to return a file-like object which bzr will
263
then read from. For now we do this via the magic of StringIO
265
# TODO: decode should be deprecated
267
mutter("FTP get: %s", self._abspath(relpath))
270
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
273
except ftplib.error_perm, e:
274
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
275
except ftplib.error_temp, e:
276
if retries > _number_of_retries:
277
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
278
% self.abspath(relpath),
281
warning("FTP temporary error: %s. Retrying.", str(e))
282
self._FTP_instance = None
283
return self.get(relpath, decode, retries+1)
285
if retries > _number_of_retries:
286
raise errors.TransportError("FTP control connection closed during GET %s."
287
% self.abspath(relpath),
290
warning("FTP control connection closed. Trying to reopen.")
291
time.sleep(_sleep_between_retries)
292
self._FTP_instance = None
293
return self.get(relpath, decode, retries+1)
295
def put_file(self, relpath, fp, mode=None, retries=0):
296
"""Copy the file-like or string object into the location.
298
:param relpath: Location to put the contents, relative to base.
299
:param fp: File-like or string object.
300
:param retries: Number of retries after temporary failures so far
303
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
305
abspath = self._abspath(relpath)
306
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
307
os.getpid(), random.randint(0,0x7FFFFFFF))
308
if getattr(fp, 'read', None) is None:
311
mutter("FTP put: %s", abspath)
314
f.storbinary('STOR '+tmp_abspath, fp)
315
f.rename(tmp_abspath, abspath)
316
except (ftplib.error_temp,EOFError), e:
317
warning("Failure during ftp PUT. Deleting temporary file.")
319
f.delete(tmp_abspath)
321
warning("Failed to delete temporary file on the"
322
" server.\nFile: %s", tmp_abspath)
325
except ftplib.error_perm, e:
326
self._translate_perm_error(e, abspath, extra='could not store')
327
except ftplib.error_temp, e:
328
if retries > _number_of_retries:
329
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
330
% self.abspath(relpath), orig_error=e)
332
warning("FTP temporary error: %s. Retrying.", str(e))
333
self._FTP_instance = None
334
self.put_file(relpath, fp, mode, retries+1)
336
if retries > _number_of_retries:
337
raise errors.TransportError("FTP control connection closed during PUT %s."
338
% self.abspath(relpath), orig_error=e)
340
warning("FTP control connection closed. Trying to reopen.")
341
time.sleep(_sleep_between_retries)
342
self._FTP_instance = None
343
self.put_file(relpath, fp, mode, retries+1)
345
def mkdir(self, relpath, mode=None):
346
"""Create a directory at the given path."""
347
abspath = self._abspath(relpath)
349
mutter("FTP mkd: %s", abspath)
352
except ftplib.error_perm, e:
353
self._translate_perm_error(e, abspath,
354
unknown_exc=errors.FileExists)
356
def rmdir(self, rel_path):
357
"""Delete the directory at rel_path"""
358
abspath = self._abspath(rel_path)
360
mutter("FTP rmd: %s", abspath)
363
except ftplib.error_perm, e:
364
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
366
def append_file(self, relpath, f, mode=None):
367
"""Append the text in the file-like object into the final
370
abspath = self._abspath(relpath)
371
if self.has(relpath):
372
ftp = self._get_FTP()
373
result = ftp.size(abspath)
377
mutter("FTP appe to %s", abspath)
378
self._try_append(relpath, f.read(), mode)
382
def _try_append(self, relpath, text, mode=None, retries=0):
383
"""Try repeatedly to append the given text to the file at relpath.
385
This is a recursive function. On errors, it will be called until the
386
number of retries is exceeded.
389
abspath = self._abspath(relpath)
390
mutter("FTP appe (try %d) to %s", retries, abspath)
391
ftp = self._get_FTP()
392
ftp.voidcmd("TYPE I")
393
cmd = "APPE %s" % abspath
394
conn = ftp.transfercmd(cmd)
398
self._setmode(relpath, mode)
400
except ftplib.error_perm, e:
401
self._translate_perm_error(e, abspath, extra='error appending',
402
unknown_exc=errors.NoSuchFile)
403
except ftplib.error_temp, e:
404
if retries > _number_of_retries:
405
raise errors.TransportError("FTP temporary error during APPEND %s." \
406
"Aborting." % abspath, orig_error=e)
408
warning("FTP temporary error: %s. Retrying.", str(e))
409
self._FTP_instance = None
410
self._try_append(relpath, text, mode, retries+1)
412
def _setmode(self, relpath, mode):
413
"""Set permissions on a path.
415
Only set permissions if the FTP server supports the 'SITE CHMOD'
419
mutter("FTP site chmod: setting permissions to %s on %s",
420
str(mode), self._abspath(relpath))
421
ftp = self._get_FTP()
422
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
424
except ftplib.error_perm, e:
425
# Command probably not available on this server
426
warning("FTP Could not set permissions to %s on %s. %s",
427
str(mode), self._abspath(relpath), str(e))
429
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
430
# to copy something to another machine. And you may be able
431
# to give it its own address as the 'to' location.
432
# So implement a fancier 'copy()'
434
def move(self, rel_from, rel_to):
435
"""Move the item at rel_from to the location at rel_to"""
436
abs_from = self._abspath(rel_from)
437
abs_to = self._abspath(rel_to)
439
mutter("FTP mv: %s => %s", abs_from, abs_to)
441
f.rename(abs_from, abs_to)
442
except ftplib.error_perm, e:
443
self._translate_perm_error(e, abs_from,
444
extra='unable to rename to %r' % (rel_to,),
445
unknown_exc=errors.PathError)
449
def delete(self, relpath):
450
"""Delete the item at relpath"""
451
abspath = self._abspath(relpath)
453
mutter("FTP rm: %s", abspath)
456
except ftplib.error_perm, e:
457
self._translate_perm_error(e, abspath, 'error deleting',
458
unknown_exc=errors.NoSuchFile)
461
"""See Transport.listable."""
464
def list_dir(self, relpath):
465
"""See Transport.list_dir."""
466
basepath = self._abspath(relpath)
467
mutter("FTP nlst: %s", basepath)
470
paths = f.nlst(basepath)
471
except ftplib.error_perm, e:
472
self._translate_perm_error(e, relpath, extra='error with list_dir')
473
# If FTP.nlst returns paths prefixed by relpath, strip 'em
474
if paths and paths[0].startswith(basepath):
475
entries = [path[len(basepath)+1:] for path in paths]
478
# Remove . and .. if present
479
return [urlutils.escape(entry) for entry in entries
480
if entry not in ('.', '..')]
482
def iter_files_recursive(self):
483
"""See Transport.iter_files_recursive.
485
This is cargo-culted from the SFTP transport"""
486
mutter("FTP iter_files_recursive")
487
queue = list(self.list_dir("."))
489
relpath = queue.pop(0)
490
st = self.stat(relpath)
491
if stat.S_ISDIR(st.st_mode):
492
for i, basename in enumerate(self.list_dir(relpath)):
493
queue.insert(i, relpath+"/"+basename)
497
def stat(self, relpath):
498
"""Return the stat information for a file."""
499
abspath = self._abspath(relpath)
501
mutter("FTP stat: %s", abspath)
503
return FtpStatResult(f, abspath)
504
except ftplib.error_perm, e:
505
self._translate_perm_error(e, abspath, extra='error w/ stat')
507
def lock_read(self, relpath):
508
"""Lock the given file for shared (read) access.
509
:return: A lock object, which should be passed to Transport.unlock()
511
# The old RemoteBranch ignore lock for reading, so we will
512
# continue that tradition and return a bogus lock object.
513
class BogusLock(object):
514
def __init__(self, path):
518
return BogusLock(relpath)
520
def lock_write(self, relpath):
521
"""Lock the given file for exclusive (write) access.
522
WARNING: many transports do not support this, so trying avoid using it
524
:return: A lock object, which should be passed to Transport.unlock()
526
return self.lock_read(relpath)
529
class FtpServer(Server):
530
"""Common code for SFTP server facilities."""
534
self._ftp_server = None
536
self._async_thread = None
541
"""Calculate an ftp url to this server."""
542
return 'ftp://foo:bar@localhost:%d/' % (self._port)
544
# def get_bogus_url(self):
545
# """Return a URL which cannot be connected to."""
546
# return 'ftp://127.0.0.1:1'
548
def log(self, message):
549
"""This is used by medusa.ftp_server to log connections, etc."""
550
self.logs.append(message)
555
raise RuntimeError('Must have medusa to run the FtpServer')
557
self._root = os.getcwdu()
558
self._ftp_server = _ftp_server(
559
authorizer=_test_authorizer(root=self._root),
561
port=0, # bind to a random port
563
logger_object=self # Use FtpServer.log() for messages
565
self._port = self._ftp_server.getsockname()[1]
566
# Don't let it loop forever, or handle an infinite number of requests.
567
# In this case it will run for 100s, or 1000 requests
568
self._async_thread = threading.Thread(target=asyncore.loop,
569
kwargs={'timeout':0.1, 'count':1000})
570
self._async_thread.setDaemon(True)
571
self._async_thread.start()
574
"""See bzrlib.transport.Server.tearDown."""
575
# have asyncore release the channel
576
self._ftp_server.del_channel()
578
self._async_thread.join()
583
_test_authorizer = None
587
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
590
import medusa.filesys
591
import medusa.ftp_server
597
class test_authorizer(object):
598
"""A custom Authorizer object for running the test suite.
600
The reason we cannot use dummy_authorizer, is because it sets the
601
channel to readonly, which we don't always want to do.
604
def __init__(self, root):
607
def authorize(self, channel, username, password):
608
"""Return (success, reply_string, filesystem)"""
610
return 0, 'No Medusa.', None
612
channel.persona = -1, -1
613
if username == 'anonymous':
614
channel.read_only = 1
616
channel.read_only = 0
618
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
621
class ftp_channel(medusa.ftp_server.ftp_channel):
622
"""Customized ftp channel"""
624
def log(self, message):
625
"""Redirect logging requests."""
626
mutter('_ftp_channel: %s', message)
628
def log_info(self, message, type='info'):
629
"""Redirect logging requests."""
630
mutter('_ftp_channel %s: %s', type, message)
632
def cmd_rnfr(self, line):
633
"""Prepare for renaming a file."""
634
self._renaming = line[1]
635
self.respond('350 Ready for RNTO')
636
# TODO: jam 20060516 in testing, the ftp server seems to
637
# check that the file already exists, or it sends
638
# 550 RNFR command failed
640
def cmd_rnto(self, line):
641
"""Rename a file based on the target given.
643
rnto must be called after calling rnfr.
645
if not self._renaming:
646
self.respond('503 RNFR required first.')
647
pfrom = self.filesystem.translate(self._renaming)
648
self._renaming = None
649
pto = self.filesystem.translate(line[1])
651
os.rename(pfrom, pto)
652
except (IOError, OSError), e:
653
# TODO: jam 20060516 return custom responses based on
654
# why the command failed
655
self.respond('550 RNTO failed: %s' % (e,))
657
self.respond('550 RNTO failed')
658
# For a test server, we will go ahead and just die
661
self.respond('250 Rename successful.')
663
def cmd_size(self, line):
664
"""Return the size of a file
666
This is overloaded to help the test suite determine if the
667
target is a directory.
670
if not self.filesystem.isfile(filename):
671
if self.filesystem.isdir(filename):
672
self.respond('550 "%s" is a directory' % (filename,))
674
self.respond('550 "%s" is not a file' % (filename,))
676
self.respond('213 %d'
677
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
679
def cmd_mkd(self, line):
680
"""Create a directory.
682
Overloaded because default implementation does not distinguish
683
*why* it cannot make a directory.
686
self.command_not_understood(''.join(line))
690
self.filesystem.mkdir (path)
691
self.respond ('257 MKD command successful.')
692
except (IOError, OSError), e:
693
self.respond ('550 error creating directory: %s' % (e,))
695
self.respond ('550 error creating directory.')
698
class ftp_server(medusa.ftp_server.ftp_server):
699
"""Customize the behavior of the Medusa ftp_server.
701
There are a few warts on the ftp_server, based on how it expects
705
ftp_channel_class = ftp_channel
707
def __init__(self, *args, **kwargs):
708
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
709
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
711
def log(self, message):
712
"""Redirect logging requests."""
713
mutter('_ftp_server: %s', message)
715
def log_info(self, message, type='info'):
716
"""Override the asyncore.log_info so we don't stipple the screen."""
717
mutter('_ftp_server %s: %s', type, message)
719
_test_authorizer = test_authorizer
720
_ftp_channel = ftp_channel
721
_ftp_server = ftp_server
726
def get_test_permutations():
727
"""Return the permutations to be used in testing."""
728
if not _setup_medusa():
729
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
732
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
