1
# Copyright (C) 2005 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
38
from warnings import warn
44
from bzrlib.trace import mutter, warning
45
from bzrlib.transport import (
55
class FtpPathError(errors.PathError):
56
"""FTP failed for path: %(path)s%(extra)s"""
60
def _find_FTP(hostname, port, username, password, is_active):
61
"""Find an ftplib.FTP instance attached to this triplet."""
62
key = (hostname, port, username, password, is_active)
63
alt_key = (hostname, port, username, '********', is_active)
64
if key not in _FTP_cache:
65
mutter("Constructing FTP instance against %r" % (alt_key,))
68
conn.connect(host=hostname, port=port)
69
if username and username != 'anonymous' and not password:
70
password = bzrlib.ui.ui_factory.get_password(
71
prompt='FTP %(user)s@%(host)s password',
72
user=username, host=hostname)
73
conn.login(user=username, passwd=password)
74
conn.set_pasv(not is_active)
76
_FTP_cache[key] = conn
78
return _FTP_cache[key]
81
class FtpStatResult(object):
82
def __init__(self, f, relpath):
84
self.st_size = f.size(relpath)
85
self.st_mode = stat.S_IFREG
86
except ftplib.error_perm:
90
self.st_mode = stat.S_IFDIR
95
_number_of_retries = 2
96
_sleep_between_retries = 5
98
class FtpTransport(Transport):
99
"""This is the transport agent for ftp:// access."""
101
def __init__(self, base, _provided_instance=None):
102
"""Set the base path where files will be stored."""
103
assert base.startswith('ftp://') or base.startswith('aftp://')
105
self.is_active = base.startswith('aftp://')
107
# urlparse won't handle aftp://
109
if not base.endswith('/'):
111
(self._proto, self._username,
112
self._password, self._host,
113
self._port, self._path) = split_url(base)
114
base = self._unparse_url()
116
super(FtpTransport, self).__init__(base)
117
self._FTP_instance = _provided_instance
119
def _unparse_url(self, path=None):
122
path = urllib.quote(path)
123
netloc = urllib.quote(self._host)
124
if self._username is not None:
125
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
126
if self._port is not None:
127
netloc = '%s:%d' % (netloc, self._port)
131
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
134
"""Return the ftplib.FTP instance for this object."""
135
if self._FTP_instance is not None:
136
return self._FTP_instance
139
self._FTP_instance = _find_FTP(self._host, self._port,
140
self._username, self._password,
142
return self._FTP_instance
143
except ftplib.error_perm, e:
144
raise errors.TransportError(msg="Error setting up connection: %s"
145
% str(e), orig_error=e)
147
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
148
"""Try to translate an ftplib.error_perm exception.
150
:param err: The error to translate into a bzr error
151
:param path: The path which had problems
152
:param extra: Extra information which can be included
153
:param unknown_exc: If None, we will just raise the original exception
154
otherwise we raise unknown_exc(path, extra=extra)
160
extra += ': ' + str(err)
161
if ('no such file' in s
162
or 'could not open' in s
163
or 'no such dir' in s
165
raise errors.NoSuchFile(path, extra=extra)
166
if ('file exists' in s):
167
raise errors.FileExists(path, extra=extra)
168
if ('not a directory' in s):
169
raise errors.PathError(path, extra=extra)
171
mutter('unable to understand error for path: %s: %s', path, err)
174
raise unknown_exc(path, extra=extra)
175
# TODO: jam 20060516 Consider re-raising the error wrapped in
176
# something like TransportError, but this loses the traceback
177
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
178
# to handle. Consider doing something like that here.
179
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
182
def should_cache(self):
183
"""Return True if the data pulled across should be cached locally.
187
def clone(self, offset=None):
188
"""Return a new FtpTransport with root at self.base + offset.
192
return FtpTransport(self.base, self._FTP_instance)
194
return FtpTransport(self.abspath(offset), self._FTP_instance)
196
def _abspath(self, relpath):
197
assert isinstance(relpath, basestring)
198
relpath = urlutils.unescape(relpath)
199
relpath_parts = relpath.split('/')
200
if len(relpath_parts) > 1:
201
if relpath_parts[0] == '':
202
raise ValueError("path %r within branch %r seems to be absolute"
203
% (relpath, self._path))
204
basepath = self._path.split('/')
205
if len(basepath) > 0 and basepath[-1] == '':
206
basepath = basepath[:-1]
207
for p in relpath_parts:
209
if len(basepath) == 0:
210
# In most filesystems, a request for the parent
211
# of root, just returns root.
214
elif p == '.' or p == '':
218
# Possibly, we could use urlparse.urljoin() here, but
219
# I'm concerned about when it chooses to strip the last
220
# portion of the path, and when it doesn't.
222
# XXX: It seems that ftplib does not handle Unicode paths
223
# at the same time, medusa won't handle utf8 paths
224
# So if we .encode(utf8) here, then we get a Server failure.
225
# while if we use str(), we get a UnicodeError, and the test suite
226
# just skips testing UnicodePaths.
227
return str('/'.join(basepath) or '/')
229
def abspath(self, relpath):
230
"""Return the full url to the given relative path.
231
This can be supplied with a string or a list
233
path = self._abspath(relpath)
234
return self._unparse_url(path)
236
def has(self, relpath):
237
"""Does the target location exist?"""
238
# FIXME jam 20060516 We *do* ask about directories in the test suite
239
# We don't seem to in the actual codebase
240
# XXX: I assume we're never asked has(dirname) and thus I use
241
# the FTP size command and assume that if it doesn't raise,
243
abspath = self._abspath(relpath)
246
mutter('FTP has check: %s => %s', relpath, abspath)
248
mutter("FTP has: %s", abspath)
250
except ftplib.error_perm, e:
251
if ('is a directory' in str(e).lower()):
252
mutter("FTP has dir: %s: %s", abspath, e)
254
mutter("FTP has not: %s: %s", abspath, e)
257
def get(self, relpath, decode=False, retries=0):
258
"""Get the file at the given relative path.
260
:param relpath: The relative path to the file
261
:param retries: Number of retries after temporary failures so far
264
We're meant to return a file-like object which bzr will
265
then read from. For now we do this via the magic of StringIO
267
# TODO: decode should be deprecated
269
mutter("FTP get: %s", self._abspath(relpath))
272
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
275
except ftplib.error_perm, e:
276
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
277
except ftplib.error_temp, e:
278
if retries > _number_of_retries:
279
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
280
% self.abspath(relpath),
283
warning("FTP temporary error: %s. Retrying.", str(e))
284
self._FTP_instance = None
285
return self.get(relpath, decode, retries+1)
287
if retries > _number_of_retries:
288
raise errors.TransportError("FTP control connection closed during GET %s."
289
% self.abspath(relpath),
292
warning("FTP control connection closed. Trying to reopen.")
293
time.sleep(_sleep_between_retries)
294
self._FTP_instance = None
295
return self.get(relpath, decode, retries+1)
297
def put_file(self, relpath, fp, mode=None, retries=0):
298
"""Copy the file-like or string object into the location.
300
:param relpath: Location to put the contents, relative to base.
301
:param fp: File-like or string object.
302
:param retries: Number of retries after temporary failures so far
305
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
307
abspath = self._abspath(relpath)
308
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
309
os.getpid(), random.randint(0,0x7FFFFFFF))
310
if getattr(fp, 'read', None) is None:
313
mutter("FTP put: %s", abspath)
316
f.storbinary('STOR '+tmp_abspath, fp)
317
f.rename(tmp_abspath, abspath)
318
except (ftplib.error_temp,EOFError), e:
319
warning("Failure during ftp PUT. Deleting temporary file.")
321
f.delete(tmp_abspath)
323
warning("Failed to delete temporary file on the"
324
" server.\nFile: %s", tmp_abspath)
327
except ftplib.error_perm, e:
328
self._translate_perm_error(e, abspath, extra='could not store')
329
except ftplib.error_temp, e:
330
if retries > _number_of_retries:
331
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
332
% self.abspath(relpath), orig_error=e)
334
warning("FTP temporary error: %s. Retrying.", str(e))
335
self._FTP_instance = None
336
self.put_file(relpath, fp, mode, retries+1)
338
if retries > _number_of_retries:
339
raise errors.TransportError("FTP control connection closed during PUT %s."
340
% self.abspath(relpath), orig_error=e)
342
warning("FTP control connection closed. Trying to reopen.")
343
time.sleep(_sleep_between_retries)
344
self._FTP_instance = None
345
self.put_file(relpath, fp, mode, retries+1)
347
def mkdir(self, relpath, mode=None):
348
"""Create a directory at the given path."""
349
abspath = self._abspath(relpath)
351
mutter("FTP mkd: %s", abspath)
354
except ftplib.error_perm, e:
355
self._translate_perm_error(e, abspath,
356
unknown_exc=errors.FileExists)
358
def rmdir(self, rel_path):
359
"""Delete the directory at rel_path"""
360
abspath = self._abspath(rel_path)
362
mutter("FTP rmd: %s", abspath)
365
except ftplib.error_perm, e:
366
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
368
def append_file(self, relpath, f, mode=None):
369
"""Append the text in the file-like object into the final
372
abspath = self._abspath(relpath)
373
if self.has(relpath):
374
ftp = self._get_FTP()
375
result = ftp.size(abspath)
379
mutter("FTP appe to %s", abspath)
380
self._try_append(relpath, f.read(), mode)
384
def _try_append(self, relpath, text, mode=None, retries=0):
385
"""Try repeatedly to append the given text to the file at relpath.
387
This is a recursive function. On errors, it will be called until the
388
number of retries is exceeded.
391
abspath = self._abspath(relpath)
392
mutter("FTP appe (try %d) to %s", retries, abspath)
393
ftp = self._get_FTP()
394
ftp.voidcmd("TYPE I")
395
cmd = "APPE %s" % abspath
396
conn = ftp.transfercmd(cmd)
400
self._setmode(relpath, mode)
402
except ftplib.error_perm, e:
403
self._translate_perm_error(e, abspath, extra='error appending',
404
unknown_exc=errors.NoSuchFile)
405
except ftplib.error_temp, e:
406
if retries > _number_of_retries:
407
raise errors.TransportError("FTP temporary error during APPEND %s." \
408
"Aborting." % abspath, orig_error=e)
410
warning("FTP temporary error: %s. Retrying.", str(e))
411
self._FTP_instance = None
412
self._try_append(relpath, text, mode, retries+1)
414
def _setmode(self, relpath, mode):
415
"""Set permissions on a path.
417
Only set permissions if the FTP server supports the 'SITE CHMOD'
421
mutter("FTP site chmod: setting permissions to %s on %s",
422
str(mode), self._abspath(relpath))
423
ftp = self._get_FTP()
424
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
426
except ftplib.error_perm, e:
427
# Command probably not available on this server
428
warning("FTP Could not set permissions to %s on %s. %s",
429
str(mode), self._abspath(relpath), str(e))
431
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
432
# to copy something to another machine. And you may be able
433
# to give it its own address as the 'to' location.
434
# So implement a fancier 'copy()'
436
def move(self, rel_from, rel_to):
437
"""Move the item at rel_from to the location at rel_to"""
438
abs_from = self._abspath(rel_from)
439
abs_to = self._abspath(rel_to)
441
mutter("FTP mv: %s => %s", abs_from, abs_to)
443
f.rename(abs_from, abs_to)
444
except ftplib.error_perm, e:
445
self._translate_perm_error(e, abs_from,
446
extra='unable to rename to %r' % (rel_to,),
447
unknown_exc=errors.PathError)
451
def delete(self, relpath):
452
"""Delete the item at relpath"""
453
abspath = self._abspath(relpath)
455
mutter("FTP rm: %s", abspath)
458
except ftplib.error_perm, e:
459
self._translate_perm_error(e, abspath, 'error deleting',
460
unknown_exc=errors.NoSuchFile)
463
"""See Transport.listable."""
466
def list_dir(self, relpath):
467
"""See Transport.list_dir."""
468
basepath = self._abspath(relpath)
469
mutter("FTP nlst: %s", basepath)
472
paths = f.nlst(basepath)
473
except ftplib.error_perm, e:
474
self._translate_perm_error(e, relpath, extra='error with list_dir')
475
# If FTP.nlst returns paths prefixed by relpath, strip 'em
476
if paths and paths[0].startswith(basepath):
477
entries = [path[len(basepath)+1:] for path in paths]
480
# Remove . and .. if present
481
return [urlutils.escape(entry) for entry in entries
482
if entry not in ('.', '..')]
484
def iter_files_recursive(self):
485
"""See Transport.iter_files_recursive.
487
This is cargo-culted from the SFTP transport"""
488
mutter("FTP iter_files_recursive")
489
queue = list(self.list_dir("."))
491
relpath = queue.pop(0)
492
st = self.stat(relpath)
493
if stat.S_ISDIR(st.st_mode):
494
for i, basename in enumerate(self.list_dir(relpath)):
495
queue.insert(i, relpath+"/"+basename)
499
def stat(self, relpath):
500
"""Return the stat information for a file."""
501
abspath = self._abspath(relpath)
503
mutter("FTP stat: %s", abspath)
505
return FtpStatResult(f, abspath)
506
except ftplib.error_perm, e:
507
self._translate_perm_error(e, abspath, extra='error w/ stat')
509
def lock_read(self, relpath):
510
"""Lock the given file for shared (read) access.
511
:return: A lock object, which should be passed to Transport.unlock()
513
# The old RemoteBranch ignore lock for reading, so we will
514
# continue that tradition and return a bogus lock object.
515
class BogusLock(object):
516
def __init__(self, path):
520
return BogusLock(relpath)
522
def lock_write(self, relpath):
523
"""Lock the given file for exclusive (write) access.
524
WARNING: many transports do not support this, so trying avoid using it
526
:return: A lock object, which should be passed to Transport.unlock()
528
return self.lock_read(relpath)
531
class FtpServer(Server):
532
"""Common code for SFTP server facilities."""
536
self._ftp_server = None
538
self._async_thread = None
543
"""Calculate an ftp url to this server."""
544
return 'ftp://foo:bar@localhost:%d/' % (self._port)
546
# def get_bogus_url(self):
547
# """Return a URL which cannot be connected to."""
548
# return 'ftp://127.0.0.1:1'
550
def log(self, message):
551
"""This is used by medusa.ftp_server to log connections, etc."""
552
self.logs.append(message)
557
raise RuntimeError('Must have medusa to run the FtpServer')
559
self._root = os.getcwdu()
560
self._ftp_server = _ftp_server(
561
authorizer=_test_authorizer(root=self._root),
563
port=0, # bind to a random port
565
logger_object=self # Use FtpServer.log() for messages
567
self._port = self._ftp_server.getsockname()[1]
568
# Don't let it loop forever, or handle an infinite number of requests.
569
# In this case it will run for 100s, or 1000 requests
570
self._async_thread = threading.Thread(target=asyncore.loop,
571
kwargs={'timeout':0.1, 'count':1000})
572
self._async_thread.setDaemon(True)
573
self._async_thread.start()
576
"""See bzrlib.transport.Server.tearDown."""
577
# have asyncore release the channel
578
self._ftp_server.del_channel()
580
self._async_thread.join()
585
_test_authorizer = None
589
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
592
import medusa.filesys
593
import medusa.ftp_server
599
class test_authorizer(object):
600
"""A custom Authorizer object for running the test suite.
602
The reason we cannot use dummy_authorizer, is because it sets the
603
channel to readonly, which we don't always want to do.
606
def __init__(self, root):
609
def authorize(self, channel, username, password):
610
"""Return (success, reply_string, filesystem)"""
612
return 0, 'No Medusa.', None
614
channel.persona = -1, -1
615
if username == 'anonymous':
616
channel.read_only = 1
618
channel.read_only = 0
620
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
623
class ftp_channel(medusa.ftp_server.ftp_channel):
624
"""Customized ftp channel"""
626
def log(self, message):
627
"""Redirect logging requests."""
628
mutter('_ftp_channel: %s', message)
630
def log_info(self, message, type='info'):
631
"""Redirect logging requests."""
632
mutter('_ftp_channel %s: %s', type, message)
634
def cmd_rnfr(self, line):
635
"""Prepare for renaming a file."""
636
self._renaming = line[1]
637
self.respond('350 Ready for RNTO')
638
# TODO: jam 20060516 in testing, the ftp server seems to
639
# check that the file already exists, or it sends
640
# 550 RNFR command failed
642
def cmd_rnto(self, line):
643
"""Rename a file based on the target given.
645
rnto must be called after calling rnfr.
647
if not self._renaming:
648
self.respond('503 RNFR required first.')
649
pfrom = self.filesystem.translate(self._renaming)
650
self._renaming = None
651
pto = self.filesystem.translate(line[1])
653
os.rename(pfrom, pto)
654
except (IOError, OSError), e:
655
# TODO: jam 20060516 return custom responses based on
656
# why the command failed
657
self.respond('550 RNTO failed: %s' % (e,))
659
self.respond('550 RNTO failed')
660
# For a test server, we will go ahead and just die
663
self.respond('250 Rename successful.')
665
def cmd_size(self, line):
666
"""Return the size of a file
668
This is overloaded to help the test suite determine if the
669
target is a directory.
672
if not self.filesystem.isfile(filename):
673
if self.filesystem.isdir(filename):
674
self.respond('550 "%s" is a directory' % (filename,))
676
self.respond('550 "%s" is not a file' % (filename,))
678
self.respond('213 %d'
679
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
681
def cmd_mkd(self, line):
682
"""Create a directory.
684
Overloaded because default implementation does not distinguish
685
*why* it cannot make a directory.
688
self.command_not_understood(''.join(line))
692
self.filesystem.mkdir (path)
693
self.respond ('257 MKD command successful.')
694
except (IOError, OSError), e:
695
self.respond ('550 error creating directory: %s' % (e,))
697
self.respond ('550 error creating directory.')
700
class ftp_server(medusa.ftp_server.ftp_server):
701
"""Customize the behavior of the Medusa ftp_server.
703
There are a few warts on the ftp_server, based on how it expects
707
ftp_channel_class = ftp_channel
709
def __init__(self, *args, **kwargs):
710
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
711
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
713
def log(self, message):
714
"""Redirect logging requests."""
715
mutter('_ftp_server: %s', message)
717
def log_info(self, message, type='info'):
718
"""Override the asyncore.log_info so we don't stipple the screen."""
719
mutter('_ftp_server %s: %s', type, message)
721
_test_authorizer = test_authorizer
722
_ftp_channel = ftp_channel
723
_ftp_server = ftp_server
728
def get_test_permutations():
729
"""Return the permutations to be used in testing."""
730
if not _setup_medusa():
731
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
734
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
