1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
40
from warnings import warn
47
from bzrlib.trace import mutter, warning
48
from bzrlib.transport import (
53
from bzrlib.transport.local import LocalURLServer
59
class FtpPathError(errors.PathError):
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
85
class FtpStatResult(object):
86
def __init__(self, f, relpath):
88
self.st_size = f.size(relpath)
89
self.st_mode = stat.S_IFREG
90
except ftplib.error_perm:
94
self.st_mode = stat.S_IFDIR
99
_number_of_retries = 2
100
_sleep_between_retries = 5
102
class FtpTransport(Transport):
103
"""This is the transport agent for ftp:// access."""
105
def __init__(self, base, _provided_instance=None):
106
"""Set the base path where files will be stored."""
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://
113
if not base.endswith('/'):
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
123
def _unparse_url(self, path=None):
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
138
"""Return the ftplib.FTP instance for this object."""
139
if self._FTP_instance is not None:
140
return self._FTP_instance
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
146
return self._FTP_instance
147
except ftplib.error_perm, e:
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
152
"""Try to translate an ftplib.error_perm exception.
154
:param err: The error to translate into a bzr error
155
:param path: The path which had problems
156
:param extra: Extra information which can be included
157
:param unknown_exc: If None, we will just raise the original exception
158
otherwise we raise unknown_exc(path, extra=extra)
164
extra += ': ' + str(err)
165
if ('no such file' in s
166
or 'could not open' in s
167
or 'no such dir' in s
168
or 'could not create file' in s # vsftpd
170
raise errors.NoSuchFile(path, extra=extra)
171
if ('file exists' in s):
172
raise errors.FileExists(path, extra=extra)
173
if ('not a directory' in s):
174
raise errors.PathError(path, extra=extra)
176
mutter('unable to understand error for path: %s: %s', path, err)
179
raise unknown_exc(path, extra=extra)
180
# TODO: jam 20060516 Consider re-raising the error wrapped in
181
# something like TransportError, but this loses the traceback
182
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
183
# to handle. Consider doing something like that here.
184
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
187
def should_cache(self):
188
"""Return True if the data pulled across should be cached locally.
192
def clone(self, offset=None):
193
"""Return a new FtpTransport with root at self.base + offset.
197
return FtpTransport(self.base, self._FTP_instance)
199
return FtpTransport(self.abspath(offset), self._FTP_instance)
201
def _abspath(self, relpath):
202
assert isinstance(relpath, basestring)
203
relpath = urlutils.unescape(relpath)
204
if relpath.startswith('/'):
207
basepath = self._path.split('/')
208
if len(basepath) > 0 and basepath[-1] == '':
209
basepath = basepath[:-1]
210
for p in relpath.split('/'):
212
if len(basepath) == 0:
213
# In most filesystems, a request for the parent
214
# of root, just returns root.
217
elif p == '.' or p == '':
221
# Possibly, we could use urlparse.urljoin() here, but
222
# I'm concerned about when it chooses to strip the last
223
# portion of the path, and when it doesn't.
225
# XXX: It seems that ftplib does not handle Unicode paths
226
# at the same time, medusa won't handle utf8 paths
227
# So if we .encode(utf8) here, then we get a Server failure.
228
# while if we use str(), we get a UnicodeError, and the test suite
229
# just skips testing UnicodePaths.
230
return str('/'.join(basepath) or '/')
232
def abspath(self, relpath):
233
"""Return the full url to the given relative path.
234
This can be supplied with a string or a list
236
path = self._abspath(relpath)
237
return self._unparse_url(path)
239
def has(self, relpath):
240
"""Does the target location exist?"""
241
# FIXME jam 20060516 We *do* ask about directories in the test suite
242
# We don't seem to in the actual codebase
243
# XXX: I assume we're never asked has(dirname) and thus I use
244
# the FTP size command and assume that if it doesn't raise,
246
abspath = self._abspath(relpath)
249
mutter('FTP has check: %s => %s', relpath, abspath)
251
mutter("FTP has: %s", abspath)
253
except ftplib.error_perm, e:
254
if ('is a directory' in str(e).lower()):
255
mutter("FTP has dir: %s: %s", abspath, e)
257
mutter("FTP has not: %s: %s", abspath, e)
260
def get(self, relpath, decode=False, retries=0):
261
"""Get the file at the given relative path.
263
:param relpath: The relative path to the file
264
:param retries: Number of retries after temporary failures so far
267
We're meant to return a file-like object which bzr will
268
then read from. For now we do this via the magic of StringIO
270
# TODO: decode should be deprecated
272
mutter("FTP get: %s", self._abspath(relpath))
275
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
278
except ftplib.error_perm, e:
279
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
280
except ftplib.error_temp, e:
281
if retries > _number_of_retries:
282
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
283
% self.abspath(relpath),
286
warning("FTP temporary error: %s. Retrying.", str(e))
287
self._FTP_instance = None
288
return self.get(relpath, decode, retries+1)
290
if retries > _number_of_retries:
291
raise errors.TransportError("FTP control connection closed during GET %s."
292
% self.abspath(relpath),
295
warning("FTP control connection closed. Trying to reopen.")
296
time.sleep(_sleep_between_retries)
297
self._FTP_instance = None
298
return self.get(relpath, decode, retries+1)
300
def put_file(self, relpath, fp, mode=None, retries=0):
301
"""Copy the file-like or string object into the location.
303
:param relpath: Location to put the contents, relative to base.
304
:param fp: File-like or string object.
305
:param retries: Number of retries after temporary failures so far
308
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
311
abspath = self._abspath(relpath)
312
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
313
os.getpid(), random.randint(0,0x7FFFFFFF))
314
if getattr(fp, 'read', None) is None:
317
mutter("FTP put: %s", abspath)
320
f.storbinary('STOR '+tmp_abspath, fp)
321
self._rename_and_overwrite(tmp_abspath, abspath, f)
322
except (ftplib.error_temp,EOFError), e:
323
warning("Failure during ftp PUT. Deleting temporary file.")
325
f.delete(tmp_abspath)
327
warning("Failed to delete temporary file on the"
328
" server.\nFile: %s", tmp_abspath)
331
except ftplib.error_perm, e:
332
self._translate_perm_error(e, abspath, extra='could not store')
333
except ftplib.error_temp, e:
334
if retries > _number_of_retries:
335
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
336
% self.abspath(relpath), orig_error=e)
338
warning("FTP temporary error: %s. Retrying.", str(e))
339
self._FTP_instance = None
340
self.put_file(relpath, fp, mode, retries+1)
342
if retries > _number_of_retries:
343
raise errors.TransportError("FTP control connection closed during PUT %s."
344
% self.abspath(relpath), orig_error=e)
346
warning("FTP control connection closed. Trying to reopen.")
347
time.sleep(_sleep_between_retries)
348
self._FTP_instance = None
349
self.put_file(relpath, fp, mode, retries+1)
351
def mkdir(self, relpath, mode=None):
352
"""Create a directory at the given path."""
353
abspath = self._abspath(relpath)
355
mutter("FTP mkd: %s", abspath)
358
except ftplib.error_perm, e:
359
self._translate_perm_error(e, abspath,
360
unknown_exc=errors.FileExists)
362
def rmdir(self, rel_path):
363
"""Delete the directory at rel_path"""
364
abspath = self._abspath(rel_path)
366
mutter("FTP rmd: %s", abspath)
369
except ftplib.error_perm, e:
370
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
372
def append_file(self, relpath, f, mode=None):
373
"""Append the text in the file-like object into the final
376
abspath = self._abspath(relpath)
377
if self.has(relpath):
378
ftp = self._get_FTP()
379
result = ftp.size(abspath)
383
mutter("FTP appe to %s", abspath)
384
self._try_append(relpath, f.read(), mode)
388
def _try_append(self, relpath, text, mode=None, retries=0):
389
"""Try repeatedly to append the given text to the file at relpath.
391
This is a recursive function. On errors, it will be called until the
392
number of retries is exceeded.
395
abspath = self._abspath(relpath)
396
mutter("FTP appe (try %d) to %s", retries, abspath)
397
ftp = self._get_FTP()
398
ftp.voidcmd("TYPE I")
399
cmd = "APPE %s" % abspath
400
conn = ftp.transfercmd(cmd)
404
self._setmode(relpath, mode)
406
except ftplib.error_perm, e:
407
self._translate_perm_error(e, abspath, extra='error appending',
408
unknown_exc=errors.NoSuchFile)
409
except ftplib.error_temp, e:
410
if retries > _number_of_retries:
411
raise errors.TransportError("FTP temporary error during APPEND %s." \
412
"Aborting." % abspath, orig_error=e)
414
warning("FTP temporary error: %s. Retrying.", str(e))
415
self._FTP_instance = None
416
self._try_append(relpath, text, mode, retries+1)
418
def _setmode(self, relpath, mode):
419
"""Set permissions on a path.
421
Only set permissions if the FTP server supports the 'SITE CHMOD'
425
mutter("FTP site chmod: setting permissions to %s on %s",
426
str(mode), self._abspath(relpath))
427
ftp = self._get_FTP()
428
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
430
except ftplib.error_perm, e:
431
# Command probably not available on this server
432
warning("FTP Could not set permissions to %s on %s. %s",
433
str(mode), self._abspath(relpath), str(e))
435
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
436
# to copy something to another machine. And you may be able
437
# to give it its own address as the 'to' location.
438
# So implement a fancier 'copy()'
440
def rename(self, rel_from, rel_to):
441
abs_from = self._abspath(rel_from)
442
abs_to = self._abspath(rel_to)
443
mutter("FTP rename: %s => %s", abs_from, abs_to)
445
return self._rename(abs_from, abs_to, f)
447
def _rename(self, abs_from, abs_to, f):
449
f.rename(abs_from, abs_to)
450
except ftplib.error_perm, e:
451
self._translate_perm_error(e, abs_from,
452
': unable to rename to %r' % (abs_to))
454
def move(self, rel_from, rel_to):
455
"""Move the item at rel_from to the location at rel_to"""
456
abs_from = self._abspath(rel_from)
457
abs_to = self._abspath(rel_to)
459
mutter("FTP mv: %s => %s", abs_from, abs_to)
461
self._rename_and_overwrite(abs_from, abs_to, f)
462
except ftplib.error_perm, e:
463
self._translate_perm_error(e, abs_from,
464
extra='unable to rename to %r' % (rel_to,),
465
unknown_exc=errors.PathError)
467
def _rename_and_overwrite(self, abs_from, abs_to, f):
468
"""Do a fancy rename on the remote server.
470
Using the implementation provided by osutils.
472
osutils.fancy_rename(abs_from, abs_to,
473
rename_func=lambda p1, p2: self._rename(p1, p2, f),
474
unlink_func=lambda p: self._delete(p, f))
476
def delete(self, relpath):
477
"""Delete the item at relpath"""
478
abspath = self._abspath(relpath)
480
self._delete(abspath, f)
482
def _delete(self, abspath, f):
484
mutter("FTP rm: %s", abspath)
486
except ftplib.error_perm, e:
487
self._translate_perm_error(e, abspath, 'error deleting',
488
unknown_exc=errors.NoSuchFile)
491
"""See Transport.listable."""
494
def list_dir(self, relpath):
495
"""See Transport.list_dir."""
496
basepath = self._abspath(relpath)
497
mutter("FTP nlst: %s", basepath)
500
paths = f.nlst(basepath)
501
except ftplib.error_perm, e:
502
self._translate_perm_error(e, relpath, extra='error with list_dir')
503
# If FTP.nlst returns paths prefixed by relpath, strip 'em
504
if paths and paths[0].startswith(basepath):
505
entries = [path[len(basepath)+1:] for path in paths]
508
# Remove . and .. if present
509
return [urlutils.escape(entry) for entry in entries
510
if entry not in ('.', '..')]
512
def iter_files_recursive(self):
513
"""See Transport.iter_files_recursive.
515
This is cargo-culted from the SFTP transport"""
516
mutter("FTP iter_files_recursive")
517
queue = list(self.list_dir("."))
519
relpath = queue.pop(0)
520
st = self.stat(relpath)
521
if stat.S_ISDIR(st.st_mode):
522
for i, basename in enumerate(self.list_dir(relpath)):
523
queue.insert(i, relpath+"/"+basename)
527
def stat(self, relpath):
528
"""Return the stat information for a file."""
529
abspath = self._abspath(relpath)
531
mutter("FTP stat: %s", abspath)
533
return FtpStatResult(f, abspath)
534
except ftplib.error_perm, e:
535
self._translate_perm_error(e, abspath, extra='error w/ stat')
537
def lock_read(self, relpath):
538
"""Lock the given file for shared (read) access.
539
:return: A lock object, which should be passed to Transport.unlock()
541
# The old RemoteBranch ignore lock for reading, so we will
542
# continue that tradition and return a bogus lock object.
543
class BogusLock(object):
544
def __init__(self, path):
548
return BogusLock(relpath)
550
def lock_write(self, relpath):
551
"""Lock the given file for exclusive (write) access.
552
WARNING: many transports do not support this, so trying avoid using it
554
:return: A lock object, which should be passed to Transport.unlock()
556
return self.lock_read(relpath)
559
class FtpServer(Server):
560
"""Common code for SFTP server facilities."""
564
self._ftp_server = None
566
self._async_thread = None
571
"""Calculate an ftp url to this server."""
572
return 'ftp://foo:bar@localhost:%d/' % (self._port)
574
# def get_bogus_url(self):
575
# """Return a URL which cannot be connected to."""
576
# return 'ftp://127.0.0.1:1'
578
def log(self, message):
579
"""This is used by medusa.ftp_server to log connections, etc."""
580
self.logs.append(message)
582
def setUp(self, vfs_server=None):
584
raise RuntimeError('Must have medusa to run the FtpServer')
586
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
587
"FtpServer currently assumes local transport, got %s" % vfs_server
589
self._root = os.getcwdu()
590
self._ftp_server = _ftp_server(
591
authorizer=_test_authorizer(root=self._root),
593
port=0, # bind to a random port
595
logger_object=self # Use FtpServer.log() for messages
597
self._port = self._ftp_server.getsockname()[1]
598
# Don't let it loop forever, or handle an infinite number of requests.
599
# In this case it will run for 100s, or 1000 requests
600
self._async_thread = threading.Thread(
601
target=FtpServer._asyncore_loop_ignore_EBADF,
602
kwargs={'timeout':0.1, 'count':1000})
603
self._async_thread.setDaemon(True)
604
self._async_thread.start()
607
"""See bzrlib.transport.Server.tearDown."""
608
# have asyncore release the channel
609
self._ftp_server.del_channel()
611
self._async_thread.join()
614
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
615
"""Ignore EBADF during server shutdown.
617
We close the socket to get the server to shutdown, but this causes
618
select.select() to raise EBADF.
621
asyncore.loop(*args, **kwargs)
622
except select.error, e:
623
if e.args[0] != errno.EBADF:
629
_test_authorizer = None
633
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
636
import medusa.filesys
637
import medusa.ftp_server
643
class test_authorizer(object):
644
"""A custom Authorizer object for running the test suite.
646
The reason we cannot use dummy_authorizer, is because it sets the
647
channel to readonly, which we don't always want to do.
650
def __init__(self, root):
653
def authorize(self, channel, username, password):
654
"""Return (success, reply_string, filesystem)"""
656
return 0, 'No Medusa.', None
658
channel.persona = -1, -1
659
if username == 'anonymous':
660
channel.read_only = 1
662
channel.read_only = 0
664
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
667
class ftp_channel(medusa.ftp_server.ftp_channel):
668
"""Customized ftp channel"""
670
def log(self, message):
671
"""Redirect logging requests."""
672
mutter('_ftp_channel: %s', message)
674
def log_info(self, message, type='info'):
675
"""Redirect logging requests."""
676
mutter('_ftp_channel %s: %s', type, message)
678
def cmd_rnfr(self, line):
679
"""Prepare for renaming a file."""
680
self._renaming = line[1]
681
self.respond('350 Ready for RNTO')
682
# TODO: jam 20060516 in testing, the ftp server seems to
683
# check that the file already exists, or it sends
684
# 550 RNFR command failed
686
def cmd_rnto(self, line):
687
"""Rename a file based on the target given.
689
rnto must be called after calling rnfr.
691
if not self._renaming:
692
self.respond('503 RNFR required first.')
693
pfrom = self.filesystem.translate(self._renaming)
694
self._renaming = None
695
pto = self.filesystem.translate(line[1])
696
if os.path.exists(pto):
697
self.respond('550 RNTO failed: file exists')
700
os.rename(pfrom, pto)
701
except (IOError, OSError), e:
702
# TODO: jam 20060516 return custom responses based on
703
# why the command failed
704
# (bialix 20070418) str(e) on Python 2.5 @ Windows
705
# sometimes don't provide expected error message;
706
# so we obtain such message via os.strerror()
707
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
709
self.respond('550 RNTO failed')
710
# For a test server, we will go ahead and just die
713
self.respond('250 Rename successful.')
715
def cmd_size(self, line):
716
"""Return the size of a file
718
This is overloaded to help the test suite determine if the
719
target is a directory.
722
if not self.filesystem.isfile(filename):
723
if self.filesystem.isdir(filename):
724
self.respond('550 "%s" is a directory' % (filename,))
726
self.respond('550 "%s" is not a file' % (filename,))
728
self.respond('213 %d'
729
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
731
def cmd_mkd(self, line):
732
"""Create a directory.
734
Overloaded because default implementation does not distinguish
735
*why* it cannot make a directory.
738
self.command_not_understood(''.join(line))
742
self.filesystem.mkdir (path)
743
self.respond ('257 MKD command successful.')
744
except (IOError, OSError), e:
745
# (bialix 20070418) str(e) on Python 2.5 @ Windows
746
# sometimes don't provide expected error message;
747
# so we obtain such message via os.strerror()
748
self.respond ('550 error creating directory: %s' %
749
os.strerror(e.errno))
751
self.respond ('550 error creating directory.')
754
class ftp_server(medusa.ftp_server.ftp_server):
755
"""Customize the behavior of the Medusa ftp_server.
757
There are a few warts on the ftp_server, based on how it expects
761
ftp_channel_class = ftp_channel
763
def __init__(self, *args, **kwargs):
764
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
765
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
767
def log(self, message):
768
"""Redirect logging requests."""
769
mutter('_ftp_server: %s', message)
771
def log_info(self, message, type='info'):
772
"""Override the asyncore.log_info so we don't stipple the screen."""
773
mutter('_ftp_server %s: %s', type, message)
775
_test_authorizer = test_authorizer
776
_ftp_channel = ftp_channel
777
_ftp_server = ftp_server
782
def get_test_permutations():
783
"""Return the permutations to be used in testing."""
784
if not _setup_medusa():
785
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
788
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
