1
# Copyright (C) 2005 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over ftp.
18
Written by Daniel Silverstone <dsilvers@digital-scurf.org> with serious
19
cargo-culting from the sftp transport and the http transport.
21
It provides the ftp:// and aftp:// protocols where ftp:// is passive ftp
22
and aftp:// is active ftp. Most people will want passive ftp for traversing
23
NAT and other firewalls, so it's best to use it unless you explicitly want
24
active, in which case aftp:// will be your friend.
27
from cStringIO import StringIO
38
from warnings import warn
40
from bzrlib.transport import (
45
import bzrlib.errors as errors
46
from bzrlib.trace import mutter, warning
52
class FtpPathError(errors.PathError):
53
"""FTP failed for path: %(path)s%(extra)s"""
57
def _find_FTP(hostname, port, username, password, is_active):
58
"""Find an ftplib.FTP instance attached to this triplet."""
59
key = (hostname, port, username, password, is_active)
60
alt_key = (hostname, port, username, '********', is_active)
61
if key not in _FTP_cache:
62
mutter("Constructing FTP instance against %r" % (alt_key,))
65
conn.connect(host=hostname, port=port)
66
if username and username != 'anonymous' and not password:
67
password = bzrlib.ui.ui_factory.get_password(
68
prompt='FTP %(user)s@%(host)s password',
69
user=username, host=hostname)
70
conn.login(user=username, passwd=password)
71
conn.set_pasv(not is_active)
73
_FTP_cache[key] = conn
75
return _FTP_cache[key]
78
class FtpStatResult(object):
79
def __init__(self, f, relpath):
81
self.st_size = f.size(relpath)
82
self.st_mode = stat.S_IFREG
83
except ftplib.error_perm:
87
self.st_mode = stat.S_IFDIR
92
_number_of_retries = 2
93
_sleep_between_retries = 5
95
class FtpTransport(Transport):
96
"""This is the transport agent for ftp:// access."""
98
def __init__(self, base, _provided_instance=None):
99
"""Set the base path where files will be stored."""
100
assert base.startswith('ftp://') or base.startswith('aftp://')
102
self.is_active = base.startswith('aftp://')
104
# urlparse won't handle aftp://
106
if not base.endswith('/'):
108
(self._proto, self._username,
109
self._password, self._host,
110
self._port, self._path) = split_url(base)
111
base = self._unparse_url()
113
super(FtpTransport, self).__init__(base)
114
self._FTP_instance = _provided_instance
116
def _unparse_url(self, path=None):
119
path = urllib.quote(path)
120
netloc = urllib.quote(self._host)
121
if self._username is not None:
122
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
123
if self._port is not None:
124
netloc = '%s:%d' % (netloc, self._port)
125
return urlparse.urlunparse(('ftp', netloc, path, '', '', ''))
128
"""Return the ftplib.FTP instance for this object."""
129
if self._FTP_instance is not None:
130
return self._FTP_instance
133
self._FTP_instance = _find_FTP(self._host, self._port,
134
self._username, self._password,
136
return self._FTP_instance
137
except ftplib.error_perm, e:
138
raise errors.TransportError(msg="Error setting up connection: %s"
139
% str(e), orig_error=e)
141
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
142
"""Try to translate an ftplib.error_perm exception.
144
:param err: The error to translate into a bzr error
145
:param path: The path which had problems
146
:param extra: Extra information which can be included
147
:param unknown_exc: If None, we will just raise the original exception
148
otherwise we raise unknown_exc(path, extra=extra)
154
extra += ': ' + str(err)
155
if ('no such file' in s
156
or 'could not open' in s
157
or 'no such dir' in s
159
raise errors.NoSuchFile(path, extra=extra)
160
if ('file exists' in s):
161
raise errors.FileExists(path, extra=extra)
162
if ('not a directory' in s):
163
raise errors.PathError(path, extra=extra)
165
mutter('unable to understand error for path: %s: %s', path, err)
168
raise unknown_exc(path, extra=extra)
169
# TODO: jam 20060516 Consider re-raising the error wrapped in
170
# something like TransportError, but this loses the traceback
171
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
172
# to handle. Consider doing something like that here.
173
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
176
def should_cache(self):
177
"""Return True if the data pulled across should be cached locally.
181
def clone(self, offset=None):
182
"""Return a new FtpTransport with root at self.base + offset.
186
return FtpTransport(self.base, self._FTP_instance)
188
return FtpTransport(self.abspath(offset), self._FTP_instance)
190
def _abspath(self, relpath):
191
assert isinstance(relpath, basestring)
192
relpath = urllib.unquote(relpath)
193
relpath_parts = relpath.split('/')
194
if len(relpath_parts) > 1:
195
if relpath_parts[0] == '':
196
raise ValueError("path %r within branch %r seems to be absolute"
197
% (relpath, self._path))
198
basepath = self._path.split('/')
199
if len(basepath) > 0 and basepath[-1] == '':
200
basepath = basepath[:-1]
201
for p in relpath_parts:
203
if len(basepath) == 0:
204
# In most filesystems, a request for the parent
205
# of root, just returns root.
208
elif p == '.' or p == '':
212
# Possibly, we could use urlparse.urljoin() here, but
213
# I'm concerned about when it chooses to strip the last
214
# portion of the path, and when it doesn't.
215
return '/'.join(basepath) or '/'
217
def abspath(self, relpath):
218
"""Return the full url to the given relative path.
219
This can be supplied with a string or a list
221
path = self._abspath(relpath)
222
return self._unparse_url(path)
224
def has(self, relpath):
225
"""Does the target location exist?"""
226
# FIXME jam 20060516 We *do* ask about directories in the test suite
227
# We don't seem to in the actual codebase
228
# XXX: I assume we're never asked has(dirname) and thus I use
229
# the FTP size command and assume that if it doesn't raise,
231
abspath = self._abspath(relpath)
234
mutter('FTP has check: %s => %s', relpath, abspath)
236
mutter("FTP has: %s", abspath)
238
except ftplib.error_perm, e:
239
if ('is a directory' in str(e).lower()):
240
mutter("FTP has dir: %s: %s", abspath, e)
242
mutter("FTP has not: %s: %s", abspath, e)
245
def get(self, relpath, decode=False, retries=0):
246
"""Get the file at the given relative path.
248
:param relpath: The relative path to the file
249
:param retries: Number of retries after temporary failures so far
252
We're meant to return a file-like object which bzr will
253
then read from. For now we do this via the magic of StringIO
255
# TODO: decode should be deprecated
257
mutter("FTP get: %s", self._abspath(relpath))
260
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
263
except ftplib.error_perm, e:
264
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
265
except ftplib.error_temp, e:
266
if retries > _number_of_retries:
267
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
268
% self.abspath(relpath),
271
warning("FTP temporary error: %s. Retrying.", str(e))
272
self._FTP_instance = None
273
return self.get(relpath, decode, retries+1)
275
if retries > _number_of_retries:
276
raise errors.TransportError("FTP control connection closed during GET %s."
277
% self.abspath(relpath),
280
warning("FTP control connection closed. Trying to reopen.")
281
time.sleep(_sleep_between_retries)
282
self._FTP_instance = None
283
return self.get(relpath, decode, retries+1)
285
def put(self, relpath, fp, mode=None, retries=0):
286
"""Copy the file-like or string object into the location.
288
:param relpath: Location to put the contents, relative to base.
289
:param fp: File-like or string object.
290
:param retries: Number of retries after temporary failures so far
293
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
295
abspath = self._abspath(relpath)
296
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
297
os.getpid(), random.randint(0,0x7FFFFFFF))
298
if not hasattr(fp, 'read'):
301
mutter("FTP put: %s", abspath)
304
f.storbinary('STOR '+tmp_abspath, fp)
305
f.rename(tmp_abspath, abspath)
306
except (ftplib.error_temp,EOFError), e:
307
warning("Failure during ftp PUT. Deleting temporary file.")
309
f.delete(tmp_abspath)
311
warning("Failed to delete temporary file on the"
312
" server.\nFile: %s", tmp_abspath)
315
except ftplib.error_perm, e:
316
self._translate_perm_error(e, abspath, extra='could not store')
317
except ftplib.error_temp, e:
318
if retries > _number_of_retries:
319
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
320
% self.abspath(relpath), orig_error=e)
322
warning("FTP temporary error: %s. Retrying.", str(e))
323
self._FTP_instance = None
324
self.put(relpath, fp, mode, retries+1)
326
if retries > _number_of_retries:
327
raise errors.TransportError("FTP control connection closed during PUT %s."
328
% self.abspath(relpath), orig_error=e)
330
warning("FTP control connection closed. Trying to reopen.")
331
time.sleep(_sleep_between_retries)
332
self._FTP_instance = None
333
self.put(relpath, fp, mode, retries+1)
335
def mkdir(self, relpath, mode=None):
336
"""Create a directory at the given path."""
337
abspath = self._abspath(relpath)
339
mutter("FTP mkd: %s", abspath)
342
except ftplib.error_perm, e:
343
self._translate_perm_error(e, abspath,
344
unknown_exc=errors.FileExists)
346
def rmdir(self, rel_path):
347
"""Delete the directory at rel_path"""
348
abspath = self._abspath(rel_path)
350
mutter("FTP rmd: %s", abspath)
353
except ftplib.error_perm, e:
354
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
356
def append(self, relpath, f, mode=None):
357
"""Append the text in the file-like object into the final
360
abspath = self._abspath(relpath)
361
if self.has(relpath):
362
ftp = self._get_FTP()
363
result = ftp.size(abspath)
367
mutter("FTP appe to %s", abspath)
368
self._try_append(relpath, f.read(), mode)
372
def _try_append(self, relpath, text, mode=None, retries=0):
373
"""Try repeatedly to append the given text to the file at relpath.
375
This is a recursive function. On errors, it will be called until the
376
number of retries is exceeded.
379
abspath = self._abspath(relpath)
380
mutter("FTP appe (try %d) to %s", retries, abspath)
381
ftp = self._get_FTP()
382
ftp.voidcmd("TYPE I")
383
cmd = "APPE %s" % abspath
384
conn = ftp.transfercmd(cmd)
388
self._setmode(relpath, mode)
390
except ftplib.error_perm, e:
391
self._translate_perm_error(e, abspath, extra='error appending',
392
unknown_exc=errors.NoSuchFile)
393
except ftplib.error_temp, e:
394
if retries > _number_of_retries:
395
raise errors.TransportError("FTP temporary error during APPEND %s." \
396
"Aborting." % abspath, orig_error=e)
398
warning("FTP temporary error: %s. Retrying.", str(e))
399
self._FTP_instance = None
400
self._try_append(relpath, text, mode, retries+1)
402
def _setmode(self, relpath, mode):
403
"""Set permissions on a path.
405
Only set permissions if the FTP server supports the 'SITE CHMOD'
409
mutter("FTP site chmod: setting permissions to %s on %s",
410
str(mode), self._abspath(relpath))
411
ftp = self._get_FTP()
412
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
414
except ftplib.error_perm, e:
415
# Command probably not available on this server
416
warning("FTP Could not set permissions to %s on %s. %s",
417
str(mode), self._abspath(relpath), str(e))
419
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
420
# to copy something to another machine. And you may be able
421
# to give it its own address as the 'to' location.
422
# So implement a fancier 'copy()'
424
def move(self, rel_from, rel_to):
425
"""Move the item at rel_from to the location at rel_to"""
426
abs_from = self._abspath(rel_from)
427
abs_to = self._abspath(rel_to)
429
mutter("FTP mv: %s => %s", abs_from, abs_to)
431
f.rename(abs_from, abs_to)
432
except ftplib.error_perm, e:
433
self._translate_perm_error(e, abs_from,
434
extra='unable to rename to %r' % (rel_to,),
435
unknown_exc=errors.PathError)
439
def delete(self, relpath):
440
"""Delete the item at relpath"""
441
abspath = self._abspath(relpath)
443
mutter("FTP rm: %s", abspath)
446
except ftplib.error_perm, e:
447
self._translate_perm_error(e, abspath, 'error deleting',
448
unknown_exc=errors.NoSuchFile)
451
"""See Transport.listable."""
454
def list_dir(self, relpath):
455
"""See Transport.list_dir."""
457
mutter("FTP nlst: %s", self._abspath(relpath))
459
basepath = self._abspath(relpath)
460
paths = f.nlst(basepath)
461
# If FTP.nlst returns paths prefixed by relpath, strip 'em
462
if paths and paths[0].startswith(basepath):
463
paths = [path[len(basepath)+1:] for path in paths]
464
# Remove . and .. if present, and return
465
return [path for path in paths if path not in (".", "..")]
466
except ftplib.error_perm, e:
467
self._translate_perm_error(e, relpath, extra='error with list_dir')
469
def iter_files_recursive(self):
470
"""See Transport.iter_files_recursive.
472
This is cargo-culted from the SFTP transport"""
473
mutter("FTP iter_files_recursive")
474
queue = list(self.list_dir("."))
476
relpath = urllib.quote(queue.pop(0))
477
st = self.stat(relpath)
478
if stat.S_ISDIR(st.st_mode):
479
for i, basename in enumerate(self.list_dir(relpath)):
480
queue.insert(i, relpath+"/"+basename)
484
def stat(self, relpath):
485
"""Return the stat information for a file."""
486
abspath = self._abspath(relpath)
488
mutter("FTP stat: %s", abspath)
490
return FtpStatResult(f, abspath)
491
except ftplib.error_perm, e:
492
self._translate_perm_error(e, abspath, extra='error w/ stat')
494
def lock_read(self, relpath):
495
"""Lock the given file for shared (read) access.
496
:return: A lock object, which should be passed to Transport.unlock()
498
# The old RemoteBranch ignore lock for reading, so we will
499
# continue that tradition and return a bogus lock object.
500
class BogusLock(object):
501
def __init__(self, path):
505
return BogusLock(relpath)
507
def lock_write(self, relpath):
508
"""Lock the given file for exclusive (write) access.
509
WARNING: many transports do not support this, so trying avoid using it
511
:return: A lock object, which should be passed to Transport.unlock()
513
return self.lock_read(relpath)
516
class FtpServer(Server):
517
"""Common code for SFTP server facilities."""
521
self._ftp_server = None
523
self._async_thread = None
528
"""Calculate an ftp url to this server."""
529
return 'ftp://foo:bar@localhost:%d/' % (self._port)
531
# def get_bogus_url(self):
532
# """Return a URL which cannot be connected to."""
533
# return 'ftp://127.0.0.1:1'
535
def log(self, message):
536
"""This is used by medusa.ftp_server to log connections, etc."""
537
self.logs.append(message)
542
raise RuntimeError('Must have medusa to run the FtpServer')
544
self._root = os.getcwdu()
545
self._ftp_server = _ftp_server(
546
authorizer=_test_authorizer(root=self._root),
548
port=0, # bind to a random port
550
logger_object=self # Use FtpServer.log() for messages
552
self._port = self._ftp_server.getsockname()[1]
553
# Don't let it loop forever, or handle an infinite number of requests.
554
# In this case it will run for 100s, or 1000 requests
555
self._async_thread = threading.Thread(target=asyncore.loop,
556
kwargs={'timeout':0.1, 'count':1000})
557
self._async_thread.setDaemon(True)
558
self._async_thread.start()
561
"""See bzrlib.transport.Server.tearDown."""
562
# have asyncore release the channel
563
self._ftp_server.del_channel()
565
self._async_thread.join()
570
_test_authorizer = None
574
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
577
import medusa.filesys
578
import medusa.ftp_server
584
class test_authorizer(object):
585
"""A custom Authorizer object for running the test suite.
587
The reason we cannot use dummy_authorizer, is because it sets the
588
channel to readonly, which we don't always want to do.
591
def __init__(self, root):
594
def authorize(self, channel, username, password):
595
"""Return (success, reply_string, filesystem)"""
597
return 0, 'No Medusa.', None
599
channel.persona = -1, -1
600
if username == 'anonymous':
601
channel.read_only = 1
603
channel.read_only = 0
605
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
608
class ftp_channel(medusa.ftp_server.ftp_channel):
609
"""Customized ftp channel"""
611
def log(self, message):
612
"""Redirect logging requests."""
613
mutter('_ftp_channel: %s', message)
615
def log_info(self, message, type='info'):
616
"""Redirect logging requests."""
617
mutter('_ftp_channel %s: %s', type, message)
619
def cmd_rnfr(self, line):
620
"""Prepare for renaming a file."""
621
self._renaming = line[1]
622
self.respond('350 Ready for RNTO')
623
# TODO: jam 20060516 in testing, the ftp server seems to
624
# check that the file already exists, or it sends
625
# 550 RNFR command failed
627
def cmd_rnto(self, line):
628
"""Rename a file based on the target given.
630
rnto must be called after calling rnfr.
632
if not self._renaming:
633
self.respond('503 RNFR required first.')
634
pfrom = self.filesystem.translate(self._renaming)
635
self._renaming = None
636
pto = self.filesystem.translate(line[1])
638
os.rename(pfrom, pto)
639
except (IOError, OSError), e:
640
# TODO: jam 20060516 return custom responses based on
641
# why the command failed
642
self.respond('550 RNTO failed: %s' % (e,))
644
self.respond('550 RNTO failed')
645
# For a test server, we will go ahead and just die
648
self.respond('250 Rename successful.')
650
def cmd_size(self, line):
651
"""Return the size of a file
653
This is overloaded to help the test suite determine if the
654
target is a directory.
657
if not self.filesystem.isfile(filename):
658
if self.filesystem.isdir(filename):
659
self.respond('550 "%s" is a directory' % (filename,))
661
self.respond('550 "%s" is not a file' % (filename,))
663
self.respond('213 %d'
664
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
666
def cmd_mkd(self, line):
667
"""Create a directory.
669
Overloaded because default implementation does not distinguish
670
*why* it cannot make a directory.
673
self.command_not_understood (string.join (line))
677
self.filesystem.mkdir (path)
678
self.respond ('257 MKD command successful.')
679
except (IOError, OSError), e:
680
self.respond ('550 error creating directory: %s' % (e,))
682
self.respond ('550 error creating directory.')
685
class ftp_server(medusa.ftp_server.ftp_server):
686
"""Customize the behavior of the Medusa ftp_server.
688
There are a few warts on the ftp_server, based on how it expects
692
ftp_channel_class = ftp_channel
694
def __init__(self, *args, **kwargs):
695
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
696
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
698
def log(self, message):
699
"""Redirect logging requests."""
700
mutter('_ftp_server: %s', message)
702
def log_info(self, message, type='info'):
703
"""Override the asyncore.log_info so we don't stipple the screen."""
704
mutter('_ftp_server %s: %s', type, message)
706
_test_authorizer = test_authorizer
707
_ftp_channel = ftp_channel
708
_ftp_server = ftp_server
713
def get_test_permutations():
714
"""Return the permutations to be used in testing."""
715
if not _setup_medusa():
716
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
719
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
