27
27
from cStringIO import StringIO
38
36
from warnings import warn
40
38
from bzrlib import (
44
44
from bzrlib.trace import mutter, warning
45
45
from bzrlib.transport import (
46
AppendBasedFileStream,
49
register_urlparse_netloc_protocol,
52
from bzrlib.transport.local import LocalURLServer
56
register_urlparse_netloc_protocol('aftp')
55
59
class FtpPathError(errors.PathError):
56
60
"""FTP failed for path: %(path)s%(extra)s"""
60
def _find_FTP(hostname, port, username, password, is_active):
61
"""Find an ftplib.FTP instance attached to this triplet."""
62
key = (hostname, port, username, password, is_active)
63
alt_key = (hostname, port, username, '********', is_active)
64
if key not in _FTP_cache:
65
mutter("Constructing FTP instance against %r" % (alt_key,))
68
conn.connect(host=hostname, port=port)
69
if username and username != 'anonymous' and not password:
70
password = bzrlib.ui.ui_factory.get_password(
71
prompt='FTP %(user)s@%(host)s password',
72
user=username, host=hostname)
73
conn.login(user=username, passwd=password)
74
conn.set_pasv(not is_active)
76
_FTP_cache[key] = conn
78
return _FTP_cache[key]
81
63
class FtpStatResult(object):
82
64
def __init__(self, f, relpath):
95
77
_number_of_retries = 2
96
78
_sleep_between_retries = 5
98
class FtpTransport(Transport):
80
# FIXME: there are inconsistencies in the way temporary errors are
81
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
82
# be taken to analyze the implications for write operations (read operations
83
# are safe to retry). Overall even some read operations are never
84
# retried. --vila 20070720 (Bug #127164)
85
class FtpTransport(ConnectedTransport):
99
86
"""This is the transport agent for ftp:// access."""
101
def __init__(self, base, _provided_instance=None):
88
def __init__(self, base, _from_transport=None):
102
89
"""Set the base path where files will be stored."""
103
90
assert base.startswith('ftp://') or base.startswith('aftp://')
105
self.is_active = base.startswith('aftp://')
107
# urlparse won't handle aftp://
109
if not base.endswith('/'):
111
(self._proto, self._username,
112
self._password, self._host,
113
self._port, self._path) = split_url(base)
114
base = self._unparse_url()
116
super(FtpTransport, self).__init__(base)
117
self._FTP_instance = _provided_instance
119
def _unparse_url(self, path=None):
122
path = urllib.quote(path)
123
netloc = urllib.quote(self._host)
124
if self._username is not None:
125
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
126
if self._port is not None:
127
netloc = '%s:%d' % (netloc, self._port)
131
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
91
super(FtpTransport, self).__init__(base,
92
_from_transport=_from_transport)
93
self._unqualified_scheme = 'ftp'
94
if self._scheme == 'aftp':
97
self.is_active = False
133
99
def _get_FTP(self):
134
100
"""Return the ftplib.FTP instance for this object."""
135
if self._FTP_instance is not None:
136
return self._FTP_instance
101
# Ensures that a connection is established
102
connection = self._get_connection()
103
if connection is None:
104
# First connection ever
105
connection, credentials = self._create_connection()
106
self._set_connection(connection, credentials)
109
def _create_connection(self, credentials=None):
110
"""Create a new connection with the provided credentials.
112
:param credentials: The credentials needed to establish the connection.
114
:return: The created connection and its associated credentials.
116
The credentials are only the password as it may have been entered
117
interactively by the user and may be different from the one provided
118
in base url at transport creation time.
120
if credentials is None:
121
user, password = self._user, self._password
123
user, password = credentials
125
auth = config.AuthenticationConfig()
127
user = auth.get_user('ftp', self._host, port=self._port)
129
# Default to local user
130
user = getpass.getuser()
132
mutter("Constructing FTP instance against %r" %
133
((self._host, self._port, user, '********',
139
self._FTP_instance = _find_FTP(self._host, self._port,
140
self._username, self._password,
142
return self._FTP_instance
136
connection = ftplib.FTP()
137
connection.connect(host=self._host, port=self._port)
138
if user and user != 'anonymous' and \
139
password is None: # '' is a valid password
140
password = auth.get_password('ftp', self._host, user,
142
connection.login(user=user, passwd=password)
143
connection.set_pasv(not self.is_active)
143
144
except ftplib.error_perm, e:
144
raise errors.TransportError(msg="Error setting up connection: %s"
145
% str(e), orig_error=e)
147
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
145
raise errors.TransportError(msg="Error setting up connection:"
146
" %s" % str(e), orig_error=e)
147
return connection, (user, password)
149
def _reconnect(self):
150
"""Create a new connection with the previously used credentials"""
151
credentials = self._get_credentials()
152
connection, credentials = self._create_connection(credentials)
153
self._set_connection(connection, credentials)
155
def _translate_perm_error(self, err, path, extra=None,
156
unknown_exc=FtpPathError):
148
157
"""Try to translate an ftplib.error_perm exception.
150
159
:param err: The error to translate into a bzr error
179
191
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
182
def should_cache(self):
183
"""Return True if the data pulled across should be cached locally.
187
def clone(self, offset=None):
188
"""Return a new FtpTransport with root at self.base + offset.
192
return FtpTransport(self.base, self._FTP_instance)
194
return FtpTransport(self.abspath(offset), self._FTP_instance)
196
def _abspath(self, relpath):
197
assert isinstance(relpath, basestring)
198
relpath = urlutils.unescape(relpath)
199
if relpath.startswith('/'):
202
basepath = self._path.split('/')
203
if len(basepath) > 0 and basepath[-1] == '':
204
basepath = basepath[:-1]
205
for p in relpath.split('/'):
207
if len(basepath) == 0:
208
# In most filesystems, a request for the parent
209
# of root, just returns root.
212
elif p == '.' or p == '':
216
# Possibly, we could use urlparse.urljoin() here, but
217
# I'm concerned about when it chooses to strip the last
218
# portion of the path, and when it doesn't.
194
def _remote_path(self, relpath):
220
195
# XXX: It seems that ftplib does not handle Unicode paths
221
# at the same time, medusa won't handle utf8 paths
222
# So if we .encode(utf8) here, then we get a Server failure.
223
# while if we use str(), we get a UnicodeError, and the test suite
224
# just skips testing UnicodePaths.
225
return str('/'.join(basepath) or '/')
227
def abspath(self, relpath):
228
"""Return the full url to the given relative path.
229
This can be supplied with a string or a list
231
path = self._abspath(relpath)
232
return self._unparse_url(path)
196
# at the same time, medusa won't handle utf8 paths So if
197
# we .encode(utf8) here (see ConnectedTransport
198
# implementation), then we get a Server failure. while
199
# if we use str(), we get a UnicodeError, and the test
200
# suite just skips testing UnicodePaths.
201
relative = str(urlutils.unescape(relpath))
202
remote_path = self._combine_paths(self._path, relative)
234
205
def has(self, relpath):
235
206
"""Does the target location exist?"""
300
271
:param retries: Number of retries after temporary failures so far
301
272
for this operation.
303
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
274
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
305
abspath = self._abspath(relpath)
277
abspath = self._remote_path(relpath)
306
278
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
307
279
os.getpid(), random.randint(0,0x7FFFFFFF))
308
281
if getattr(fp, 'read', None) is None:
282
# hand in a string IO
286
# capture the byte count; .read() may be read only so
288
class byte_counter(object):
289
def __init__(self, fp):
291
self.counted_bytes = 0
292
def read(self, count):
293
result = self.fp.read(count)
294
self.counted_bytes += len(result)
296
fp = byte_counter(fp)
311
298
mutter("FTP put: %s", abspath)
312
299
f = self._get_FTP()
314
301
f.storbinary('STOR '+tmp_abspath, fp)
315
f.rename(tmp_abspath, abspath)
302
self._rename_and_overwrite(tmp_abspath, abspath, f)
303
if bytes is not None:
306
return fp.counted_bytes
316
307
except (ftplib.error_temp,EOFError), e:
317
308
warning("Failure during ftp PUT. Deleting temporary file.")
419
426
mutter("FTP site chmod: setting permissions to %s on %s",
420
str(mode), self._abspath(relpath))
427
str(mode), self._remote_path(relpath))
421
428
ftp = self._get_FTP()
422
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
429
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
424
431
except ftplib.error_perm, e:
425
432
# Command probably not available on this server
426
433
warning("FTP Could not set permissions to %s on %s. %s",
427
str(mode), self._abspath(relpath), str(e))
434
str(mode), self._remote_path(relpath), str(e))
429
436
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
430
437
# to copy something to another machine. And you may be able
431
438
# to give it its own address as the 'to' location.
432
439
# So implement a fancier 'copy()'
441
def rename(self, rel_from, rel_to):
442
abs_from = self._remote_path(rel_from)
443
abs_to = self._remote_path(rel_to)
444
mutter("FTP rename: %s => %s", abs_from, abs_to)
446
return self._rename(abs_from, abs_to, f)
448
def _rename(self, abs_from, abs_to, f):
450
f.rename(abs_from, abs_to)
451
except ftplib.error_perm, e:
452
self._translate_perm_error(e, abs_from,
453
': unable to rename to %r' % (abs_to))
434
455
def move(self, rel_from, rel_to):
435
456
"""Move the item at rel_from to the location at rel_to"""
436
abs_from = self._abspath(rel_from)
437
abs_to = self._abspath(rel_to)
457
abs_from = self._remote_path(rel_from)
458
abs_to = self._remote_path(rel_to)
439
460
mutter("FTP mv: %s => %s", abs_from, abs_to)
440
461
f = self._get_FTP()
441
f.rename(abs_from, abs_to)
462
self._rename_and_overwrite(abs_from, abs_to, f)
442
463
except ftplib.error_perm, e:
443
464
self._translate_perm_error(e, abs_from,
444
465
extra='unable to rename to %r' % (rel_to,),
445
466
unknown_exc=errors.PathError)
468
def _rename_and_overwrite(self, abs_from, abs_to, f):
469
"""Do a fancy rename on the remote server.
471
Using the implementation provided by osutils.
473
osutils.fancy_rename(abs_from, abs_to,
474
rename_func=lambda p1, p2: self._rename(p1, p2, f),
475
unlink_func=lambda p: self._delete(p, f))
449
477
def delete(self, relpath):
450
478
"""Delete the item at relpath"""
451
abspath = self._abspath(relpath)
479
abspath = self._remote_path(relpath)
481
self._delete(abspath, f)
483
def _delete(self, abspath, f):
453
485
mutter("FTP rm: %s", abspath)
455
486
f.delete(abspath)
456
487
except ftplib.error_perm, e:
457
488
self._translate_perm_error(e, abspath, 'error deleting',
458
489
unknown_exc=errors.NoSuchFile)
491
def external_url(self):
492
"""See bzrlib.transport.Transport.external_url."""
493
# FTP URL's are externally usable.
460
496
def listable(self):
461
497
"""See Transport.listable."""
464
500
def list_dir(self, relpath):
465
501
"""See Transport.list_dir."""
466
basepath = self._abspath(relpath)
502
basepath = self._remote_path(relpath)
467
503
mutter("FTP nlst: %s", basepath)
468
504
f = self._get_FTP()
526
562
return self.lock_read(relpath)
529
class FtpServer(Server):
530
"""Common code for SFTP server facilities."""
534
self._ftp_server = None
536
self._async_thread = None
541
"""Calculate an ftp url to this server."""
542
return 'ftp://foo:bar@localhost:%d/' % (self._port)
544
# def get_bogus_url(self):
545
# """Return a URL which cannot be connected to."""
546
# return 'ftp://127.0.0.1:1'
548
def log(self, message):
549
"""This is used by medusa.ftp_server to log connections, etc."""
550
self.logs.append(message)
555
raise RuntimeError('Must have medusa to run the FtpServer')
557
self._root = os.getcwdu()
558
self._ftp_server = _ftp_server(
559
authorizer=_test_authorizer(root=self._root),
561
port=0, # bind to a random port
563
logger_object=self # Use FtpServer.log() for messages
565
self._port = self._ftp_server.getsockname()[1]
566
# Don't let it loop forever, or handle an infinite number of requests.
567
# In this case it will run for 100s, or 1000 requests
568
self._async_thread = threading.Thread(target=asyncore.loop,
569
kwargs={'timeout':0.1, 'count':1000})
570
self._async_thread.setDaemon(True)
571
self._async_thread.start()
574
"""See bzrlib.transport.Server.tearDown."""
575
# have asyncore release the channel
576
self._ftp_server.del_channel()
578
self._async_thread.join()
583
_test_authorizer = None
587
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
590
import medusa.filesys
591
import medusa.ftp_server
597
class test_authorizer(object):
598
"""A custom Authorizer object for running the test suite.
600
The reason we cannot use dummy_authorizer, is because it sets the
601
channel to readonly, which we don't always want to do.
604
def __init__(self, root):
607
def authorize(self, channel, username, password):
608
"""Return (success, reply_string, filesystem)"""
610
return 0, 'No Medusa.', None
612
channel.persona = -1, -1
613
if username == 'anonymous':
614
channel.read_only = 1
616
channel.read_only = 0
618
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
621
class ftp_channel(medusa.ftp_server.ftp_channel):
622
"""Customized ftp channel"""
624
def log(self, message):
625
"""Redirect logging requests."""
626
mutter('_ftp_channel: %s', message)
628
def log_info(self, message, type='info'):
629
"""Redirect logging requests."""
630
mutter('_ftp_channel %s: %s', type, message)
632
def cmd_rnfr(self, line):
633
"""Prepare for renaming a file."""
634
self._renaming = line[1]
635
self.respond('350 Ready for RNTO')
636
# TODO: jam 20060516 in testing, the ftp server seems to
637
# check that the file already exists, or it sends
638
# 550 RNFR command failed
640
def cmd_rnto(self, line):
641
"""Rename a file based on the target given.
643
rnto must be called after calling rnfr.
645
if not self._renaming:
646
self.respond('503 RNFR required first.')
647
pfrom = self.filesystem.translate(self._renaming)
648
self._renaming = None
649
pto = self.filesystem.translate(line[1])
651
os.rename(pfrom, pto)
652
except (IOError, OSError), e:
653
# TODO: jam 20060516 return custom responses based on
654
# why the command failed
655
self.respond('550 RNTO failed: %s' % (e,))
657
self.respond('550 RNTO failed')
658
# For a test server, we will go ahead and just die
661
self.respond('250 Rename successful.')
663
def cmd_size(self, line):
664
"""Return the size of a file
666
This is overloaded to help the test suite determine if the
667
target is a directory.
670
if not self.filesystem.isfile(filename):
671
if self.filesystem.isdir(filename):
672
self.respond('550 "%s" is a directory' % (filename,))
674
self.respond('550 "%s" is not a file' % (filename,))
676
self.respond('213 %d'
677
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
679
def cmd_mkd(self, line):
680
"""Create a directory.
682
Overloaded because default implementation does not distinguish
683
*why* it cannot make a directory.
686
self.command_not_understood(''.join(line))
690
self.filesystem.mkdir (path)
691
self.respond ('257 MKD command successful.')
692
except (IOError, OSError), e:
693
self.respond ('550 error creating directory: %s' % (e,))
695
self.respond ('550 error creating directory.')
698
class ftp_server(medusa.ftp_server.ftp_server):
699
"""Customize the behavior of the Medusa ftp_server.
701
There are a few warts on the ftp_server, based on how it expects
705
ftp_channel_class = ftp_channel
707
def __init__(self, *args, **kwargs):
708
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
709
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
711
def log(self, message):
712
"""Redirect logging requests."""
713
mutter('_ftp_server: %s', message)
715
def log_info(self, message, type='info'):
716
"""Override the asyncore.log_info so we don't stipple the screen."""
717
mutter('_ftp_server %s: %s', type, message)
719
_test_authorizer = test_authorizer
720
_ftp_channel = ftp_channel
721
_ftp_server = ftp_server
726
565
def get_test_permutations():
727
566
"""Return the permutations to be used in testing."""
728
if not _setup_medusa():
729
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
567
from bzrlib import tests
568
if tests.FTPServerFeature.available():
569
from bzrlib.tests import ftp_server
570
return [(FtpTransport, ftp_server.FTPServer)]
732
return [(FtpTransport, FtpServer)]
572
# Dummy server to have the test suite report the number of tests
573
# needing that feature.
574
class UnavailableFTPServer(object):
576
raise tests.UnavailableFeature(tests.FTPServerFeature)
578
return [(FtpTransport, UnavailableFTPServer)]
added
removed
bzrlib/transport/ftp.py
