27
27
from cStringIO import StringIO
38
40
from warnings import warn
40
42
from bzrlib import (
46
47
from bzrlib.trace import mutter, warning
47
48
from bzrlib.transport import (
48
AppendBasedFileStream,
51
register_urlparse_netloc_protocol,
54
53
from bzrlib.transport.local import LocalURLServer
58
register_urlparse_netloc_protocol('aftp')
61
59
class FtpPathError(errors.PathError):
62
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
65
85
class FtpStatResult(object):
66
86
def __init__(self, f, relpath):
79
99
_number_of_retries = 2
80
100
_sleep_between_retries = 5
82
# FIXME: there are inconsistencies in the way temporary errors are
83
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
84
# be taken to analyze the implications for write operations (read operations
85
# are safe to retry). Overall even some read operations are never
86
# retried. --vila 20070720 (Bug #127164)
87
class FtpTransport(ConnectedTransport):
102
class FtpTransport(Transport):
88
103
"""This is the transport agent for ftp:// access."""
90
def __init__(self, base, _from_transport=None):
105
def __init__(self, base, _provided_instance=None):
91
106
"""Set the base path where files will be stored."""
92
if not (base.startswith('ftp://') or base.startswith('aftp://')):
93
raise ValueError(base)
94
super(FtpTransport, self).__init__(base,
95
_from_transport=_from_transport)
96
self._unqualified_scheme = 'ftp'
97
if self._scheme == 'aftp':
100
self.is_active = False
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://
113
if not base.endswith('/'):
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
123
def _unparse_url(self, path=None):
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
102
137
def _get_FTP(self):
103
138
"""Return the ftplib.FTP instance for this object."""
104
# Ensures that a connection is established
105
connection = self._get_connection()
106
if connection is None:
107
# First connection ever
108
connection, credentials = self._create_connection()
109
self._set_connection(connection, credentials)
112
def _create_connection(self, credentials=None):
113
"""Create a new connection with the provided credentials.
115
:param credentials: The credentials needed to establish the connection.
117
:return: The created connection and its associated credentials.
119
The input credentials are only the password as it may have been
120
entered interactively by the user and may be different from the one
121
provided in base url at transport creation time. The returned
122
credentials are username, password.
124
if credentials is None:
125
user, password = self._user, self._password
127
user, password = credentials
129
auth = config.AuthenticationConfig()
131
user = auth.get_user('ftp', self._host, port=self._port)
133
# Default to local user
134
user = getpass.getuser()
136
mutter("Constructing FTP instance against %r" %
137
((self._host, self._port, user, '********',
139
if self._FTP_instance is not None:
140
return self._FTP_instance
140
connection = ftplib.FTP()
141
connection.connect(host=self._host, port=self._port)
142
if user and user != 'anonymous' and \
143
password is None: # '' is a valid password
144
password = auth.get_password('ftp', self._host, user,
146
connection.login(user=user, passwd=password)
147
connection.set_pasv(not self.is_active)
148
except socket.error, e:
149
raise errors.SocketConnectionError(self._host, self._port,
150
msg='Unable to connect to',
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
146
return self._FTP_instance
152
147
except ftplib.error_perm, e:
153
raise errors.TransportError(msg="Error setting up connection:"
154
" %s" % str(e), orig_error=e)
155
return connection, (user, password)
157
def _reconnect(self):
158
"""Create a new connection with the previously used credentials"""
159
credentials = self._get_credentials()
160
connection, credentials = self._create_connection(credentials)
161
self._set_connection(connection, credentials)
163
def _translate_perm_error(self, err, path, extra=None,
164
unknown_exc=FtpPathError):
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
165
152
"""Try to translate an ftplib.error_perm exception.
167
154
:param err: The error to translate into a bzr error
202
184
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
205
def _remote_path(self, relpath):
187
def should_cache(self):
188
"""Return True if the data pulled across should be cached locally.
192
def clone(self, offset=None):
193
"""Return a new FtpTransport with root at self.base + offset.
197
return FtpTransport(self.base, self._FTP_instance)
199
return FtpTransport(self.abspath(offset), self._FTP_instance)
201
def _abspath(self, relpath):
202
assert isinstance(relpath, basestring)
203
relpath = urlutils.unescape(relpath)
204
if relpath.startswith('/'):
207
basepath = self._path.split('/')
208
if len(basepath) > 0 and basepath[-1] == '':
209
basepath = basepath[:-1]
210
for p in relpath.split('/'):
212
if len(basepath) == 0:
213
# In most filesystems, a request for the parent
214
# of root, just returns root.
217
elif p == '.' or p == '':
221
# Possibly, we could use urlparse.urljoin() here, but
222
# I'm concerned about when it chooses to strip the last
223
# portion of the path, and when it doesn't.
206
225
# XXX: It seems that ftplib does not handle Unicode paths
207
# at the same time, medusa won't handle utf8 paths So if
208
# we .encode(utf8) here (see ConnectedTransport
209
# implementation), then we get a Server failure. while
210
# if we use str(), we get a UnicodeError, and the test
211
# suite just skips testing UnicodePaths.
212
relative = str(urlutils.unescape(relpath))
213
remote_path = self._combine_paths(self._path, relative)
226
# at the same time, medusa won't handle utf8 paths
227
# So if we .encode(utf8) here, then we get a Server failure.
228
# while if we use str(), we get a UnicodeError, and the test suite
229
# just skips testing UnicodePaths.
230
return str('/'.join(basepath) or '/')
232
def abspath(self, relpath):
233
"""Return the full url to the given relative path.
234
This can be supplied with a string or a list
236
path = self._abspath(relpath)
237
return self._unparse_url(path)
216
239
def has(self, relpath):
217
240
"""Does the target location exist?"""
344
346
warning("FTP control connection closed. Trying to reopen.")
345
347
time.sleep(_sleep_between_retries)
348
self._FTP_instance = None
347
349
self.put_file(relpath, fp, mode, retries+1)
349
351
def mkdir(self, relpath, mode=None):
350
352
"""Create a directory at the given path."""
351
abspath = self._remote_path(relpath)
353
abspath = self._abspath(relpath)
353
355
mutter("FTP mkd: %s", abspath)
354
356
f = self._get_FTP()
356
self._setmode(relpath, mode)
357
358
except ftplib.error_perm, e:
358
359
self._translate_perm_error(e, abspath,
359
360
unknown_exc=errors.FileExists)
361
def open_write_stream(self, relpath, mode=None):
362
"""See Transport.open_write_stream."""
363
self.put_bytes(relpath, "", mode)
364
result = AppendBasedFileStream(self, relpath)
365
_file_streams[self.abspath(relpath)] = result
368
def recommended_page_size(self):
369
"""See Transport.recommended_page_size().
371
For FTP we suggest a large page size to reduce the overhead
372
introduced by latency.
376
362
def rmdir(self, rel_path):
377
363
"""Delete the directory at rel_path"""
378
abspath = self._remote_path(rel_path)
364
abspath = self._abspath(rel_path)
380
366
mutter("FTP rmd: %s", abspath)
381
367
f = self._get_FTP()
586
556
return self.lock_read(relpath)
559
class FtpServer(Server):
560
"""Common code for SFTP server facilities."""
564
self._ftp_server = None
566
self._async_thread = None
571
"""Calculate an ftp url to this server."""
572
return 'ftp://foo:bar@localhost:%d/' % (self._port)
574
# def get_bogus_url(self):
575
# """Return a URL which cannot be connected to."""
576
# return 'ftp://127.0.0.1:1'
578
def log(self, message):
579
"""This is used by medusa.ftp_server to log connections, etc."""
580
self.logs.append(message)
582
def setUp(self, vfs_server=None):
584
raise RuntimeError('Must have medusa to run the FtpServer')
586
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
587
"FtpServer currently assumes local transport, got %s" % vfs_server
589
self._root = os.getcwdu()
590
self._ftp_server = _ftp_server(
591
authorizer=_test_authorizer(root=self._root),
593
port=0, # bind to a random port
595
logger_object=self # Use FtpServer.log() for messages
597
self._port = self._ftp_server.getsockname()[1]
598
# Don't let it loop forever, or handle an infinite number of requests.
599
# In this case it will run for 100s, or 1000 requests
600
self._async_thread = threading.Thread(
601
target=FtpServer._asyncore_loop_ignore_EBADF,
602
kwargs={'timeout':0.1, 'count':1000})
603
self._async_thread.setDaemon(True)
604
self._async_thread.start()
607
"""See bzrlib.transport.Server.tearDown."""
608
# have asyncore release the channel
609
self._ftp_server.del_channel()
611
self._async_thread.join()
614
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
615
"""Ignore EBADF during server shutdown.
617
We close the socket to get the server to shutdown, but this causes
618
select.select() to raise EBADF.
621
asyncore.loop(*args, **kwargs)
622
except select.error, e:
623
if e.args[0] != errno.EBADF:
629
_test_authorizer = None
633
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
636
import medusa.filesys
637
import medusa.ftp_server
643
class test_authorizer(object):
644
"""A custom Authorizer object for running the test suite.
646
The reason we cannot use dummy_authorizer, is because it sets the
647
channel to readonly, which we don't always want to do.
650
def __init__(self, root):
653
def authorize(self, channel, username, password):
654
"""Return (success, reply_string, filesystem)"""
656
return 0, 'No Medusa.', None
658
channel.persona = -1, -1
659
if username == 'anonymous':
660
channel.read_only = 1
662
channel.read_only = 0
664
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
667
class ftp_channel(medusa.ftp_server.ftp_channel):
668
"""Customized ftp channel"""
670
def log(self, message):
671
"""Redirect logging requests."""
672
mutter('_ftp_channel: %s', message)
674
def log_info(self, message, type='info'):
675
"""Redirect logging requests."""
676
mutter('_ftp_channel %s: %s', type, message)
678
def cmd_rnfr(self, line):
679
"""Prepare for renaming a file."""
680
self._renaming = line[1]
681
self.respond('350 Ready for RNTO')
682
# TODO: jam 20060516 in testing, the ftp server seems to
683
# check that the file already exists, or it sends
684
# 550 RNFR command failed
686
def cmd_rnto(self, line):
687
"""Rename a file based on the target given.
689
rnto must be called after calling rnfr.
691
if not self._renaming:
692
self.respond('503 RNFR required first.')
693
pfrom = self.filesystem.translate(self._renaming)
694
self._renaming = None
695
pto = self.filesystem.translate(line[1])
696
if os.path.exists(pto):
697
self.respond('550 RNTO failed: file exists')
700
os.rename(pfrom, pto)
701
except (IOError, OSError), e:
702
# TODO: jam 20060516 return custom responses based on
703
# why the command failed
704
self.respond('550 RNTO failed: %s' % (e,))
706
self.respond('550 RNTO failed')
707
# For a test server, we will go ahead and just die
710
self.respond('250 Rename successful.')
712
def cmd_size(self, line):
713
"""Return the size of a file
715
This is overloaded to help the test suite determine if the
716
target is a directory.
719
if not self.filesystem.isfile(filename):
720
if self.filesystem.isdir(filename):
721
self.respond('550 "%s" is a directory' % (filename,))
723
self.respond('550 "%s" is not a file' % (filename,))
725
self.respond('213 %d'
726
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
728
def cmd_mkd(self, line):
729
"""Create a directory.
731
Overloaded because default implementation does not distinguish
732
*why* it cannot make a directory.
735
self.command_not_understood(''.join(line))
739
self.filesystem.mkdir (path)
740
self.respond ('257 MKD command successful.')
741
except (IOError, OSError), e:
742
self.respond ('550 error creating directory: %s' % (e,))
744
self.respond ('550 error creating directory.')
747
class ftp_server(medusa.ftp_server.ftp_server):
748
"""Customize the behavior of the Medusa ftp_server.
750
There are a few warts on the ftp_server, based on how it expects
754
ftp_channel_class = ftp_channel
756
def __init__(self, *args, **kwargs):
757
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
758
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
760
def log(self, message):
761
"""Redirect logging requests."""
762
mutter('_ftp_server: %s', message)
764
def log_info(self, message, type='info'):
765
"""Override the asyncore.log_info so we don't stipple the screen."""
766
mutter('_ftp_server %s: %s', type, message)
768
_test_authorizer = test_authorizer
769
_ftp_channel = ftp_channel
770
_ftp_server = ftp_server
589
775
def get_test_permutations():
590
776
"""Return the permutations to be used in testing."""
591
from bzrlib import tests
592
if tests.FTPServerFeature.available():
593
from bzrlib.tests import ftp_server
594
return [(FtpTransport, ftp_server.FTPServer)]
777
if not _setup_medusa():
778
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
596
# Dummy server to have the test suite report the number of tests
597
# needing that feature. We raise UnavailableFeature from methods before
598
# the test server is being used. Doing so in the setUp method has bad
599
# side-effects (tearDown is never called).
600
class UnavailableFTPServer(object):
609
raise tests.UnavailableFeature(tests.FTPServerFeature)
611
def get_bogus_url(self):
612
raise tests.UnavailableFeature(tests.FTPServerFeature)
614
return [(FtpTransport, UnavailableFTPServer)]
781
return [(FtpTransport, FtpServer)]