27
27
from cStringIO import StringIO
40
38
from warnings import warn
42
40
from bzrlib import (
47
46
from bzrlib.trace import mutter, warning
48
47
from bzrlib.transport import (
48
AppendBasedFileStream,
51
register_urlparse_netloc_protocol,
53
54
from bzrlib.transport.local import LocalURLServer
58
register_urlparse_netloc_protocol('aftp')
59
61
class FtpPathError(errors.PathError):
60
62
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
85
65
class FtpStatResult(object):
86
66
def __init__(self, f, relpath):
99
79
_number_of_retries = 2
100
80
_sleep_between_retries = 5
102
class FtpTransport(Transport):
82
# FIXME: there are inconsistencies in the way temporary errors are
83
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
84
# be taken to analyze the implications for write operations (read operations
85
# are safe to retry). Overall even some read operations are never
86
# retried. --vila 20070720 (Bug #127164)
87
class FtpTransport(ConnectedTransport):
103
88
"""This is the transport agent for ftp:// access."""
105
def __init__(self, base, _provided_instance=None):
90
def __init__(self, base, _from_transport=None):
106
91
"""Set the base path where files will be stored."""
107
92
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://
113
if not base.endswith('/'):
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
123
def _unparse_url(self, path=None):
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
93
super(FtpTransport, self).__init__(base,
94
_from_transport=_from_transport)
95
self._unqualified_scheme = 'ftp'
96
if self._scheme == 'aftp':
99
self.is_active = False
137
101
def _get_FTP(self):
138
102
"""Return the ftplib.FTP instance for this object."""
139
if self._FTP_instance is not None:
140
return self._FTP_instance
103
# Ensures that a connection is established
104
connection = self._get_connection()
105
if connection is None:
106
# First connection ever
107
connection, credentials = self._create_connection()
108
self._set_connection(connection, credentials)
111
def _create_connection(self, credentials=None):
112
"""Create a new connection with the provided credentials.
114
:param credentials: The credentials needed to establish the connection.
116
:return: The created connection and its associated credentials.
118
The credentials are only the password as it may have been entered
119
interactively by the user and may be different from the one provided
120
in base url at transport creation time.
122
if credentials is None:
123
user, password = self._user, self._password
125
user, password = credentials
127
auth = config.AuthenticationConfig()
129
user = auth.get_user('ftp', self._host, port=self._port)
131
# Default to local user
132
user = getpass.getuser()
134
mutter("Constructing FTP instance against %r" %
135
((self._host, self._port, user, '********',
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
146
return self._FTP_instance
138
connection = ftplib.FTP()
139
connection.connect(host=self._host, port=self._port)
140
if user and user != 'anonymous' and \
141
password is None: # '' is a valid password
142
password = auth.get_password('ftp', self._host, user,
144
connection.login(user=user, passwd=password)
145
connection.set_pasv(not self.is_active)
146
except socket.error, e:
147
raise errors.SocketConnectionError(self._host, self._port,
148
msg='Unable to connect to',
147
150
except ftplib.error_perm, e:
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
151
raise errors.TransportError(msg="Error setting up connection:"
152
" %s" % str(e), orig_error=e)
153
return connection, (user, password)
155
def _reconnect(self):
156
"""Create a new connection with the previously used credentials"""
157
credentials = self._get_credentials()
158
connection, credentials = self._create_connection(credentials)
159
self._set_connection(connection, credentials)
161
def _translate_perm_error(self, err, path, extra=None,
162
unknown_exc=FtpPathError):
152
163
"""Try to translate an ftplib.error_perm exception.
154
165
:param err: The error to translate into a bzr error
184
197
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
187
def should_cache(self):
188
"""Return True if the data pulled across should be cached locally.
192
def clone(self, offset=None):
193
"""Return a new FtpTransport with root at self.base + offset.
197
return FtpTransport(self.base, self._FTP_instance)
199
return FtpTransport(self.abspath(offset), self._FTP_instance)
201
def _abspath(self, relpath):
202
assert isinstance(relpath, basestring)
203
relpath = urlutils.unescape(relpath)
204
if relpath.startswith('/'):
207
basepath = self._path.split('/')
208
if len(basepath) > 0 and basepath[-1] == '':
209
basepath = basepath[:-1]
210
for p in relpath.split('/'):
212
if len(basepath) == 0:
213
# In most filesystems, a request for the parent
214
# of root, just returns root.
217
elif p == '.' or p == '':
221
# Possibly, we could use urlparse.urljoin() here, but
222
# I'm concerned about when it chooses to strip the last
223
# portion of the path, and when it doesn't.
200
def _remote_path(self, relpath):
225
201
# XXX: It seems that ftplib does not handle Unicode paths
226
# at the same time, medusa won't handle utf8 paths
227
# So if we .encode(utf8) here, then we get a Server failure.
228
# while if we use str(), we get a UnicodeError, and the test suite
229
# just skips testing UnicodePaths.
230
return str('/'.join(basepath) or '/')
232
def abspath(self, relpath):
233
"""Return the full url to the given relative path.
234
This can be supplied with a string or a list
236
path = self._abspath(relpath)
237
return self._unparse_url(path)
202
# at the same time, medusa won't handle utf8 paths So if
203
# we .encode(utf8) here (see ConnectedTransport
204
# implementation), then we get a Server failure. while
205
# if we use str(), we get a UnicodeError, and the test
206
# suite just skips testing UnicodePaths.
207
relative = str(urlutils.unescape(relpath))
208
remote_path = self._combine_paths(self._path, relative)
239
211
def has(self, relpath):
240
212
"""Does the target location exist?"""
556
568
return self.lock_read(relpath)
559
class FtpServer(Server):
560
"""Common code for SFTP server facilities."""
564
self._ftp_server = None
566
self._async_thread = None
571
"""Calculate an ftp url to this server."""
572
return 'ftp://foo:bar@localhost:%d/' % (self._port)
574
# def get_bogus_url(self):
575
# """Return a URL which cannot be connected to."""
576
# return 'ftp://127.0.0.1:1'
578
def log(self, message):
579
"""This is used by medusa.ftp_server to log connections, etc."""
580
self.logs.append(message)
582
def setUp(self, vfs_server=None):
584
raise RuntimeError('Must have medusa to run the FtpServer')
586
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
587
"FtpServer currently assumes local transport, got %s" % vfs_server
589
self._root = os.getcwdu()
590
self._ftp_server = _ftp_server(
591
authorizer=_test_authorizer(root=self._root),
593
port=0, # bind to a random port
595
logger_object=self # Use FtpServer.log() for messages
597
self._port = self._ftp_server.getsockname()[1]
598
# Don't let it loop forever, or handle an infinite number of requests.
599
# In this case it will run for 100s, or 1000 requests
600
self._async_thread = threading.Thread(
601
target=FtpServer._asyncore_loop_ignore_EBADF,
602
kwargs={'timeout':0.1, 'count':1000})
603
self._async_thread.setDaemon(True)
604
self._async_thread.start()
607
"""See bzrlib.transport.Server.tearDown."""
608
# have asyncore release the channel
609
self._ftp_server.del_channel()
611
self._async_thread.join()
614
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
615
"""Ignore EBADF during server shutdown.
617
We close the socket to get the server to shutdown, but this causes
618
select.select() to raise EBADF.
621
asyncore.loop(*args, **kwargs)
622
except select.error, e:
623
if e.args[0] != errno.EBADF:
629
_test_authorizer = None
633
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
636
import medusa.filesys
637
import medusa.ftp_server
643
class test_authorizer(object):
644
"""A custom Authorizer object for running the test suite.
646
The reason we cannot use dummy_authorizer, is because it sets the
647
channel to readonly, which we don't always want to do.
650
def __init__(self, root):
653
def authorize(self, channel, username, password):
654
"""Return (success, reply_string, filesystem)"""
656
return 0, 'No Medusa.', None
658
channel.persona = -1, -1
659
if username == 'anonymous':
660
channel.read_only = 1
662
channel.read_only = 0
664
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
667
class ftp_channel(medusa.ftp_server.ftp_channel):
668
"""Customized ftp channel"""
670
def log(self, message):
671
"""Redirect logging requests."""
672
mutter('_ftp_channel: %s', message)
674
def log_info(self, message, type='info'):
675
"""Redirect logging requests."""
676
mutter('_ftp_channel %s: %s', type, message)
678
def cmd_rnfr(self, line):
679
"""Prepare for renaming a file."""
680
self._renaming = line[1]
681
self.respond('350 Ready for RNTO')
682
# TODO: jam 20060516 in testing, the ftp server seems to
683
# check that the file already exists, or it sends
684
# 550 RNFR command failed
686
def cmd_rnto(self, line):
687
"""Rename a file based on the target given.
689
rnto must be called after calling rnfr.
691
if not self._renaming:
692
self.respond('503 RNFR required first.')
693
pfrom = self.filesystem.translate(self._renaming)
694
self._renaming = None
695
pto = self.filesystem.translate(line[1])
696
if os.path.exists(pto):
697
self.respond('550 RNTO failed: file exists')
700
os.rename(pfrom, pto)
701
except (IOError, OSError), e:
702
# TODO: jam 20060516 return custom responses based on
703
# why the command failed
704
# (bialix 20070418) str(e) on Python 2.5 @ Windows
705
# sometimes don't provide expected error message;
706
# so we obtain such message via os.strerror()
707
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
709
self.respond('550 RNTO failed')
710
# For a test server, we will go ahead and just die
713
self.respond('250 Rename successful.')
715
def cmd_size(self, line):
716
"""Return the size of a file
718
This is overloaded to help the test suite determine if the
719
target is a directory.
722
if not self.filesystem.isfile(filename):
723
if self.filesystem.isdir(filename):
724
self.respond('550 "%s" is a directory' % (filename,))
726
self.respond('550 "%s" is not a file' % (filename,))
728
self.respond('213 %d'
729
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
731
def cmd_mkd(self, line):
732
"""Create a directory.
734
Overloaded because default implementation does not distinguish
735
*why* it cannot make a directory.
738
self.command_not_understood(''.join(line))
742
self.filesystem.mkdir (path)
743
self.respond ('257 MKD command successful.')
744
except (IOError, OSError), e:
745
# (bialix 20070418) str(e) on Python 2.5 @ Windows
746
# sometimes don't provide expected error message;
747
# so we obtain such message via os.strerror()
748
self.respond ('550 error creating directory: %s' %
749
os.strerror(e.errno))
751
self.respond ('550 error creating directory.')
754
class ftp_server(medusa.ftp_server.ftp_server):
755
"""Customize the behavior of the Medusa ftp_server.
757
There are a few warts on the ftp_server, based on how it expects
761
ftp_channel_class = ftp_channel
763
def __init__(self, *args, **kwargs):
764
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
765
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
767
def log(self, message):
768
"""Redirect logging requests."""
769
mutter('_ftp_server: %s', message)
771
def log_info(self, message, type='info'):
772
"""Override the asyncore.log_info so we don't stipple the screen."""
773
mutter('_ftp_server %s: %s', type, message)
775
_test_authorizer = test_authorizer
776
_ftp_channel = ftp_channel
777
_ftp_server = ftp_server
782
571
def get_test_permutations():
783
572
"""Return the permutations to be used in testing."""
784
if not _setup_medusa():
785
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
573
from bzrlib import tests
574
if tests.FTPServerFeature.available():
575
from bzrlib.tests import ftp_server
576
return [(FtpTransport, ftp_server.FTPServer)]
788
return [(FtpTransport, FtpServer)]
578
# Dummy server to have the test suite report the number of tests
579
# needing that feature.
580
class UnavailableFTPServer(object):
582
raise tests.UnavailableFeature(tests.FTPServerFeature)
584
return [(FtpTransport, UnavailableFTPServer)]
added
removed
bzrlib/transport/ftp.py
