27
27
from cStringIO import StringIO
40
from warnings import warn
36
42
from bzrlib import (
42
47
from bzrlib.trace import mutter, warning
43
48
from bzrlib.transport import (
44
AppendBasedFileStream,
47
register_urlparse_netloc_protocol,
52
register_urlparse_netloc_protocol('aftp')
53
from bzrlib.transport.local import LocalURLServer
55
59
class FtpPathError(errors.PathError):
56
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
59
85
class FtpStatResult(object):
61
def __init__(self, f, abspath):
86
def __init__(self, f, relpath):
63
self.st_size = f.size(abspath)
88
self.st_size = f.size(relpath)
64
89
self.st_mode = stat.S_IFREG
65
90
except ftplib.error_perm:
69
94
self.st_mode = stat.S_IFDIR
74
99
_number_of_retries = 2
75
100
_sleep_between_retries = 5
77
# FIXME: there are inconsistencies in the way temporary errors are
78
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
79
# be taken to analyze the implications for write operations (read operations
80
# are safe to retry). Overall even some read operations are never
81
# retried. --vila 20070720 (Bug #127164)
82
class FtpTransport(ConnectedTransport):
102
class FtpTransport(Transport):
83
103
"""This is the transport agent for ftp:// access."""
85
def __init__(self, base, _from_transport=None):
105
def __init__(self, base, _provided_instance=None):
86
106
"""Set the base path where files will be stored."""
87
if not (base.startswith('ftp://') or base.startswith('aftp://')):
88
raise ValueError(base)
89
super(FtpTransport, self).__init__(base,
90
_from_transport=_from_transport)
91
self._unqualified_scheme = 'ftp'
92
if self._scheme == 'aftp':
95
self.is_active = False
97
# Most modern FTP servers support the APPE command. If ours doesn't, we
98
# (re)set this flag accordingly later.
99
self._has_append = True
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://
113
if not base.endswith('/'):
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
123
def _unparse_url(self, path=None):
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
101
137
def _get_FTP(self):
102
138
"""Return the ftplib.FTP instance for this object."""
103
# Ensures that a connection is established
104
connection = self._get_connection()
105
if connection is None:
106
# First connection ever
107
connection, credentials = self._create_connection()
108
self._set_connection(connection, credentials)
111
connection_class = ftplib.FTP
113
def _create_connection(self, credentials=None):
114
"""Create a new connection with the provided credentials.
116
:param credentials: The credentials needed to establish the connection.
118
:return: The created connection and its associated credentials.
120
The input credentials are only the password as it may have been
121
entered interactively by the user and may be different from the one
122
provided in base url at transport creation time. The returned
123
credentials are username, password.
125
if credentials is None:
126
user, password = self._user, self._password
128
user, password = credentials
130
auth = config.AuthenticationConfig()
132
user = auth.get_user('ftp', self._host, port=self._port,
133
default=getpass.getuser())
134
mutter("Constructing FTP instance against %r" %
135
((self._host, self._port, user, '********',
139
if self._FTP_instance is not None:
140
return self._FTP_instance
138
connection = self.connection_class()
139
connection.connect(host=self._host, port=self._port)
140
self._login(connection, auth, user, password)
141
connection.set_pasv(not self.is_active)
142
# binary mode is the default
143
connection.voidcmd('TYPE I')
144
except socket.error, e:
145
raise errors.SocketConnectionError(self._host, self._port,
146
msg='Unable to connect to',
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
146
return self._FTP_instance
148
147
except ftplib.error_perm, e:
149
raise errors.TransportError(msg="Error setting up connection:"
150
" %s" % str(e), orig_error=e)
151
return connection, (user, password)
153
def _login(self, connection, auth, user, password):
154
# '' is a valid password
155
if user and user != 'anonymous' and password is None:
156
password = auth.get_password('ftp', self._host,
157
user, port=self._port)
158
connection.login(user=user, passwd=password)
160
def _reconnect(self):
161
"""Create a new connection with the previously used credentials"""
162
credentials = self._get_credentials()
163
connection, credentials = self._create_connection(credentials)
164
self._set_connection(connection, credentials)
166
def _translate_perm_error(self, err, path, extra=None,
167
unknown_exc=FtpPathError):
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
168
152
"""Try to translate an ftplib.error_perm exception.
170
154
:param err: The error to translate into a bzr error
200
179
raise unknown_exc(path, extra=extra)
201
# TODO: jam 20060516 Consider re-raising the error wrapped in
180
# TODO: jam 20060516 Consider re-raising the error wrapped in
202
181
# something like TransportError, but this loses the traceback
203
182
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
204
183
# to handle. Consider doing something like that here.
205
184
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
187
def should_cache(self):
188
"""Return True if the data pulled across should be cached locally.
192
def clone(self, offset=None):
193
"""Return a new FtpTransport with root at self.base + offset.
197
return FtpTransport(self.base, self._FTP_instance)
199
return FtpTransport(self.abspath(offset), self._FTP_instance)
201
def _abspath(self, relpath):
202
assert isinstance(relpath, basestring)
203
relpath = urlutils.unescape(relpath)
204
if relpath.startswith('/'):
207
basepath = self._path.split('/')
208
if len(basepath) > 0 and basepath[-1] == '':
209
basepath = basepath[:-1]
210
for p in relpath.split('/'):
212
if len(basepath) == 0:
213
# In most filesystems, a request for the parent
214
# of root, just returns root.
217
elif p == '.' or p == '':
221
# Possibly, we could use urlparse.urljoin() here, but
222
# I'm concerned about when it chooses to strip the last
223
# portion of the path, and when it doesn't.
225
# XXX: It seems that ftplib does not handle Unicode paths
226
# at the same time, medusa won't handle utf8 paths
227
# So if we .encode(utf8) here, then we get a Server failure.
228
# while if we use str(), we get a UnicodeError, and the test suite
229
# just skips testing UnicodePaths.
230
return str('/'.join(basepath) or '/')
232
def abspath(self, relpath):
233
"""Return the full url to the given relative path.
234
This can be supplied with a string or a list
236
path = self._abspath(relpath)
237
return self._unparse_url(path)
208
239
def has(self, relpath):
209
240
"""Does the target location exist?"""
210
241
# FIXME jam 20060516 We *do* ask about directories in the test suite
336
346
warning("FTP control connection closed. Trying to reopen.")
337
347
time.sleep(_sleep_between_retries)
348
self._FTP_instance = None
339
349
self.put_file(relpath, fp, mode, retries+1)
341
351
def mkdir(self, relpath, mode=None):
342
352
"""Create a directory at the given path."""
343
abspath = self._remote_path(relpath)
353
abspath = self._abspath(relpath)
345
355
mutter("FTP mkd: %s", abspath)
346
356
f = self._get_FTP()
348
self._setmode(relpath, mode)
349
358
except ftplib.error_perm, e:
350
359
self._translate_perm_error(e, abspath,
351
360
unknown_exc=errors.FileExists)
353
def open_write_stream(self, relpath, mode=None):
354
"""See Transport.open_write_stream."""
355
self.put_bytes(relpath, "", mode)
356
result = AppendBasedFileStream(self, relpath)
357
_file_streams[self.abspath(relpath)] = result
360
def recommended_page_size(self):
361
"""See Transport.recommended_page_size().
363
For FTP we suggest a large page size to reduce the overhead
364
introduced by latency.
368
362
def rmdir(self, rel_path):
369
363
"""Delete the directory at rel_path"""
370
abspath = self._remote_path(rel_path)
364
abspath = self._abspath(rel_path)
372
366
mutter("FTP rmd: %s", abspath)
373
367
f = self._get_FTP()
379
373
"""Append the text in the file-like object into the final
383
abspath = self._remote_path(relpath)
376
abspath = self._abspath(relpath)
384
377
if self.has(relpath):
385
378
ftp = self._get_FTP()
386
379
result = ftp.size(abspath)
391
mutter("FTP appe to %s", abspath)
392
self._try_append(relpath, text, mode)
394
self._fallback_append(relpath, text, mode)
383
mutter("FTP appe to %s", abspath)
384
self._try_append(relpath, f.read(), mode)
398
388
def _try_append(self, relpath, text, mode=None, retries=0):
399
389
"""Try repeatedly to append the given text to the file at relpath.
401
391
This is a recursive function. On errors, it will be called until the
402
392
number of retries is exceeded.
405
abspath = self._remote_path(relpath)
395
abspath = self._abspath(relpath)
406
396
mutter("FTP appe (try %d) to %s", retries, abspath)
407
397
ftp = self._get_FTP()
398
ftp.voidcmd("TYPE I")
408
399
cmd = "APPE %s" % abspath
409
400
conn = ftp.transfercmd(cmd)
410
401
conn.sendall(text)
412
self._setmode(relpath, mode)
404
self._setmode(relpath, mode)
414
406
except ftplib.error_perm, e:
415
# Check whether the command is not supported (reply code 502)
416
if str(e).startswith('502 '):
417
warning("FTP server does not support file appending natively. "
418
"Performance may be severely degraded! (%s)", e)
419
self._has_append = False
420
self._fallback_append(relpath, text, mode)
422
self._translate_perm_error(e, abspath, extra='error appending',
423
unknown_exc=errors.NoSuchFile)
407
self._translate_perm_error(e, abspath, extra='error appending',
408
unknown_exc=errors.NoSuchFile)
424
409
except ftplib.error_temp, e:
425
410
if retries > _number_of_retries:
426
raise errors.TransportError(
427
"FTP temporary error during APPEND %s. Aborting."
428
% abspath, orig_error=e)
411
raise errors.TransportError("FTP temporary error during APPEND %s." \
412
"Aborting." % abspath, orig_error=e)
430
414
warning("FTP temporary error: %s. Retrying.", str(e))
415
self._FTP_instance = None
432
416
self._try_append(relpath, text, mode, retries+1)
434
def _fallback_append(self, relpath, text, mode = None):
435
remote = self.get(relpath)
436
remote.seek(0, os.SEEK_END)
439
return self.put_file(relpath, remote, mode)
441
418
def _setmode(self, relpath, mode):
442
419
"""Set permissions on a path.
444
421
Only set permissions if the FTP server supports the 'SITE CHMOD'
449
mutter("FTP site chmod: setting permissions to %s on %s",
450
oct(mode), self._remote_path(relpath))
451
ftp = self._get_FTP()
452
cmd = "SITE CHMOD %s %s" % (oct(mode),
453
self._remote_path(relpath))
455
except ftplib.error_perm, e:
456
# Command probably not available on this server
457
warning("FTP Could not set permissions to %s on %s. %s",
458
oct(mode), self._remote_path(relpath), str(e))
425
mutter("FTP site chmod: setting permissions to %s on %s",
426
str(mode), self._abspath(relpath))
427
ftp = self._get_FTP()
428
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
430
except ftplib.error_perm, e:
431
# Command probably not available on this server
432
warning("FTP Could not set permissions to %s on %s. %s",
433
str(mode), self._abspath(relpath), str(e))
460
435
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
461
436
# to copy something to another machine. And you may be able
512
487
self._translate_perm_error(e, abspath, 'error deleting',
513
488
unknown_exc=errors.NoSuchFile)
515
def external_url(self):
516
"""See bzrlib.transport.Transport.external_url."""
517
# FTP URL's are externally usable.
520
490
def listable(self):
521
491
"""See Transport.listable."""
524
494
def list_dir(self, relpath):
525
495
"""See Transport.list_dir."""
526
basepath = self._remote_path(relpath)
496
basepath = self._abspath(relpath)
527
497
mutter("FTP nlst: %s", basepath)
528
498
f = self._get_FTP()
531
paths = f.nlst(basepath)
532
except ftplib.error_perm, e:
533
self._translate_perm_error(e, relpath,
534
extra='error with list_dir')
535
except ftplib.error_temp, e:
536
# xs4all's ftp server raises a 450 temp error when listing an
537
# empty directory. Check for that and just return an empty list
538
# in that case. See bug #215522
539
if str(e).lower().startswith('450 no files found'):
540
mutter('FTP Server returned "%s" for nlst.'
541
' Assuming it means empty directory',
546
# Restore binary mode as nlst switch to ascii mode to retrieve file
500
paths = f.nlst(basepath)
501
except ftplib.error_perm, e:
502
self._translate_perm_error(e, relpath, extra='error with list_dir')
550
503
# If FTP.nlst returns paths prefixed by relpath, strip 'em
551
504
if paths and paths[0].startswith(basepath):
552
505
entries = [path[len(basepath)+1:] for path in paths]
603
556
return self.lock_read(relpath)
559
class FtpServer(Server):
560
"""Common code for SFTP server facilities."""
564
self._ftp_server = None
566
self._async_thread = None
571
"""Calculate an ftp url to this server."""
572
return 'ftp://foo:bar@localhost:%d/' % (self._port)
574
# def get_bogus_url(self):
575
# """Return a URL which cannot be connected to."""
576
# return 'ftp://127.0.0.1:1'
578
def log(self, message):
579
"""This is used by medusa.ftp_server to log connections, etc."""
580
self.logs.append(message)
582
def setUp(self, vfs_server=None):
584
raise RuntimeError('Must have medusa to run the FtpServer')
586
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
587
"FtpServer currently assumes local transport, got %s" % vfs_server
589
self._root = os.getcwdu()
590
self._ftp_server = _ftp_server(
591
authorizer=_test_authorizer(root=self._root),
593
port=0, # bind to a random port
595
logger_object=self # Use FtpServer.log() for messages
597
self._port = self._ftp_server.getsockname()[1]
598
# Don't let it loop forever, or handle an infinite number of requests.
599
# In this case it will run for 100s, or 1000 requests
600
self._async_thread = threading.Thread(
601
target=FtpServer._asyncore_loop_ignore_EBADF,
602
kwargs={'timeout':0.1, 'count':1000})
603
self._async_thread.setDaemon(True)
604
self._async_thread.start()
607
"""See bzrlib.transport.Server.tearDown."""
608
# have asyncore release the channel
609
self._ftp_server.del_channel()
611
self._async_thread.join()
614
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
615
"""Ignore EBADF during server shutdown.
617
We close the socket to get the server to shutdown, but this causes
618
select.select() to raise EBADF.
621
asyncore.loop(*args, **kwargs)
622
except select.error, e:
623
if e.args[0] != errno.EBADF:
629
_test_authorizer = None
633
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
636
import medusa.filesys
637
import medusa.ftp_server
643
class test_authorizer(object):
644
"""A custom Authorizer object for running the test suite.
646
The reason we cannot use dummy_authorizer, is because it sets the
647
channel to readonly, which we don't always want to do.
650
def __init__(self, root):
653
def authorize(self, channel, username, password):
654
"""Return (success, reply_string, filesystem)"""
656
return 0, 'No Medusa.', None
658
channel.persona = -1, -1
659
if username == 'anonymous':
660
channel.read_only = 1
662
channel.read_only = 0
664
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
667
class ftp_channel(medusa.ftp_server.ftp_channel):
668
"""Customized ftp channel"""
670
def log(self, message):
671
"""Redirect logging requests."""
672
mutter('_ftp_channel: %s', message)
674
def log_info(self, message, type='info'):
675
"""Redirect logging requests."""
676
mutter('_ftp_channel %s: %s', type, message)
678
def cmd_rnfr(self, line):
679
"""Prepare for renaming a file."""
680
self._renaming = line[1]
681
self.respond('350 Ready for RNTO')
682
# TODO: jam 20060516 in testing, the ftp server seems to
683
# check that the file already exists, or it sends
684
# 550 RNFR command failed
686
def cmd_rnto(self, line):
687
"""Rename a file based on the target given.
689
rnto must be called after calling rnfr.
691
if not self._renaming:
692
self.respond('503 RNFR required first.')
693
pfrom = self.filesystem.translate(self._renaming)
694
self._renaming = None
695
pto = self.filesystem.translate(line[1])
696
if os.path.exists(pto):
697
self.respond('550 RNTO failed: file exists')
700
os.rename(pfrom, pto)
701
except (IOError, OSError), e:
702
# TODO: jam 20060516 return custom responses based on
703
# why the command failed
704
self.respond('550 RNTO failed: %s' % (e,))
706
self.respond('550 RNTO failed')
707
# For a test server, we will go ahead and just die
710
self.respond('250 Rename successful.')
712
def cmd_size(self, line):
713
"""Return the size of a file
715
This is overloaded to help the test suite determine if the
716
target is a directory.
719
if not self.filesystem.isfile(filename):
720
if self.filesystem.isdir(filename):
721
self.respond('550 "%s" is a directory' % (filename,))
723
self.respond('550 "%s" is not a file' % (filename,))
725
self.respond('213 %d'
726
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
728
def cmd_mkd(self, line):
729
"""Create a directory.
731
Overloaded because default implementation does not distinguish
732
*why* it cannot make a directory.
735
self.command_not_understood(''.join(line))
739
self.filesystem.mkdir (path)
740
self.respond ('257 MKD command successful.')
741
except (IOError, OSError), e:
742
self.respond ('550 error creating directory: %s' % (e,))
744
self.respond ('550 error creating directory.')
747
class ftp_server(medusa.ftp_server.ftp_server):
748
"""Customize the behavior of the Medusa ftp_server.
750
There are a few warts on the ftp_server, based on how it expects
754
ftp_channel_class = ftp_channel
756
def __init__(self, *args, **kwargs):
757
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
758
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
760
def log(self, message):
761
"""Redirect logging requests."""
762
mutter('_ftp_server: %s', message)
764
def log_info(self, message, type='info'):
765
"""Override the asyncore.log_info so we don't stipple the screen."""
766
mutter('_ftp_server %s: %s', type, message)
768
_test_authorizer = test_authorizer
769
_ftp_channel = ftp_channel
770
_ftp_server = ftp_server
606
775
def get_test_permutations():
607
776
"""Return the permutations to be used in testing."""
608
from bzrlib.tests import ftp_server
609
return [(FtpTransport, ftp_server.FTPTestServer)]
777
if not _setup_medusa():
778
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
781
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
