28
27
from cStringIO import StringIO
40
from warnings import warn
37
42
from bzrlib import (
43
from bzrlib.symbol_versioning import (
49
47
from bzrlib.trace import mutter, warning
50
48
from bzrlib.transport import (
51
AppendBasedFileStream,
54
register_urlparse_netloc_protocol,
59
register_urlparse_netloc_protocol('aftp')
53
from bzrlib.transport.local import LocalURLServer
62
59
class FtpPathError(errors.PathError):
63
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
66
85
class FtpStatResult(object):
68
def __init__(self, f, abspath):
86
def __init__(self, f, relpath):
70
self.st_size = f.size(abspath)
88
self.st_size = f.size(relpath)
71
89
self.st_mode = stat.S_IFREG
72
90
except ftplib.error_perm:
76
94
self.st_mode = stat.S_IFDIR
81
99
_number_of_retries = 2
82
100
_sleep_between_retries = 5
84
# FIXME: there are inconsistencies in the way temporary errors are
85
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
86
# be taken to analyze the implications for write operations (read operations
87
# are safe to retry). Overall even some read operations are never
88
# retried. --vila 20070720 (Bug #127164)
89
class FtpTransport(ConnectedTransport):
102
class FtpTransport(Transport):
90
103
"""This is the transport agent for ftp:// access."""
92
def __init__(self, base, _from_transport=None):
105
def __init__(self, base, _provided_instance=None):
93
106
"""Set the base path where files will be stored."""
94
if not (base.startswith('ftp://') or base.startswith('aftp://')):
95
raise ValueError(base)
96
super(FtpTransport, self).__init__(base,
97
_from_transport=_from_transport)
98
self._unqualified_scheme = 'ftp'
99
if self._parsed_url.scheme == 'aftp':
100
self.is_active = True
102
self.is_active = False
104
# Most modern FTP servers support the APPE command. If ours doesn't, we
105
# (re)set this flag accordingly later.
106
self._has_append = True
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://
113
if not base.endswith('/'):
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
123
def _unparse_url(self, path=None):
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
108
137
def _get_FTP(self):
109
138
"""Return the ftplib.FTP instance for this object."""
110
# Ensures that a connection is established
111
connection = self._get_connection()
112
if connection is None:
113
# First connection ever
114
connection, credentials = self._create_connection()
115
self._set_connection(connection, credentials)
118
connection_class = ftplib.FTP
120
def _create_connection(self, credentials=None):
121
"""Create a new connection with the provided credentials.
123
:param credentials: The credentials needed to establish the connection.
125
:return: The created connection and its associated credentials.
127
The input credentials are only the password as it may have been
128
entered interactively by the user and may be different from the one
129
provided in base url at transport creation time. The returned
130
credentials are username, password.
132
if credentials is None:
133
user, password = self._user, self._password
135
user, password = credentials
137
auth = config.AuthenticationConfig()
139
user = auth.get_user('ftp', self._host, port=self._port,
140
default=getpass.getuser())
141
mutter("Constructing FTP instance against %r" %
142
((self._host, self._port, user, '********',
139
if self._FTP_instance is not None:
140
return self._FTP_instance
145
connection = self.connection_class()
146
connection.connect(host=self._host, port=self._port)
147
self._login(connection, auth, user, password)
148
connection.set_pasv(not self.is_active)
149
# binary mode is the default
150
connection.voidcmd('TYPE I')
151
except socket.error, e:
152
raise errors.SocketConnectionError(self._host, self._port,
153
msg='Unable to connect to',
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
146
return self._FTP_instance
155
147
except ftplib.error_perm, e:
156
raise errors.TransportError(msg="Error setting up connection:"
157
" %s" % str(e), orig_error=e)
158
return connection, (user, password)
160
def _login(self, connection, auth, user, password):
161
# '' is a valid password
162
if user and user != 'anonymous' and password is None:
163
password = auth.get_password('ftp', self._host,
164
user, port=self._port)
165
connection.login(user=user, passwd=password)
167
def _reconnect(self):
168
"""Create a new connection with the previously used credentials"""
169
credentials = self._get_credentials()
170
connection, credentials = self._create_connection(credentials)
171
self._set_connection(connection, credentials)
173
def disconnect(self):
174
connection = self._get_connection()
175
if connection is not None:
178
def _translate_ftp_error(self, err, path, extra=None,
179
unknown_exc=FtpPathError):
180
"""Try to translate an ftplib exception to a bzrlib exception.
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
152
"""Try to translate an ftplib.error_perm exception.
182
154
:param err: The error to translate into a bzr error
183
155
:param path: The path which had problems
198
167
or 'no such dir' in s
199
168
or 'could not create file' in s # vsftpd
200
169
or 'file doesn\'t exist' in s
201
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
202
or 'file/directory not found' in s # filezilla server
203
# Microsoft FTP-Service RNFR reply if file not found
204
or (s.startswith('550 ') and 'unable to rename to' in extra)
205
# if containing directory doesn't exist, suggested by
206
# <https://bugs.launchpad.net/bzr/+bug/224373>
207
or (s.startswith('550 ') and "can't find folder" in s)
209
171
raise errors.NoSuchFile(path, extra=extra)
210
elif ('file exists' in s):
172
if ('file exists' in s):
211
173
raise errors.FileExists(path, extra=extra)
212
elif ('not a directory' in s):
174
if ('not a directory' in s):
213
175
raise errors.PathError(path, extra=extra)
214
elif 'directory not empty' in s:
215
raise errors.DirectoryNotEmpty(path, extra=extra)
217
177
mutter('unable to understand error for path: %s: %s', path, err)
220
180
raise unknown_exc(path, extra=extra)
221
# TODO: jam 20060516 Consider re-raising the error wrapped in
181
# TODO: jam 20060516 Consider re-raising the error wrapped in
222
182
# something like TransportError, but this loses the traceback
223
183
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
224
184
# to handle. Consider doing something like that here.
225
185
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
188
def should_cache(self):
189
"""Return True if the data pulled across should be cached locally.
193
def clone(self, offset=None):
194
"""Return a new FtpTransport with root at self.base + offset.
198
return FtpTransport(self.base, self._FTP_instance)
200
return FtpTransport(self.abspath(offset), self._FTP_instance)
202
def _abspath(self, relpath):
203
assert isinstance(relpath, basestring)
204
relpath = urlutils.unescape(relpath)
205
if relpath.startswith('/'):
208
basepath = self._path.split('/')
209
if len(basepath) > 0 and basepath[-1] == '':
210
basepath = basepath[:-1]
211
for p in relpath.split('/'):
213
if len(basepath) == 0:
214
# In most filesystems, a request for the parent
215
# of root, just returns root.
218
elif p == '.' or p == '':
222
# Possibly, we could use urlparse.urljoin() here, but
223
# I'm concerned about when it chooses to strip the last
224
# portion of the path, and when it doesn't.
226
# XXX: It seems that ftplib does not handle Unicode paths
227
# at the same time, medusa won't handle utf8 paths
228
# So if we .encode(utf8) here, then we get a Server failure.
229
# while if we use str(), we get a UnicodeError, and the test suite
230
# just skips testing UnicodePaths.
231
return str('/'.join(basepath) or '/')
233
def abspath(self, relpath):
234
"""Return the full url to the given relative path.
235
This can be supplied with a string or a list
237
path = self._abspath(relpath)
238
return self._unparse_url(path)
228
240
def has(self, relpath):
229
241
"""Does the target location exist?"""
230
242
# FIXME jam 20060516 We *do* ask about directories in the test suite
361
348
warning("FTP control connection closed. Trying to reopen.")
362
349
time.sleep(_sleep_between_retries)
350
self._FTP_instance = None
364
351
self.put_file(relpath, fp, mode, retries+1)
366
353
def mkdir(self, relpath, mode=None):
367
354
"""Create a directory at the given path."""
368
abspath = self._remote_path(relpath)
355
abspath = self._abspath(relpath)
370
357
mutter("FTP mkd: %s", abspath)
371
358
f = self._get_FTP()
374
except ftplib.error_reply, e:
375
# <https://bugs.launchpad.net/bzr/+bug/224373> Microsoft FTP
376
# server returns "250 Directory created." which is kind of
377
# reasonable, 250 meaning "requested file action OK", but not what
378
# Python's ftplib expects.
379
if e[0][:3] == '250':
383
self._setmode(relpath, mode)
384
360
except ftplib.error_perm, e:
385
self._translate_ftp_error(e, abspath,
361
self._translate_perm_error(e, abspath,
386
362
unknown_exc=errors.FileExists)
388
def open_write_stream(self, relpath, mode=None):
389
"""See Transport.open_write_stream."""
390
self.put_bytes(relpath, "", mode)
391
result = AppendBasedFileStream(self, relpath)
392
_file_streams[self.abspath(relpath)] = result
395
def recommended_page_size(self):
396
"""See Transport.recommended_page_size().
398
For FTP we suggest a large page size to reduce the overhead
399
introduced by latency.
403
364
def rmdir(self, rel_path):
404
365
"""Delete the directory at rel_path"""
405
abspath = self._remote_path(rel_path)
366
abspath = self._abspath(rel_path)
407
368
mutter("FTP rmd: %s", abspath)
408
369
f = self._get_FTP()
410
371
except ftplib.error_perm, e:
411
self._translate_ftp_error(e, abspath, unknown_exc=errors.PathError)
372
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
413
374
def append_file(self, relpath, f, mode=None):
414
375
"""Append the text in the file-like object into the final
418
abspath = self._remote_path(relpath)
378
abspath = self._abspath(relpath)
419
379
if self.has(relpath):
420
380
ftp = self._get_FTP()
421
381
result = ftp.size(abspath)
426
mutter("FTP appe to %s", abspath)
427
self._try_append(relpath, text, mode)
429
self._fallback_append(relpath, text, mode)
385
mutter("FTP appe to %s", abspath)
386
self._try_append(relpath, f.read(), mode)
433
390
def _try_append(self, relpath, text, mode=None, retries=0):
434
391
"""Try repeatedly to append the given text to the file at relpath.
436
393
This is a recursive function. On errors, it will be called until the
437
394
number of retries is exceeded.
440
abspath = self._remote_path(relpath)
397
abspath = self._abspath(relpath)
441
398
mutter("FTP appe (try %d) to %s", retries, abspath)
442
399
ftp = self._get_FTP()
400
ftp.voidcmd("TYPE I")
443
401
cmd = "APPE %s" % abspath
444
402
conn = ftp.transfercmd(cmd)
445
403
conn.sendall(text)
447
self._setmode(relpath, mode)
406
self._setmode(relpath, mode)
449
408
except ftplib.error_perm, e:
450
# Check whether the command is not supported (reply code 502)
451
if str(e).startswith('502 '):
452
warning("FTP server does not support file appending natively. "
453
"Performance may be severely degraded! (%s)", e)
454
self._has_append = False
455
self._fallback_append(relpath, text, mode)
457
self._translate_ftp_error(e, abspath, extra='error appending',
458
unknown_exc=errors.NoSuchFile)
409
self._translate_perm_error(e, abspath, extra='error appending',
410
unknown_exc=errors.NoSuchFile)
459
411
except ftplib.error_temp, e:
460
412
if retries > _number_of_retries:
461
raise errors.TransportError(
462
"FTP temporary error during APPEND %s. Aborting."
463
% abspath, orig_error=e)
413
raise errors.TransportError("FTP temporary error during APPEND %s." \
414
"Aborting." % abspath, orig_error=e)
465
416
warning("FTP temporary error: %s. Retrying.", str(e))
417
self._FTP_instance = None
467
418
self._try_append(relpath, text, mode, retries+1)
469
def _fallback_append(self, relpath, text, mode = None):
470
remote = self.get(relpath)
471
remote.seek(0, os.SEEK_END)
474
return self.put_file(relpath, remote, mode)
476
420
def _setmode(self, relpath, mode):
477
421
"""Set permissions on a path.
479
423
Only set permissions if the FTP server supports the 'SITE CHMOD'
484
mutter("FTP site chmod: setting permissions to %s on %s",
485
oct(mode), self._remote_path(relpath))
486
ftp = self._get_FTP()
487
cmd = "SITE CHMOD %s %s" % (oct(mode),
488
self._remote_path(relpath))
490
except ftplib.error_perm, e:
491
# Command probably not available on this server
492
warning("FTP Could not set permissions to %s on %s. %s",
493
oct(mode), self._remote_path(relpath), str(e))
427
mutter("FTP site chmod: setting permissions to %s on %s",
428
str(mode), self._abspath(relpath))
429
ftp = self._get_FTP()
430
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
432
except ftplib.error_perm, e:
433
# Command probably not available on this server
434
warning("FTP Could not set permissions to %s on %s. %s",
435
str(mode), self._abspath(relpath), str(e))
495
437
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
496
438
# to copy something to another machine. And you may be able
507
449
def _rename(self, abs_from, abs_to, f):
509
451
f.rename(abs_from, abs_to)
510
except (ftplib.error_temp, ftplib.error_perm), e:
511
self._translate_ftp_error(e, abs_from,
452
except ftplib.error_perm, e:
453
self._translate_perm_error(e, abs_from,
512
454
': unable to rename to %r' % (abs_to))
514
456
def move(self, rel_from, rel_to):
515
457
"""Move the item at rel_from to the location at rel_to"""
516
abs_from = self._remote_path(rel_from)
517
abs_to = self._remote_path(rel_to)
458
abs_from = self._abspath(rel_from)
459
abs_to = self._abspath(rel_to)
519
461
mutter("FTP mv: %s => %s", abs_from, abs_to)
520
462
f = self._get_FTP()
521
463
self._rename_and_overwrite(abs_from, abs_to, f)
522
464
except ftplib.error_perm, e:
523
self._translate_ftp_error(e, abs_from,
524
extra='unable to rename to %r' % (rel_to,),
465
self._translate_perm_error(e, abs_from,
466
extra='unable to rename to %r' % (rel_to,),
525
467
unknown_exc=errors.PathError)
527
469
def _rename_and_overwrite(self, abs_from, abs_to, f):
544
486
mutter("FTP rm: %s", abspath)
545
487
f.delete(abspath)
546
488
except ftplib.error_perm, e:
547
self._translate_ftp_error(e, abspath, 'error deleting',
489
self._translate_perm_error(e, abspath, 'error deleting',
548
490
unknown_exc=errors.NoSuchFile)
550
def external_url(self):
551
"""See bzrlib.transport.Transport.external_url."""
552
# FTP URL's are externally usable.
555
492
def listable(self):
556
493
"""See Transport.listable."""
559
496
def list_dir(self, relpath):
560
497
"""See Transport.list_dir."""
561
basepath = self._remote_path(relpath)
498
basepath = self._abspath(relpath)
562
499
mutter("FTP nlst: %s", basepath)
563
500
f = self._get_FTP()
566
paths = f.nlst(basepath)
567
except ftplib.error_perm, e:
568
self._translate_ftp_error(e, relpath,
569
extra='error with list_dir')
570
except ftplib.error_temp, e:
571
# xs4all's ftp server raises a 450 temp error when listing an
572
# empty directory. Check for that and just return an empty list
573
# in that case. See bug #215522
574
if str(e).lower().startswith('450 no files found'):
575
mutter('FTP Server returned "%s" for nlst.'
576
' Assuming it means empty directory',
581
# Restore binary mode as nlst switch to ascii mode to retrieve file
502
paths = f.nlst(basepath)
503
except ftplib.error_perm, e:
504
self._translate_perm_error(e, relpath, extra='error with list_dir')
585
505
# If FTP.nlst returns paths prefixed by relpath, strip 'em
586
506
if paths and paths[0].startswith(basepath):
587
507
entries = [path[len(basepath)+1:] for path in paths]
638
558
return self.lock_read(relpath)
561
class FtpServer(Server):
562
"""Common code for FTP server facilities."""
566
self._ftp_server = None
568
self._async_thread = None
573
"""Calculate an ftp url to this server."""
574
return 'ftp://foo:bar@localhost:%d/' % (self._port)
576
# def get_bogus_url(self):
577
# """Return a URL which cannot be connected to."""
578
# return 'ftp://127.0.0.1:1'
580
def log(self, message):
581
"""This is used by medusa.ftp_server to log connections, etc."""
582
self.logs.append(message)
584
def setUp(self, vfs_server=None):
586
raise RuntimeError('Must have medusa to run the FtpServer')
588
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
589
"FtpServer currently assumes local transport, got %s" % vfs_server
591
self._root = os.getcwdu()
592
self._ftp_server = _ftp_server(
593
authorizer=_test_authorizer(root=self._root),
595
port=0, # bind to a random port
597
logger_object=self # Use FtpServer.log() for messages
599
self._port = self._ftp_server.getsockname()[1]
600
# Don't let it loop forever, or handle an infinite number of requests.
601
# In this case it will run for 100s, or 1000 requests
602
self._async_thread = threading.Thread(
603
target=FtpServer._asyncore_loop_ignore_EBADF,
604
kwargs={'timeout':0.1, 'count':1000})
605
self._async_thread.setDaemon(True)
606
self._async_thread.start()
609
"""See bzrlib.transport.Server.tearDown."""
610
# have asyncore release the channel
611
self._ftp_server.del_channel()
613
self._async_thread.join()
616
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
617
"""Ignore EBADF during server shutdown.
619
We close the socket to get the server to shutdown, but this causes
620
select.select() to raise EBADF.
623
asyncore.loop(*args, **kwargs)
624
except select.error, e:
625
if e.args[0] != errno.EBADF:
631
_test_authorizer = None
635
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
638
import medusa.filesys
639
import medusa.ftp_server
645
class test_authorizer(object):
646
"""A custom Authorizer object for running the test suite.
648
The reason we cannot use dummy_authorizer, is because it sets the
649
channel to readonly, which we don't always want to do.
652
def __init__(self, root):
655
def authorize(self, channel, username, password):
656
"""Return (success, reply_string, filesystem)"""
658
return 0, 'No Medusa.', None
660
channel.persona = -1, -1
661
if username == 'anonymous':
662
channel.read_only = 1
664
channel.read_only = 0
666
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
669
class ftp_channel(medusa.ftp_server.ftp_channel):
670
"""Customized ftp channel"""
672
def log(self, message):
673
"""Redirect logging requests."""
674
mutter('_ftp_channel: %s', message)
676
def log_info(self, message, type='info'):
677
"""Redirect logging requests."""
678
mutter('_ftp_channel %s: %s', type, message)
680
def cmd_rnfr(self, line):
681
"""Prepare for renaming a file."""
682
self._renaming = line[1]
683
self.respond('350 Ready for RNTO')
684
# TODO: jam 20060516 in testing, the ftp server seems to
685
# check that the file already exists, or it sends
686
# 550 RNFR command failed
688
def cmd_rnto(self, line):
689
"""Rename a file based on the target given.
691
rnto must be called after calling rnfr.
693
if not self._renaming:
694
self.respond('503 RNFR required first.')
695
pfrom = self.filesystem.translate(self._renaming)
696
self._renaming = None
697
pto = self.filesystem.translate(line[1])
698
if os.path.exists(pto):
699
self.respond('550 RNTO failed: file exists')
702
os.rename(pfrom, pto)
703
except (IOError, OSError), e:
704
# TODO: jam 20060516 return custom responses based on
705
# why the command failed
706
# (bialix 20070418) str(e) on Python 2.5 @ Windows
707
# sometimes don't provide expected error message;
708
# so we obtain such message via os.strerror()
709
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
711
self.respond('550 RNTO failed')
712
# For a test server, we will go ahead and just die
715
self.respond('250 Rename successful.')
717
def cmd_size(self, line):
718
"""Return the size of a file
720
This is overloaded to help the test suite determine if the
721
target is a directory.
724
if not self.filesystem.isfile(filename):
725
if self.filesystem.isdir(filename):
726
self.respond('550 "%s" is a directory' % (filename,))
728
self.respond('550 "%s" is not a file' % (filename,))
730
self.respond('213 %d'
731
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
733
def cmd_mkd(self, line):
734
"""Create a directory.
736
Overloaded because default implementation does not distinguish
737
*why* it cannot make a directory.
740
self.command_not_understood(''.join(line))
744
self.filesystem.mkdir (path)
745
self.respond ('257 MKD command successful.')
746
except (IOError, OSError), e:
747
# (bialix 20070418) str(e) on Python 2.5 @ Windows
748
# sometimes don't provide expected error message;
749
# so we obtain such message via os.strerror()
750
self.respond ('550 error creating directory: %s' %
751
os.strerror(e.errno))
753
self.respond ('550 error creating directory.')
756
class ftp_server(medusa.ftp_server.ftp_server):
757
"""Customize the behavior of the Medusa ftp_server.
759
There are a few warts on the ftp_server, based on how it expects
763
ftp_channel_class = ftp_channel
765
def __init__(self, *args, **kwargs):
766
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
767
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
769
def log(self, message):
770
"""Redirect logging requests."""
771
mutter('_ftp_server: %s', message)
773
def log_info(self, message, type='info'):
774
"""Override the asyncore.log_info so we don't stipple the screen."""
775
mutter('_ftp_server %s: %s', type, message)
777
_test_authorizer = test_authorizer
778
_ftp_channel = ftp_channel
779
_ftp_server = ftp_server
641
784
def get_test_permutations():
642
785
"""Return the permutations to be used in testing."""
643
from bzrlib.tests import ftp_server
644
return [(FtpTransport, ftp_server.FTPTestServer)]
786
if not _setup_medusa():
787
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
790
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
