28
27
from cStringIO import StringIO
40
from warnings import warn
37
42
from bzrlib import (
43
from bzrlib.symbol_versioning import (
49
47
from bzrlib.trace import mutter, warning
50
48
from bzrlib.transport import (
51
AppendBasedFileStream,
54
register_urlparse_netloc_protocol,
59
register_urlparse_netloc_protocol('aftp')
53
from bzrlib.transport.local import LocalURLServer
62
59
class FtpPathError(errors.PathError):
63
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
66
85
class FtpStatResult(object):
68
def __init__(self, f, abspath):
86
def __init__(self, f, relpath):
70
self.st_size = f.size(abspath)
88
self.st_size = f.size(relpath)
71
89
self.st_mode = stat.S_IFREG
72
90
except ftplib.error_perm:
76
94
self.st_mode = stat.S_IFDIR
81
99
_number_of_retries = 2
82
100
_sleep_between_retries = 5
84
# FIXME: there are inconsistencies in the way temporary errors are
85
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
86
# be taken to analyze the implications for write operations (read operations
87
# are safe to retry). Overall even some read operations are never
88
# retried. --vila 20070720 (Bug #127164)
89
class FtpTransport(ConnectedTransport):
102
class FtpTransport(Transport):
90
103
"""This is the transport agent for ftp:// access."""
92
def __init__(self, base, _from_transport=None):
105
def __init__(self, base, _provided_instance=None):
93
106
"""Set the base path where files will be stored."""
94
if not (base.startswith('ftp://') or base.startswith('aftp://')):
95
raise ValueError(base)
96
super(FtpTransport, self).__init__(base,
97
_from_transport=_from_transport)
98
self._unqualified_scheme = 'ftp'
99
if self._parsed_url.scheme == 'aftp':
100
self.is_active = True
102
self.is_active = False
104
# Most modern FTP servers support the APPE command. If ours doesn't, we
105
# (re)set this flag accordingly later.
106
self._has_append = True
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://
113
if not base.endswith('/'):
115
(self._proto, self._username,
116
self._password, self._host,
117
self._port, self._path) = split_url(base)
118
base = self._unparse_url()
120
super(FtpTransport, self).__init__(base)
121
self._FTP_instance = _provided_instance
123
def _unparse_url(self, path=None):
126
path = urllib.quote(path)
127
netloc = urllib.quote(self._host)
128
if self._username is not None:
129
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
130
if self._port is not None:
131
netloc = '%s:%d' % (netloc, self._port)
135
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
108
137
def _get_FTP(self):
109
138
"""Return the ftplib.FTP instance for this object."""
110
# Ensures that a connection is established
111
connection = self._get_connection()
112
if connection is None:
113
# First connection ever
114
connection, credentials = self._create_connection()
115
self._set_connection(connection, credentials)
118
connection_class = ftplib.FTP
120
def _create_connection(self, credentials=None):
121
"""Create a new connection with the provided credentials.
123
:param credentials: The credentials needed to establish the connection.
125
:return: The created connection and its associated credentials.
127
The input credentials are only the password as it may have been
128
entered interactively by the user and may be different from the one
129
provided in base url at transport creation time. The returned
130
credentials are username, password.
132
if credentials is None:
133
user, password = self._user, self._password
135
user, password = credentials
137
auth = config.AuthenticationConfig()
139
user = auth.get_user('ftp', self._host, port=self._port,
140
default=getpass.getuser())
141
mutter("Constructing FTP instance against %r" %
142
((self._host, self._port, user, '********',
139
if self._FTP_instance is not None:
140
return self._FTP_instance
145
connection = self.connection_class()
146
connection.connect(host=self._host, port=self._port)
147
self._login(connection, auth, user, password)
148
connection.set_pasv(not self.is_active)
149
# binary mode is the default
150
connection.voidcmd('TYPE I')
151
except socket.error, e:
152
raise errors.SocketConnectionError(self._host, self._port,
153
msg='Unable to connect to',
143
self._FTP_instance = _find_FTP(self._host, self._port,
144
self._username, self._password,
146
return self._FTP_instance
155
147
except ftplib.error_perm, e:
156
raise errors.TransportError(msg="Error setting up connection:"
157
" %s" % str(e), orig_error=e)
158
return connection, (user, password)
160
def _login(self, connection, auth, user, password):
161
# '' is a valid password
162
if user and user != 'anonymous' and password is None:
163
password = auth.get_password('ftp', self._host,
164
user, port=self._port)
165
connection.login(user=user, passwd=password)
167
def _reconnect(self):
168
"""Create a new connection with the previously used credentials"""
169
credentials = self._get_credentials()
170
connection, credentials = self._create_connection(credentials)
171
self._set_connection(connection, credentials)
173
def disconnect(self):
174
connection = self._get_connection()
175
if connection is not None:
178
def _translate_ftp_error(self, err, path, extra=None,
179
unknown_exc=FtpPathError):
180
"""Try to translate an ftplib exception to a bzrlib exception.
148
raise errors.TransportError(msg="Error setting up connection: %s"
149
% str(e), orig_error=e)
151
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
152
"""Try to translate an ftplib.error_perm exception.
182
154
:param err: The error to translate into a bzr error
183
155
:param path: The path which had problems
197
166
or 'could not open' in s
198
167
or 'no such dir' in s
199
168
or 'could not create file' in s # vsftpd
200
or 'file doesn\'t exist' in s
201
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
202
or 'file/directory not found' in s # filezilla server
203
# Microsoft FTP-Service RNFR reply if file not found
204
or (s.startswith('550 ') and 'unable to rename to' in extra)
205
# if containing directory doesn't exist, suggested by
206
# <https://bugs.launchpad.net/bzr/+bug/224373>
207
or (s.startswith('550 ') and "can't find folder" in s)
209
170
raise errors.NoSuchFile(path, extra=extra)
210
elif ('file exists' in s):
171
if ('file exists' in s):
211
172
raise errors.FileExists(path, extra=extra)
212
elif ('not a directory' in s):
173
if ('not a directory' in s):
213
174
raise errors.PathError(path, extra=extra)
214
elif 'directory not empty' in s:
215
raise errors.DirectoryNotEmpty(path, extra=extra)
217
176
mutter('unable to understand error for path: %s: %s', path, err)
220
179
raise unknown_exc(path, extra=extra)
221
# TODO: jam 20060516 Consider re-raising the error wrapped in
180
# TODO: jam 20060516 Consider re-raising the error wrapped in
222
181
# something like TransportError, but this loses the traceback
223
182
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
224
183
# to handle. Consider doing something like that here.
225
184
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
187
def should_cache(self):
188
"""Return True if the data pulled across should be cached locally.
192
def clone(self, offset=None):
193
"""Return a new FtpTransport with root at self.base + offset.
197
return FtpTransport(self.base, self._FTP_instance)
199
return FtpTransport(self.abspath(offset), self._FTP_instance)
201
def _abspath(self, relpath):
202
assert isinstance(relpath, basestring)
203
relpath = urlutils.unescape(relpath)
204
if relpath.startswith('/'):
207
basepath = self._path.split('/')
208
if len(basepath) > 0 and basepath[-1] == '':
209
basepath = basepath[:-1]
210
for p in relpath.split('/'):
212
if len(basepath) == 0:
213
# In most filesystems, a request for the parent
214
# of root, just returns root.
217
elif p == '.' or p == '':
221
# Possibly, we could use urlparse.urljoin() here, but
222
# I'm concerned about when it chooses to strip the last
223
# portion of the path, and when it doesn't.
225
# XXX: It seems that ftplib does not handle Unicode paths
226
# at the same time, medusa won't handle utf8 paths
227
# So if we .encode(utf8) here, then we get a Server failure.
228
# while if we use str(), we get a UnicodeError, and the test suite
229
# just skips testing UnicodePaths.
230
return str('/'.join(basepath) or '/')
232
def abspath(self, relpath):
233
"""Return the full url to the given relative path.
234
This can be supplied with a string or a list
236
path = self._abspath(relpath)
237
return self._unparse_url(path)
228
239
def has(self, relpath):
229
240
"""Does the target location exist?"""
230
241
# FIXME jam 20060516 We *do* ask about directories in the test suite
361
346
warning("FTP control connection closed. Trying to reopen.")
362
347
time.sleep(_sleep_between_retries)
348
self._FTP_instance = None
364
349
self.put_file(relpath, fp, mode, retries+1)
366
351
def mkdir(self, relpath, mode=None):
367
352
"""Create a directory at the given path."""
368
abspath = self._remote_path(relpath)
353
abspath = self._abspath(relpath)
370
355
mutter("FTP mkd: %s", abspath)
371
356
f = self._get_FTP()
374
except ftplib.error_reply, e:
375
# <https://bugs.launchpad.net/bzr/+bug/224373> Microsoft FTP
376
# server returns "250 Directory created." which is kind of
377
# reasonable, 250 meaning "requested file action OK", but not what
378
# Python's ftplib expects.
379
if e[0][:3] == '250':
383
self._setmode(relpath, mode)
384
358
except ftplib.error_perm, e:
385
self._translate_ftp_error(e, abspath,
359
self._translate_perm_error(e, abspath,
386
360
unknown_exc=errors.FileExists)
388
def open_write_stream(self, relpath, mode=None):
389
"""See Transport.open_write_stream."""
390
self.put_bytes(relpath, "", mode)
391
result = AppendBasedFileStream(self, relpath)
392
_file_streams[self.abspath(relpath)] = result
395
def recommended_page_size(self):
396
"""See Transport.recommended_page_size().
398
For FTP we suggest a large page size to reduce the overhead
399
introduced by latency.
403
362
def rmdir(self, rel_path):
404
363
"""Delete the directory at rel_path"""
405
abspath = self._remote_path(rel_path)
364
abspath = self._abspath(rel_path)
407
366
mutter("FTP rmd: %s", abspath)
408
367
f = self._get_FTP()
410
369
except ftplib.error_perm, e:
411
self._translate_ftp_error(e, abspath, unknown_exc=errors.PathError)
370
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
413
372
def append_file(self, relpath, f, mode=None):
414
373
"""Append the text in the file-like object into the final
418
abspath = self._remote_path(relpath)
376
abspath = self._abspath(relpath)
419
377
if self.has(relpath):
420
378
ftp = self._get_FTP()
421
379
result = ftp.size(abspath)
426
mutter("FTP appe to %s", abspath)
427
self._try_append(relpath, text, mode)
429
self._fallback_append(relpath, text, mode)
383
mutter("FTP appe to %s", abspath)
384
self._try_append(relpath, f.read(), mode)
433
388
def _try_append(self, relpath, text, mode=None, retries=0):
434
389
"""Try repeatedly to append the given text to the file at relpath.
436
391
This is a recursive function. On errors, it will be called until the
437
392
number of retries is exceeded.
440
abspath = self._remote_path(relpath)
395
abspath = self._abspath(relpath)
441
396
mutter("FTP appe (try %d) to %s", retries, abspath)
442
397
ftp = self._get_FTP()
398
ftp.voidcmd("TYPE I")
443
399
cmd = "APPE %s" % abspath
444
400
conn = ftp.transfercmd(cmd)
445
401
conn.sendall(text)
447
self._setmode(relpath, mode)
404
self._setmode(relpath, mode)
449
406
except ftplib.error_perm, e:
450
# Check whether the command is not supported (reply code 502)
451
if str(e).startswith('502 '):
452
warning("FTP server does not support file appending natively. "
453
"Performance may be severely degraded! (%s)", e)
454
self._has_append = False
455
self._fallback_append(relpath, text, mode)
457
self._translate_ftp_error(e, abspath, extra='error appending',
458
unknown_exc=errors.NoSuchFile)
407
self._translate_perm_error(e, abspath, extra='error appending',
408
unknown_exc=errors.NoSuchFile)
459
409
except ftplib.error_temp, e:
460
410
if retries > _number_of_retries:
461
raise errors.TransportError(
462
"FTP temporary error during APPEND %s. Aborting."
463
% abspath, orig_error=e)
411
raise errors.TransportError("FTP temporary error during APPEND %s." \
412
"Aborting." % abspath, orig_error=e)
465
414
warning("FTP temporary error: %s. Retrying.", str(e))
415
self._FTP_instance = None
467
416
self._try_append(relpath, text, mode, retries+1)
469
def _fallback_append(self, relpath, text, mode = None):
470
remote = self.get(relpath)
471
remote.seek(0, os.SEEK_END)
474
return self.put_file(relpath, remote, mode)
476
418
def _setmode(self, relpath, mode):
477
419
"""Set permissions on a path.
479
421
Only set permissions if the FTP server supports the 'SITE CHMOD'
484
mutter("FTP site chmod: setting permissions to %s on %s",
485
oct(mode), self._remote_path(relpath))
486
ftp = self._get_FTP()
487
cmd = "SITE CHMOD %s %s" % (oct(mode),
488
self._remote_path(relpath))
490
except ftplib.error_perm, e:
491
# Command probably not available on this server
492
warning("FTP Could not set permissions to %s on %s. %s",
493
oct(mode), self._remote_path(relpath), str(e))
425
mutter("FTP site chmod: setting permissions to %s on %s",
426
str(mode), self._abspath(relpath))
427
ftp = self._get_FTP()
428
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
430
except ftplib.error_perm, e:
431
# Command probably not available on this server
432
warning("FTP Could not set permissions to %s on %s. %s",
433
str(mode), self._abspath(relpath), str(e))
495
435
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
496
436
# to copy something to another machine. And you may be able
507
447
def _rename(self, abs_from, abs_to, f):
509
449
f.rename(abs_from, abs_to)
510
except (ftplib.error_temp, ftplib.error_perm), e:
511
self._translate_ftp_error(e, abs_from,
450
except ftplib.error_perm, e:
451
self._translate_perm_error(e, abs_from,
512
452
': unable to rename to %r' % (abs_to))
514
454
def move(self, rel_from, rel_to):
515
455
"""Move the item at rel_from to the location at rel_to"""
516
abs_from = self._remote_path(rel_from)
517
abs_to = self._remote_path(rel_to)
456
abs_from = self._abspath(rel_from)
457
abs_to = self._abspath(rel_to)
519
459
mutter("FTP mv: %s => %s", abs_from, abs_to)
520
460
f = self._get_FTP()
521
461
self._rename_and_overwrite(abs_from, abs_to, f)
522
462
except ftplib.error_perm, e:
523
self._translate_ftp_error(e, abs_from,
524
extra='unable to rename to %r' % (rel_to,),
463
self._translate_perm_error(e, abs_from,
464
extra='unable to rename to %r' % (rel_to,),
525
465
unknown_exc=errors.PathError)
527
467
def _rename_and_overwrite(self, abs_from, abs_to, f):
544
484
mutter("FTP rm: %s", abspath)
545
485
f.delete(abspath)
546
486
except ftplib.error_perm, e:
547
self._translate_ftp_error(e, abspath, 'error deleting',
487
self._translate_perm_error(e, abspath, 'error deleting',
548
488
unknown_exc=errors.NoSuchFile)
550
def external_url(self):
551
"""See bzrlib.transport.Transport.external_url."""
552
# FTP URL's are externally usable.
555
490
def listable(self):
556
491
"""See Transport.listable."""
559
494
def list_dir(self, relpath):
560
495
"""See Transport.list_dir."""
561
basepath = self._remote_path(relpath)
496
basepath = self._abspath(relpath)
562
497
mutter("FTP nlst: %s", basepath)
563
498
f = self._get_FTP()
566
paths = f.nlst(basepath)
567
except ftplib.error_perm, e:
568
self._translate_ftp_error(e, relpath,
569
extra='error with list_dir')
570
except ftplib.error_temp, e:
571
# xs4all's ftp server raises a 450 temp error when listing an
572
# empty directory. Check for that and just return an empty list
573
# in that case. See bug #215522
574
if str(e).lower().startswith('450 no files found'):
575
mutter('FTP Server returned "%s" for nlst.'
576
' Assuming it means empty directory',
581
# Restore binary mode as nlst switch to ascii mode to retrieve file
500
paths = f.nlst(basepath)
501
except ftplib.error_perm, e:
502
self._translate_perm_error(e, relpath, extra='error with list_dir')
585
503
# If FTP.nlst returns paths prefixed by relpath, strip 'em
586
504
if paths and paths[0].startswith(basepath):
587
505
entries = [path[len(basepath)+1:] for path in paths]
638
556
return self.lock_read(relpath)
559
class FtpServer(Server):
560
"""Common code for SFTP server facilities."""
564
self._ftp_server = None
566
self._async_thread = None
571
"""Calculate an ftp url to this server."""
572
return 'ftp://foo:bar@localhost:%d/' % (self._port)
574
# def get_bogus_url(self):
575
# """Return a URL which cannot be connected to."""
576
# return 'ftp://127.0.0.1:1'
578
def log(self, message):
579
"""This is used by medusa.ftp_server to log connections, etc."""
580
self.logs.append(message)
582
def setUp(self, vfs_server=None):
584
raise RuntimeError('Must have medusa to run the FtpServer')
586
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
587
"FtpServer currently assumes local transport, got %s" % vfs_server
589
self._root = os.getcwdu()
590
self._ftp_server = _ftp_server(
591
authorizer=_test_authorizer(root=self._root),
593
port=0, # bind to a random port
595
logger_object=self # Use FtpServer.log() for messages
597
self._port = self._ftp_server.getsockname()[1]
598
# Don't let it loop forever, or handle an infinite number of requests.
599
# In this case it will run for 100s, or 1000 requests
600
self._async_thread = threading.Thread(
601
target=FtpServer._asyncore_loop_ignore_EBADF,
602
kwargs={'timeout':0.1, 'count':1000})
603
self._async_thread.setDaemon(True)
604
self._async_thread.start()
607
"""See bzrlib.transport.Server.tearDown."""
608
# have asyncore release the channel
609
self._ftp_server.del_channel()
611
self._async_thread.join()
614
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
615
"""Ignore EBADF during server shutdown.
617
We close the socket to get the server to shutdown, but this causes
618
select.select() to raise EBADF.
621
asyncore.loop(*args, **kwargs)
622
except select.error, e:
623
if e.args[0] != errno.EBADF:
629
_test_authorizer = None
633
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
636
import medusa.filesys
637
import medusa.ftp_server
643
class test_authorizer(object):
644
"""A custom Authorizer object for running the test suite.
646
The reason we cannot use dummy_authorizer, is because it sets the
647
channel to readonly, which we don't always want to do.
650
def __init__(self, root):
653
def authorize(self, channel, username, password):
654
"""Return (success, reply_string, filesystem)"""
656
return 0, 'No Medusa.', None
658
channel.persona = -1, -1
659
if username == 'anonymous':
660
channel.read_only = 1
662
channel.read_only = 0
664
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
667
class ftp_channel(medusa.ftp_server.ftp_channel):
668
"""Customized ftp channel"""
670
def log(self, message):
671
"""Redirect logging requests."""
672
mutter('_ftp_channel: %s', message)
674
def log_info(self, message, type='info'):
675
"""Redirect logging requests."""
676
mutter('_ftp_channel %s: %s', type, message)
678
def cmd_rnfr(self, line):
679
"""Prepare for renaming a file."""
680
self._renaming = line[1]
681
self.respond('350 Ready for RNTO')
682
# TODO: jam 20060516 in testing, the ftp server seems to
683
# check that the file already exists, or it sends
684
# 550 RNFR command failed
686
def cmd_rnto(self, line):
687
"""Rename a file based on the target given.
689
rnto must be called after calling rnfr.
691
if not self._renaming:
692
self.respond('503 RNFR required first.')
693
pfrom = self.filesystem.translate(self._renaming)
694
self._renaming = None
695
pto = self.filesystem.translate(line[1])
696
if os.path.exists(pto):
697
self.respond('550 RNTO failed: file exists')
700
os.rename(pfrom, pto)
701
except (IOError, OSError), e:
702
# TODO: jam 20060516 return custom responses based on
703
# why the command failed
704
# (bialix 20070418) str(e) on Python 2.5 @ Windows
705
# sometimes don't provide expected error message;
706
# so we obtain such message via os.strerror()
707
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
709
self.respond('550 RNTO failed')
710
# For a test server, we will go ahead and just die
713
self.respond('250 Rename successful.')
715
def cmd_size(self, line):
716
"""Return the size of a file
718
This is overloaded to help the test suite determine if the
719
target is a directory.
722
if not self.filesystem.isfile(filename):
723
if self.filesystem.isdir(filename):
724
self.respond('550 "%s" is a directory' % (filename,))
726
self.respond('550 "%s" is not a file' % (filename,))
728
self.respond('213 %d'
729
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
731
def cmd_mkd(self, line):
732
"""Create a directory.
734
Overloaded because default implementation does not distinguish
735
*why* it cannot make a directory.
738
self.command_not_understood(''.join(line))
742
self.filesystem.mkdir (path)
743
self.respond ('257 MKD command successful.')
744
except (IOError, OSError), e:
745
# (bialix 20070418) str(e) on Python 2.5 @ Windows
746
# sometimes don't provide expected error message;
747
# so we obtain such message via os.strerror()
748
self.respond ('550 error creating directory: %s' %
749
os.strerror(e.errno))
751
self.respond ('550 error creating directory.')
754
class ftp_server(medusa.ftp_server.ftp_server):
755
"""Customize the behavior of the Medusa ftp_server.
757
There are a few warts on the ftp_server, based on how it expects
761
ftp_channel_class = ftp_channel
763
def __init__(self, *args, **kwargs):
764
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
765
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
767
def log(self, message):
768
"""Redirect logging requests."""
769
mutter('_ftp_server: %s', message)
771
def log_info(self, message, type='info'):
772
"""Override the asyncore.log_info so we don't stipple the screen."""
773
mutter('_ftp_server %s: %s', type, message)
775
_test_authorizer = test_authorizer
776
_ftp_channel = ftp_channel
777
_ftp_server = ftp_server
641
782
def get_test_permutations():
642
783
"""Return the permutations to be used in testing."""
643
from bzrlib.tests import ftp_server
644
return [(FtpTransport, ftp_server.FTPTestServer)]
784
if not _setup_medusa():
785
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
788
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
