28
27
from cStringIO import StringIO
40
from warnings import warn
37
42
from bzrlib import (
43
from bzrlib.symbol_versioning import (
49
47
from bzrlib.trace import mutter, warning
50
48
from bzrlib.transport import (
51
AppendBasedFileStream,
54
register_urlparse_netloc_protocol,
59
register_urlparse_netloc_protocol('aftp')
53
from bzrlib.transport.local import LocalURLServer
62
59
class FtpPathError(errors.PathError):
63
60
"""FTP failed for path: %(path)s%(extra)s"""
64
def _find_FTP(hostname, port, username, password, is_active):
65
"""Find an ftplib.FTP instance attached to this triplet."""
66
key = (hostname, port, username, password, is_active)
67
alt_key = (hostname, port, username, '********', is_active)
68
if key not in _FTP_cache:
69
mutter("Constructing FTP instance against %r" % (alt_key,))
72
conn.connect(host=hostname, port=port)
73
if username and username != 'anonymous' and not password:
74
password = bzrlib.ui.ui_factory.get_password(
75
prompt='FTP %(user)s@%(host)s password',
76
user=username, host=hostname)
77
conn.login(user=username, passwd=password)
78
conn.set_pasv(not is_active)
80
_FTP_cache[key] = conn
82
return _FTP_cache[key]
66
85
class FtpStatResult(object):
68
def __init__(self, f, abspath):
86
def __init__(self, f, relpath):
70
self.st_size = f.size(abspath)
88
self.st_size = f.size(relpath)
71
89
self.st_mode = stat.S_IFREG
72
90
except ftplib.error_perm:
76
94
self.st_mode = stat.S_IFDIR
81
99
_number_of_retries = 2
82
100
_sleep_between_retries = 5
84
# FIXME: there are inconsistencies in the way temporary errors are
85
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
86
# be taken to analyze the implications for write operations (read operations
87
# are safe to retry). Overall even some read operations are never
88
# retried. --vila 20070720 (Bug #127164)
89
class FtpTransport(ConnectedTransport):
102
class FtpTransport(Transport):
90
103
"""This is the transport agent for ftp:// access."""
92
def __init__(self, base, _from_transport=None):
105
def __init__(self, base, _provided_instance=None):
93
106
"""Set the base path where files will be stored."""
94
if not (base.startswith('ftp://') or base.startswith('aftp://')):
95
raise ValueError(base)
96
super(FtpTransport, self).__init__(base,
97
_from_transport=_from_transport)
98
self._unqualified_scheme = 'ftp'
99
if self._parsed_url.scheme == 'aftp':
100
self.is_active = True
102
self.is_active = False
104
# Most modern FTP servers support the APPE command. If ours doesn't, we
105
# (re)set this flag accordingly later.
106
self._has_append = True
107
assert base.startswith('ftp://') or base.startswith('aftp://')
109
self.is_active = base.startswith('aftp://')
111
# urlparse won't handle aftp://, delete the leading 'a'
113
# FIXME: This breaks even hopes of connection sharing
114
# by modifying the the url coming from the user.
116
if not base.endswith('/'):
118
(self._proto, self._username,
119
self._password, self._host,
120
self._port, self._path) = split_url(base)
121
base = self._unparse_url()
123
super(FtpTransport, self).__init__(base)
124
self._FTP_instance = _provided_instance
126
def _unparse_url(self, path=None):
129
path = urllib.quote(path)
130
netloc = urllib.quote(self._host)
131
if self._username is not None:
132
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
133
if self._port is not None:
134
netloc = '%s:%d' % (netloc, self._port)
138
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
108
140
def _get_FTP(self):
109
141
"""Return the ftplib.FTP instance for this object."""
110
# Ensures that a connection is established
111
connection = self._get_connection()
112
if connection is None:
113
# First connection ever
114
connection, credentials = self._create_connection()
115
self._set_connection(connection, credentials)
118
connection_class = ftplib.FTP
120
def _create_connection(self, credentials=None):
121
"""Create a new connection with the provided credentials.
123
:param credentials: The credentials needed to establish the connection.
125
:return: The created connection and its associated credentials.
127
The input credentials are only the password as it may have been
128
entered interactively by the user and may be different from the one
129
provided in base url at transport creation time. The returned
130
credentials are username, password.
132
if credentials is None:
133
user, password = self._user, self._password
135
user, password = credentials
137
auth = config.AuthenticationConfig()
139
user = auth.get_user('ftp', self._host, port=self._port,
140
default=getpass.getuser())
141
mutter("Constructing FTP instance against %r" %
142
((self._host, self._port, user, '********',
142
if self._FTP_instance is not None:
143
return self._FTP_instance
145
connection = self.connection_class()
146
connection.connect(host=self._host, port=self._port)
147
self._login(connection, auth, user, password)
148
connection.set_pasv(not self.is_active)
149
# binary mode is the default
150
connection.voidcmd('TYPE I')
151
except socket.error, e:
152
raise errors.SocketConnectionError(self._host, self._port,
153
msg='Unable to connect to',
146
self._FTP_instance = _find_FTP(self._host, self._port,
147
self._username, self._password,
149
return self._FTP_instance
155
150
except ftplib.error_perm, e:
156
raise errors.TransportError(msg="Error setting up connection:"
157
" %s" % str(e), orig_error=e)
158
return connection, (user, password)
160
def _login(self, connection, auth, user, password):
161
# '' is a valid password
162
if user and user != 'anonymous' and password is None:
163
password = auth.get_password('ftp', self._host,
164
user, port=self._port)
165
connection.login(user=user, passwd=password)
167
def _reconnect(self):
168
"""Create a new connection with the previously used credentials"""
169
credentials = self._get_credentials()
170
connection, credentials = self._create_connection(credentials)
171
self._set_connection(connection, credentials)
173
def disconnect(self):
174
connection = self._get_connection()
175
if connection is not None:
178
def _translate_ftp_error(self, err, path, extra=None,
179
unknown_exc=FtpPathError):
180
"""Try to translate an ftplib exception to a bzrlib exception.
151
raise errors.TransportError(msg="Error setting up connection: %s"
152
% str(e), orig_error=e)
154
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
155
"""Try to translate an ftplib.error_perm exception.
182
157
:param err: The error to translate into a bzr error
183
158
:param path: The path which had problems
197
169
or 'could not open' in s
198
170
or 'no such dir' in s
199
171
or 'could not create file' in s # vsftpd
200
or 'file doesn\'t exist' in s
201
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
202
or 'file/directory not found' in s # filezilla server
203
# Microsoft FTP-Service RNFR reply if file not found
204
or (s.startswith('550 ') and 'unable to rename to' in extra)
205
# if containing directory doesn't exist, suggested by
206
# <https://bugs.launchpad.net/bzr/+bug/224373>
207
or (s.startswith('550 ') and "can't find folder" in s)
209
173
raise errors.NoSuchFile(path, extra=extra)
210
elif ('file exists' in s):
174
if ('file exists' in s):
211
175
raise errors.FileExists(path, extra=extra)
212
elif ('not a directory' in s):
176
if ('not a directory' in s):
213
177
raise errors.PathError(path, extra=extra)
214
elif 'directory not empty' in s:
215
raise errors.DirectoryNotEmpty(path, extra=extra)
217
179
mutter('unable to understand error for path: %s: %s', path, err)
220
182
raise unknown_exc(path, extra=extra)
221
# TODO: jam 20060516 Consider re-raising the error wrapped in
183
# TODO: jam 20060516 Consider re-raising the error wrapped in
222
184
# something like TransportError, but this loses the traceback
223
185
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
224
186
# to handle. Consider doing something like that here.
225
187
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
190
def should_cache(self):
191
"""Return True if the data pulled across should be cached locally.
195
def clone(self, offset=None):
196
"""Return a new FtpTransport with root at self.base + offset.
200
return FtpTransport(self.base, self._FTP_instance)
202
return FtpTransport(self.abspath(offset), self._FTP_instance)
204
def _abspath(self, relpath):
205
assert isinstance(relpath, basestring)
206
relpath = urlutils.unescape(relpath)
207
if relpath.startswith('/'):
210
basepath = self._path.split('/')
211
if len(basepath) > 0 and basepath[-1] == '':
212
basepath = basepath[:-1]
213
for p in relpath.split('/'):
215
if len(basepath) == 0:
216
# In most filesystems, a request for the parent
217
# of root, just returns root.
220
elif p == '.' or p == '':
224
# Possibly, we could use urlparse.urljoin() here, but
225
# I'm concerned about when it chooses to strip the last
226
# portion of the path, and when it doesn't.
228
# XXX: It seems that ftplib does not handle Unicode paths
229
# at the same time, medusa won't handle utf8 paths
230
# So if we .encode(utf8) here, then we get a Server failure.
231
# while if we use str(), we get a UnicodeError, and the test suite
232
# just skips testing UnicodePaths.
233
return str('/'.join(basepath) or '/')
235
def abspath(self, relpath):
236
"""Return the full url to the given relative path.
237
This can be supplied with a string or a list
239
path = self._abspath(relpath)
240
return self._unparse_url(path)
228
242
def has(self, relpath):
229
243
"""Does the target location exist?"""
230
244
# FIXME jam 20060516 We *do* ask about directories in the test suite
361
349
warning("FTP control connection closed. Trying to reopen.")
362
350
time.sleep(_sleep_between_retries)
351
self._FTP_instance = None
364
352
self.put_file(relpath, fp, mode, retries+1)
366
354
def mkdir(self, relpath, mode=None):
367
355
"""Create a directory at the given path."""
368
abspath = self._remote_path(relpath)
356
abspath = self._abspath(relpath)
370
358
mutter("FTP mkd: %s", abspath)
371
359
f = self._get_FTP()
374
except ftplib.error_reply, e:
375
# <https://bugs.launchpad.net/bzr/+bug/224373> Microsoft FTP
376
# server returns "250 Directory created." which is kind of
377
# reasonable, 250 meaning "requested file action OK", but not what
378
# Python's ftplib expects.
379
if e[0][:3] == '250':
383
self._setmode(relpath, mode)
384
361
except ftplib.error_perm, e:
385
self._translate_ftp_error(e, abspath,
362
self._translate_perm_error(e, abspath,
386
363
unknown_exc=errors.FileExists)
388
def open_write_stream(self, relpath, mode=None):
389
"""See Transport.open_write_stream."""
390
self.put_bytes(relpath, "", mode)
391
result = AppendBasedFileStream(self, relpath)
392
_file_streams[self.abspath(relpath)] = result
395
def recommended_page_size(self):
396
"""See Transport.recommended_page_size().
398
For FTP we suggest a large page size to reduce the overhead
399
introduced by latency.
403
365
def rmdir(self, rel_path):
404
366
"""Delete the directory at rel_path"""
405
abspath = self._remote_path(rel_path)
367
abspath = self._abspath(rel_path)
407
369
mutter("FTP rmd: %s", abspath)
408
370
f = self._get_FTP()
410
372
except ftplib.error_perm, e:
411
self._translate_ftp_error(e, abspath, unknown_exc=errors.PathError)
373
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
413
375
def append_file(self, relpath, f, mode=None):
414
376
"""Append the text in the file-like object into the final
418
abspath = self._remote_path(relpath)
379
abspath = self._abspath(relpath)
419
380
if self.has(relpath):
420
381
ftp = self._get_FTP()
421
382
result = ftp.size(abspath)
426
mutter("FTP appe to %s", abspath)
427
self._try_append(relpath, text, mode)
429
self._fallback_append(relpath, text, mode)
386
mutter("FTP appe to %s", abspath)
387
self._try_append(relpath, f.read(), mode)
433
391
def _try_append(self, relpath, text, mode=None, retries=0):
434
392
"""Try repeatedly to append the given text to the file at relpath.
436
394
This is a recursive function. On errors, it will be called until the
437
395
number of retries is exceeded.
440
abspath = self._remote_path(relpath)
398
abspath = self._abspath(relpath)
441
399
mutter("FTP appe (try %d) to %s", retries, abspath)
442
400
ftp = self._get_FTP()
401
ftp.voidcmd("TYPE I")
443
402
cmd = "APPE %s" % abspath
444
403
conn = ftp.transfercmd(cmd)
445
404
conn.sendall(text)
447
self._setmode(relpath, mode)
407
self._setmode(relpath, mode)
449
409
except ftplib.error_perm, e:
450
# Check whether the command is not supported (reply code 502)
451
if str(e).startswith('502 '):
452
warning("FTP server does not support file appending natively. "
453
"Performance may be severely degraded! (%s)", e)
454
self._has_append = False
455
self._fallback_append(relpath, text, mode)
457
self._translate_ftp_error(e, abspath, extra='error appending',
458
unknown_exc=errors.NoSuchFile)
410
self._translate_perm_error(e, abspath, extra='error appending',
411
unknown_exc=errors.NoSuchFile)
459
412
except ftplib.error_temp, e:
460
413
if retries > _number_of_retries:
461
raise errors.TransportError(
462
"FTP temporary error during APPEND %s. Aborting."
463
% abspath, orig_error=e)
414
raise errors.TransportError("FTP temporary error during APPEND %s." \
415
"Aborting." % abspath, orig_error=e)
465
417
warning("FTP temporary error: %s. Retrying.", str(e))
418
self._FTP_instance = None
467
419
self._try_append(relpath, text, mode, retries+1)
469
def _fallback_append(self, relpath, text, mode = None):
470
remote = self.get(relpath)
471
remote.seek(0, os.SEEK_END)
474
return self.put_file(relpath, remote, mode)
476
421
def _setmode(self, relpath, mode):
477
422
"""Set permissions on a path.
479
424
Only set permissions if the FTP server supports the 'SITE CHMOD'
484
mutter("FTP site chmod: setting permissions to %s on %s",
485
oct(mode), self._remote_path(relpath))
486
ftp = self._get_FTP()
487
cmd = "SITE CHMOD %s %s" % (oct(mode),
488
self._remote_path(relpath))
490
except ftplib.error_perm, e:
491
# Command probably not available on this server
492
warning("FTP Could not set permissions to %s on %s. %s",
493
oct(mode), self._remote_path(relpath), str(e))
428
mutter("FTP site chmod: setting permissions to %s on %s",
429
str(mode), self._abspath(relpath))
430
ftp = self._get_FTP()
431
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
433
except ftplib.error_perm, e:
434
# Command probably not available on this server
435
warning("FTP Could not set permissions to %s on %s. %s",
436
str(mode), self._abspath(relpath), str(e))
495
438
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
496
439
# to copy something to another machine. And you may be able
507
450
def _rename(self, abs_from, abs_to, f):
509
452
f.rename(abs_from, abs_to)
510
except (ftplib.error_temp, ftplib.error_perm), e:
511
self._translate_ftp_error(e, abs_from,
453
except ftplib.error_perm, e:
454
self._translate_perm_error(e, abs_from,
512
455
': unable to rename to %r' % (abs_to))
514
457
def move(self, rel_from, rel_to):
515
458
"""Move the item at rel_from to the location at rel_to"""
516
abs_from = self._remote_path(rel_from)
517
abs_to = self._remote_path(rel_to)
459
abs_from = self._abspath(rel_from)
460
abs_to = self._abspath(rel_to)
519
462
mutter("FTP mv: %s => %s", abs_from, abs_to)
520
463
f = self._get_FTP()
521
464
self._rename_and_overwrite(abs_from, abs_to, f)
522
465
except ftplib.error_perm, e:
523
self._translate_ftp_error(e, abs_from,
524
extra='unable to rename to %r' % (rel_to,),
466
self._translate_perm_error(e, abs_from,
467
extra='unable to rename to %r' % (rel_to,),
525
468
unknown_exc=errors.PathError)
527
470
def _rename_and_overwrite(self, abs_from, abs_to, f):
544
487
mutter("FTP rm: %s", abspath)
545
488
f.delete(abspath)
546
489
except ftplib.error_perm, e:
547
self._translate_ftp_error(e, abspath, 'error deleting',
490
self._translate_perm_error(e, abspath, 'error deleting',
548
491
unknown_exc=errors.NoSuchFile)
550
def external_url(self):
551
"""See bzrlib.transport.Transport.external_url."""
552
# FTP URL's are externally usable.
555
493
def listable(self):
556
494
"""See Transport.listable."""
559
497
def list_dir(self, relpath):
560
498
"""See Transport.list_dir."""
561
basepath = self._remote_path(relpath)
499
basepath = self._abspath(relpath)
562
500
mutter("FTP nlst: %s", basepath)
563
501
f = self._get_FTP()
566
paths = f.nlst(basepath)
567
except ftplib.error_perm, e:
568
self._translate_ftp_error(e, relpath,
569
extra='error with list_dir')
570
except ftplib.error_temp, e:
571
# xs4all's ftp server raises a 450 temp error when listing an
572
# empty directory. Check for that and just return an empty list
573
# in that case. See bug #215522
574
if str(e).lower().startswith('450 no files found'):
575
mutter('FTP Server returned "%s" for nlst.'
576
' Assuming it means empty directory',
581
# Restore binary mode as nlst switch to ascii mode to retrieve file
503
paths = f.nlst(basepath)
504
except ftplib.error_perm, e:
505
self._translate_perm_error(e, relpath, extra='error with list_dir')
585
506
# If FTP.nlst returns paths prefixed by relpath, strip 'em
586
507
if paths and paths[0].startswith(basepath):
587
508
entries = [path[len(basepath)+1:] for path in paths]
638
559
return self.lock_read(relpath)
562
class FtpServer(Server):
563
"""Common code for SFTP server facilities."""
567
self._ftp_server = None
569
self._async_thread = None
574
"""Calculate an ftp url to this server."""
575
return 'ftp://foo:bar@localhost:%d/' % (self._port)
577
# def get_bogus_url(self):
578
# """Return a URL which cannot be connected to."""
579
# return 'ftp://127.0.0.1:1'
581
def log(self, message):
582
"""This is used by medusa.ftp_server to log connections, etc."""
583
self.logs.append(message)
585
def setUp(self, vfs_server=None):
587
raise RuntimeError('Must have medusa to run the FtpServer')
589
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
590
"FtpServer currently assumes local transport, got %s" % vfs_server
592
self._root = os.getcwdu()
593
self._ftp_server = _ftp_server(
594
authorizer=_test_authorizer(root=self._root),
596
port=0, # bind to a random port
598
logger_object=self # Use FtpServer.log() for messages
600
self._port = self._ftp_server.getsockname()[1]
601
# Don't let it loop forever, or handle an infinite number of requests.
602
# In this case it will run for 100s, or 1000 requests
603
self._async_thread = threading.Thread(
604
target=FtpServer._asyncore_loop_ignore_EBADF,
605
kwargs={'timeout':0.1, 'count':1000})
606
self._async_thread.setDaemon(True)
607
self._async_thread.start()
610
"""See bzrlib.transport.Server.tearDown."""
611
# have asyncore release the channel
612
self._ftp_server.del_channel()
614
self._async_thread.join()
617
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
618
"""Ignore EBADF during server shutdown.
620
We close the socket to get the server to shutdown, but this causes
621
select.select() to raise EBADF.
624
asyncore.loop(*args, **kwargs)
625
except select.error, e:
626
if e.args[0] != errno.EBADF:
632
_test_authorizer = None
636
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
639
import medusa.filesys
640
import medusa.ftp_server
646
class test_authorizer(object):
647
"""A custom Authorizer object for running the test suite.
649
The reason we cannot use dummy_authorizer, is because it sets the
650
channel to readonly, which we don't always want to do.
653
def __init__(self, root):
656
def authorize(self, channel, username, password):
657
"""Return (success, reply_string, filesystem)"""
659
return 0, 'No Medusa.', None
661
channel.persona = -1, -1
662
if username == 'anonymous':
663
channel.read_only = 1
665
channel.read_only = 0
667
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
670
class ftp_channel(medusa.ftp_server.ftp_channel):
671
"""Customized ftp channel"""
673
def log(self, message):
674
"""Redirect logging requests."""
675
mutter('_ftp_channel: %s', message)
677
def log_info(self, message, type='info'):
678
"""Redirect logging requests."""
679
mutter('_ftp_channel %s: %s', type, message)
681
def cmd_rnfr(self, line):
682
"""Prepare for renaming a file."""
683
self._renaming = line[1]
684
self.respond('350 Ready for RNTO')
685
# TODO: jam 20060516 in testing, the ftp server seems to
686
# check that the file already exists, or it sends
687
# 550 RNFR command failed
689
def cmd_rnto(self, line):
690
"""Rename a file based on the target given.
692
rnto must be called after calling rnfr.
694
if not self._renaming:
695
self.respond('503 RNFR required first.')
696
pfrom = self.filesystem.translate(self._renaming)
697
self._renaming = None
698
pto = self.filesystem.translate(line[1])
699
if os.path.exists(pto):
700
self.respond('550 RNTO failed: file exists')
703
os.rename(pfrom, pto)
704
except (IOError, OSError), e:
705
# TODO: jam 20060516 return custom responses based on
706
# why the command failed
707
# (bialix 20070418) str(e) on Python 2.5 @ Windows
708
# sometimes don't provide expected error message;
709
# so we obtain such message via os.strerror()
710
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
712
self.respond('550 RNTO failed')
713
# For a test server, we will go ahead and just die
716
self.respond('250 Rename successful.')
718
def cmd_size(self, line):
719
"""Return the size of a file
721
This is overloaded to help the test suite determine if the
722
target is a directory.
725
if not self.filesystem.isfile(filename):
726
if self.filesystem.isdir(filename):
727
self.respond('550 "%s" is a directory' % (filename,))
729
self.respond('550 "%s" is not a file' % (filename,))
731
self.respond('213 %d'
732
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
734
def cmd_mkd(self, line):
735
"""Create a directory.
737
Overloaded because default implementation does not distinguish
738
*why* it cannot make a directory.
741
self.command_not_understood(''.join(line))
745
self.filesystem.mkdir (path)
746
self.respond ('257 MKD command successful.')
747
except (IOError, OSError), e:
748
# (bialix 20070418) str(e) on Python 2.5 @ Windows
749
# sometimes don't provide expected error message;
750
# so we obtain such message via os.strerror()
751
self.respond ('550 error creating directory: %s' %
752
os.strerror(e.errno))
754
self.respond ('550 error creating directory.')
757
class ftp_server(medusa.ftp_server.ftp_server):
758
"""Customize the behavior of the Medusa ftp_server.
760
There are a few warts on the ftp_server, based on how it expects
764
ftp_channel_class = ftp_channel
766
def __init__(self, *args, **kwargs):
767
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
768
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
770
def log(self, message):
771
"""Redirect logging requests."""
772
mutter('_ftp_server: %s', message)
774
def log_info(self, message, type='info'):
775
"""Override the asyncore.log_info so we don't stipple the screen."""
776
mutter('_ftp_server %s: %s', type, message)
778
_test_authorizer = test_authorizer
779
_ftp_channel = ftp_channel
780
_ftp_server = ftp_server
641
785
def get_test_permutations():
642
786
"""Return the permutations to be used in testing."""
643
from bzrlib.tests import ftp_server
644
return [(FtpTransport, ftp_server.FTPTestServer)]
787
if not _setup_medusa():
788
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
791
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
