27
27
from cStringIO import StringIO
38
38
from warnings import warn
46
from bzrlib.trace import mutter, warning
40
47
from bzrlib.transport import (
48
AppendBasedFileStream,
51
register_urlparse_netloc_protocol,
45
import bzrlib.errors as errors
46
from bzrlib.trace import mutter, warning
54
from bzrlib.transport.local import LocalURLServer
58
register_urlparse_netloc_protocol('aftp')
52
61
class FtpPathError(errors.PathError):
53
62
"""FTP failed for path: %(path)s%(extra)s"""
57
def _find_FTP(hostname, port, username, password, is_active):
58
"""Find an ftplib.FTP instance attached to this triplet."""
59
key = (hostname, port, username, password, is_active)
60
alt_key = (hostname, port, username, '********', is_active)
61
if key not in _FTP_cache:
62
mutter("Constructing FTP instance against %r" % (alt_key,))
65
conn.connect(host=hostname, port=port)
66
if username and username != 'anonymous' and not password:
67
password = bzrlib.ui.ui_factory.get_password(
68
prompt='FTP %(user)s@%(host)s password',
69
user=username, host=hostname)
70
conn.login(user=username, passwd=password)
71
conn.set_pasv(not is_active)
73
_FTP_cache[key] = conn
75
return _FTP_cache[key]
78
65
class FtpStatResult(object):
79
66
def __init__(self, f, relpath):
92
79
_number_of_retries = 2
93
80
_sleep_between_retries = 5
95
class FtpTransport(Transport):
82
# FIXME: there are inconsistencies in the way temporary errors are
83
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
84
# be taken to analyze the implications for write operations (read operations
85
# are safe to retry). Overall even some read operations are never
86
# retried. --vila 20070720 (Bug #127164)
87
class FtpTransport(ConnectedTransport):
96
88
"""This is the transport agent for ftp:// access."""
98
def __init__(self, base, _provided_instance=None):
90
def __init__(self, base, _from_transport=None):
99
91
"""Set the base path where files will be stored."""
100
assert base.startswith('ftp://') or base.startswith('aftp://')
102
self.is_active = base.startswith('aftp://')
104
# urlparse won't handle aftp://
106
if not base.endswith('/'):
108
(self._proto, self._username,
109
self._password, self._host,
110
self._port, self._path) = split_url(base)
111
base = self._unparse_url()
113
super(FtpTransport, self).__init__(base)
114
self._FTP_instance = _provided_instance
116
def _unparse_url(self, path=None):
119
path = urllib.quote(path)
120
netloc = urllib.quote(self._host)
121
if self._username is not None:
122
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
123
if self._port is not None:
124
netloc = '%s:%d' % (netloc, self._port)
125
return urlparse.urlunparse(('ftp', netloc, path, '', '', ''))
92
if not (base.startswith('ftp://') or base.startswith('aftp://')):
93
raise ValueError(base)
94
super(FtpTransport, self).__init__(base,
95
_from_transport=_from_transport)
96
self._unqualified_scheme = 'ftp'
97
if self._scheme == 'aftp':
100
self.is_active = False
127
102
def _get_FTP(self):
128
103
"""Return the ftplib.FTP instance for this object."""
129
if self._FTP_instance is not None:
130
return self._FTP_instance
104
# Ensures that a connection is established
105
connection = self._get_connection()
106
if connection is None:
107
# First connection ever
108
connection, credentials = self._create_connection()
109
self._set_connection(connection, credentials)
112
def _create_connection(self, credentials=None):
113
"""Create a new connection with the provided credentials.
115
:param credentials: The credentials needed to establish the connection.
117
:return: The created connection and its associated credentials.
119
The credentials are only the password as it may have been entered
120
interactively by the user and may be different from the one provided
121
in base url at transport creation time.
123
if credentials is None:
124
user, password = self._user, self._password
126
user, password = credentials
128
auth = config.AuthenticationConfig()
130
user = auth.get_user('ftp', self._host, port=self._port)
132
# Default to local user
133
user = getpass.getuser()
135
mutter("Constructing FTP instance against %r" %
136
((self._host, self._port, user, '********',
133
self._FTP_instance = _find_FTP(self._host, self._port,
134
self._username, self._password,
136
return self._FTP_instance
139
connection = ftplib.FTP()
140
connection.connect(host=self._host, port=self._port)
141
if user and user != 'anonymous' and \
142
password is None: # '' is a valid password
143
password = auth.get_password('ftp', self._host, user,
145
connection.login(user=user, passwd=password)
146
connection.set_pasv(not self.is_active)
147
except socket.error, e:
148
raise errors.SocketConnectionError(self._host, self._port,
149
msg='Unable to connect to',
137
151
except ftplib.error_perm, e:
138
raise errors.TransportError(msg="Error setting up connection: %s"
139
% str(e), orig_error=e)
141
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
152
raise errors.TransportError(msg="Error setting up connection:"
153
" %s" % str(e), orig_error=e)
154
return connection, (user, password)
156
def _reconnect(self):
157
"""Create a new connection with the previously used credentials"""
158
credentials = self._get_credentials()
159
connection, credentials = self._create_connection(credentials)
160
self._set_connection(connection, credentials)
162
def _translate_perm_error(self, err, path, extra=None,
163
unknown_exc=FtpPathError):
142
164
"""Try to translate an ftplib.error_perm exception.
144
166
:param err: The error to translate into a bzr error
173
201
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
176
def should_cache(self):
177
"""Return True if the data pulled across should be cached locally.
181
def clone(self, offset=None):
182
"""Return a new FtpTransport with root at self.base + offset.
186
return FtpTransport(self.base, self._FTP_instance)
188
return FtpTransport(self.abspath(offset), self._FTP_instance)
190
def _abspath(self, relpath):
191
assert isinstance(relpath, basestring)
192
relpath = urllib.unquote(relpath)
193
relpath_parts = relpath.split('/')
194
if len(relpath_parts) > 1:
195
if relpath_parts[0] == '':
196
raise ValueError("path %r within branch %r seems to be absolute"
197
% (relpath, self._path))
198
basepath = self._path.split('/')
199
if len(basepath) > 0 and basepath[-1] == '':
200
basepath = basepath[:-1]
201
for p in relpath_parts:
203
if len(basepath) == 0:
204
# In most filesystems, a request for the parent
205
# of root, just returns root.
208
elif p == '.' or p == '':
212
# Possibly, we could use urlparse.urljoin() here, but
213
# I'm concerned about when it chooses to strip the last
214
# portion of the path, and when it doesn't.
215
return '/'.join(basepath) or '/'
217
def abspath(self, relpath):
218
"""Return the full url to the given relative path.
219
This can be supplied with a string or a list
221
path = self._abspath(relpath)
222
return self._unparse_url(path)
204
def _remote_path(self, relpath):
205
# XXX: It seems that ftplib does not handle Unicode paths
206
# at the same time, medusa won't handle utf8 paths So if
207
# we .encode(utf8) here (see ConnectedTransport
208
# implementation), then we get a Server failure. while
209
# if we use str(), we get a UnicodeError, and the test
210
# suite just skips testing UnicodePaths.
211
relative = str(urlutils.unescape(relpath))
212
remote_path = self._combine_paths(self._path, relative)
224
215
def has(self, relpath):
225
216
"""Does the target location exist?"""
290
281
:param retries: Number of retries after temporary failures so far
291
282
for this operation.
293
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
284
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
295
abspath = self._abspath(relpath)
287
abspath = self._remote_path(relpath)
296
288
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
297
289
os.getpid(), random.randint(0,0x7FFFFFFF))
298
if not hasattr(fp, 'read'):
291
if getattr(fp, 'read', None) is None:
292
# hand in a string IO
296
# capture the byte count; .read() may be read only so
298
class byte_counter(object):
299
def __init__(self, fp):
301
self.counted_bytes = 0
302
def read(self, count):
303
result = self.fp.read(count)
304
self.counted_bytes += len(result)
306
fp = byte_counter(fp)
301
308
mutter("FTP put: %s", abspath)
302
309
f = self._get_FTP()
304
311
f.storbinary('STOR '+tmp_abspath, fp)
305
f.rename(tmp_abspath, abspath)
312
self._rename_and_overwrite(tmp_abspath, abspath, f)
313
if bytes is not None:
316
return fp.counted_bytes
306
317
except (ftplib.error_temp,EOFError), e:
307
318
warning("Failure during ftp PUT. Deleting temporary file.")
409
436
mutter("FTP site chmod: setting permissions to %s on %s",
410
str(mode), self._abspath(relpath))
437
str(mode), self._remote_path(relpath))
411
438
ftp = self._get_FTP()
412
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
439
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
414
441
except ftplib.error_perm, e:
415
442
# Command probably not available on this server
416
443
warning("FTP Could not set permissions to %s on %s. %s",
417
str(mode), self._abspath(relpath), str(e))
444
str(mode), self._remote_path(relpath), str(e))
419
446
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
420
447
# to copy something to another machine. And you may be able
421
448
# to give it its own address as the 'to' location.
422
449
# So implement a fancier 'copy()'
451
def rename(self, rel_from, rel_to):
452
abs_from = self._remote_path(rel_from)
453
abs_to = self._remote_path(rel_to)
454
mutter("FTP rename: %s => %s", abs_from, abs_to)
456
return self._rename(abs_from, abs_to, f)
458
def _rename(self, abs_from, abs_to, f):
460
f.rename(abs_from, abs_to)
461
except ftplib.error_perm, e:
462
self._translate_perm_error(e, abs_from,
463
': unable to rename to %r' % (abs_to))
424
465
def move(self, rel_from, rel_to):
425
466
"""Move the item at rel_from to the location at rel_to"""
426
abs_from = self._abspath(rel_from)
427
abs_to = self._abspath(rel_to)
467
abs_from = self._remote_path(rel_from)
468
abs_to = self._remote_path(rel_to)
429
470
mutter("FTP mv: %s => %s", abs_from, abs_to)
430
471
f = self._get_FTP()
431
f.rename(abs_from, abs_to)
472
self._rename_and_overwrite(abs_from, abs_to, f)
432
473
except ftplib.error_perm, e:
433
474
self._translate_perm_error(e, abs_from,
434
475
extra='unable to rename to %r' % (rel_to,),
435
476
unknown_exc=errors.PathError)
478
def _rename_and_overwrite(self, abs_from, abs_to, f):
479
"""Do a fancy rename on the remote server.
481
Using the implementation provided by osutils.
483
osutils.fancy_rename(abs_from, abs_to,
484
rename_func=lambda p1, p2: self._rename(p1, p2, f),
485
unlink_func=lambda p: self._delete(p, f))
439
487
def delete(self, relpath):
440
488
"""Delete the item at relpath"""
441
abspath = self._abspath(relpath)
489
abspath = self._remote_path(relpath)
491
self._delete(abspath, f)
493
def _delete(self, abspath, f):
443
495
mutter("FTP rm: %s", abspath)
445
496
f.delete(abspath)
446
497
except ftplib.error_perm, e:
447
498
self._translate_perm_error(e, abspath, 'error deleting',
448
499
unknown_exc=errors.NoSuchFile)
501
def external_url(self):
502
"""See bzrlib.transport.Transport.external_url."""
503
# FTP URL's are externally usable.
450
506
def listable(self):
451
507
"""See Transport.listable."""
454
510
def list_dir(self, relpath):
455
511
"""See Transport.list_dir."""
512
basepath = self._remote_path(relpath)
513
mutter("FTP nlst: %s", basepath)
457
mutter("FTP nlst: %s", self._abspath(relpath))
459
basepath = self._abspath(relpath)
460
516
paths = f.nlst(basepath)
461
# If FTP.nlst returns paths prefixed by relpath, strip 'em
462
if paths and paths[0].startswith(basepath):
463
paths = [path[len(basepath)+1:] for path in paths]
464
# Remove . and .. if present, and return
465
return [path for path in paths if path not in (".", "..")]
466
517
except ftplib.error_perm, e:
467
518
self._translate_perm_error(e, relpath, extra='error with list_dir')
519
except ftplib.error_temp, e:
520
# xs4all's ftp server raises a 450 temp error when listing an empty
521
# directory. Check for that and just return an empty list in that
522
# case. See bug #215522
523
if str(e).lower().startswith('450 no files found'):
524
mutter('FTP Server returned "%s" for nlst.'
525
' Assuming it means empty directory',
529
# If FTP.nlst returns paths prefixed by relpath, strip 'em
530
if paths and paths[0].startswith(basepath):
531
entries = [path[len(basepath)+1:] for path in paths]
534
# Remove . and .. if present
535
return [urlutils.escape(entry) for entry in entries
536
if entry not in ('.', '..')]
469
538
def iter_files_recursive(self):
470
539
"""See Transport.iter_files_recursive.
513
582
return self.lock_read(relpath)
516
class FtpServer(Server):
517
"""Common code for SFTP server facilities."""
521
self._ftp_server = None
523
self._async_thread = None
528
"""Calculate an ftp url to this server."""
529
return 'ftp://foo:bar@localhost:%d/' % (self._port)
531
# def get_bogus_url(self):
532
# """Return a URL which cannot be connected to."""
533
# return 'ftp://127.0.0.1:1'
535
def log(self, message):
536
"""This is used by medusa.ftp_server to log connections, etc."""
537
self.logs.append(message)
542
raise RuntimeError('Must have medusa to run the FtpServer')
544
self._root = os.getcwdu()
545
self._ftp_server = _ftp_server(
546
authorizer=_test_authorizer(root=self._root),
548
port=0, # bind to a random port
550
logger_object=self # Use FtpServer.log() for messages
552
self._port = self._ftp_server.getsockname()[1]
553
# Don't let it loop forever, or handle an infinite number of requests.
554
# In this case it will run for 100s, or 1000 requests
555
self._async_thread = threading.Thread(target=asyncore.loop,
556
kwargs={'timeout':0.1, 'count':1000})
557
self._async_thread.setDaemon(True)
558
self._async_thread.start()
561
"""See bzrlib.transport.Server.tearDown."""
562
# have asyncore release the channel
563
self._ftp_server.del_channel()
565
self._async_thread.join()
570
_test_authorizer = None
574
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
577
import medusa.filesys
578
import medusa.ftp_server
584
class test_authorizer(object):
585
"""A custom Authorizer object for running the test suite.
587
The reason we cannot use dummy_authorizer, is because it sets the
588
channel to readonly, which we don't always want to do.
591
def __init__(self, root):
594
def authorize(self, channel, username, password):
595
"""Return (success, reply_string, filesystem)"""
597
return 0, 'No Medusa.', None
599
channel.persona = -1, -1
600
if username == 'anonymous':
601
channel.read_only = 1
603
channel.read_only = 0
605
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
608
class ftp_channel(medusa.ftp_server.ftp_channel):
609
"""Customized ftp channel"""
611
def log(self, message):
612
"""Redirect logging requests."""
613
mutter('_ftp_channel: %s', message)
615
def log_info(self, message, type='info'):
616
"""Redirect logging requests."""
617
mutter('_ftp_channel %s: %s', type, message)
619
def cmd_rnfr(self, line):
620
"""Prepare for renaming a file."""
621
self._renaming = line[1]
622
self.respond('350 Ready for RNTO')
623
# TODO: jam 20060516 in testing, the ftp server seems to
624
# check that the file already exists, or it sends
625
# 550 RNFR command failed
627
def cmd_rnto(self, line):
628
"""Rename a file based on the target given.
630
rnto must be called after calling rnfr.
632
if not self._renaming:
633
self.respond('503 RNFR required first.')
634
pfrom = self.filesystem.translate(self._renaming)
635
self._renaming = None
636
pto = self.filesystem.translate(line[1])
638
os.rename(pfrom, pto)
639
except (IOError, OSError), e:
640
# TODO: jam 20060516 return custom responses based on
641
# why the command failed
642
self.respond('550 RNTO failed: %s' % (e,))
644
self.respond('550 RNTO failed')
645
# For a test server, we will go ahead and just die
648
self.respond('250 Rename successful.')
650
def cmd_size(self, line):
651
"""Return the size of a file
653
This is overloaded to help the test suite determine if the
654
target is a directory.
657
if not self.filesystem.isfile(filename):
658
if self.filesystem.isdir(filename):
659
self.respond('550 "%s" is a directory' % (filename,))
661
self.respond('550 "%s" is not a file' % (filename,))
663
self.respond('213 %d'
664
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
666
def cmd_mkd(self, line):
667
"""Create a directory.
669
Overloaded because default implementation does not distinguish
670
*why* it cannot make a directory.
673
self.command_not_understood (string.join (line))
677
self.filesystem.mkdir (path)
678
self.respond ('257 MKD command successful.')
679
except (IOError, OSError), e:
680
self.respond ('550 error creating directory: %s' % (e,))
682
self.respond ('550 error creating directory.')
685
class ftp_server(medusa.ftp_server.ftp_server):
686
"""Customize the behavior of the Medusa ftp_server.
688
There are a few warts on the ftp_server, based on how it expects
692
ftp_channel_class = ftp_channel
694
def __init__(self, *args, **kwargs):
695
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
696
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
698
def log(self, message):
699
"""Redirect logging requests."""
700
mutter('_ftp_server: %s', message)
702
def log_info(self, message, type='info'):
703
"""Override the asyncore.log_info so we don't stipple the screen."""
704
mutter('_ftp_server %s: %s', type, message)
706
_test_authorizer = test_authorizer
707
_ftp_channel = ftp_channel
708
_ftp_server = ftp_server
713
585
def get_test_permutations():
714
586
"""Return the permutations to be used in testing."""
715
if not _setup_medusa():
716
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
587
from bzrlib import tests
588
if tests.FTPServerFeature.available():
589
from bzrlib.tests import ftp_server
590
return [(FtpTransport, ftp_server.FTPServer)]
719
return [(FtpTransport, FtpServer)]
592
# Dummy server to have the test suite report the number of tests
593
# needing that feature. We raise UnavailableFeature from methods before
594
# the test server is being used. Doing so in the setUp method has bad
595
# side-effects (tearDown is never called).
596
class UnavailableFTPServer(object):
605
raise tests.UnavailableFeature(tests.FTPServerFeature)
607
def get_bogus_url(self):
608
raise tests.UnavailableFeature(tests.FTPServerFeature)
610
return [(FtpTransport, UnavailableFTPServer)]
added
removed
bzrlib/transport/ftp.py
