27
27
from cStringIO import StringIO
38
38
from warnings import warn
40
40
from bzrlib import (
46
44
from bzrlib.trace import mutter, warning
47
45
from bzrlib.transport import (
48
AppendBasedFileStream,
51
register_urlparse_netloc_protocol,
54
from bzrlib.transport.local import LocalURLServer
58
register_urlparse_netloc_protocol('aftp')
61
55
class FtpPathError(errors.PathError):
62
56
"""FTP failed for path: %(path)s%(extra)s"""
60
def _find_FTP(hostname, port, username, password, is_active):
61
"""Find an ftplib.FTP instance attached to this triplet."""
62
key = (hostname, port, username, password, is_active)
63
alt_key = (hostname, port, username, '********', is_active)
64
if key not in _FTP_cache:
65
mutter("Constructing FTP instance against %r" % (alt_key,))
68
conn.connect(host=hostname, port=port)
69
if username and username != 'anonymous' and not password:
70
password = bzrlib.ui.ui_factory.get_password(
71
prompt='FTP %(user)s@%(host)s password',
72
user=username, host=hostname)
73
conn.login(user=username, passwd=password)
74
conn.set_pasv(not is_active)
76
_FTP_cache[key] = conn
78
return _FTP_cache[key]
65
81
class FtpStatResult(object):
66
82
def __init__(self, f, relpath):
79
95
_number_of_retries = 2
80
96
_sleep_between_retries = 5
82
# FIXME: there are inconsistencies in the way temporary errors are
83
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
84
# be taken to analyze the implications for write operations (read operations
85
# are safe to retry). Overall even some read operations are never
86
# retried. --vila 20070720 (Bug #127164)
87
class FtpTransport(ConnectedTransport):
98
class FtpTransport(Transport):
88
99
"""This is the transport agent for ftp:// access."""
90
def __init__(self, base, _from_transport=None):
101
def __init__(self, base, _provided_instance=None):
91
102
"""Set the base path where files will be stored."""
92
if not (base.startswith('ftp://') or base.startswith('aftp://')):
93
raise ValueError(base)
94
super(FtpTransport, self).__init__(base,
95
_from_transport=_from_transport)
96
self._unqualified_scheme = 'ftp'
97
if self._scheme == 'aftp':
100
self.is_active = False
103
assert base.startswith('ftp://') or base.startswith('aftp://')
105
self.is_active = base.startswith('aftp://')
107
# urlparse won't handle aftp://
109
if not base.endswith('/'):
111
(self._proto, self._username,
112
self._password, self._host,
113
self._port, self._path) = split_url(base)
114
base = self._unparse_url()
116
super(FtpTransport, self).__init__(base)
117
self._FTP_instance = _provided_instance
119
def _unparse_url(self, path=None):
122
path = urllib.quote(path)
123
netloc = urllib.quote(self._host)
124
if self._username is not None:
125
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
126
if self._port is not None:
127
netloc = '%s:%d' % (netloc, self._port)
131
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
102
133
def _get_FTP(self):
103
134
"""Return the ftplib.FTP instance for this object."""
104
# Ensures that a connection is established
105
connection = self._get_connection()
106
if connection is None:
107
# First connection ever
108
connection, credentials = self._create_connection()
109
self._set_connection(connection, credentials)
112
def _create_connection(self, credentials=None):
113
"""Create a new connection with the provided credentials.
115
:param credentials: The credentials needed to establish the connection.
117
:return: The created connection and its associated credentials.
119
The credentials are only the password as it may have been entered
120
interactively by the user and may be different from the one provided
121
in base url at transport creation time.
123
if credentials is None:
124
user, password = self._user, self._password
126
user, password = credentials
128
auth = config.AuthenticationConfig()
130
user = auth.get_user('ftp', self._host, port=self._port)
132
# Default to local user
133
user = getpass.getuser()
135
mutter("Constructing FTP instance against %r" %
136
((self._host, self._port, user, '********',
135
if self._FTP_instance is not None:
136
return self._FTP_instance
139
connection = ftplib.FTP()
140
connection.connect(host=self._host, port=self._port)
141
if user and user != 'anonymous' and \
142
password is None: # '' is a valid password
143
password = auth.get_password('ftp', self._host, user,
145
connection.login(user=user, passwd=password)
146
connection.set_pasv(not self.is_active)
147
except socket.error, e:
148
raise errors.SocketConnectionError(self._host, self._port,
149
msg='Unable to connect to',
139
self._FTP_instance = _find_FTP(self._host, self._port,
140
self._username, self._password,
142
return self._FTP_instance
151
143
except ftplib.error_perm, e:
152
raise errors.TransportError(msg="Error setting up connection:"
153
" %s" % str(e), orig_error=e)
154
return connection, (user, password)
156
def _reconnect(self):
157
"""Create a new connection with the previously used credentials"""
158
credentials = self._get_credentials()
159
connection, credentials = self._create_connection(credentials)
160
self._set_connection(connection, credentials)
162
def _translate_perm_error(self, err, path, extra=None,
163
unknown_exc=FtpPathError):
144
raise errors.TransportError(msg="Error setting up connection: %s"
145
% str(e), orig_error=e)
147
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
164
148
"""Try to translate an ftplib.error_perm exception.
166
150
:param err: The error to translate into a bzr error
201
180
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
204
def _remote_path(self, relpath):
183
def should_cache(self):
184
"""Return True if the data pulled across should be cached locally.
188
def clone(self, offset=None):
189
"""Return a new FtpTransport with root at self.base + offset.
193
return FtpTransport(self.base, self._FTP_instance)
195
return FtpTransport(self.abspath(offset), self._FTP_instance)
197
def _abspath(self, relpath):
198
assert isinstance(relpath, basestring)
199
relpath = urlutils.unescape(relpath)
200
if relpath.startswith('/'):
203
basepath = self._path.split('/')
204
if len(basepath) > 0 and basepath[-1] == '':
205
basepath = basepath[:-1]
206
for p in relpath.split('/'):
208
if len(basepath) == 0:
209
# In most filesystems, a request for the parent
210
# of root, just returns root.
213
elif p == '.' or p == '':
217
# Possibly, we could use urlparse.urljoin() here, but
218
# I'm concerned about when it chooses to strip the last
219
# portion of the path, and when it doesn't.
205
221
# XXX: It seems that ftplib does not handle Unicode paths
206
# at the same time, medusa won't handle utf8 paths So if
207
# we .encode(utf8) here (see ConnectedTransport
208
# implementation), then we get a Server failure. while
209
# if we use str(), we get a UnicodeError, and the test
210
# suite just skips testing UnicodePaths.
211
relative = str(urlutils.unescape(relpath))
212
remote_path = self._combine_paths(self._path, relative)
222
# at the same time, medusa won't handle utf8 paths
223
# So if we .encode(utf8) here, then we get a Server failure.
224
# while if we use str(), we get a UnicodeError, and the test suite
225
# just skips testing UnicodePaths.
226
return str('/'.join(basepath) or '/')
228
def abspath(self, relpath):
229
"""Return the full url to the given relative path.
230
This can be supplied with a string or a list
232
path = self._abspath(relpath)
233
return self._unparse_url(path)
215
235
def has(self, relpath):
216
236
"""Does the target location exist?"""
281
301
:param retries: Number of retries after temporary failures so far
282
302
for this operation.
284
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
304
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
287
abspath = self._remote_path(relpath)
306
abspath = self._abspath(relpath)
288
307
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
289
308
os.getpid(), random.randint(0,0x7FFFFFFF))
291
309
if getattr(fp, 'read', None) is None:
292
# hand in a string IO
296
# capture the byte count; .read() may be read only so
298
class byte_counter(object):
299
def __init__(self, fp):
301
self.counted_bytes = 0
302
def read(self, count):
303
result = self.fp.read(count)
304
self.counted_bytes += len(result)
306
fp = byte_counter(fp)
308
312
mutter("FTP put: %s", abspath)
309
313
f = self._get_FTP()
311
315
f.storbinary('STOR '+tmp_abspath, fp)
312
self._rename_and_overwrite(tmp_abspath, abspath, f)
313
self._setmode(relpath, mode)
314
if bytes is not None:
317
return fp.counted_bytes
316
f.rename(tmp_abspath, abspath)
318
317
except (ftplib.error_temp,EOFError), e:
319
318
warning("Failure during ftp PUT. Deleting temporary file.")
343
341
warning("FTP control connection closed. Trying to reopen.")
344
342
time.sleep(_sleep_between_retries)
343
self._FTP_instance = None
346
344
self.put_file(relpath, fp, mode, retries+1)
348
346
def mkdir(self, relpath, mode=None):
349
347
"""Create a directory at the given path."""
350
abspath = self._remote_path(relpath)
348
abspath = self._abspath(relpath)
352
350
mutter("FTP mkd: %s", abspath)
353
351
f = self._get_FTP()
355
self._setmode(relpath, mode)
356
353
except ftplib.error_perm, e:
357
354
self._translate_perm_error(e, abspath,
358
355
unknown_exc=errors.FileExists)
360
def open_write_stream(self, relpath, mode=None):
361
"""See Transport.open_write_stream."""
362
self.put_bytes(relpath, "", mode)
363
result = AppendBasedFileStream(self, relpath)
364
_file_streams[self.abspath(relpath)] = result
367
def recommended_page_size(self):
368
"""See Transport.recommended_page_size().
370
For FTP we suggest a large page size to reduce the overhead
371
introduced by latency.
375
357
def rmdir(self, rel_path):
376
358
"""Delete the directory at rel_path"""
377
abspath = self._remote_path(rel_path)
359
abspath = self._abspath(rel_path)
379
361
mutter("FTP rmd: %s", abspath)
380
362
f = self._get_FTP()
433
416
Only set permissions if the FTP server supports the 'SITE CHMOD'
438
mutter("FTP site chmod: setting permissions to %s on %s",
439
str(mode), self._remote_path(relpath))
440
ftp = self._get_FTP()
441
cmd = "SITE CHMOD %s %s" % (oct(mode),
442
self._remote_path(relpath))
444
except ftplib.error_perm, e:
445
# Command probably not available on this server
446
warning("FTP Could not set permissions to %s on %s. %s",
447
str(mode), self._remote_path(relpath), str(e))
420
mutter("FTP site chmod: setting permissions to %s on %s",
421
str(mode), self._abspath(relpath))
422
ftp = self._get_FTP()
423
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
425
except ftplib.error_perm, e:
426
# Command probably not available on this server
427
warning("FTP Could not set permissions to %s on %s. %s",
428
str(mode), self._abspath(relpath), str(e))
449
430
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
450
431
# to copy something to another machine. And you may be able
451
432
# to give it its own address as the 'to' location.
452
433
# So implement a fancier 'copy()'
454
def rename(self, rel_from, rel_to):
455
abs_from = self._remote_path(rel_from)
456
abs_to = self._remote_path(rel_to)
457
mutter("FTP rename: %s => %s", abs_from, abs_to)
459
return self._rename(abs_from, abs_to, f)
461
def _rename(self, abs_from, abs_to, f):
463
f.rename(abs_from, abs_to)
464
except ftplib.error_perm, e:
465
self._translate_perm_error(e, abs_from,
466
': unable to rename to %r' % (abs_to))
468
435
def move(self, rel_from, rel_to):
469
436
"""Move the item at rel_from to the location at rel_to"""
470
abs_from = self._remote_path(rel_from)
471
abs_to = self._remote_path(rel_to)
437
abs_from = self._abspath(rel_from)
438
abs_to = self._abspath(rel_to)
473
440
mutter("FTP mv: %s => %s", abs_from, abs_to)
474
441
f = self._get_FTP()
475
self._rename_and_overwrite(abs_from, abs_to, f)
442
f.rename(abs_from, abs_to)
476
443
except ftplib.error_perm, e:
477
444
self._translate_perm_error(e, abs_from,
478
445
extra='unable to rename to %r' % (rel_to,),
479
446
unknown_exc=errors.PathError)
481
def _rename_and_overwrite(self, abs_from, abs_to, f):
482
"""Do a fancy rename on the remote server.
484
Using the implementation provided by osutils.
486
osutils.fancy_rename(abs_from, abs_to,
487
rename_func=lambda p1, p2: self._rename(p1, p2, f),
488
unlink_func=lambda p: self._delete(p, f))
490
450
def delete(self, relpath):
491
451
"""Delete the item at relpath"""
492
abspath = self._remote_path(relpath)
494
self._delete(abspath, f)
496
def _delete(self, abspath, f):
452
abspath = self._abspath(relpath)
498
454
mutter("FTP rm: %s", abspath)
499
456
f.delete(abspath)
500
457
except ftplib.error_perm, e:
501
458
self._translate_perm_error(e, abspath, 'error deleting',
502
459
unknown_exc=errors.NoSuchFile)
504
def external_url(self):
505
"""See bzrlib.transport.Transport.external_url."""
506
# FTP URL's are externally usable.
509
461
def listable(self):
510
462
"""See Transport.listable."""
513
465
def list_dir(self, relpath):
514
466
"""See Transport.list_dir."""
515
basepath = self._remote_path(relpath)
467
basepath = self._abspath(relpath)
516
468
mutter("FTP nlst: %s", basepath)
517
469
f = self._get_FTP()
519
471
paths = f.nlst(basepath)
520
472
except ftplib.error_perm, e:
521
473
self._translate_perm_error(e, relpath, extra='error with list_dir')
522
except ftplib.error_temp, e:
523
# xs4all's ftp server raises a 450 temp error when listing an empty
524
# directory. Check for that and just return an empty list in that
525
# case. See bug #215522
526
if str(e).lower().startswith('450 no files found'):
527
mutter('FTP Server returned "%s" for nlst.'
528
' Assuming it means empty directory',
532
474
# If FTP.nlst returns paths prefixed by relpath, strip 'em
533
475
if paths and paths[0].startswith(basepath):
534
476
entries = [path[len(basepath)+1:] for path in paths]
585
527
return self.lock_read(relpath)
530
class FtpServer(Server):
531
"""Common code for SFTP server facilities."""
535
self._ftp_server = None
537
self._async_thread = None
542
"""Calculate an ftp url to this server."""
543
return 'ftp://foo:bar@localhost:%d/' % (self._port)
545
# def get_bogus_url(self):
546
# """Return a URL which cannot be connected to."""
547
# return 'ftp://127.0.0.1:1'
549
def log(self, message):
550
"""This is used by medusa.ftp_server to log connections, etc."""
551
self.logs.append(message)
556
raise RuntimeError('Must have medusa to run the FtpServer')
558
self._root = os.getcwdu()
559
self._ftp_server = _ftp_server(
560
authorizer=_test_authorizer(root=self._root),
562
port=0, # bind to a random port
564
logger_object=self # Use FtpServer.log() for messages
566
self._port = self._ftp_server.getsockname()[1]
567
# Don't let it loop forever, or handle an infinite number of requests.
568
# In this case it will run for 100s, or 1000 requests
569
self._async_thread = threading.Thread(target=asyncore.loop,
570
kwargs={'timeout':0.1, 'count':1000})
571
self._async_thread.setDaemon(True)
572
self._async_thread.start()
575
"""See bzrlib.transport.Server.tearDown."""
576
# have asyncore release the channel
577
self._ftp_server.del_channel()
579
self._async_thread.join()
584
_test_authorizer = None
588
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
591
import medusa.filesys
592
import medusa.ftp_server
598
class test_authorizer(object):
599
"""A custom Authorizer object for running the test suite.
601
The reason we cannot use dummy_authorizer, is because it sets the
602
channel to readonly, which we don't always want to do.
605
def __init__(self, root):
608
def authorize(self, channel, username, password):
609
"""Return (success, reply_string, filesystem)"""
611
return 0, 'No Medusa.', None
613
channel.persona = -1, -1
614
if username == 'anonymous':
615
channel.read_only = 1
617
channel.read_only = 0
619
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
622
class ftp_channel(medusa.ftp_server.ftp_channel):
623
"""Customized ftp channel"""
625
def log(self, message):
626
"""Redirect logging requests."""
627
mutter('_ftp_channel: %s', message)
629
def log_info(self, message, type='info'):
630
"""Redirect logging requests."""
631
mutter('_ftp_channel %s: %s', type, message)
633
def cmd_rnfr(self, line):
634
"""Prepare for renaming a file."""
635
self._renaming = line[1]
636
self.respond('350 Ready for RNTO')
637
# TODO: jam 20060516 in testing, the ftp server seems to
638
# check that the file already exists, or it sends
639
# 550 RNFR command failed
641
def cmd_rnto(self, line):
642
"""Rename a file based on the target given.
644
rnto must be called after calling rnfr.
646
if not self._renaming:
647
self.respond('503 RNFR required first.')
648
pfrom = self.filesystem.translate(self._renaming)
649
self._renaming = None
650
pto = self.filesystem.translate(line[1])
652
os.rename(pfrom, pto)
653
except (IOError, OSError), e:
654
# TODO: jam 20060516 return custom responses based on
655
# why the command failed
656
self.respond('550 RNTO failed: %s' % (e,))
658
self.respond('550 RNTO failed')
659
# For a test server, we will go ahead and just die
662
self.respond('250 Rename successful.')
664
def cmd_size(self, line):
665
"""Return the size of a file
667
This is overloaded to help the test suite determine if the
668
target is a directory.
671
if not self.filesystem.isfile(filename):
672
if self.filesystem.isdir(filename):
673
self.respond('550 "%s" is a directory' % (filename,))
675
self.respond('550 "%s" is not a file' % (filename,))
677
self.respond('213 %d'
678
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
680
def cmd_mkd(self, line):
681
"""Create a directory.
683
Overloaded because default implementation does not distinguish
684
*why* it cannot make a directory.
687
self.command_not_understood(''.join(line))
691
self.filesystem.mkdir (path)
692
self.respond ('257 MKD command successful.')
693
except (IOError, OSError), e:
694
self.respond ('550 error creating directory: %s' % (e,))
696
self.respond ('550 error creating directory.')
699
class ftp_server(medusa.ftp_server.ftp_server):
700
"""Customize the behavior of the Medusa ftp_server.
702
There are a few warts on the ftp_server, based on how it expects
706
ftp_channel_class = ftp_channel
708
def __init__(self, *args, **kwargs):
709
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
710
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
712
def log(self, message):
713
"""Redirect logging requests."""
714
mutter('_ftp_server: %s', message)
716
def log_info(self, message, type='info'):
717
"""Override the asyncore.log_info so we don't stipple the screen."""
718
mutter('_ftp_server %s: %s', type, message)
720
_test_authorizer = test_authorizer
721
_ftp_channel = ftp_channel
722
_ftp_server = ftp_server
588
727
def get_test_permutations():
589
728
"""Return the permutations to be used in testing."""
590
from bzrlib import tests
591
if tests.FTPServerFeature.available():
592
from bzrlib.tests import ftp_server
593
return [(FtpTransport, ftp_server.FTPServer)]
729
if not _setup_medusa():
730
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
595
# Dummy server to have the test suite report the number of tests
596
# needing that feature. We raise UnavailableFeature from methods before
597
# the test server is being used. Doing so in the setUp method has bad
598
# side-effects (tearDown is never called).
599
class UnavailableFTPServer(object):
608
raise tests.UnavailableFeature(tests.FTPServerFeature)
610
def get_bogus_url(self):
611
raise tests.UnavailableFeature(tests.FTPServerFeature)
613
return [(FtpTransport, UnavailableFTPServer)]
733
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
