27
27
from cStringIO import StringIO
38
38
from warnings import warn
40
40
from bzrlib import (
46
44
from bzrlib.trace import mutter, warning
47
45
from bzrlib.transport import (
48
AppendBasedFileStream,
51
register_urlparse_netloc_protocol,
54
from bzrlib.transport.local import LocalURLServer
58
register_urlparse_netloc_protocol('aftp')
61
55
class FtpPathError(errors.PathError):
62
56
"""FTP failed for path: %(path)s%(extra)s"""
60
def _find_FTP(hostname, port, username, password, is_active):
61
"""Find an ftplib.FTP instance attached to this triplet."""
62
key = (hostname, port, username, password, is_active)
63
alt_key = (hostname, port, username, '********', is_active)
64
if key not in _FTP_cache:
65
mutter("Constructing FTP instance against %r" % (alt_key,))
68
conn.connect(host=hostname, port=port)
69
if username and username != 'anonymous' and not password:
70
password = bzrlib.ui.ui_factory.get_password(
71
prompt='FTP %(user)s@%(host)s password',
72
user=username, host=hostname)
73
conn.login(user=username, passwd=password)
74
conn.set_pasv(not is_active)
76
_FTP_cache[key] = conn
78
return _FTP_cache[key]
65
81
class FtpStatResult(object):
67
def __init__(self, f, abspath):
82
def __init__(self, f, relpath):
69
self.st_size = f.size(abspath)
84
self.st_size = f.size(relpath)
70
85
self.st_mode = stat.S_IFREG
71
86
except ftplib.error_perm:
75
90
self.st_mode = stat.S_IFDIR
80
95
_number_of_retries = 2
81
96
_sleep_between_retries = 5
83
# FIXME: there are inconsistencies in the way temporary errors are
84
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
85
# be taken to analyze the implications for write operations (read operations
86
# are safe to retry). Overall even some read operations are never
87
# retried. --vila 20070720 (Bug #127164)
88
class FtpTransport(ConnectedTransport):
98
class FtpTransport(Transport):
89
99
"""This is the transport agent for ftp:// access."""
91
def __init__(self, base, _from_transport=None):
101
def __init__(self, base, _provided_instance=None):
92
102
"""Set the base path where files will be stored."""
93
if not (base.startswith('ftp://') or base.startswith('aftp://')):
94
raise ValueError(base)
95
super(FtpTransport, self).__init__(base,
96
_from_transport=_from_transport)
97
self._unqualified_scheme = 'ftp'
98
if self._scheme == 'aftp':
101
self.is_active = False
103
assert base.startswith('ftp://') or base.startswith('aftp://')
105
self.is_active = base.startswith('aftp://')
107
# urlparse won't handle aftp://
109
if not base.endswith('/'):
111
(self._proto, self._username,
112
self._password, self._host,
113
self._port, self._path) = split_url(base)
114
base = self._unparse_url()
116
super(FtpTransport, self).__init__(base)
117
self._FTP_instance = _provided_instance
119
def _unparse_url(self, path=None):
122
path = urllib.quote(path)
123
netloc = urllib.quote(self._host)
124
if self._username is not None:
125
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
126
if self._port is not None:
127
netloc = '%s:%d' % (netloc, self._port)
131
return urlparse.urlunparse((proto, netloc, path, '', '', ''))
103
133
def _get_FTP(self):
104
134
"""Return the ftplib.FTP instance for this object."""
105
# Ensures that a connection is established
106
connection = self._get_connection()
107
if connection is None:
108
# First connection ever
109
connection, credentials = self._create_connection()
110
self._set_connection(connection, credentials)
113
def _create_connection(self, credentials=None):
114
"""Create a new connection with the provided credentials.
116
:param credentials: The credentials needed to establish the connection.
118
:return: The created connection and its associated credentials.
120
The input credentials are only the password as it may have been
121
entered interactively by the user and may be different from the one
122
provided in base url at transport creation time. The returned
123
credentials are username, password.
125
if credentials is None:
126
user, password = self._user, self._password
128
user, password = credentials
130
auth = config.AuthenticationConfig()
132
user = auth.get_user('ftp', self._host, port=self._port)
134
# Default to local user
135
user = getpass.getuser()
137
mutter("Constructing FTP instance against %r" %
138
((self._host, self._port, user, '********',
135
if self._FTP_instance is not None:
136
return self._FTP_instance
141
connection = ftplib.FTP()
142
connection.connect(host=self._host, port=self._port)
143
if user and user != 'anonymous' and \
144
password is None: # '' is a valid password
145
password = auth.get_password('ftp', self._host, user,
147
connection.login(user=user, passwd=password)
148
connection.set_pasv(not self.is_active)
149
# binary mode is the default
150
connection.voidcmd('TYPE I')
151
except socket.error, e:
152
raise errors.SocketConnectionError(self._host, self._port,
153
msg='Unable to connect to',
139
self._FTP_instance = _find_FTP(self._host, self._port,
140
self._username, self._password,
142
return self._FTP_instance
155
143
except ftplib.error_perm, e:
156
raise errors.TransportError(msg="Error setting up connection:"
157
" %s" % str(e), orig_error=e)
158
return connection, (user, password)
160
def _reconnect(self):
161
"""Create a new connection with the previously used credentials"""
162
credentials = self._get_credentials()
163
connection, credentials = self._create_connection(credentials)
164
self._set_connection(connection, credentials)
166
def _translate_perm_error(self, err, path, extra=None,
167
unknown_exc=FtpPathError):
144
raise errors.TransportError(msg="Error setting up connection: %s"
145
% str(e), orig_error=e)
147
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
168
148
"""Try to translate an ftplib.error_perm exception.
170
150
:param err: The error to translate into a bzr error
200
174
raise unknown_exc(path, extra=extra)
201
# TODO: jam 20060516 Consider re-raising the error wrapped in
175
# TODO: jam 20060516 Consider re-raising the error wrapped in
202
176
# something like TransportError, but this loses the traceback
203
177
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
204
178
# to handle. Consider doing something like that here.
205
179
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
208
def _remote_path(self, relpath):
182
def should_cache(self):
183
"""Return True if the data pulled across should be cached locally.
187
def clone(self, offset=None):
188
"""Return a new FtpTransport with root at self.base + offset.
192
return FtpTransport(self.base, self._FTP_instance)
194
return FtpTransport(self.abspath(offset), self._FTP_instance)
196
def _abspath(self, relpath):
197
assert isinstance(relpath, basestring)
198
relpath = urlutils.unescape(relpath)
199
if relpath.startswith('/'):
202
basepath = self._path.split('/')
203
if len(basepath) > 0 and basepath[-1] == '':
204
basepath = basepath[:-1]
205
for p in relpath.split('/'):
207
if len(basepath) == 0:
208
# In most filesystems, a request for the parent
209
# of root, just returns root.
212
elif p == '.' or p == '':
216
# Possibly, we could use urlparse.urljoin() here, but
217
# I'm concerned about when it chooses to strip the last
218
# portion of the path, and when it doesn't.
209
220
# XXX: It seems that ftplib does not handle Unicode paths
210
# at the same time, medusa won't handle utf8 paths So if
211
# we .encode(utf8) here (see ConnectedTransport
212
# implementation), then we get a Server failure. while
213
# if we use str(), we get a UnicodeError, and the test
214
# suite just skips testing UnicodePaths.
215
relative = str(urlutils.unescape(relpath))
216
remote_path = self._combine_paths(self._path, relative)
221
# at the same time, medusa won't handle utf8 paths
222
# So if we .encode(utf8) here, then we get a Server failure.
223
# while if we use str(), we get a UnicodeError, and the test suite
224
# just skips testing UnicodePaths.
225
return str('/'.join(basepath) or '/')
227
def abspath(self, relpath):
228
"""Return the full url to the given relative path.
229
This can be supplied with a string or a list
231
path = self._abspath(relpath)
232
return self._unparse_url(path)
219
234
def has(self, relpath):
220
235
"""Does the target location exist?"""
285
300
:param retries: Number of retries after temporary failures so far
286
301
for this operation.
288
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
303
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
291
abspath = self._remote_path(relpath)
305
abspath = self._abspath(relpath)
292
306
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
293
307
os.getpid(), random.randint(0,0x7FFFFFFF))
295
308
if getattr(fp, 'read', None) is None:
296
# hand in a string IO
300
# capture the byte count; .read() may be read only so
302
class byte_counter(object):
303
def __init__(self, fp):
305
self.counted_bytes = 0
306
def read(self, count):
307
result = self.fp.read(count)
308
self.counted_bytes += len(result)
310
fp = byte_counter(fp)
312
311
mutter("FTP put: %s", abspath)
313
312
f = self._get_FTP()
315
314
f.storbinary('STOR '+tmp_abspath, fp)
316
self._rename_and_overwrite(tmp_abspath, abspath, f)
317
self._setmode(relpath, mode)
318
if bytes is not None:
321
return fp.counted_bytes
315
f.rename(tmp_abspath, abspath)
322
316
except (ftplib.error_temp,EOFError), e:
323
317
warning("Failure during ftp PUT. Deleting temporary file.")
347
340
warning("FTP control connection closed. Trying to reopen.")
348
341
time.sleep(_sleep_between_retries)
342
self._FTP_instance = None
350
343
self.put_file(relpath, fp, mode, retries+1)
352
345
def mkdir(self, relpath, mode=None):
353
346
"""Create a directory at the given path."""
354
abspath = self._remote_path(relpath)
347
abspath = self._abspath(relpath)
356
349
mutter("FTP mkd: %s", abspath)
357
350
f = self._get_FTP()
359
self._setmode(relpath, mode)
360
352
except ftplib.error_perm, e:
361
353
self._translate_perm_error(e, abspath,
362
354
unknown_exc=errors.FileExists)
364
def open_write_stream(self, relpath, mode=None):
365
"""See Transport.open_write_stream."""
366
self.put_bytes(relpath, "", mode)
367
result = AppendBasedFileStream(self, relpath)
368
_file_streams[self.abspath(relpath)] = result
371
def recommended_page_size(self):
372
"""See Transport.recommended_page_size().
374
For FTP we suggest a large page size to reduce the overhead
375
introduced by latency.
379
356
def rmdir(self, rel_path):
380
357
"""Delete the directory at rel_path"""
381
abspath = self._remote_path(rel_path)
358
abspath = self._abspath(rel_path)
383
360
mutter("FTP rmd: %s", abspath)
384
361
f = self._get_FTP()
436
415
Only set permissions if the FTP server supports the 'SITE CHMOD'
441
mutter("FTP site chmod: setting permissions to %s on %s",
442
oct(mode), self._remote_path(relpath))
443
ftp = self._get_FTP()
444
cmd = "SITE CHMOD %s %s" % (oct(mode),
445
self._remote_path(relpath))
447
except ftplib.error_perm, e:
448
# Command probably not available on this server
449
warning("FTP Could not set permissions to %s on %s. %s",
450
oct(mode), self._remote_path(relpath), str(e))
419
mutter("FTP site chmod: setting permissions to %s on %s",
420
str(mode), self._abspath(relpath))
421
ftp = self._get_FTP()
422
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
424
except ftplib.error_perm, e:
425
# Command probably not available on this server
426
warning("FTP Could not set permissions to %s on %s. %s",
427
str(mode), self._abspath(relpath), str(e))
452
429
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
453
430
# to copy something to another machine. And you may be able
454
431
# to give it its own address as the 'to' location.
455
432
# So implement a fancier 'copy()'
457
def rename(self, rel_from, rel_to):
458
abs_from = self._remote_path(rel_from)
459
abs_to = self._remote_path(rel_to)
460
mutter("FTP rename: %s => %s", abs_from, abs_to)
462
return self._rename(abs_from, abs_to, f)
464
def _rename(self, abs_from, abs_to, f):
466
f.rename(abs_from, abs_to)
467
except ftplib.error_perm, e:
468
self._translate_perm_error(e, abs_from,
469
': unable to rename to %r' % (abs_to))
471
434
def move(self, rel_from, rel_to):
472
435
"""Move the item at rel_from to the location at rel_to"""
473
abs_from = self._remote_path(rel_from)
474
abs_to = self._remote_path(rel_to)
436
abs_from = self._abspath(rel_from)
437
abs_to = self._abspath(rel_to)
476
439
mutter("FTP mv: %s => %s", abs_from, abs_to)
477
440
f = self._get_FTP()
478
self._rename_and_overwrite(abs_from, abs_to, f)
441
f.rename(abs_from, abs_to)
479
442
except ftplib.error_perm, e:
480
443
self._translate_perm_error(e, abs_from,
481
extra='unable to rename to %r' % (rel_to,),
444
extra='unable to rename to %r' % (rel_to,),
482
445
unknown_exc=errors.PathError)
484
def _rename_and_overwrite(self, abs_from, abs_to, f):
485
"""Do a fancy rename on the remote server.
487
Using the implementation provided by osutils.
489
osutils.fancy_rename(abs_from, abs_to,
490
rename_func=lambda p1, p2: self._rename(p1, p2, f),
491
unlink_func=lambda p: self._delete(p, f))
493
449
def delete(self, relpath):
494
450
"""Delete the item at relpath"""
495
abspath = self._remote_path(relpath)
497
self._delete(abspath, f)
499
def _delete(self, abspath, f):
451
abspath = self._abspath(relpath)
501
453
mutter("FTP rm: %s", abspath)
502
455
f.delete(abspath)
503
456
except ftplib.error_perm, e:
504
457
self._translate_perm_error(e, abspath, 'error deleting',
505
458
unknown_exc=errors.NoSuchFile)
507
def external_url(self):
508
"""See bzrlib.transport.Transport.external_url."""
509
# FTP URL's are externally usable.
512
460
def listable(self):
513
461
"""See Transport.listable."""
516
464
def list_dir(self, relpath):
517
465
"""See Transport.list_dir."""
518
basepath = self._remote_path(relpath)
466
basepath = self._abspath(relpath)
519
467
mutter("FTP nlst: %s", basepath)
520
468
f = self._get_FTP()
523
paths = f.nlst(basepath)
524
except ftplib.error_perm, e:
525
self._translate_perm_error(e, relpath,
526
extra='error with list_dir')
527
except ftplib.error_temp, e:
528
# xs4all's ftp server raises a 450 temp error when listing an
529
# empty directory. Check for that and just return an empty list
530
# in that case. See bug #215522
531
if str(e).lower().startswith('450 no files found'):
532
mutter('FTP Server returned "%s" for nlst.'
533
' Assuming it means empty directory',
538
# Restore binary mode as nlst switch to ascii mode to retrieve file
470
paths = f.nlst(basepath)
471
except ftplib.error_perm, e:
472
self._translate_perm_error(e, relpath, extra='error with list_dir')
542
473
# If FTP.nlst returns paths prefixed by relpath, strip 'em
543
474
if paths and paths[0].startswith(basepath):
544
475
entries = [path[len(basepath)+1:] for path in paths]
595
526
return self.lock_read(relpath)
529
class FtpServer(Server):
530
"""Common code for SFTP server facilities."""
534
self._ftp_server = None
536
self._async_thread = None
541
"""Calculate an ftp url to this server."""
542
return 'ftp://foo:bar@localhost:%d/' % (self._port)
544
# def get_bogus_url(self):
545
# """Return a URL which cannot be connected to."""
546
# return 'ftp://127.0.0.1:1'
548
def log(self, message):
549
"""This is used by medusa.ftp_server to log connections, etc."""
550
self.logs.append(message)
555
raise RuntimeError('Must have medusa to run the FtpServer')
557
self._root = os.getcwdu()
558
self._ftp_server = _ftp_server(
559
authorizer=_test_authorizer(root=self._root),
561
port=0, # bind to a random port
563
logger_object=self # Use FtpServer.log() for messages
565
self._port = self._ftp_server.getsockname()[1]
566
# Don't let it loop forever, or handle an infinite number of requests.
567
# In this case it will run for 100s, or 1000 requests
568
self._async_thread = threading.Thread(target=asyncore.loop,
569
kwargs={'timeout':0.1, 'count':1000})
570
self._async_thread.setDaemon(True)
571
self._async_thread.start()
574
"""See bzrlib.transport.Server.tearDown."""
575
# have asyncore release the channel
576
self._ftp_server.del_channel()
578
self._async_thread.join()
583
_test_authorizer = None
587
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
590
import medusa.filesys
591
import medusa.ftp_server
597
class test_authorizer(object):
598
"""A custom Authorizer object for running the test suite.
600
The reason we cannot use dummy_authorizer, is because it sets the
601
channel to readonly, which we don't always want to do.
604
def __init__(self, root):
607
def authorize(self, channel, username, password):
608
"""Return (success, reply_string, filesystem)"""
610
return 0, 'No Medusa.', None
612
channel.persona = -1, -1
613
if username == 'anonymous':
614
channel.read_only = 1
616
channel.read_only = 0
618
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
621
class ftp_channel(medusa.ftp_server.ftp_channel):
622
"""Customized ftp channel"""
624
def log(self, message):
625
"""Redirect logging requests."""
626
mutter('_ftp_channel: %s', message)
628
def log_info(self, message, type='info'):
629
"""Redirect logging requests."""
630
mutter('_ftp_channel %s: %s', type, message)
632
def cmd_rnfr(self, line):
633
"""Prepare for renaming a file."""
634
self._renaming = line[1]
635
self.respond('350 Ready for RNTO')
636
# TODO: jam 20060516 in testing, the ftp server seems to
637
# check that the file already exists, or it sends
638
# 550 RNFR command failed
640
def cmd_rnto(self, line):
641
"""Rename a file based on the target given.
643
rnto must be called after calling rnfr.
645
if not self._renaming:
646
self.respond('503 RNFR required first.')
647
pfrom = self.filesystem.translate(self._renaming)
648
self._renaming = None
649
pto = self.filesystem.translate(line[1])
651
os.rename(pfrom, pto)
652
except (IOError, OSError), e:
653
# TODO: jam 20060516 return custom responses based on
654
# why the command failed
655
self.respond('550 RNTO failed: %s' % (e,))
657
self.respond('550 RNTO failed')
658
# For a test server, we will go ahead and just die
661
self.respond('250 Rename successful.')
663
def cmd_size(self, line):
664
"""Return the size of a file
666
This is overloaded to help the test suite determine if the
667
target is a directory.
670
if not self.filesystem.isfile(filename):
671
if self.filesystem.isdir(filename):
672
self.respond('550 "%s" is a directory' % (filename,))
674
self.respond('550 "%s" is not a file' % (filename,))
676
self.respond('213 %d'
677
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
679
def cmd_mkd(self, line):
680
"""Create a directory.
682
Overloaded because default implementation does not distinguish
683
*why* it cannot make a directory.
686
self.command_not_understood(''.join(line))
690
self.filesystem.mkdir (path)
691
self.respond ('257 MKD command successful.')
692
except (IOError, OSError), e:
693
self.respond ('550 error creating directory: %s' % (e,))
695
self.respond ('550 error creating directory.')
698
class ftp_server(medusa.ftp_server.ftp_server):
699
"""Customize the behavior of the Medusa ftp_server.
701
There are a few warts on the ftp_server, based on how it expects
705
ftp_channel_class = ftp_channel
707
def __init__(self, *args, **kwargs):
708
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
709
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
711
def log(self, message):
712
"""Redirect logging requests."""
713
mutter('_ftp_server: %s', message)
715
def log_info(self, message, type='info'):
716
"""Override the asyncore.log_info so we don't stipple the screen."""
717
mutter('_ftp_server %s: %s', type, message)
719
_test_authorizer = test_authorizer
720
_ftp_channel = ftp_channel
721
_ftp_server = ftp_server
598
726
def get_test_permutations():
599
727
"""Return the permutations to be used in testing."""
600
from bzrlib.tests import ftp_server
601
return [(FtpTransport, ftp_server.FTPTestServer)]
728
if not _setup_medusa():
729
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
732
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
