76
74
_number_of_retries = 2
77
75
_sleep_between_retries = 5
79
# FIXME: there are inconsistencies in the way temporary errors are
80
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
81
# be taken to analyze the implications for write operations (read operations
82
# are safe to retry). Overall even some read operations are never retried.
83
class FtpTransport(ConnectedTransport):
77
class FtpTransport(Transport):
84
78
"""This is the transport agent for ftp:// access."""
86
def __init__(self, base, from_transport=None):
80
def __init__(self, base, _provided_instance=None):
87
81
"""Set the base path where files will be stored."""
88
82
assert base.startswith('ftp://') or base.startswith('aftp://')
89
super(FtpTransport, self).__init__(base, from_transport)
90
self._unqualified_scheme = 'ftp'
91
if self._scheme == 'aftp':
94
self.is_active = False
83
super(FtpTransport, self).__init__(base)
84
self.is_active = base.startswith('aftp://')
87
(self._proto, self._host,
88
self._path, self._parameters,
89
self._query, self._fragment) = urlparse.urlparse(self.base)
90
self._FTP_instance = _provided_instance
96
92
def _get_FTP(self):
97
93
"""Return the ftplib.FTP instance for this object."""
98
# Ensures that a connection is established
99
connection = self._get_connection()
100
if connection is None:
101
# First connection ever
102
connection, credentials = self._create_connection()
103
self._set_connection(connection, credentials)
106
def _create_connection(self, credentials=None):
107
"""Create a new connection with the provided credentials.
109
:param credentials: The credentials needed to establish the connection.
111
:return: The created connection and its associated credentials.
113
The credentials are only the password as it may have been entered
114
interactively by the user and may be different from the one provided
115
in base url at transport creation time.
117
if credentials is None:
118
password = self._password
120
password = credentials
122
mutter("Constructing FTP instance against %r" %
123
((self._host, self._port, self._user, '********',
94
if self._FTP_instance is not None:
95
return self._FTP_instance
126
connection = ftplib.FTP()
127
connection.connect(host=self._host, port=self._port)
128
if self._user and self._user != 'anonymous' and \
129
password is not None: # '' is a valid password
130
get_password = bzrlib.ui.ui_factory.get_password
131
password = get_password(prompt='FTP %(user)s@%(host)s password',
132
user=self._user, host=self._host)
133
connection.login(user=self._user, passwd=password)
134
connection.set_pasv(not self.is_active)
100
hostname = self._host
102
username, hostname = hostname.split("@", 1)
104
username, password = username.split(":", 1)
106
self._FTP_instance = _find_FTP(hostname, username, password,
108
return self._FTP_instance
135
109
except ftplib.error_perm, e:
136
raise errors.TransportError(msg="Error setting up connection:"
137
" %s" % str(e), orig_error=e)
138
return connection, password
140
def _reconnect(self):
141
"""Create a new connection with the previously used credentials"""
142
credentials = self.get_credentials()
143
connection, credentials = self._create_connection(credentials)
144
self._set_connection(connection, credentials)
146
def _translate_perm_error(self, err, path, extra=None,
147
unknown_exc=FtpPathError):
148
"""Try to translate an ftplib.error_perm exception.
150
:param err: The error to translate into a bzr error
151
:param path: The path which had problems
152
:param extra: Extra information which can be included
153
:param unknown_exc: If None, we will just raise the original exception
154
otherwise we raise unknown_exc(path, extra=extra)
160
extra += ': ' + str(err)
161
if ('no such file' in s
162
or 'could not open' in s
163
or 'no such dir' in s
164
or 'could not create file' in s # vsftpd
165
or 'file doesn\'t exist' in s
167
raise errors.NoSuchFile(path, extra=extra)
168
if ('file exists' in s):
169
raise errors.FileExists(path, extra=extra)
170
if ('not a directory' in s):
171
raise errors.PathError(path, extra=extra)
173
mutter('unable to understand error for path: %s: %s', path, err)
176
raise unknown_exc(path, extra=extra)
177
# TODO: jam 20060516 Consider re-raising the error wrapped in
178
# something like TransportError, but this loses the traceback
179
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
180
# to handle. Consider doing something like that here.
181
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
110
raise TransportError(msg="Error setting up connection: %s"
111
% str(e), orig_error=e)
184
113
def should_cache(self):
185
114
"""Return True if the data pulled across should be cached locally.
189
def _remote_path(self, relpath):
190
# XXX: It seems that ftplib does not handle Unicode paths
191
# at the same time, medusa won't handle utf8 paths So if
192
# we .encode(utf8) here (see ConnectedTransport
193
# implementation), then we get a Server failure. while
194
# if we use str(), we get a UnicodeError, and the test
195
# suite just skips testing UnicodePaths.
196
relative = str(urlutils.unescape(relpath))
197
remote_path = self._combine_paths(self._path, relative)
118
def clone(self, offset=None):
119
"""Return a new FtpTransport with root at self.base + offset.
123
return FtpTransport(self.base, self._FTP_instance)
125
return FtpTransport(self.abspath(offset), self._FTP_instance)
127
def _abspath(self, relpath):
128
assert isinstance(relpath, basestring)
129
relpath = urllib.unquote(relpath)
130
if isinstance(relpath, basestring):
131
relpath_parts = relpath.split('/')
133
# TODO: Don't call this with an array - no magic interfaces
134
relpath_parts = relpath[:]
135
if len(relpath_parts) > 1:
136
if relpath_parts[0] == '':
137
raise ValueError("path %r within branch %r seems to be absolute"
138
% (relpath, self._path))
139
basepath = self._path.split('/')
140
if len(basepath) > 0 and basepath[-1] == '':
141
basepath = basepath[:-1]
142
for p in relpath_parts:
144
if len(basepath) == 0:
145
# In most filesystems, a request for the parent
146
# of root, just returns root.
149
elif p == '.' or p == '':
153
# Possibly, we could use urlparse.urljoin() here, but
154
# I'm concerned about when it chooses to strip the last
155
# portion of the path, and when it doesn't.
156
return '/'.join(basepath)
158
def abspath(self, relpath):
159
"""Return the full url to the given relative path.
160
This can be supplied with a string or a list
162
path = self._abspath(relpath)
163
return urlparse.urlunparse((self._proto,
164
self._host, path, '', '', ''))
200
166
def has(self, relpath):
201
"""Does the target location exist?"""
202
# FIXME jam 20060516 We *do* ask about directories in the test suite
203
# We don't seem to in the actual codebase
204
# XXX: I assume we're never asked has(dirname) and thus I use
205
# the FTP size command and assume that if it doesn't raise,
207
abspath = self._remote_path(relpath)
167
"""Does the target location exist?
169
XXX: I assume we're never asked has(dirname) and thus I use
170
the FTP size command and assume that if it doesn't raise,
209
174
f = self._get_FTP()
210
mutter('FTP has check: %s => %s', relpath, abspath)
212
mutter("FTP has: %s", abspath)
175
s = f.size(self._abspath(relpath))
176
mutter("FTP has: %s" % self._abspath(relpath))
214
except ftplib.error_perm, e:
215
if ('is a directory' in str(e).lower()):
216
mutter("FTP has dir: %s: %s", abspath, e)
218
mutter("FTP has not: %s: %s", abspath, e)
178
except ftplib.error_perm:
179
mutter("FTP has not: %s" % self._abspath(relpath))
221
182
def get(self, relpath, decode=False, retries=0):
266
227
:param retries: Number of retries after temporary failures so far
267
228
for this operation.
269
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
230
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
272
abspath = self._remote_path(relpath)
273
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
232
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (self._abspath(relpath), time.time(),
274
233
os.getpid(), random.randint(0,0x7FFFFFFF))
275
if getattr(fp, 'read', None) is None:
234
if not hasattr(fp, 'read'):
276
235
fp = StringIO(fp)
278
mutter("FTP put: %s", abspath)
237
mutter("FTP put: %s" % self._abspath(relpath))
279
238
f = self._get_FTP()
281
240
f.storbinary('STOR '+tmp_abspath, fp)
282
self._rename_and_overwrite(tmp_abspath, abspath, f)
241
f.rename(tmp_abspath, self._abspath(relpath))
283
242
except (ftplib.error_temp,EOFError), e:
284
243
warning("Failure during ftp PUT. Deleting temporary file.")
286
245
f.delete(tmp_abspath)
288
warning("Failed to delete temporary file on the"
289
" server.\nFile: %s", tmp_abspath)
247
warning("Failed to delete temporary file on the server.\nFile: %s"
292
251
except ftplib.error_perm, e:
293
self._translate_perm_error(e, abspath, extra='could not store',
294
unknown_exc=errors.NoSuchFile)
252
if "no such file" in str(e).lower():
253
raise NoSuchFile("Error storing %s: %s"
254
% (self.abspath(relpath), str(e)), extra=e)
256
raise FtpTransportError(orig_error=e)
295
257
except ftplib.error_temp, e:
296
258
if retries > _number_of_retries:
297
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
259
raise TransportError("FTP temporary error during PUT %s. Aborting."
298
260
% self.abspath(relpath), orig_error=e)
300
warning("FTP temporary error: %s. Retrying.", str(e))
302
self.put_file(relpath, fp, mode, retries+1)
262
warning("FTP temporary error: %s. Retrying." % str(e))
263
self._FTP_instance = None
264
self.put(relpath, fp, mode, retries+1)
304
266
if retries > _number_of_retries:
305
raise errors.TransportError("FTP control connection closed during PUT %s."
267
raise TransportError("FTP control connection closed during PUT %s."
306
268
% self.abspath(relpath), orig_error=e)
308
270
warning("FTP control connection closed. Trying to reopen.")
309
271
time.sleep(_sleep_between_retries)
311
self.put_file(relpath, fp, mode, retries+1)
272
self._FTP_instance = None
273
self.put(relpath, fp, mode, retries+1)
313
276
def mkdir(self, relpath, mode=None):
314
277
"""Create a directory at the given path."""
315
abspath = self._remote_path(relpath)
317
mutter("FTP mkd: %s", abspath)
320
except ftplib.error_perm, e:
321
self._translate_perm_error(e, abspath,
322
unknown_exc=errors.FileExists)
324
def rmdir(self, rel_path):
325
"""Delete the directory at rel_path"""
326
abspath = self._remote_path(rel_path)
328
mutter("FTP rmd: %s", abspath)
331
except ftplib.error_perm, e:
332
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
334
def append_file(self, relpath, f, mode=None):
279
mutter("FTP mkd: %s" % self._abspath(relpath))
282
f.mkd(self._abspath(relpath))
283
except ftplib.error_perm, e:
285
if 'File exists' in s:
286
raise FileExists(self.abspath(relpath), extra=s)
289
except ftplib.error_perm, e:
290
raise TransportError(orig_error=e)
292
def append(self, relpath, f):
335
293
"""Append the text in the file-like object into the final
338
abspath = self._remote_path(relpath)
339
if self.has(relpath):
340
ftp = self._get_FTP()
341
result = ftp.size(abspath)
345
mutter("FTP appe to %s", abspath)
346
self._try_append(relpath, f.read(), mode)
350
def _try_append(self, relpath, text, mode=None, retries=0):
351
"""Try repeatedly to append the given text to the file at relpath.
353
This is a recursive function. On errors, it will be called until the
354
number of retries is exceeded.
357
abspath = self._remote_path(relpath)
358
mutter("FTP appe (try %d) to %s", retries, abspath)
359
ftp = self._get_FTP()
360
ftp.voidcmd("TYPE I")
361
cmd = "APPE %s" % abspath
362
conn = ftp.transfercmd(cmd)
366
self._setmode(relpath, mode)
368
except ftplib.error_perm, e:
369
self._translate_perm_error(e, abspath, extra='error appending',
370
unknown_exc=errors.NoSuchFile)
371
except ftplib.error_temp, e:
372
if retries > _number_of_retries:
373
raise errors.TransportError("FTP temporary error during APPEND %s." \
374
"Aborting." % abspath, orig_error=e)
376
warning("FTP temporary error: %s. Retrying.", str(e))
378
self._try_append(relpath, text, mode, retries+1)
380
def _setmode(self, relpath, mode):
381
"""Set permissions on a path.
383
Only set permissions if the FTP server supports the 'SITE CHMOD'
387
mutter("FTP site chmod: setting permissions to %s on %s",
388
str(mode), self._remote_path(relpath))
389
ftp = self._get_FTP()
390
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
392
except ftplib.error_perm, e:
393
# Command probably not available on this server
394
warning("FTP Could not set permissions to %s on %s. %s",
395
str(mode), self._remote_path(relpath), str(e))
397
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
398
# to copy something to another machine. And you may be able
399
# to give it its own address as the 'to' location.
400
# So implement a fancier 'copy()'
402
def rename(self, rel_from, rel_to):
403
abs_from = self._remote_path(rel_from)
404
abs_to = self._remote_path(rel_to)
405
mutter("FTP rename: %s => %s", abs_from, abs_to)
407
return self._rename(abs_from, abs_to, f)
409
def _rename(self, abs_from, abs_to, f):
411
f.rename(abs_from, abs_to)
412
except ftplib.error_perm, e:
413
self._translate_perm_error(e, abs_from,
414
': unable to rename to %r' % (abs_to))
296
raise TransportNotPossible('ftp does not support append()')
298
def copy(self, rel_from, rel_to):
299
"""Copy the item at rel_from to the location at rel_to"""
300
raise TransportNotPossible('ftp does not (yet) support copy()')
416
302
def move(self, rel_from, rel_to):
417
303
"""Move the item at rel_from to the location at rel_to"""
418
abs_from = self._remote_path(rel_from)
419
abs_to = self._remote_path(rel_to)
421
mutter("FTP mv: %s => %s", abs_from, abs_to)
305
mutter("FTP mv: %s => %s" % (self._abspath(rel_from),
306
self._abspath(rel_to)))
422
307
f = self._get_FTP()
423
self._rename_and_overwrite(abs_from, abs_to, f)
308
f.rename(self._abspath(rel_from), self._abspath(rel_to))
424
309
except ftplib.error_perm, e:
425
self._translate_perm_error(e, abs_from,
426
extra='unable to rename to %r' % (rel_to,),
427
unknown_exc=errors.PathError)
429
def _rename_and_overwrite(self, abs_from, abs_to, f):
430
"""Do a fancy rename on the remote server.
432
Using the implementation provided by osutils.
434
osutils.fancy_rename(abs_from, abs_to,
435
rename_func=lambda p1, p2: self._rename(p1, p2, f),
436
unlink_func=lambda p: self._delete(p, f))
310
raise TransportError(orig_error=e)
438
312
def delete(self, relpath):
439
313
"""Delete the item at relpath"""
440
abspath = self._remote_path(relpath)
442
self._delete(abspath, f)
444
def _delete(self, abspath, f):
446
mutter("FTP rm: %s", abspath)
315
mutter("FTP rm: %s" % self._abspath(relpath))
317
f.delete(self._abspath(relpath))
448
318
except ftplib.error_perm, e:
449
self._translate_perm_error(e, abspath, 'error deleting',
450
unknown_exc=errors.NoSuchFile)
319
raise TransportError(orig_error=e)
452
321
def listable(self):
453
322
"""See Transport.listable."""
518
387
return self.lock_read(relpath)
521
class FtpServer(Server):
522
"""Common code for FTP server facilities."""
526
self._ftp_server = None
528
self._async_thread = None
533
"""Calculate an ftp url to this server."""
534
return 'ftp://foo:bar@localhost:%d/' % (self._port)
536
# def get_bogus_url(self):
537
# """Return a URL which cannot be connected to."""
538
# return 'ftp://127.0.0.1:1'
540
def log(self, message):
541
"""This is used by medusa.ftp_server to log connections, etc."""
542
self.logs.append(message)
544
def setUp(self, vfs_server=None):
546
raise RuntimeError('Must have medusa to run the FtpServer')
548
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
549
"FtpServer currently assumes local transport, got %s" % vfs_server
551
self._root = os.getcwdu()
552
self._ftp_server = _ftp_server(
553
authorizer=_test_authorizer(root=self._root),
555
port=0, # bind to a random port
557
logger_object=self # Use FtpServer.log() for messages
559
self._port = self._ftp_server.getsockname()[1]
560
# Don't let it loop forever, or handle an infinite number of requests.
561
# In this case it will run for 1000s, or 10000 requests
562
self._async_thread = threading.Thread(
563
target=FtpServer._asyncore_loop_ignore_EBADF,
564
kwargs={'timeout':0.1, 'count':10000})
565
self._async_thread.setDaemon(True)
566
self._async_thread.start()
569
"""See bzrlib.transport.Server.tearDown."""
570
# have asyncore release the channel
571
self._ftp_server.del_channel()
573
self._async_thread.join()
576
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
577
"""Ignore EBADF during server shutdown.
579
We close the socket to get the server to shutdown, but this causes
580
select.select() to raise EBADF.
583
asyncore.loop(*args, **kwargs)
584
# FIXME: If we reach that point, we should raise an exception
585
# explaining that the 'count' parameter in setUp is too low or
586
# testers may wonder why their test just sits there waiting for a
587
# server that is already dead. Note that if the tester waits too
588
# long under pdb the server will also die.
589
except select.error, e:
590
if e.args[0] != errno.EBADF:
596
_test_authorizer = None
600
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
603
import medusa.filesys
604
import medusa.ftp_server
610
class test_authorizer(object):
611
"""A custom Authorizer object for running the test suite.
613
The reason we cannot use dummy_authorizer, is because it sets the
614
channel to readonly, which we don't always want to do.
617
def __init__(self, root):
620
def authorize(self, channel, username, password):
621
"""Return (success, reply_string, filesystem)"""
623
return 0, 'No Medusa.', None
625
channel.persona = -1, -1
626
if username == 'anonymous':
627
channel.read_only = 1
629
channel.read_only = 0
631
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
634
class ftp_channel(medusa.ftp_server.ftp_channel):
635
"""Customized ftp channel"""
637
def log(self, message):
638
"""Redirect logging requests."""
639
mutter('_ftp_channel: %s', message)
641
def log_info(self, message, type='info'):
642
"""Redirect logging requests."""
643
mutter('_ftp_channel %s: %s', type, message)
645
def cmd_rnfr(self, line):
646
"""Prepare for renaming a file."""
647
self._renaming = line[1]
648
self.respond('350 Ready for RNTO')
649
# TODO: jam 20060516 in testing, the ftp server seems to
650
# check that the file already exists, or it sends
651
# 550 RNFR command failed
653
def cmd_rnto(self, line):
654
"""Rename a file based on the target given.
656
rnto must be called after calling rnfr.
658
if not self._renaming:
659
self.respond('503 RNFR required first.')
660
pfrom = self.filesystem.translate(self._renaming)
661
self._renaming = None
662
pto = self.filesystem.translate(line[1])
663
if os.path.exists(pto):
664
self.respond('550 RNTO failed: file exists')
667
os.rename(pfrom, pto)
668
except (IOError, OSError), e:
669
# TODO: jam 20060516 return custom responses based on
670
# why the command failed
671
# (bialix 20070418) str(e) on Python 2.5 @ Windows
672
# sometimes don't provide expected error message;
673
# so we obtain such message via os.strerror()
674
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
676
self.respond('550 RNTO failed')
677
# For a test server, we will go ahead and just die
680
self.respond('250 Rename successful.')
682
def cmd_size(self, line):
683
"""Return the size of a file
685
This is overloaded to help the test suite determine if the
686
target is a directory.
689
if not self.filesystem.isfile(filename):
690
if self.filesystem.isdir(filename):
691
self.respond('550 "%s" is a directory' % (filename,))
693
self.respond('550 "%s" is not a file' % (filename,))
695
self.respond('213 %d'
696
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
698
def cmd_mkd(self, line):
699
"""Create a directory.
701
Overloaded because default implementation does not distinguish
702
*why* it cannot make a directory.
705
self.command_not_understood(''.join(line))
709
self.filesystem.mkdir (path)
710
self.respond ('257 MKD command successful.')
711
except (IOError, OSError), e:
712
# (bialix 20070418) str(e) on Python 2.5 @ Windows
713
# sometimes don't provide expected error message;
714
# so we obtain such message via os.strerror()
715
self.respond ('550 error creating directory: %s' %
716
os.strerror(e.errno))
718
self.respond ('550 error creating directory.')
721
class ftp_server(medusa.ftp_server.ftp_server):
722
"""Customize the behavior of the Medusa ftp_server.
724
There are a few warts on the ftp_server, based on how it expects
728
ftp_channel_class = ftp_channel
730
def __init__(self, *args, **kwargs):
731
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
732
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
734
def log(self, message):
735
"""Redirect logging requests."""
736
mutter('_ftp_server: %s', message)
738
def log_info(self, message, type='info'):
739
"""Override the asyncore.log_info so we don't stipple the screen."""
740
mutter('_ftp_server %s: %s', type, message)
742
_test_authorizer = test_authorizer
743
_ftp_channel = ftp_channel
744
_ftp_server = ftp_server
749
390
def get_test_permutations():
750
391
"""Return the permutations to be used in testing."""
751
if not _setup_medusa():
752
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
755
return [(FtpTransport, FtpServer)]
392
warn("There are no FTP transport provider tests yet.")
added
removed
bzrlib/transport/ftp.py
