57
class FtpTransport(Transport):
76
_number_of_retries = 2
77
_sleep_between_retries = 5
79
# FIXME: there are inconsistencies in the way temporary errors are
80
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
81
# be taken to analyze the implications for write operations (read operations
82
# are safe to retry). Overall even some read operations are never
83
# retried. --vila 20070720 (Bug #127164)
84
class FtpTransport(ConnectedTransport):
58
85
"""This is the transport agent for ftp:// access."""
60
def __init__(self, base, _provided_instance=None):
87
def __init__(self, base, _from_transport=None):
61
88
"""Set the base path where files will be stored."""
62
89
assert base.startswith('ftp://') or base.startswith('aftp://')
63
super(FtpTransport, self).__init__(base)
64
self.is_active = base.startswith('aftp://')
67
(self._proto, self._host,
68
self._path, self._parameters,
69
self._query, self._fragment) = urlparse.urlparse(self.base)
70
self._FTP_instance = _provided_instance
90
super(FtpTransport, self).__init__(base,
91
_from_transport=_from_transport)
92
self._unqualified_scheme = 'ftp'
93
if self._scheme == 'aftp':
96
self.is_active = False
73
98
def _get_FTP(self):
74
99
"""Return the ftplib.FTP instance for this object."""
75
if self._FTP_instance is not None:
76
return self._FTP_instance
100
# Ensures that a connection is established
101
connection = self._get_connection()
102
if connection is None:
103
# First connection ever
104
connection, credentials = self._create_connection()
105
self._set_connection(connection, credentials)
108
def _create_connection(self, credentials=None):
109
"""Create a new connection with the provided credentials.
111
:param credentials: The credentials needed to establish the connection.
113
:return: The created connection and its associated credentials.
115
The credentials are only the password as it may have been entered
116
interactively by the user and may be different from the one provided
117
in base url at transport creation time.
119
if credentials is None:
120
password = self._password
122
password = credentials
124
mutter("Constructing FTP instance against %r" %
125
((self._host, self._port, self._user, '********',
83
username, hostname = hostname.split("@", 1)
85
username, password = username.split(":", 1)
87
mutter("Constructing FTP instance")
88
self._FTP_instance = ftplib.FTP(hostname, username, password)
89
self._FTP_instance.set_pasv(not self.is_active)
90
return self._FTP_instance
128
connection = ftplib.FTP()
129
connection.connect(host=self._host, port=self._port)
130
if self._user and self._user != 'anonymous' and \
131
password is not None: # '' is a valid password
132
get_password = bzrlib.ui.ui_factory.get_password
133
password = get_password(prompt='FTP %(user)s@%(host)s password',
134
user=self._user, host=self._host)
135
connection.login(user=self._user, passwd=password)
136
connection.set_pasv(not self.is_active)
91
137
except ftplib.error_perm, e:
92
raise TransportError(msg="Error setting up connection: %s"
93
% str(e), orig_error=e)
138
raise errors.TransportError(msg="Error setting up connection:"
139
" %s" % str(e), orig_error=e)
140
return connection, password
142
def _reconnect(self):
143
"""Create a new connection with the previously used credentials"""
144
credentials = self.get_credentials()
145
connection, credentials = self._create_connection(credentials)
146
self._set_connection(connection, credentials)
148
def _translate_perm_error(self, err, path, extra=None,
149
unknown_exc=FtpPathError):
150
"""Try to translate an ftplib.error_perm exception.
152
:param err: The error to translate into a bzr error
153
:param path: The path which had problems
154
:param extra: Extra information which can be included
155
:param unknown_exc: If None, we will just raise the original exception
156
otherwise we raise unknown_exc(path, extra=extra)
162
extra += ': ' + str(err)
163
if ('no such file' in s
164
or 'could not open' in s
165
or 'no such dir' in s
166
or 'could not create file' in s # vsftpd
167
or 'file doesn\'t exist' in s
169
raise errors.NoSuchFile(path, extra=extra)
170
if ('file exists' in s):
171
raise errors.FileExists(path, extra=extra)
172
if ('not a directory' in s):
173
raise errors.PathError(path, extra=extra)
175
mutter('unable to understand error for path: %s: %s', path, err)
178
raise unknown_exc(path, extra=extra)
179
# TODO: jam 20060516 Consider re-raising the error wrapped in
180
# something like TransportError, but this loses the traceback
181
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
182
# to handle. Consider doing something like that here.
183
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
95
186
def should_cache(self):
96
187
"""Return True if the data pulled across should be cached locally.
100
def clone(self, offset=None):
101
"""Return a new FtpTransport with root at self.base + offset.
105
return FtpTransport(self.base, self._FTP_instance)
107
return FtpTransport(self.abspath(offset), self._FTP_instance)
109
def _abspath(self, relpath):
110
assert isinstance(relpath, basestring)
111
relpath = urllib.unquote(relpath)
112
if isinstance(relpath, basestring):
113
relpath_parts = relpath.split('/')
115
# TODO: Don't call this with an array - no magic interfaces
116
relpath_parts = relpath[:]
117
if len(relpath_parts) > 1:
118
if relpath_parts[0] == '':
119
raise ValueError("path %r within branch %r seems to be absolute"
120
% (relpath, self._path))
121
basepath = self._path.split('/')
122
if len(basepath) > 0 and basepath[-1] == '':
123
basepath = basepath[:-1]
124
for p in relpath_parts:
126
if len(basepath) == 0:
127
# In most filesystems, a request for the parent
128
# of root, just returns root.
131
elif p == '.' or p == '':
135
# Possibly, we could use urlparse.urljoin() here, but
136
# I'm concerned about when it chooses to strip the last
137
# portion of the path, and when it doesn't.
138
return '/'.join(basepath)
140
def abspath(self, relpath):
141
"""Return the full url to the given relative path.
142
This can be supplied with a string or a list
144
path = self._abspath(relpath)
145
return urlparse.urlunparse((self._proto,
146
self._host, path, '', '', ''))
191
def _remote_path(self, relpath):
192
# XXX: It seems that ftplib does not handle Unicode paths
193
# at the same time, medusa won't handle utf8 paths So if
194
# we .encode(utf8) here (see ConnectedTransport
195
# implementation), then we get a Server failure. while
196
# if we use str(), we get a UnicodeError, and the test
197
# suite just skips testing UnicodePaths.
198
relative = str(urlutils.unescape(relpath))
199
remote_path = self._combine_paths(self._path, relative)
148
202
def has(self, relpath):
149
"""Does the target location exist?
151
XXX: I assume we're never asked has(dirname) and thus I use
152
the FTP size command and assume that if it doesn't raise,
203
"""Does the target location exist?"""
204
# FIXME jam 20060516 We *do* ask about directories in the test suite
205
# We don't seem to in the actual codebase
206
# XXX: I assume we're never asked has(dirname) and thus I use
207
# the FTP size command and assume that if it doesn't raise,
209
abspath = self._remote_path(relpath)
156
211
f = self._get_FTP()
157
s = f.size(self._abspath(relpath))
158
mutter("FTP has: %s" % self._abspath(relpath))
212
mutter('FTP has check: %s => %s', relpath, abspath)
214
mutter("FTP has: %s", abspath)
160
except ftplib.error_perm:
161
mutter("FTP has not: %s" % self._abspath(relpath))
216
except ftplib.error_perm, e:
217
if ('is a directory' in str(e).lower()):
218
mutter("FTP has dir: %s: %s", abspath, e)
220
mutter("FTP has not: %s: %s", abspath, e)
164
def get(self, relpath, decode=False):
223
def get(self, relpath, decode=False, retries=0):
165
224
"""Get the file at the given relative path.
167
226
:param relpath: The relative path to the file
227
:param retries: Number of retries after temporary failures so far
169
230
We're meant to return a file-like object which bzr will
170
231
then read from. For now we do this via the magic of StringIO
233
# TODO: decode should be deprecated
173
mutter("FTP get: %s" % self._abspath(relpath))
235
mutter("FTP get: %s", self._remote_path(relpath))
174
236
f = self._get_FTP()
176
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
238
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
179
241
except ftplib.error_perm, e:
180
raise NoSuchFile(self.abspath(relpath), extra=extra)
242
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
243
except ftplib.error_temp, e:
244
if retries > _number_of_retries:
245
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
246
% self.abspath(relpath),
249
warning("FTP temporary error: %s. Retrying.", str(e))
251
return self.get(relpath, decode, retries+1)
253
if retries > _number_of_retries:
254
raise errors.TransportError("FTP control connection closed during GET %s."
255
% self.abspath(relpath),
258
warning("FTP control connection closed. Trying to reopen.")
259
time.sleep(_sleep_between_retries)
261
return self.get(relpath, decode, retries+1)
182
def put(self, relpath, fp, mode=None):
263
def put_file(self, relpath, fp, mode=None, retries=0):
183
264
"""Copy the file-like or string object into the location.
185
266
:param relpath: Location to put the contents, relative to base.
186
:param f: File-like or string object.
187
TODO: jam 20051215 This should be an atomic put, not overwritting files in place
188
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
267
:param fp: File-like or string object.
268
:param retries: Number of retries after temporary failures so far
271
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
190
if not hasattr(fp, 'read'):
274
abspath = self._remote_path(relpath)
275
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
276
os.getpid(), random.randint(0,0x7FFFFFFF))
277
if getattr(fp, 'read', None) is None:
191
278
fp = StringIO(fp)
193
mutter("FTP put: %s" % self._abspath(relpath))
280
mutter("FTP put: %s", abspath)
194
281
f = self._get_FTP()
195
f.storbinary('STOR '+self._abspath(relpath), fp, 8192)
283
f.storbinary('STOR '+tmp_abspath, fp)
284
self._rename_and_overwrite(tmp_abspath, abspath, f)
285
except (ftplib.error_temp,EOFError), e:
286
warning("Failure during ftp PUT. Deleting temporary file.")
288
f.delete(tmp_abspath)
290
warning("Failed to delete temporary file on the"
291
" server.\nFile: %s", tmp_abspath)
196
294
except ftplib.error_perm, e:
197
raise TransportError(orig_error=e)
295
self._translate_perm_error(e, abspath, extra='could not store',
296
unknown_exc=errors.NoSuchFile)
297
except ftplib.error_temp, e:
298
if retries > _number_of_retries:
299
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
300
% self.abspath(relpath), orig_error=e)
302
warning("FTP temporary error: %s. Retrying.", str(e))
304
self.put_file(relpath, fp, mode, retries+1)
306
if retries > _number_of_retries:
307
raise errors.TransportError("FTP control connection closed during PUT %s."
308
% self.abspath(relpath), orig_error=e)
310
warning("FTP control connection closed. Trying to reopen.")
311
time.sleep(_sleep_between_retries)
313
self.put_file(relpath, fp, mode, retries+1)
199
315
def mkdir(self, relpath, mode=None):
200
316
"""Create a directory at the given path."""
202
mutter("FTP mkd: %s" % self._abspath(relpath))
205
f.mkd(self._abspath(relpath))
206
except ftplib.error_perm, e:
208
if 'File exists' in s:
209
raise FileExists(self.abspath(relpath), extra=s)
212
except ftplib.error_perm, e:
213
raise TransportError(orig_error=e)
215
def append(self, relpath, f):
317
abspath = self._remote_path(relpath)
319
mutter("FTP mkd: %s", abspath)
322
except ftplib.error_perm, e:
323
self._translate_perm_error(e, abspath,
324
unknown_exc=errors.FileExists)
326
def rmdir(self, rel_path):
327
"""Delete the directory at rel_path"""
328
abspath = self._remote_path(rel_path)
330
mutter("FTP rmd: %s", abspath)
333
except ftplib.error_perm, e:
334
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
336
def append_file(self, relpath, f, mode=None):
216
337
"""Append the text in the file-like object into the final
219
raise TransportNotPossible('ftp does not support append()')
221
def copy(self, rel_from, rel_to):
222
"""Copy the item at rel_from to the location at rel_to"""
223
raise TransportNotPossible('ftp does not (yet) support copy()')
340
abspath = self._remote_path(relpath)
341
if self.has(relpath):
342
ftp = self._get_FTP()
343
result = ftp.size(abspath)
347
mutter("FTP appe to %s", abspath)
348
self._try_append(relpath, f.read(), mode)
352
def _try_append(self, relpath, text, mode=None, retries=0):
353
"""Try repeatedly to append the given text to the file at relpath.
355
This is a recursive function. On errors, it will be called until the
356
number of retries is exceeded.
359
abspath = self._remote_path(relpath)
360
mutter("FTP appe (try %d) to %s", retries, abspath)
361
ftp = self._get_FTP()
362
ftp.voidcmd("TYPE I")
363
cmd = "APPE %s" % abspath
364
conn = ftp.transfercmd(cmd)
368
self._setmode(relpath, mode)
370
except ftplib.error_perm, e:
371
self._translate_perm_error(e, abspath, extra='error appending',
372
unknown_exc=errors.NoSuchFile)
373
except ftplib.error_temp, e:
374
if retries > _number_of_retries:
375
raise errors.TransportError("FTP temporary error during APPEND %s." \
376
"Aborting." % abspath, orig_error=e)
378
warning("FTP temporary error: %s. Retrying.", str(e))
380
self._try_append(relpath, text, mode, retries+1)
382
def _setmode(self, relpath, mode):
383
"""Set permissions on a path.
385
Only set permissions if the FTP server supports the 'SITE CHMOD'
389
mutter("FTP site chmod: setting permissions to %s on %s",
390
str(mode), self._remote_path(relpath))
391
ftp = self._get_FTP()
392
cmd = "SITE CHMOD %s %s" % (self._remote_path(relpath), str(mode))
394
except ftplib.error_perm, e:
395
# Command probably not available on this server
396
warning("FTP Could not set permissions to %s on %s. %s",
397
str(mode), self._remote_path(relpath), str(e))
399
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
400
# to copy something to another machine. And you may be able
401
# to give it its own address as the 'to' location.
402
# So implement a fancier 'copy()'
404
def rename(self, rel_from, rel_to):
405
abs_from = self._remote_path(rel_from)
406
abs_to = self._remote_path(rel_to)
407
mutter("FTP rename: %s => %s", abs_from, abs_to)
409
return self._rename(abs_from, abs_to, f)
411
def _rename(self, abs_from, abs_to, f):
413
f.rename(abs_from, abs_to)
414
except ftplib.error_perm, e:
415
self._translate_perm_error(e, abs_from,
416
': unable to rename to %r' % (abs_to))
225
418
def move(self, rel_from, rel_to):
226
419
"""Move the item at rel_from to the location at rel_to"""
420
abs_from = self._remote_path(rel_from)
421
abs_to = self._remote_path(rel_to)
228
mutter("FTP mv: %s => %s" % (self._abspath(rel_from),
229
self._abspath(rel_to)))
423
mutter("FTP mv: %s => %s", abs_from, abs_to)
230
424
f = self._get_FTP()
231
f.rename(self._abspath(rel_from), self._abspath(rel_to))
425
self._rename_and_overwrite(abs_from, abs_to, f)
232
426
except ftplib.error_perm, e:
233
raise TransportError(orig_error=e)
427
self._translate_perm_error(e, abs_from,
428
extra='unable to rename to %r' % (rel_to,),
429
unknown_exc=errors.PathError)
431
def _rename_and_overwrite(self, abs_from, abs_to, f):
432
"""Do a fancy rename on the remote server.
434
Using the implementation provided by osutils.
436
osutils.fancy_rename(abs_from, abs_to,
437
rename_func=lambda p1, p2: self._rename(p1, p2, f),
438
unlink_func=lambda p: self._delete(p, f))
235
440
def delete(self, relpath):
236
441
"""Delete the item at relpath"""
442
abspath = self._remote_path(relpath)
444
self._delete(abspath, f)
446
def _delete(self, abspath, f):
238
mutter("FTP rm: %s" % self._abspath(relpath))
240
f.delete(self._abspath(relpath))
448
mutter("FTP rm: %s", abspath)
241
450
except ftplib.error_perm, e:
242
raise TransportError(orig_error=e)
451
self._translate_perm_error(e, abspath, 'error deleting',
452
unknown_exc=errors.NoSuchFile)
454
def external_url(self):
455
"""See bzrlib.transport.Transport.external_url."""
456
# FTP URL's are externally usable.
244
459
def listable(self):
245
460
"""See Transport.listable."""
304
523
:return: A lock object, which should be passed to Transport.unlock()
306
525
return self.lock_read(relpath)
528
class FtpServer(Server):
529
"""Common code for FTP server facilities."""
533
self._ftp_server = None
535
self._async_thread = None
540
"""Calculate an ftp url to this server."""
541
return 'ftp://foo:bar@localhost:%d/' % (self._port)
543
# def get_bogus_url(self):
544
# """Return a URL which cannot be connected to."""
545
# return 'ftp://127.0.0.1:1'
547
def log(self, message):
548
"""This is used by medusa.ftp_server to log connections, etc."""
549
self.logs.append(message)
551
def setUp(self, vfs_server=None):
553
raise RuntimeError('Must have medusa to run the FtpServer')
555
assert vfs_server is None or isinstance(vfs_server, LocalURLServer), \
556
"FtpServer currently assumes local transport, got %s" % vfs_server
558
self._root = os.getcwdu()
559
self._ftp_server = _ftp_server(
560
authorizer=_test_authorizer(root=self._root),
562
port=0, # bind to a random port
564
logger_object=self # Use FtpServer.log() for messages
566
self._port = self._ftp_server.getsockname()[1]
567
# Don't let it loop forever, or handle an infinite number of requests.
568
# In this case it will run for 1000s, or 10000 requests
569
self._async_thread = threading.Thread(
570
target=FtpServer._asyncore_loop_ignore_EBADF,
571
kwargs={'timeout':0.1, 'count':10000})
572
self._async_thread.setDaemon(True)
573
self._async_thread.start()
576
"""See bzrlib.transport.Server.tearDown."""
577
# have asyncore release the channel
578
self._ftp_server.del_channel()
580
self._async_thread.join()
583
def _asyncore_loop_ignore_EBADF(*args, **kwargs):
584
"""Ignore EBADF during server shutdown.
586
We close the socket to get the server to shutdown, but this causes
587
select.select() to raise EBADF.
590
asyncore.loop(*args, **kwargs)
591
# FIXME: If we reach that point, we should raise an exception
592
# explaining that the 'count' parameter in setUp is too low or
593
# testers may wonder why their test just sits there waiting for a
594
# server that is already dead. Note that if the tester waits too
595
# long under pdb the server will also die.
596
except select.error, e:
597
if e.args[0] != errno.EBADF:
603
_test_authorizer = None
607
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
610
import medusa.filesys
611
import medusa.ftp_server
617
class test_authorizer(object):
618
"""A custom Authorizer object for running the test suite.
620
The reason we cannot use dummy_authorizer, is because it sets the
621
channel to readonly, which we don't always want to do.
624
def __init__(self, root):
627
def authorize(self, channel, username, password):
628
"""Return (success, reply_string, filesystem)"""
630
return 0, 'No Medusa.', None
632
channel.persona = -1, -1
633
if username == 'anonymous':
634
channel.read_only = 1
636
channel.read_only = 0
638
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
641
class ftp_channel(medusa.ftp_server.ftp_channel):
642
"""Customized ftp channel"""
644
def log(self, message):
645
"""Redirect logging requests."""
646
mutter('_ftp_channel: %s', message)
648
def log_info(self, message, type='info'):
649
"""Redirect logging requests."""
650
mutter('_ftp_channel %s: %s', type, message)
652
def cmd_rnfr(self, line):
653
"""Prepare for renaming a file."""
654
self._renaming = line[1]
655
self.respond('350 Ready for RNTO')
656
# TODO: jam 20060516 in testing, the ftp server seems to
657
# check that the file already exists, or it sends
658
# 550 RNFR command failed
660
def cmd_rnto(self, line):
661
"""Rename a file based on the target given.
663
rnto must be called after calling rnfr.
665
if not self._renaming:
666
self.respond('503 RNFR required first.')
667
pfrom = self.filesystem.translate(self._renaming)
668
self._renaming = None
669
pto = self.filesystem.translate(line[1])
670
if os.path.exists(pto):
671
self.respond('550 RNTO failed: file exists')
674
os.rename(pfrom, pto)
675
except (IOError, OSError), e:
676
# TODO: jam 20060516 return custom responses based on
677
# why the command failed
678
# (bialix 20070418) str(e) on Python 2.5 @ Windows
679
# sometimes don't provide expected error message;
680
# so we obtain such message via os.strerror()
681
self.respond('550 RNTO failed: %s' % os.strerror(e.errno))
683
self.respond('550 RNTO failed')
684
# For a test server, we will go ahead and just die
687
self.respond('250 Rename successful.')
689
def cmd_size(self, line):
690
"""Return the size of a file
692
This is overloaded to help the test suite determine if the
693
target is a directory.
696
if not self.filesystem.isfile(filename):
697
if self.filesystem.isdir(filename):
698
self.respond('550 "%s" is a directory' % (filename,))
700
self.respond('550 "%s" is not a file' % (filename,))
702
self.respond('213 %d'
703
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
705
def cmd_mkd(self, line):
706
"""Create a directory.
708
Overloaded because default implementation does not distinguish
709
*why* it cannot make a directory.
712
self.command_not_understood(''.join(line))
716
self.filesystem.mkdir (path)
717
self.respond ('257 MKD command successful.')
718
except (IOError, OSError), e:
719
# (bialix 20070418) str(e) on Python 2.5 @ Windows
720
# sometimes don't provide expected error message;
721
# so we obtain such message via os.strerror()
722
self.respond ('550 error creating directory: %s' %
723
os.strerror(e.errno))
725
self.respond ('550 error creating directory.')
728
class ftp_server(medusa.ftp_server.ftp_server):
729
"""Customize the behavior of the Medusa ftp_server.
731
There are a few warts on the ftp_server, based on how it expects
735
ftp_channel_class = ftp_channel
737
def __init__(self, *args, **kwargs):
738
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
739
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
741
def log(self, message):
742
"""Redirect logging requests."""
743
mutter('_ftp_server: %s', message)
745
def log_info(self, message, type='info'):
746
"""Override the asyncore.log_info so we don't stipple the screen."""
747
mutter('_ftp_server %s: %s', type, message)
749
_test_authorizer = test_authorizer
750
_ftp_channel = ftp_channel
751
_ftp_server = ftp_server
756
def get_test_permutations():
757
"""Return the permutations to be used in testing."""
758
if not _setup_medusa():
759
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
762
return [(FtpTransport, FtpServer)]
added
removed
bzrlib/transport/ftp.py
