79
74
_number_of_retries = 2
80
75
_sleep_between_retries = 5
82
# FIXME: there are inconsistencies in the way temporary errors are
83
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
84
# be taken to analyze the implications for write operations (read operations
85
# are safe to retry). Overall even some read operations are never
86
# retried. --vila 20070720 (Bug #127164)
87
class FtpTransport(ConnectedTransport):
77
class FtpTransport(Transport):
88
78
"""This is the transport agent for ftp:// access."""
90
def __init__(self, base, _from_transport=None):
80
def __init__(self, base, _provided_instance=None):
91
81
"""Set the base path where files will be stored."""
92
if not (base.startswith('ftp://') or base.startswith('aftp://')):
93
raise ValueError(base)
94
super(FtpTransport, self).__init__(base,
95
_from_transport=_from_transport)
96
self._unqualified_scheme = 'ftp'
97
if self._scheme == 'aftp':
100
self.is_active = False
82
assert base.startswith('ftp://') or base.startswith('aftp://')
83
super(FtpTransport, self).__init__(base)
84
self.is_active = base.startswith('aftp://')
87
(self._proto, self._host,
88
self._path, self._parameters,
89
self._query, self._fragment) = urlparse.urlparse(self.base)
90
self._FTP_instance = _provided_instance
102
92
def _get_FTP(self):
103
93
"""Return the ftplib.FTP instance for this object."""
104
# Ensures that a connection is established
105
connection = self._get_connection()
106
if connection is None:
107
# First connection ever
108
connection, credentials = self._create_connection()
109
self._set_connection(connection, credentials)
112
def _create_connection(self, credentials=None):
113
"""Create a new connection with the provided credentials.
115
:param credentials: The credentials needed to establish the connection.
117
:return: The created connection and its associated credentials.
119
The input credentials are only the password as it may have been
120
entered interactively by the user and may be different from the one
121
provided in base url at transport creation time. The returned
122
credentials are username, password.
124
if credentials is None:
125
user, password = self._user, self._password
127
user, password = credentials
129
auth = config.AuthenticationConfig()
131
user = auth.get_user('ftp', self._host, port=self._port)
133
# Default to local user
134
user = getpass.getuser()
136
mutter("Constructing FTP instance against %r" %
137
((self._host, self._port, user, '********',
94
if self._FTP_instance is not None:
95
return self._FTP_instance
140
connection = ftplib.FTP()
141
connection.connect(host=self._host, port=self._port)
142
if user and user != 'anonymous' and \
143
password is None: # '' is a valid password
144
password = auth.get_password('ftp', self._host, user,
146
connection.login(user=user, passwd=password)
147
connection.set_pasv(not self.is_active)
148
except socket.error, e:
149
raise errors.SocketConnectionError(self._host, self._port,
150
msg='Unable to connect to',
100
hostname = self._host
102
username, hostname = hostname.split("@", 1)
104
username, password = username.split(":", 1)
106
self._FTP_instance = _find_FTP(hostname, username, password,
108
return self._FTP_instance
152
109
except ftplib.error_perm, e:
153
raise errors.TransportError(msg="Error setting up connection:"
154
" %s" % str(e), orig_error=e)
155
return connection, (user, password)
157
def _reconnect(self):
158
"""Create a new connection with the previously used credentials"""
159
credentials = self._get_credentials()
160
connection, credentials = self._create_connection(credentials)
161
self._set_connection(connection, credentials)
163
def _translate_perm_error(self, err, path, extra=None,
164
unknown_exc=FtpPathError):
165
"""Try to translate an ftplib.error_perm exception.
167
:param err: The error to translate into a bzr error
168
:param path: The path which had problems
169
:param extra: Extra information which can be included
170
:param unknown_exc: If None, we will just raise the original exception
171
otherwise we raise unknown_exc(path, extra=extra)
177
extra += ': ' + str(err)
178
if ('no such file' in s
179
or 'could not open' in s
180
or 'no such dir' in s
181
or 'could not create file' in s # vsftpd
182
or 'file doesn\'t exist' in s
183
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
184
or 'file/directory not found' in s # filezilla server
185
# Microsoft FTP-Service RNFR reply if file not found
186
or (s.startswith('550 ') and 'unable to rename to' in extra)
188
raise errors.NoSuchFile(path, extra=extra)
189
if ('file exists' in s):
190
raise errors.FileExists(path, extra=extra)
191
if ('not a directory' in s):
192
raise errors.PathError(path, extra=extra)
194
mutter('unable to understand error for path: %s: %s', path, err)
197
raise unknown_exc(path, extra=extra)
198
# TODO: jam 20060516 Consider re-raising the error wrapped in
199
# something like TransportError, but this loses the traceback
200
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
201
# to handle. Consider doing something like that here.
202
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
205
def _remote_path(self, relpath):
206
# XXX: It seems that ftplib does not handle Unicode paths
207
# at the same time, medusa won't handle utf8 paths So if
208
# we .encode(utf8) here (see ConnectedTransport
209
# implementation), then we get a Server failure. while
210
# if we use str(), we get a UnicodeError, and the test
211
# suite just skips testing UnicodePaths.
212
relative = str(urlutils.unescape(relpath))
213
remote_path = self._combine_paths(self._path, relative)
110
raise TransportError(msg="Error setting up connection: %s"
111
% str(e), orig_error=e)
113
def should_cache(self):
114
"""Return True if the data pulled across should be cached locally.
118
def clone(self, offset=None):
119
"""Return a new FtpTransport with root at self.base + offset.
123
return FtpTransport(self.base, self._FTP_instance)
125
return FtpTransport(self.abspath(offset), self._FTP_instance)
127
def _abspath(self, relpath):
128
assert isinstance(relpath, basestring)
129
relpath = urllib.unquote(relpath)
130
if isinstance(relpath, basestring):
131
relpath_parts = relpath.split('/')
133
# TODO: Don't call this with an array - no magic interfaces
134
relpath_parts = relpath[:]
135
if len(relpath_parts) > 1:
136
if relpath_parts[0] == '':
137
raise ValueError("path %r within branch %r seems to be absolute"
138
% (relpath, self._path))
139
basepath = self._path.split('/')
140
if len(basepath) > 0 and basepath[-1] == '':
141
basepath = basepath[:-1]
142
for p in relpath_parts:
144
if len(basepath) == 0:
145
# In most filesystems, a request for the parent
146
# of root, just returns root.
149
elif p == '.' or p == '':
153
# Possibly, we could use urlparse.urljoin() here, but
154
# I'm concerned about when it chooses to strip the last
155
# portion of the path, and when it doesn't.
156
return '/'.join(basepath)
158
def abspath(self, relpath):
159
"""Return the full url to the given relative path.
160
This can be supplied with a string or a list
162
path = self._abspath(relpath)
163
return urlparse.urlunparse((self._proto,
164
self._host, path, '', '', ''))
216
166
def has(self, relpath):
217
"""Does the target location exist?"""
218
# FIXME jam 20060516 We *do* ask about directories in the test suite
219
# We don't seem to in the actual codebase
220
# XXX: I assume we're never asked has(dirname) and thus I use
221
# the FTP size command and assume that if it doesn't raise,
223
abspath = self._remote_path(relpath)
167
"""Does the target location exist?
169
XXX: I assume we're never asked has(dirname) and thus I use
170
the FTP size command and assume that if it doesn't raise,
225
174
f = self._get_FTP()
226
mutter('FTP has check: %s => %s', relpath, abspath)
228
mutter("FTP has: %s", abspath)
175
s = f.size(self._abspath(relpath))
176
mutter("FTP has: %s" % self._abspath(relpath))
230
except ftplib.error_perm, e:
231
if ('is a directory' in str(e).lower()):
232
mutter("FTP has dir: %s: %s", abspath, e)
234
mutter("FTP has not: %s: %s", abspath, e)
178
except ftplib.error_perm:
179
mutter("FTP has not: %s" % self._abspath(relpath))
237
182
def get(self, relpath, decode=False, retries=0):
282
227
:param retries: Number of retries after temporary failures so far
283
228
for this operation.
285
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
230
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
288
abspath = self._remote_path(relpath)
289
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
232
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (self._abspath(relpath), time.time(),
290
233
os.getpid(), random.randint(0,0x7FFFFFFF))
292
if getattr(fp, 'read', None) is None:
293
# hand in a string IO
297
# capture the byte count; .read() may be read only so
299
class byte_counter(object):
300
def __init__(self, fp):
302
self.counted_bytes = 0
303
def read(self, count):
304
result = self.fp.read(count)
305
self.counted_bytes += len(result)
307
fp = byte_counter(fp)
234
if not hasattr(fp, 'read'):
309
mutter("FTP put: %s", abspath)
237
mutter("FTP put: %s" % self._abspath(relpath))
310
238
f = self._get_FTP()
312
240
f.storbinary('STOR '+tmp_abspath, fp)
313
self._rename_and_overwrite(tmp_abspath, abspath, f)
314
self._setmode(relpath, mode)
315
if bytes is not None:
318
return fp.counted_bytes
241
f.rename(tmp_abspath, self._abspath(relpath))
319
242
except (ftplib.error_temp,EOFError), e:
320
243
warning("Failure during ftp PUT. Deleting temporary file.")
322
245
f.delete(tmp_abspath)
324
warning("Failed to delete temporary file on the"
325
" server.\nFile: %s", tmp_abspath)
247
warning("Failed to delete temporary file on the server.\nFile: %s"
328
251
except ftplib.error_perm, e:
329
self._translate_perm_error(e, abspath, extra='could not store',
330
unknown_exc=errors.NoSuchFile)
252
if "no such file" in str(e).lower():
253
raise NoSuchFile("Error storing %s: %s"
254
% (self.abspath(relpath), str(e)), extra=e)
256
raise FtpTransportError(orig_error=e)
331
257
except ftplib.error_temp, e:
332
258
if retries > _number_of_retries:
333
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
259
raise TransportError("FTP temporary error during PUT %s. Aborting."
334
260
% self.abspath(relpath), orig_error=e)
336
warning("FTP temporary error: %s. Retrying.", str(e))
338
self.put_file(relpath, fp, mode, retries+1)
262
warning("FTP temporary error: %s. Retrying." % str(e))
263
self._FTP_instance = None
264
self.put(relpath, fp, mode, retries+1)
340
266
if retries > _number_of_retries:
341
raise errors.TransportError("FTP control connection closed during PUT %s."
267
raise TransportError("FTP control connection closed during PUT %s."
342
268
% self.abspath(relpath), orig_error=e)
344
270
warning("FTP control connection closed. Trying to reopen.")
345
271
time.sleep(_sleep_between_retries)
347
self.put_file(relpath, fp, mode, retries+1)
272
self._FTP_instance = None
273
self.put(relpath, fp, mode, retries+1)
349
276
def mkdir(self, relpath, mode=None):
350
277
"""Create a directory at the given path."""
351
abspath = self._remote_path(relpath)
353
mutter("FTP mkd: %s", abspath)
279
mutter("FTP mkd: %s" % self._abspath(relpath))
354
280
f = self._get_FTP()
356
self._setmode(relpath, mode)
282
f.mkd(self._abspath(relpath))
283
except ftplib.error_perm, e:
285
if 'File exists' in s:
286
raise FileExists(self.abspath(relpath), extra=s)
357
289
except ftplib.error_perm, e:
358
self._translate_perm_error(e, abspath,
359
unknown_exc=errors.FileExists)
361
def open_write_stream(self, relpath, mode=None):
362
"""See Transport.open_write_stream."""
363
self.put_bytes(relpath, "", mode)
364
result = AppendBasedFileStream(self, relpath)
365
_file_streams[self.abspath(relpath)] = result
368
def recommended_page_size(self):
369
"""See Transport.recommended_page_size().
371
For FTP we suggest a large page size to reduce the overhead
372
introduced by latency.
290
raise TransportError(orig_error=e)
376
292
def rmdir(self, rel_path):
377
293
"""Delete the directory at rel_path"""
378
abspath = self._remote_path(rel_path)
380
mutter("FTP rmd: %s", abspath)
295
mutter("FTP rmd: %s" % self._abspath(rel_path))
381
297
f = self._get_FTP()
298
f.rmd(self._abspath(rel_path))
383
299
except ftplib.error_perm, e:
384
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
300
if str(e).endswith("Directory not empty"):
301
raise DirectoryNotEmpty(self._abspath(rel_path), extra=str(e))
303
raise TransportError(msg="Cannot remove directory at %s" % \
304
self._abspath(rel_path), extra=str(e))
386
def append_file(self, relpath, f, mode=None):
306
def append(self, relpath, f):
387
307
"""Append the text in the file-like object into the final
390
abspath = self._remote_path(relpath)
391
if self.has(relpath):
392
ftp = self._get_FTP()
393
result = ftp.size(abspath)
397
mutter("FTP appe to %s", abspath)
398
self._try_append(relpath, f.read(), mode)
402
def _try_append(self, relpath, text, mode=None, retries=0):
403
"""Try repeatedly to append the given text to the file at relpath.
405
This is a recursive function. On errors, it will be called until the
406
number of retries is exceeded.
409
abspath = self._remote_path(relpath)
410
mutter("FTP appe (try %d) to %s", retries, abspath)
411
ftp = self._get_FTP()
412
ftp.voidcmd("TYPE I")
413
cmd = "APPE %s" % abspath
414
conn = ftp.transfercmd(cmd)
417
self._setmode(relpath, mode)
419
except ftplib.error_perm, e:
420
self._translate_perm_error(e, abspath, extra='error appending',
421
unknown_exc=errors.NoSuchFile)
422
except ftplib.error_temp, e:
423
if retries > _number_of_retries:
424
raise errors.TransportError("FTP temporary error during APPEND %s." \
425
"Aborting." % abspath, orig_error=e)
427
warning("FTP temporary error: %s. Retrying.", str(e))
429
self._try_append(relpath, text, mode, retries+1)
431
def _setmode(self, relpath, mode):
432
"""Set permissions on a path.
434
Only set permissions if the FTP server supports the 'SITE CHMOD'
439
mutter("FTP site chmod: setting permissions to %s on %s",
440
str(mode), self._remote_path(relpath))
441
ftp = self._get_FTP()
442
cmd = "SITE CHMOD %s %s" % (oct(mode),
443
self._remote_path(relpath))
445
except ftplib.error_perm, e:
446
# Command probably not available on this server
447
warning("FTP Could not set permissions to %s on %s. %s",
448
str(mode), self._remote_path(relpath), str(e))
450
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
451
# to copy something to another machine. And you may be able
452
# to give it its own address as the 'to' location.
453
# So implement a fancier 'copy()'
455
def rename(self, rel_from, rel_to):
456
abs_from = self._remote_path(rel_from)
457
abs_to = self._remote_path(rel_to)
458
mutter("FTP rename: %s => %s", abs_from, abs_to)
460
return self._rename(abs_from, abs_to, f)
462
def _rename(self, abs_from, abs_to, f):
464
f.rename(abs_from, abs_to)
465
except ftplib.error_perm, e:
466
self._translate_perm_error(e, abs_from,
467
': unable to rename to %r' % (abs_to))
310
raise TransportNotPossible('ftp does not support append()')
312
def copy(self, rel_from, rel_to):
313
"""Copy the item at rel_from to the location at rel_to"""
314
raise TransportNotPossible('ftp does not (yet) support copy()')
469
316
def move(self, rel_from, rel_to):
470
317
"""Move the item at rel_from to the location at rel_to"""
471
abs_from = self._remote_path(rel_from)
472
abs_to = self._remote_path(rel_to)
474
mutter("FTP mv: %s => %s", abs_from, abs_to)
319
mutter("FTP mv: %s => %s" % (self._abspath(rel_from),
320
self._abspath(rel_to)))
475
321
f = self._get_FTP()
476
self._rename_and_overwrite(abs_from, abs_to, f)
322
f.rename(self._abspath(rel_from), self._abspath(rel_to))
477
323
except ftplib.error_perm, e:
478
self._translate_perm_error(e, abs_from,
479
extra='unable to rename to %r' % (rel_to,),
480
unknown_exc=errors.PathError)
482
def _rename_and_overwrite(self, abs_from, abs_to, f):
483
"""Do a fancy rename on the remote server.
485
Using the implementation provided by osutils.
487
osutils.fancy_rename(abs_from, abs_to,
488
rename_func=lambda p1, p2: self._rename(p1, p2, f),
489
unlink_func=lambda p: self._delete(p, f))
324
raise TransportError(orig_error=e)
491
328
def delete(self, relpath):
492
329
"""Delete the item at relpath"""
493
abspath = self._remote_path(relpath)
495
self._delete(abspath, f)
497
def _delete(self, abspath, f):
499
mutter("FTP rm: %s", abspath)
331
mutter("FTP rm: %s" % self._abspath(relpath))
333
f.delete(self._abspath(relpath))
501
334
except ftplib.error_perm, e:
502
self._translate_perm_error(e, abspath, 'error deleting',
503
unknown_exc=errors.NoSuchFile)
505
def external_url(self):
506
"""See bzrlib.transport.Transport.external_url."""
507
# FTP URL's are externally usable.
335
if str(e).endswith("No such file or directory"):
336
raise NoSuchFile(self._abspath(relpath), extra=str(e))
338
raise TransportError(orig_error=e)
510
340
def listable(self):
511
341
"""See Transport.listable."""
added
removed
bzrlib/transport/ftp.py
