83
74
_number_of_retries = 2
84
75
_sleep_between_retries = 5
86
# FIXME: there are inconsistencies in the way temporary errors are
87
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
88
# be taken to analyze the implications for write operations (read operations
89
# are safe to retry). Overall even some read operations are never
90
# retried. --vila 20070720 (Bug #127164)
91
class FtpTransport(ConnectedTransport):
77
class FtpTransport(Transport):
92
78
"""This is the transport agent for ftp:// access."""
94
def __init__(self, base, _from_transport=None):
80
def __init__(self, base, _provided_instance=None):
95
81
"""Set the base path where files will be stored."""
96
if not (base.startswith('ftp://') or base.startswith('aftp://')):
97
raise ValueError(base)
98
super(FtpTransport, self).__init__(base,
99
_from_transport=_from_transport)
100
self._unqualified_scheme = 'ftp'
101
if self._parsed_url.scheme == 'aftp':
102
self.is_active = True
104
self.is_active = False
106
# Most modern FTP servers support the APPE command. If ours doesn't, we
107
# (re)set this flag accordingly later.
108
self._has_append = True
82
assert base.startswith('ftp://') or base.startswith('aftp://')
83
super(FtpTransport, self).__init__(base)
84
self.is_active = base.startswith('aftp://')
87
(self._proto, self._host,
88
self._path, self._parameters,
89
self._query, self._fragment) = urlparse.urlparse(self.base)
90
self._FTP_instance = _provided_instance
110
92
def _get_FTP(self):
111
93
"""Return the ftplib.FTP instance for this object."""
112
# Ensures that a connection is established
113
connection = self._get_connection()
114
if connection is None:
115
# First connection ever
116
connection, credentials = self._create_connection()
117
self._set_connection(connection, credentials)
120
connection_class = ftplib.FTP
122
def _create_connection(self, credentials=None):
123
"""Create a new connection with the provided credentials.
125
:param credentials: The credentials needed to establish the connection.
127
:return: The created connection and its associated credentials.
129
The input credentials are only the password as it may have been
130
entered interactively by the user and may be different from the one
131
provided in base url at transport creation time. The returned
132
credentials are username, password.
134
if credentials is None:
135
user, password = self._user, self._password
137
user, password = credentials
139
auth = config.AuthenticationConfig()
141
user = auth.get_user('ftp', self._host, port=self._port,
142
default=getpass.getuser())
143
mutter("Constructing FTP instance against %r" %
144
((self._host, self._port, user, '********',
94
if self._FTP_instance is not None:
95
return self._FTP_instance
147
connection = self.connection_class()
148
connection.connect(host=self._host, port=self._port)
149
self._login(connection, auth, user, password)
150
connection.set_pasv(not self.is_active)
151
# binary mode is the default
152
connection.voidcmd('TYPE I')
153
except socket.error, e:
154
raise errors.SocketConnectionError(self._host, self._port,
155
msg='Unable to connect to',
100
hostname = self._host
102
username, hostname = hostname.split("@", 1)
104
username, password = username.split(":", 1)
106
self._FTP_instance = _find_FTP(hostname, username, password,
108
return self._FTP_instance
157
109
except ftplib.error_perm, e:
158
raise errors.TransportError(msg="Error setting up connection:"
159
" %s" % str(e), orig_error=e)
160
return connection, (user, password)
162
def _login(self, connection, auth, user, password):
163
# '' is a valid password
164
if user and user != 'anonymous' and password is None:
165
password = auth.get_password('ftp', self._host,
166
user, port=self._port)
167
connection.login(user=user, passwd=password)
169
def _reconnect(self):
170
"""Create a new connection with the previously used credentials"""
171
credentials = self._get_credentials()
172
connection, credentials = self._create_connection(credentials)
173
self._set_connection(connection, credentials)
175
def disconnect(self):
176
connection = self._get_connection()
177
if connection is not None:
180
def _translate_ftp_error(self, err, path, extra=None,
181
unknown_exc=FtpPathError):
182
"""Try to translate an ftplib exception to a bzrlib exception.
184
:param err: The error to translate into a bzr error
185
:param path: The path which had problems
186
:param extra: Extra information which can be included
187
:param unknown_exc: If None, we will just raise the original exception
188
otherwise we raise unknown_exc(path, extra=extra)
190
# ftp error numbers are very generic, like "451: Requested action aborted,
191
# local error in processing" so unfortunately we have to match by
197
extra += ': ' + str(err)
198
if ('no such file' in s
199
or 'could not open' in s
200
or 'no such dir' in s
201
or 'could not create file' in s # vsftpd
202
or 'file doesn\'t exist' in s
203
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
204
or 'file/directory not found' in s # filezilla server
205
# Microsoft FTP-Service RNFR reply if file not found
206
or (s.startswith('550 ') and 'unable to rename to' in extra)
207
# if containing directory doesn't exist, suggested by
208
# <https://bugs.launchpad.net/bzr/+bug/224373>
209
or (s.startswith('550 ') and "can't find folder" in s)
211
raise errors.NoSuchFile(path, extra=extra)
212
elif ('file exists' in s):
213
raise errors.FileExists(path, extra=extra)
214
elif ('not a directory' in s):
215
raise errors.PathError(path, extra=extra)
216
elif 'directory not empty' in s:
217
raise errors.DirectoryNotEmpty(path, extra=extra)
219
mutter('unable to understand error for path: %s: %s', path, err)
222
raise unknown_exc(path, extra=extra)
223
# TODO: jam 20060516 Consider re-raising the error wrapped in
224
# something like TransportError, but this loses the traceback
225
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
226
# to handle. Consider doing something like that here.
227
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
110
raise TransportError(msg="Error setting up connection: %s"
111
% str(e), orig_error=e)
113
def should_cache(self):
114
"""Return True if the data pulled across should be cached locally.
118
def clone(self, offset=None):
119
"""Return a new FtpTransport with root at self.base + offset.
123
return FtpTransport(self.base, self._FTP_instance)
125
return FtpTransport(self.abspath(offset), self._FTP_instance)
127
def _abspath(self, relpath):
128
assert isinstance(relpath, basestring)
129
relpath = urllib.unquote(relpath)
130
if isinstance(relpath, basestring):
131
relpath_parts = relpath.split('/')
133
# TODO: Don't call this with an array - no magic interfaces
134
relpath_parts = relpath[:]
135
if len(relpath_parts) > 1:
136
if relpath_parts[0] == '':
137
raise ValueError("path %r within branch %r seems to be absolute"
138
% (relpath, self._path))
139
basepath = self._path.split('/')
140
if len(basepath) > 0 and basepath[-1] == '':
141
basepath = basepath[:-1]
142
for p in relpath_parts:
144
if len(basepath) == 0:
145
# In most filesystems, a request for the parent
146
# of root, just returns root.
149
elif p == '.' or p == '':
153
# Possibly, we could use urlparse.urljoin() here, but
154
# I'm concerned about when it chooses to strip the last
155
# portion of the path, and when it doesn't.
156
return '/'.join(basepath)
158
def abspath(self, relpath):
159
"""Return the full url to the given relative path.
160
This can be supplied with a string or a list
162
path = self._abspath(relpath)
163
return urlparse.urlunparse((self._proto,
164
self._host, path, '', '', ''))
230
166
def has(self, relpath):
231
"""Does the target location exist?"""
232
# FIXME jam 20060516 We *do* ask about directories in the test suite
233
# We don't seem to in the actual codebase
234
# XXX: I assume we're never asked has(dirname) and thus I use
235
# the FTP size command and assume that if it doesn't raise,
237
abspath = self._remote_path(relpath)
167
"""Does the target location exist?
169
XXX: I assume we're never asked has(dirname) and thus I use
170
the FTP size command and assume that if it doesn't raise,
239
174
f = self._get_FTP()
240
mutter('FTP has check: %s => %s', relpath, abspath)
242
mutter("FTP has: %s", abspath)
175
s = f.size(self._abspath(relpath))
176
mutter("FTP has: %s" % self._abspath(relpath))
244
except ftplib.error_perm, e:
245
if ('is a directory' in str(e).lower()):
246
mutter("FTP has dir: %s: %s", abspath, e)
248
mutter("FTP has not: %s: %s", abspath, e)
178
except ftplib.error_perm:
179
mutter("FTP has not: %s" % self._abspath(relpath))
251
def get(self, relpath, retries=0):
182
def get(self, relpath, decode=False, retries=0):
252
183
"""Get the file at the given relative path.
254
185
:param relpath: The relative path to the file
295
227
:param retries: Number of retries after temporary failures so far
296
228
for this operation.
298
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
230
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
301
abspath = self._remote_path(relpath)
302
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
232
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (self._abspath(relpath), time.time(),
303
233
os.getpid(), random.randint(0,0x7FFFFFFF))
305
if getattr(fp, 'read', None) is None:
306
# hand in a string IO
310
# capture the byte count; .read() may be read only so
312
class byte_counter(object):
313
def __init__(self, fp):
315
self.counted_bytes = 0
316
def read(self, count):
317
result = self.fp.read(count)
318
self.counted_bytes += len(result)
320
fp = byte_counter(fp)
234
if not hasattr(fp, 'read'):
322
mutter("FTP put: %s", abspath)
237
mutter("FTP put: %s" % self._abspath(relpath))
323
238
f = self._get_FTP()
325
240
f.storbinary('STOR '+tmp_abspath, fp)
326
self._rename_and_overwrite(tmp_abspath, abspath, f)
327
self._setmode(relpath, mode)
328
if bytes is not None:
331
return fp.counted_bytes
332
except (ftplib.error_temp, EOFError), e:
333
warning("Failure during ftp PUT of %s: %s. Deleting temporary file."
334
% (tmp_abspath, e, ))
241
f.rename(tmp_abspath, self._abspath(relpath))
242
except (ftplib.error_temp,EOFError), e:
243
warning("Failure during ftp PUT. Deleting temporary file.")
336
245
f.delete(tmp_abspath)
338
warning("Failed to delete temporary file on the"
339
" server.\nFile: %s", tmp_abspath)
247
warning("Failed to delete temporary file on the server.\nFile: %s"
342
251
except ftplib.error_perm, e:
343
self._translate_ftp_error(e, abspath, extra='could not store',
344
unknown_exc=errors.NoSuchFile)
252
if "no such file" in str(e).lower():
253
raise NoSuchFile("Error storing %s: %s"
254
% (self.abspath(relpath), str(e)), extra=e)
256
raise FtpTransportError(orig_error=e)
345
257
except ftplib.error_temp, e:
346
258
if retries > _number_of_retries:
347
raise errors.TransportError(
348
"FTP temporary error during PUT %s: %s. Aborting."
349
% (self.abspath(relpath), e), orig_error=e)
259
raise TransportError("FTP temporary error during PUT %s. Aborting."
260
% self.abspath(relpath), orig_error=e)
351
warning("FTP temporary error: %s. Retrying.", str(e))
353
self.put_file(relpath, fp, mode, retries+1)
262
warning("FTP temporary error: %s. Retrying." % str(e))
263
self._FTP_instance = None
264
self.put(relpath, fp, mode, retries+1)
355
266
if retries > _number_of_retries:
356
raise errors.TransportError("FTP control connection closed during PUT %s."
267
raise TransportError("FTP control connection closed during PUT %s."
357
268
% self.abspath(relpath), orig_error=e)
359
270
warning("FTP control connection closed. Trying to reopen.")
360
271
time.sleep(_sleep_between_retries)
362
self.put_file(relpath, fp, mode, retries+1)
272
self._FTP_instance = None
273
self.put(relpath, fp, mode, retries+1)
364
276
def mkdir(self, relpath, mode=None):
365
277
"""Create a directory at the given path."""
366
abspath = self._remote_path(relpath)
368
mutter("FTP mkd: %s", abspath)
279
mutter("FTP mkd: %s" % self._abspath(relpath))
369
280
f = self._get_FTP()
372
except ftplib.error_reply, e:
373
# <https://bugs.launchpad.net/bzr/+bug/224373> Microsoft FTP
374
# server returns "250 Directory created." which is kind of
375
# reasonable, 250 meaning "requested file action OK", but not what
376
# Python's ftplib expects.
377
if e[0][:3] == '250':
282
f.mkd(self._abspath(relpath))
283
except ftplib.error_perm, e:
285
if 'File exists' in s:
286
raise FileExists(self.abspath(relpath), extra=s)
381
self._setmode(relpath, mode)
382
289
except ftplib.error_perm, e:
383
self._translate_ftp_error(e, abspath,
384
unknown_exc=errors.FileExists)
386
def open_write_stream(self, relpath, mode=None):
387
"""See Transport.open_write_stream."""
388
self.put_bytes(relpath, "", mode)
389
result = AppendBasedFileStream(self, relpath)
390
_file_streams[self.abspath(relpath)] = result
393
def recommended_page_size(self):
394
"""See Transport.recommended_page_size().
396
For FTP we suggest a large page size to reduce the overhead
397
introduced by latency.
290
raise TransportError(orig_error=e)
401
292
def rmdir(self, rel_path):
402
293
"""Delete the directory at rel_path"""
403
abspath = self._remote_path(rel_path)
405
mutter("FTP rmd: %s", abspath)
295
mutter("FTP rmd: %s" % self._abspath(rel_path))
406
297
f = self._get_FTP()
298
f.rmd(self._abspath(rel_path))
408
299
except ftplib.error_perm, e:
409
self._translate_ftp_error(e, abspath, unknown_exc=errors.PathError)
300
if str(e).endswith("Directory not empty"):
301
raise DirectoryNotEmpty(self._abspath(rel_path), extra=str(e))
303
raise TransportError(msg="Cannot remove directory at %s" % \
304
self._abspath(rel_path), extra=str(e))
411
def append_file(self, relpath, f, mode=None):
306
def append(self, relpath, f):
412
307
"""Append the text in the file-like object into the final
416
abspath = self._remote_path(relpath)
417
if self.has(relpath):
418
ftp = self._get_FTP()
419
result = ftp.size(abspath)
424
mutter("FTP appe to %s", abspath)
425
self._try_append(relpath, text, mode)
427
self._fallback_append(relpath, text, mode)
431
def _try_append(self, relpath, text, mode=None, retries=0):
432
"""Try repeatedly to append the given text to the file at relpath.
434
This is a recursive function. On errors, it will be called until the
435
number of retries is exceeded.
438
abspath = self._remote_path(relpath)
439
mutter("FTP appe (try %d) to %s", retries, abspath)
440
ftp = self._get_FTP()
441
cmd = "APPE %s" % abspath
442
conn = ftp.transfercmd(cmd)
445
self._setmode(relpath, mode)
447
except ftplib.error_perm, e:
448
# Check whether the command is not supported (reply code 502)
449
if str(e).startswith('502 '):
450
warning("FTP server does not support file appending natively. "
451
"Performance may be severely degraded! (%s)", e)
452
self._has_append = False
453
self._fallback_append(relpath, text, mode)
455
self._translate_ftp_error(e, abspath, extra='error appending',
456
unknown_exc=errors.NoSuchFile)
457
except ftplib.error_temp, e:
458
if retries > _number_of_retries:
459
raise errors.TransportError(
460
"FTP temporary error during APPEND %s. Aborting."
461
% abspath, orig_error=e)
463
warning("FTP temporary error: %s. Retrying.", str(e))
465
self._try_append(relpath, text, mode, retries+1)
467
def _fallback_append(self, relpath, text, mode = None):
468
remote = self.get(relpath)
469
remote.seek(0, os.SEEK_END)
472
return self.put_file(relpath, remote, mode)
474
def _setmode(self, relpath, mode):
475
"""Set permissions on a path.
477
Only set permissions if the FTP server supports the 'SITE CHMOD'
482
mutter("FTP site chmod: setting permissions to %s on %s",
483
oct(mode), self._remote_path(relpath))
484
ftp = self._get_FTP()
485
cmd = "SITE CHMOD %s %s" % (oct(mode),
486
self._remote_path(relpath))
488
except ftplib.error_perm, e:
489
# Command probably not available on this server
490
warning("FTP Could not set permissions to %s on %s. %s",
491
oct(mode), self._remote_path(relpath), str(e))
493
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
494
# to copy something to another machine. And you may be able
495
# to give it its own address as the 'to' location.
496
# So implement a fancier 'copy()'
498
def rename(self, rel_from, rel_to):
499
abs_from = self._remote_path(rel_from)
500
abs_to = self._remote_path(rel_to)
501
mutter("FTP rename: %s => %s", abs_from, abs_to)
503
return self._rename(abs_from, abs_to, f)
505
def _rename(self, abs_from, abs_to, f):
507
f.rename(abs_from, abs_to)
508
except (ftplib.error_temp, ftplib.error_perm), e:
509
self._translate_ftp_error(e, abs_from,
510
': unable to rename to %r' % (abs_to))
310
raise TransportNotPossible('ftp does not support append()')
312
def copy(self, rel_from, rel_to):
313
"""Copy the item at rel_from to the location at rel_to"""
314
raise TransportNotPossible('ftp does not (yet) support copy()')
512
316
def move(self, rel_from, rel_to):
513
317
"""Move the item at rel_from to the location at rel_to"""
514
abs_from = self._remote_path(rel_from)
515
abs_to = self._remote_path(rel_to)
517
mutter("FTP mv: %s => %s", abs_from, abs_to)
319
mutter("FTP mv: %s => %s" % (self._abspath(rel_from),
320
self._abspath(rel_to)))
518
321
f = self._get_FTP()
519
self._rename_and_overwrite(abs_from, abs_to, f)
322
f.rename(self._abspath(rel_from), self._abspath(rel_to))
520
323
except ftplib.error_perm, e:
521
self._translate_ftp_error(e, abs_from,
522
extra='unable to rename to %r' % (rel_to,),
523
unknown_exc=errors.PathError)
525
def _rename_and_overwrite(self, abs_from, abs_to, f):
526
"""Do a fancy rename on the remote server.
528
Using the implementation provided by osutils.
530
osutils.fancy_rename(abs_from, abs_to,
531
rename_func=lambda p1, p2: self._rename(p1, p2, f),
532
unlink_func=lambda p: self._delete(p, f))
324
raise TransportError(orig_error=e)
534
328
def delete(self, relpath):
535
329
"""Delete the item at relpath"""
536
abspath = self._remote_path(relpath)
538
self._delete(abspath, f)
540
def _delete(self, abspath, f):
542
mutter("FTP rm: %s", abspath)
331
mutter("FTP rm: %s" % self._abspath(relpath))
333
f.delete(self._abspath(relpath))
544
334
except ftplib.error_perm, e:
545
self._translate_ftp_error(e, abspath, 'error deleting',
546
unknown_exc=errors.NoSuchFile)
548
def external_url(self):
549
"""See bzrlib.transport.Transport.external_url."""
550
# FTP URL's are externally usable.
335
if str(e).endswith("No such file or directory"):
336
raise NoSuchFile(self._abspath(relpath), extra=str(e))
338
raise TransportError(orig_error=e)
553
340
def listable(self):
554
341
"""See Transport.listable."""
added
removed
bzrlib/transport/ftp.py
