81
74
_number_of_retries = 2
82
75
_sleep_between_retries = 5
84
# FIXME: there are inconsistencies in the way temporary errors are
85
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
86
# be taken to analyze the implications for write operations (read operations
87
# are safe to retry). Overall even some read operations are never
88
# retried. --vila 20070720 (Bug #127164)
89
class FtpTransport(ConnectedTransport):
77
class FtpTransport(Transport):
90
78
"""This is the transport agent for ftp:// access."""
92
def __init__(self, base, _from_transport=None):
80
def __init__(self, base, _provided_instance=None):
93
81
"""Set the base path where files will be stored."""
94
if not (base.startswith('ftp://') or base.startswith('aftp://')):
95
raise ValueError(base)
96
super(FtpTransport, self).__init__(base,
97
_from_transport=_from_transport)
98
self._unqualified_scheme = 'ftp'
99
if self._parsed_url.scheme == 'aftp':
100
self.is_active = True
102
self.is_active = False
104
# Most modern FTP servers support the APPE command. If ours doesn't, we
105
# (re)set this flag accordingly later.
106
self._has_append = True
82
assert base.startswith('ftp://') or base.startswith('aftp://')
83
super(FtpTransport, self).__init__(base)
84
self.is_active = base.startswith('aftp://')
87
(self._proto, self._host,
88
self._path, self._parameters,
89
self._query, self._fragment) = urlparse.urlparse(self.base)
90
self._FTP_instance = _provided_instance
108
92
def _get_FTP(self):
109
93
"""Return the ftplib.FTP instance for this object."""
110
# Ensures that a connection is established
111
connection = self._get_connection()
112
if connection is None:
113
# First connection ever
114
connection, credentials = self._create_connection()
115
self._set_connection(connection, credentials)
118
connection_class = ftplib.FTP
120
def _create_connection(self, credentials=None):
121
"""Create a new connection with the provided credentials.
123
:param credentials: The credentials needed to establish the connection.
125
:return: The created connection and its associated credentials.
127
The input credentials are only the password as it may have been
128
entered interactively by the user and may be different from the one
129
provided in base url at transport creation time. The returned
130
credentials are username, password.
132
if credentials is None:
133
user, password = self._user, self._password
135
user, password = credentials
137
auth = config.AuthenticationConfig()
139
user = auth.get_user('ftp', self._host, port=self._port,
140
default=getpass.getuser())
141
mutter("Constructing FTP instance against %r" %
142
((self._host, self._port, user, '********',
94
if self._FTP_instance is not None:
95
return self._FTP_instance
145
connection = self.connection_class()
146
connection.connect(host=self._host, port=self._port)
147
self._login(connection, auth, user, password)
148
connection.set_pasv(not self.is_active)
149
# binary mode is the default
150
connection.voidcmd('TYPE I')
151
except socket.error, e:
152
raise errors.SocketConnectionError(self._host, self._port,
153
msg='Unable to connect to',
100
hostname = self._host
102
username, hostname = hostname.split("@", 1)
104
username, password = username.split(":", 1)
106
self._FTP_instance = _find_FTP(hostname, username, password,
108
return self._FTP_instance
155
109
except ftplib.error_perm, e:
156
raise errors.TransportError(msg="Error setting up connection:"
157
" %s" % str(e), orig_error=e)
158
return connection, (user, password)
160
def _login(self, connection, auth, user, password):
161
# '' is a valid password
162
if user and user != 'anonymous' and password is None:
163
password = auth.get_password('ftp', self._host,
164
user, port=self._port)
165
connection.login(user=user, passwd=password)
167
def _reconnect(self):
168
"""Create a new connection with the previously used credentials"""
169
credentials = self._get_credentials()
170
connection, credentials = self._create_connection(credentials)
171
self._set_connection(connection, credentials)
173
def disconnect(self):
174
connection = self._get_connection()
175
if connection is not None:
178
def _translate_ftp_error(self, err, path, extra=None,
179
unknown_exc=FtpPathError):
180
"""Try to translate an ftplib exception to a bzrlib exception.
182
:param err: The error to translate into a bzr error
183
:param path: The path which had problems
184
:param extra: Extra information which can be included
185
:param unknown_exc: If None, we will just raise the original exception
186
otherwise we raise unknown_exc(path, extra=extra)
188
# ftp error numbers are very generic, like "451: Requested action aborted,
189
# local error in processing" so unfortunately we have to match by
195
extra += ': ' + str(err)
196
if ('no such file' in s
197
or 'could not open' in s
198
or 'no such dir' in s
199
or 'could not create file' in s # vsftpd
200
or 'file doesn\'t exist' in s
201
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
202
or 'file/directory not found' in s # filezilla server
203
# Microsoft FTP-Service RNFR reply if file not found
204
or (s.startswith('550 ') and 'unable to rename to' in extra)
205
# if containing directory doesn't exist, suggested by
206
# <https://bugs.launchpad.net/bzr/+bug/224373>
207
or (s.startswith('550 ') and "can't find folder" in s)
209
raise errors.NoSuchFile(path, extra=extra)
210
elif ('file exists' in s):
211
raise errors.FileExists(path, extra=extra)
212
elif ('not a directory' in s):
213
raise errors.PathError(path, extra=extra)
214
elif 'directory not empty' in s:
215
raise errors.DirectoryNotEmpty(path, extra=extra)
217
mutter('unable to understand error for path: %s: %s', path, err)
220
raise unknown_exc(path, extra=extra)
221
# TODO: jam 20060516 Consider re-raising the error wrapped in
222
# something like TransportError, but this loses the traceback
223
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
224
# to handle. Consider doing something like that here.
225
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
110
raise TransportError(msg="Error setting up connection: %s"
111
% str(e), orig_error=e)
113
def should_cache(self):
114
"""Return True if the data pulled across should be cached locally.
118
def clone(self, offset=None):
119
"""Return a new FtpTransport with root at self.base + offset.
123
return FtpTransport(self.base, self._FTP_instance)
125
return FtpTransport(self.abspath(offset), self._FTP_instance)
127
def _abspath(self, relpath):
128
assert isinstance(relpath, basestring)
129
relpath = urllib.unquote(relpath)
130
if isinstance(relpath, basestring):
131
relpath_parts = relpath.split('/')
133
# TODO: Don't call this with an array - no magic interfaces
134
relpath_parts = relpath[:]
135
if len(relpath_parts) > 1:
136
if relpath_parts[0] == '':
137
raise ValueError("path %r within branch %r seems to be absolute"
138
% (relpath, self._path))
139
basepath = self._path.split('/')
140
if len(basepath) > 0 and basepath[-1] == '':
141
basepath = basepath[:-1]
142
for p in relpath_parts:
144
if len(basepath) == 0:
145
# In most filesystems, a request for the parent
146
# of root, just returns root.
149
elif p == '.' or p == '':
153
# Possibly, we could use urlparse.urljoin() here, but
154
# I'm concerned about when it chooses to strip the last
155
# portion of the path, and when it doesn't.
156
return '/'.join(basepath)
158
def abspath(self, relpath):
159
"""Return the full url to the given relative path.
160
This can be supplied with a string or a list
162
path = self._abspath(relpath)
163
return urlparse.urlunparse((self._proto,
164
self._host, path, '', '', ''))
228
166
def has(self, relpath):
229
"""Does the target location exist?"""
230
# FIXME jam 20060516 We *do* ask about directories in the test suite
231
# We don't seem to in the actual codebase
232
# XXX: I assume we're never asked has(dirname) and thus I use
233
# the FTP size command and assume that if it doesn't raise,
235
abspath = self._remote_path(relpath)
167
"""Does the target location exist?
169
XXX: I assume we're never asked has(dirname) and thus I use
170
the FTP size command and assume that if it doesn't raise,
237
174
f = self._get_FTP()
238
mutter('FTP has check: %s => %s', relpath, abspath)
240
mutter("FTP has: %s", abspath)
175
s = f.size(self._abspath(relpath))
176
mutter("FTP has: %s" % self._abspath(relpath))
242
except ftplib.error_perm, e:
243
if ('is a directory' in str(e).lower()):
244
mutter("FTP has dir: %s: %s", abspath, e)
246
mutter("FTP has not: %s: %s", abspath, e)
178
except ftplib.error_perm:
179
mutter("FTP has not: %s" % self._abspath(relpath))
249
def get(self, relpath, decode=DEPRECATED_PARAMETER, retries=0):
182
def get(self, relpath, decode=False, retries=0):
250
183
"""Get the file at the given relative path.
252
185
:param relpath: The relative path to the file
297
226
:param retries: Number of retries after temporary failures so far
298
227
for this operation.
300
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
229
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
303
abspath = self._remote_path(relpath)
304
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
231
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (self._abspath(relpath), time.time(),
305
232
os.getpid(), random.randint(0,0x7FFFFFFF))
307
if getattr(fp, 'read', None) is None:
308
# hand in a string IO
312
# capture the byte count; .read() may be read only so
314
class byte_counter(object):
315
def __init__(self, fp):
317
self.counted_bytes = 0
318
def read(self, count):
319
result = self.fp.read(count)
320
self.counted_bytes += len(result)
322
fp = byte_counter(fp)
233
if not hasattr(fp, 'read'):
324
mutter("FTP put: %s", abspath)
236
mutter("FTP put: %s" % self._abspath(relpath))
325
237
f = self._get_FTP()
327
239
f.storbinary('STOR '+tmp_abspath, fp)
328
self._rename_and_overwrite(tmp_abspath, abspath, f)
329
self._setmode(relpath, mode)
330
if bytes is not None:
333
return fp.counted_bytes
334
except (ftplib.error_temp, EOFError), e:
335
warning("Failure during ftp PUT of %s: %s. Deleting temporary file."
336
% (tmp_abspath, e, ))
240
f.rename(tmp_abspath, self._abspath(relpath))
241
except (ftplib.error_temp,EOFError), e:
242
warning("Failure during ftp PUT. Deleting temporary file.")
338
244
f.delete(tmp_abspath)
340
warning("Failed to delete temporary file on the"
341
" server.\nFile: %s", tmp_abspath)
246
warning("Failed to delete temporary file on the server.\nFile: %s"
344
250
except ftplib.error_perm, e:
345
self._translate_ftp_error(e, abspath, extra='could not store',
346
unknown_exc=errors.NoSuchFile)
251
if "no such file" in str(e).lower():
252
raise NoSuchFile("Error storing %s: %s"
253
% (self.abspath(relpath), str(e)), extra=e)
255
raise FtpTransportError(orig_error=e)
347
256
except ftplib.error_temp, e:
348
257
if retries > _number_of_retries:
349
raise errors.TransportError(
350
"FTP temporary error during PUT %s: %s. Aborting."
351
% (self.abspath(relpath), e), orig_error=e)
258
raise TransportError("FTP temporary error during PUT %s. Aborting."
259
% self.abspath(relpath), orig_error=e)
353
warning("FTP temporary error: %s. Retrying.", str(e))
355
self.put_file(relpath, fp, mode, retries+1)
261
warning("FTP temporary error: %s. Retrying." % str(e))
262
self._FTP_instance = None
263
self.put(relpath, fp, mode, retries+1)
357
265
if retries > _number_of_retries:
358
raise errors.TransportError("FTP control connection closed during PUT %s."
266
raise TransportError("FTP control connection closed during PUT %s."
359
267
% self.abspath(relpath), orig_error=e)
361
269
warning("FTP control connection closed. Trying to reopen.")
362
270
time.sleep(_sleep_between_retries)
364
self.put_file(relpath, fp, mode, retries+1)
271
self._FTP_instance = None
272
self.put(relpath, fp, mode, retries+1)
366
275
def mkdir(self, relpath, mode=None):
367
276
"""Create a directory at the given path."""
368
abspath = self._remote_path(relpath)
370
mutter("FTP mkd: %s", abspath)
278
mutter("FTP mkd: %s" % self._abspath(relpath))
371
279
f = self._get_FTP()
374
except ftplib.error_reply, e:
375
# <https://bugs.launchpad.net/bzr/+bug/224373> Microsoft FTP
376
# server returns "250 Directory created." which is kind of
377
# reasonable, 250 meaning "requested file action OK", but not what
378
# Python's ftplib expects.
379
if e[0][:3] == '250':
281
f.mkd(self._abspath(relpath))
282
except ftplib.error_perm, e:
284
if 'File exists' in s:
285
raise FileExists(self.abspath(relpath), extra=s)
383
self._setmode(relpath, mode)
384
except ftplib.error_perm, e:
385
self._translate_ftp_error(e, abspath,
386
unknown_exc=errors.FileExists)
388
def open_write_stream(self, relpath, mode=None):
389
"""See Transport.open_write_stream."""
390
self.put_bytes(relpath, "", mode)
391
result = AppendBasedFileStream(self, relpath)
392
_file_streams[self.abspath(relpath)] = result
395
def recommended_page_size(self):
396
"""See Transport.recommended_page_size().
398
For FTP we suggest a large page size to reduce the overhead
399
introduced by latency.
403
def rmdir(self, rel_path):
404
"""Delete the directory at rel_path"""
405
abspath = self._remote_path(rel_path)
407
mutter("FTP rmd: %s", abspath)
410
except ftplib.error_perm, e:
411
self._translate_ftp_error(e, abspath, unknown_exc=errors.PathError)
413
def append_file(self, relpath, f, mode=None):
288
except ftplib.error_perm, e:
289
raise TransportError(orig_error=e)
291
def append(self, relpath, f):
414
292
"""Append the text in the file-like object into the final
418
abspath = self._remote_path(relpath)
419
if self.has(relpath):
420
ftp = self._get_FTP()
421
result = ftp.size(abspath)
426
mutter("FTP appe to %s", abspath)
427
self._try_append(relpath, text, mode)
429
self._fallback_append(relpath, text, mode)
433
def _try_append(self, relpath, text, mode=None, retries=0):
434
"""Try repeatedly to append the given text to the file at relpath.
436
This is a recursive function. On errors, it will be called until the
437
number of retries is exceeded.
440
abspath = self._remote_path(relpath)
441
mutter("FTP appe (try %d) to %s", retries, abspath)
442
ftp = self._get_FTP()
443
cmd = "APPE %s" % abspath
444
conn = ftp.transfercmd(cmd)
447
self._setmode(relpath, mode)
449
except ftplib.error_perm, e:
450
# Check whether the command is not supported (reply code 502)
451
if str(e).startswith('502 '):
452
warning("FTP server does not support file appending natively. "
453
"Performance may be severely degraded! (%s)", e)
454
self._has_append = False
455
self._fallback_append(relpath, text, mode)
457
self._translate_ftp_error(e, abspath, extra='error appending',
458
unknown_exc=errors.NoSuchFile)
459
except ftplib.error_temp, e:
460
if retries > _number_of_retries:
461
raise errors.TransportError(
462
"FTP temporary error during APPEND %s. Aborting."
463
% abspath, orig_error=e)
465
warning("FTP temporary error: %s. Retrying.", str(e))
467
self._try_append(relpath, text, mode, retries+1)
469
def _fallback_append(self, relpath, text, mode = None):
470
remote = self.get(relpath)
471
remote.seek(0, os.SEEK_END)
474
return self.put_file(relpath, remote, mode)
476
def _setmode(self, relpath, mode):
477
"""Set permissions on a path.
479
Only set permissions if the FTP server supports the 'SITE CHMOD'
484
mutter("FTP site chmod: setting permissions to %s on %s",
485
oct(mode), self._remote_path(relpath))
486
ftp = self._get_FTP()
487
cmd = "SITE CHMOD %s %s" % (oct(mode),
488
self._remote_path(relpath))
490
except ftplib.error_perm, e:
491
# Command probably not available on this server
492
warning("FTP Could not set permissions to %s on %s. %s",
493
oct(mode), self._remote_path(relpath), str(e))
495
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
496
# to copy something to another machine. And you may be able
497
# to give it its own address as the 'to' location.
498
# So implement a fancier 'copy()'
500
def rename(self, rel_from, rel_to):
501
abs_from = self._remote_path(rel_from)
502
abs_to = self._remote_path(rel_to)
503
mutter("FTP rename: %s => %s", abs_from, abs_to)
505
return self._rename(abs_from, abs_to, f)
507
def _rename(self, abs_from, abs_to, f):
509
f.rename(abs_from, abs_to)
510
except (ftplib.error_temp, ftplib.error_perm), e:
511
self._translate_ftp_error(e, abs_from,
512
': unable to rename to %r' % (abs_to))
295
raise TransportNotPossible('ftp does not support append()')
297
def copy(self, rel_from, rel_to):
298
"""Copy the item at rel_from to the location at rel_to"""
299
raise TransportNotPossible('ftp does not (yet) support copy()')
514
301
def move(self, rel_from, rel_to):
515
302
"""Move the item at rel_from to the location at rel_to"""
516
abs_from = self._remote_path(rel_from)
517
abs_to = self._remote_path(rel_to)
519
mutter("FTP mv: %s => %s", abs_from, abs_to)
304
mutter("FTP mv: %s => %s" % (self._abspath(rel_from),
305
self._abspath(rel_to)))
520
306
f = self._get_FTP()
521
self._rename_and_overwrite(abs_from, abs_to, f)
307
f.rename(self._abspath(rel_from), self._abspath(rel_to))
522
308
except ftplib.error_perm, e:
523
self._translate_ftp_error(e, abs_from,
524
extra='unable to rename to %r' % (rel_to,),
525
unknown_exc=errors.PathError)
527
def _rename_and_overwrite(self, abs_from, abs_to, f):
528
"""Do a fancy rename on the remote server.
530
Using the implementation provided by osutils.
532
osutils.fancy_rename(abs_from, abs_to,
533
rename_func=lambda p1, p2: self._rename(p1, p2, f),
534
unlink_func=lambda p: self._delete(p, f))
309
raise TransportError(orig_error=e)
536
311
def delete(self, relpath):
537
312
"""Delete the item at relpath"""
538
abspath = self._remote_path(relpath)
540
self._delete(abspath, f)
542
def _delete(self, abspath, f):
544
mutter("FTP rm: %s", abspath)
314
mutter("FTP rm: %s" % self._abspath(relpath))
316
f.delete(self._abspath(relpath))
546
317
except ftplib.error_perm, e:
547
self._translate_ftp_error(e, abspath, 'error deleting',
548
unknown_exc=errors.NoSuchFile)
550
def external_url(self):
551
"""See bzrlib.transport.Transport.external_url."""
552
# FTP URL's are externally usable.
318
raise TransportError(orig_error=e)
555
320
def listable(self):
556
321
"""See Transport.listable."""
added
removed
bzrlib/transport/ftp.py
