28
27
from cStringIO import StringIO
43
from bzrlib.trace import mutter, warning
44
from bzrlib.transport import (
45
AppendBasedFileStream,
48
register_urlparse_netloc_protocol,
53
register_urlparse_netloc_protocol('aftp')
56
class FtpPathError(errors.PathError):
57
"""FTP failed for path: %(path)s%(extra)s"""
34
from warnings import warn
37
from bzrlib.transport import Transport
38
from bzrlib.errors import (TransportNotPossible, TransportError,
39
NoSuchFile, FileExists)
40
from bzrlib.trace import mutter
60
43
class FtpStatResult(object):
62
def __init__(self, f, abspath):
44
def __init__(self, f, relpath):
64
self.st_size = f.size(abspath)
46
self.st_size = f.size(relpath)
65
47
self.st_mode = stat.S_IFREG
66
48
except ftplib.error_perm:
70
52
self.st_mode = stat.S_IFDIR
75
_number_of_retries = 2
76
_sleep_between_retries = 5
78
# FIXME: there are inconsistencies in the way temporary errors are
79
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
80
# be taken to analyze the implications for write operations (read operations
81
# are safe to retry). Overall even some read operations are never
82
# retried. --vila 20070720 (Bug #127164)
83
class FtpTransport(ConnectedTransport):
57
class FtpTransport(Transport):
84
58
"""This is the transport agent for ftp:// access."""
86
def __init__(self, base, _from_transport=None):
60
def __init__(self, base, _provided_instance=None):
87
61
"""Set the base path where files will be stored."""
88
if not (base.startswith('ftp://') or base.startswith('aftp://')):
89
raise ValueError(base)
90
super(FtpTransport, self).__init__(base,
91
_from_transport=_from_transport)
92
self._unqualified_scheme = 'ftp'
93
if self._scheme == 'aftp':
96
self.is_active = False
62
assert base.startswith('ftp://') or base.startswith('aftp://')
63
super(FtpTransport, self).__init__(base)
64
self.is_active = base.startswith('aftp://')
67
(self._proto, self._host,
68
self._path, self._parameters,
69
self._query, self._fragment) = urlparse.urlparse(self.base)
70
self._FTP_instance = _provided_instance
98
# Most modern FTP servers support the APPE command. If ours doesn't, we
99
# (re)set this flag accordingly later.
100
self._has_append = True
102
73
def _get_FTP(self):
103
74
"""Return the ftplib.FTP instance for this object."""
104
# Ensures that a connection is established
105
connection = self._get_connection()
106
if connection is None:
107
# First connection ever
108
connection, credentials = self._create_connection()
109
self._set_connection(connection, credentials)
112
connection_class = ftplib.FTP
114
def _create_connection(self, credentials=None):
115
"""Create a new connection with the provided credentials.
117
:param credentials: The credentials needed to establish the connection.
119
:return: The created connection and its associated credentials.
121
The input credentials are only the password as it may have been
122
entered interactively by the user and may be different from the one
123
provided in base url at transport creation time. The returned
124
credentials are username, password.
126
if credentials is None:
127
user, password = self._user, self._password
129
user, password = credentials
131
auth = config.AuthenticationConfig()
133
user = auth.get_user('ftp', self._host, port=self._port,
134
default=getpass.getuser())
135
mutter("Constructing FTP instance against %r" %
136
((self._host, self._port, user, '********',
75
if self._FTP_instance is not None:
76
return self._FTP_instance
139
connection = self.connection_class()
140
connection.connect(host=self._host, port=self._port)
141
self._login(connection, auth, user, password)
142
connection.set_pasv(not self.is_active)
143
# binary mode is the default
144
connection.voidcmd('TYPE I')
145
except socket.error, e:
146
raise errors.SocketConnectionError(self._host, self._port,
147
msg='Unable to connect to',
83
username, hostname = hostname.split("@", 1)
85
username, password = username.split(":", 1)
87
mutter("Constructing FTP instance")
88
self._FTP_instance = ftplib.FTP(hostname, username, password)
89
self._FTP_instance.set_pasv(not self.is_active)
90
return self._FTP_instance
149
91
except ftplib.error_perm, e:
150
raise errors.TransportError(msg="Error setting up connection:"
151
" %s" % str(e), orig_error=e)
152
return connection, (user, password)
154
def _login(self, connection, auth, user, password):
155
# '' is a valid password
156
if user and user != 'anonymous' and password is None:
157
password = auth.get_password('ftp', self._host,
158
user, port=self._port)
159
connection.login(user=user, passwd=password)
161
def _reconnect(self):
162
"""Create a new connection with the previously used credentials"""
163
credentials = self._get_credentials()
164
connection, credentials = self._create_connection(credentials)
165
self._set_connection(connection, credentials)
167
def _translate_ftp_error(self, err, path, extra=None,
168
unknown_exc=FtpPathError):
169
"""Try to translate an ftplib exception to a bzrlib exception.
171
:param err: The error to translate into a bzr error
172
:param path: The path which had problems
173
:param extra: Extra information which can be included
174
:param unknown_exc: If None, we will just raise the original exception
175
otherwise we raise unknown_exc(path, extra=extra)
177
# ftp error numbers are very generic, like "451: Requested action aborted,
178
# local error in processing" so unfortunately we have to match by
184
extra += ': ' + str(err)
185
if ('no such file' in s
186
or 'could not open' in s
187
or 'no such dir' in s
188
or 'could not create file' in s # vsftpd
189
or 'file doesn\'t exist' in s
190
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
191
or 'file/directory not found' in s # filezilla server
192
# Microsoft FTP-Service RNFR reply if file not found
193
or (s.startswith('550 ') and 'unable to rename to' in extra)
195
raise errors.NoSuchFile(path, extra=extra)
196
elif ('file exists' in s):
197
raise errors.FileExists(path, extra=extra)
198
elif ('not a directory' in s):
199
raise errors.PathError(path, extra=extra)
200
elif 'directory not empty' in s:
201
raise errors.DirectoryNotEmpty(path, extra=extra)
203
mutter('unable to understand error for path: %s: %s', path, err)
206
raise unknown_exc(path, extra=extra)
207
# TODO: jam 20060516 Consider re-raising the error wrapped in
208
# something like TransportError, but this loses the traceback
209
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
210
# to handle. Consider doing something like that here.
211
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
92
raise TransportError(msg="Error setting up connection: %s"
93
% str(e), orig_error=e)
95
def should_cache(self):
96
"""Return True if the data pulled across should be cached locally.
100
def clone(self, offset=None):
101
"""Return a new FtpTransport with root at self.base + offset.
105
return FtpTransport(self.base, self._FTP_instance)
107
return FtpTransport(self.abspath(offset), self._FTP_instance)
109
def _abspath(self, relpath):
110
assert isinstance(relpath, basestring)
111
relpath = urllib.unquote(relpath)
112
if isinstance(relpath, basestring):
113
relpath_parts = relpath.split('/')
115
# TODO: Don't call this with an array - no magic interfaces
116
relpath_parts = relpath[:]
117
if len(relpath_parts) > 1:
118
if relpath_parts[0] == '':
119
raise ValueError("path %r within branch %r seems to be absolute"
120
% (relpath, self._path))
121
basepath = self._path.split('/')
122
if len(basepath) > 0 and basepath[-1] == '':
123
basepath = basepath[:-1]
124
for p in relpath_parts:
126
if len(basepath) == 0:
127
# In most filesystems, a request for the parent
128
# of root, just returns root.
131
elif p == '.' or p == '':
135
# Possibly, we could use urlparse.urljoin() here, but
136
# I'm concerned about when it chooses to strip the last
137
# portion of the path, and when it doesn't.
138
return '/'.join(basepath)
140
def abspath(self, relpath):
141
"""Return the full url to the given relative path.
142
This can be supplied with a string or a list
144
path = self._abspath(relpath)
145
return urlparse.urlunparse((self._proto,
146
self._host, path, '', '', ''))
214
148
def has(self, relpath):
215
"""Does the target location exist?"""
216
# FIXME jam 20060516 We *do* ask about directories in the test suite
217
# We don't seem to in the actual codebase
218
# XXX: I assume we're never asked has(dirname) and thus I use
219
# the FTP size command and assume that if it doesn't raise,
221
abspath = self._remote_path(relpath)
149
"""Does the target location exist?
151
XXX: I assume we're never asked has(dirname) and thus I use
152
the FTP size command and assume that if it doesn't raise,
223
156
f = self._get_FTP()
224
mutter('FTP has check: %s => %s', relpath, abspath)
226
mutter("FTP has: %s", abspath)
157
s = f.size(self._abspath(relpath))
158
mutter("FTP has: %s" % self._abspath(relpath))
228
except ftplib.error_perm, e:
229
if ('is a directory' in str(e).lower()):
230
mutter("FTP has dir: %s: %s", abspath, e)
232
mutter("FTP has not: %s: %s", abspath, e)
160
except ftplib.error_perm:
161
mutter("FTP has not: %s" % self._abspath(relpath))
235
def get(self, relpath, decode=False, retries=0):
164
def get(self, relpath, decode=False):
236
165
"""Get the file at the given relative path.
238
167
:param relpath: The relative path to the file
239
:param retries: Number of retries after temporary failures so far
242
169
We're meant to return a file-like object which bzr will
243
170
then read from. For now we do this via the magic of StringIO
245
# TODO: decode should be deprecated
247
mutter("FTP get: %s", self._remote_path(relpath))
173
mutter("FTP get: %s" % self._abspath(relpath))
248
174
f = self._get_FTP()
250
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
176
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
253
179
except ftplib.error_perm, e:
254
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
255
except ftplib.error_temp, e:
256
if retries > _number_of_retries:
257
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
258
% self.abspath(relpath),
261
warning("FTP temporary error: %s. Retrying.", str(e))
263
return self.get(relpath, decode, retries+1)
265
if retries > _number_of_retries:
266
raise errors.TransportError("FTP control connection closed during GET %s."
267
% self.abspath(relpath),
270
warning("FTP control connection closed. Trying to reopen.")
271
time.sleep(_sleep_between_retries)
273
return self.get(relpath, decode, retries+1)
180
raise NoSuchFile(self.abspath(relpath), extra=extra)
275
def put_file(self, relpath, fp, mode=None, retries=0):
182
def put(self, relpath, fp, mode=None):
276
183
"""Copy the file-like or string object into the location.
278
185
:param relpath: Location to put the contents, relative to base.
279
:param fp: File-like or string object.
280
:param retries: Number of retries after temporary failures so far
283
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
186
:param f: File-like or string object.
187
TODO: jam 20051215 This should be an atomic put, not overwritting files in place
188
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
286
abspath = self._remote_path(relpath)
287
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
288
os.getpid(), random.randint(0,0x7FFFFFFF))
290
if getattr(fp, 'read', None) is None:
291
# hand in a string IO
295
# capture the byte count; .read() may be read only so
297
class byte_counter(object):
298
def __init__(self, fp):
300
self.counted_bytes = 0
301
def read(self, count):
302
result = self.fp.read(count)
303
self.counted_bytes += len(result)
305
fp = byte_counter(fp)
190
if not hasattr(fp, 'read'):
307
mutter("FTP put: %s", abspath)
193
mutter("FTP put: %s" % self._abspath(relpath))
308
194
f = self._get_FTP()
310
f.storbinary('STOR '+tmp_abspath, fp)
311
self._rename_and_overwrite(tmp_abspath, abspath, f)
312
self._setmode(relpath, mode)
313
if bytes is not None:
316
return fp.counted_bytes
317
except (ftplib.error_temp,EOFError), e:
318
warning("Failure during ftp PUT. Deleting temporary file.")
320
f.delete(tmp_abspath)
322
warning("Failed to delete temporary file on the"
323
" server.\nFile: %s", tmp_abspath)
195
f.storbinary('STOR '+self._abspath(relpath), fp, 8192)
326
196
except ftplib.error_perm, e:
327
self._translate_ftp_error(e, abspath, extra='could not store',
328
unknown_exc=errors.NoSuchFile)
329
except ftplib.error_temp, e:
330
if retries > _number_of_retries:
331
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
332
% self.abspath(relpath), orig_error=e)
334
warning("FTP temporary error: %s. Retrying.", str(e))
336
self.put_file(relpath, fp, mode, retries+1)
338
if retries > _number_of_retries:
339
raise errors.TransportError("FTP control connection closed during PUT %s."
340
% self.abspath(relpath), orig_error=e)
342
warning("FTP control connection closed. Trying to reopen.")
343
time.sleep(_sleep_between_retries)
345
self.put_file(relpath, fp, mode, retries+1)
197
raise TransportError(orig_error=e)
347
199
def mkdir(self, relpath, mode=None):
348
200
"""Create a directory at the given path."""
349
abspath = self._remote_path(relpath)
351
mutter("FTP mkd: %s", abspath)
354
self._setmode(relpath, mode)
355
except ftplib.error_perm, e:
356
self._translate_ftp_error(e, abspath,
357
unknown_exc=errors.FileExists)
359
def open_write_stream(self, relpath, mode=None):
360
"""See Transport.open_write_stream."""
361
self.put_bytes(relpath, "", mode)
362
result = AppendBasedFileStream(self, relpath)
363
_file_streams[self.abspath(relpath)] = result
366
def recommended_page_size(self):
367
"""See Transport.recommended_page_size().
369
For FTP we suggest a large page size to reduce the overhead
370
introduced by latency.
374
def rmdir(self, rel_path):
375
"""Delete the directory at rel_path"""
376
abspath = self._remote_path(rel_path)
378
mutter("FTP rmd: %s", abspath)
381
except ftplib.error_perm, e:
382
self._translate_ftp_error(e, abspath, unknown_exc=errors.PathError)
384
def append_file(self, relpath, f, mode=None):
202
mutter("FTP mkd: %s" % self._abspath(relpath))
205
f.mkd(self._abspath(relpath))
206
except ftplib.error_perm, e:
208
if 'File exists' in s:
209
raise FileExists(self.abspath(relpath), extra=s)
212
except ftplib.error_perm, e:
213
raise TransportError(orig_error=e)
215
def append(self, relpath, f):
385
216
"""Append the text in the file-like object into the final
389
abspath = self._remote_path(relpath)
390
if self.has(relpath):
391
ftp = self._get_FTP()
392
result = ftp.size(abspath)
397
mutter("FTP appe to %s", abspath)
398
self._try_append(relpath, text, mode)
400
self._fallback_append(relpath, text, mode)
404
def _try_append(self, relpath, text, mode=None, retries=0):
405
"""Try repeatedly to append the given text to the file at relpath.
407
This is a recursive function. On errors, it will be called until the
408
number of retries is exceeded.
411
abspath = self._remote_path(relpath)
412
mutter("FTP appe (try %d) to %s", retries, abspath)
413
ftp = self._get_FTP()
414
cmd = "APPE %s" % abspath
415
conn = ftp.transfercmd(cmd)
418
self._setmode(relpath, mode)
420
except ftplib.error_perm, e:
421
# Check whether the command is not supported (reply code 502)
422
if str(e).startswith('502 '):
423
warning("FTP server does not support file appending natively. "
424
"Performance may be severely degraded! (%s)", e)
425
self._has_append = False
426
self._fallback_append(relpath, text, mode)
428
self._translate_ftp_error(e, abspath, extra='error appending',
429
unknown_exc=errors.NoSuchFile)
430
except ftplib.error_temp, e:
431
if retries > _number_of_retries:
432
raise errors.TransportError(
433
"FTP temporary error during APPEND %s. Aborting."
434
% abspath, orig_error=e)
436
warning("FTP temporary error: %s. Retrying.", str(e))
438
self._try_append(relpath, text, mode, retries+1)
440
def _fallback_append(self, relpath, text, mode = None):
441
remote = self.get(relpath)
442
remote.seek(0, os.SEEK_END)
445
return self.put_file(relpath, remote, mode)
447
def _setmode(self, relpath, mode):
448
"""Set permissions on a path.
450
Only set permissions if the FTP server supports the 'SITE CHMOD'
455
mutter("FTP site chmod: setting permissions to %s on %s",
456
oct(mode), self._remote_path(relpath))
457
ftp = self._get_FTP()
458
cmd = "SITE CHMOD %s %s" % (oct(mode),
459
self._remote_path(relpath))
461
except ftplib.error_perm, e:
462
# Command probably not available on this server
463
warning("FTP Could not set permissions to %s on %s. %s",
464
oct(mode), self._remote_path(relpath), str(e))
466
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
467
# to copy something to another machine. And you may be able
468
# to give it its own address as the 'to' location.
469
# So implement a fancier 'copy()'
471
def rename(self, rel_from, rel_to):
472
abs_from = self._remote_path(rel_from)
473
abs_to = self._remote_path(rel_to)
474
mutter("FTP rename: %s => %s", abs_from, abs_to)
476
return self._rename(abs_from, abs_to, f)
478
def _rename(self, abs_from, abs_to, f):
480
f.rename(abs_from, abs_to)
481
except (ftplib.error_temp, ftplib.error_perm), e:
482
self._translate_ftp_error(e, abs_from,
483
': unable to rename to %r' % (abs_to))
219
raise TransportNotPossible('ftp does not support append()')
221
def copy(self, rel_from, rel_to):
222
"""Copy the item at rel_from to the location at rel_to"""
223
raise TransportNotPossible('ftp does not (yet) support copy()')
485
225
def move(self, rel_from, rel_to):
486
226
"""Move the item at rel_from to the location at rel_to"""
487
abs_from = self._remote_path(rel_from)
488
abs_to = self._remote_path(rel_to)
490
mutter("FTP mv: %s => %s", abs_from, abs_to)
228
mutter("FTP mv: %s => %s" % (self._abspath(rel_from),
229
self._abspath(rel_to)))
491
230
f = self._get_FTP()
492
self._rename_and_overwrite(abs_from, abs_to, f)
231
f.rename(self._abspath(rel_from), self._abspath(rel_to))
493
232
except ftplib.error_perm, e:
494
self._translate_ftp_error(e, abs_from,
495
extra='unable to rename to %r' % (rel_to,),
496
unknown_exc=errors.PathError)
498
def _rename_and_overwrite(self, abs_from, abs_to, f):
499
"""Do a fancy rename on the remote server.
501
Using the implementation provided by osutils.
503
osutils.fancy_rename(abs_from, abs_to,
504
rename_func=lambda p1, p2: self._rename(p1, p2, f),
505
unlink_func=lambda p: self._delete(p, f))
233
raise TransportError(orig_error=e)
507
235
def delete(self, relpath):
508
236
"""Delete the item at relpath"""
509
abspath = self._remote_path(relpath)
511
self._delete(abspath, f)
513
def _delete(self, abspath, f):
515
mutter("FTP rm: %s", abspath)
238
mutter("FTP rm: %s" % self._abspath(relpath))
240
f.delete(self._abspath(relpath))
517
241
except ftplib.error_perm, e:
518
self._translate_ftp_error(e, abspath, 'error deleting',
519
unknown_exc=errors.NoSuchFile)
521
def external_url(self):
522
"""See bzrlib.transport.Transport.external_url."""
523
# FTP URL's are externally usable.
242
raise TransportError(orig_error=e)
526
244
def listable(self):
527
245
"""See Transport.listable."""
added
removed
bzrlib/transport/ftp.py
