25
24
active, in which case aftp:// will be your friend.
27
from bzrlib.transport import Transport
29
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
30
NonRelativePath, TransportError, ConnectionError)
28
33
from cStringIO import StringIO
43
from bzrlib.trace import mutter, warning
44
from bzrlib.transport import (
45
AppendBasedFileStream,
48
register_urlparse_netloc_protocol,
53
register_urlparse_netloc_protocol('aftp')
56
class FtpPathError(errors.PathError):
57
"""FTP failed for path: %(path)s%(extra)s"""
39
from bzrlib.errors import BzrError, BzrCheckError
40
from bzrlib.branch import Branch
41
from bzrlib.trace import mutter
44
class FtpTransportError(TransportError):
60
48
class FtpStatResult(object):
62
def __init__(self, f, abspath):
49
def __init__(self, f, relpath):
64
self.st_size = f.size(abspath)
51
self.st_size = f.size(relpath)
65
52
self.st_mode = stat.S_IFREG
66
53
except ftplib.error_perm:
70
57
self.st_mode = stat.S_IFDIR
75
_number_of_retries = 2
76
_sleep_between_retries = 5
78
# FIXME: there are inconsistencies in the way temporary errors are
79
# handled. Sometimes we reconnect, sometimes we raise an exception. Care should
80
# be taken to analyze the implications for write operations (read operations
81
# are safe to retry). Overall even some read operations are never
82
# retried. --vila 20070720 (Bug #127164)
83
class FtpTransport(ConnectedTransport):
62
class FtpTransport(Transport):
84
63
"""This is the transport agent for ftp:// access."""
86
def __init__(self, base, _from_transport=None):
65
def __init__(self, base, _provided_instance=None):
87
66
"""Set the base path where files will be stored."""
88
if not (base.startswith('ftp://') or base.startswith('aftp://')):
89
raise ValueError(base)
90
super(FtpTransport, self).__init__(base,
91
_from_transport=_from_transport)
92
self._unqualified_scheme = 'ftp'
93
if self._scheme == 'aftp':
96
self.is_active = False
67
assert base.startswith('ftp://') or base.startswith('aftp://')
68
super(FtpTransport, self).__init__(base)
69
self.is_active = base.startswith('aftp://')
72
(self._proto, self._host,
73
self._path, self._parameters,
74
self._query, self._fragment) = urlparse.urlparse(self.base)
75
self._FTP_instance = _provided_instance
98
# Most modern FTP servers support the APPE command. If ours doesn't, we
99
# (re)set this flag accordingly later.
100
self._has_append = True
102
78
def _get_FTP(self):
103
79
"""Return the ftplib.FTP instance for this object."""
104
# Ensures that a connection is established
105
connection = self._get_connection()
106
if connection is None:
107
# First connection ever
108
connection, credentials = self._create_connection()
109
self._set_connection(connection, credentials)
112
connection_class = ftplib.FTP
114
def _create_connection(self, credentials=None):
115
"""Create a new connection with the provided credentials.
117
:param credentials: The credentials needed to establish the connection.
119
:return: The created connection and its associated credentials.
121
The input credentials are only the password as it may have been
122
entered interactively by the user and may be different from the one
123
provided in base url at transport creation time. The returned
124
credentials are username, password.
126
if credentials is None:
127
user, password = self._user, self._password
129
user, password = credentials
131
auth = config.AuthenticationConfig()
133
user = auth.get_user('ftp', self._host, port=self._port,
134
default=getpass.getuser())
135
mutter("Constructing FTP instance against %r" %
136
((self._host, self._port, user, '********',
80
if self._FTP_instance is not None:
81
return self._FTP_instance
139
connection = self.connection_class()
140
connection.connect(host=self._host, port=self._port)
141
self._login(connection, auth, user, password)
142
connection.set_pasv(not self.is_active)
143
# binary mode is the default
144
connection.voidcmd('TYPE I')
145
except socket.error, e:
146
raise errors.SocketConnectionError(self._host, self._port,
147
msg='Unable to connect to',
88
username, hostname = hostname.split("@", 1)
90
username, password = username.split(":", 1)
92
mutter("Constructing FTP instance")
93
self._FTP_instance = ftplib.FTP(hostname, username, password)
94
self._FTP_instance.set_pasv(not self.is_active)
95
return self._FTP_instance
149
96
except ftplib.error_perm, e:
150
raise errors.TransportError(msg="Error setting up connection:"
151
" %s" % str(e), orig_error=e)
152
return connection, (user, password)
154
def _login(self, connection, auth, user, password):
155
# '' is a valid password
156
if user and user != 'anonymous' and password is None:
157
password = auth.get_password('ftp', self._host,
158
user, port=self._port)
159
connection.login(user=user, passwd=password)
161
def _reconnect(self):
162
"""Create a new connection with the previously used credentials"""
163
credentials = self._get_credentials()
164
connection, credentials = self._create_connection(credentials)
165
self._set_connection(connection, credentials)
167
def _translate_ftp_error(self, err, path, extra=None,
168
unknown_exc=FtpPathError):
169
"""Try to translate an ftplib exception to a bzrlib exception.
171
:param err: The error to translate into a bzr error
172
:param path: The path which had problems
173
:param extra: Extra information which can be included
174
:param unknown_exc: If None, we will just raise the original exception
175
otherwise we raise unknown_exc(path, extra=extra)
177
# ftp error numbers are very generic, like "451: Requested action aborted,
178
# local error in processing" so unfortunately we have to match by
184
extra += ': ' + str(err)
185
if ('no such file' in s
186
or 'could not open' in s
187
or 'no such dir' in s
188
or 'could not create file' in s # vsftpd
189
or 'file doesn\'t exist' in s
190
or 'rnfr command failed.' in s # vsftpd RNFR reply if file not found
191
or 'file/directory not found' in s # filezilla server
192
# Microsoft FTP-Service RNFR reply if file not found
193
or (s.startswith('550 ') and 'unable to rename to' in extra)
195
raise errors.NoSuchFile(path, extra=extra)
196
elif ('file exists' in s):
197
raise errors.FileExists(path, extra=extra)
198
elif ('not a directory' in s):
199
raise errors.PathError(path, extra=extra)
200
elif 'directory not empty' in s:
201
raise errors.DirectoryNotEmpty(path, extra=extra)
203
mutter('unable to understand error for path: %s: %s', path, err)
206
raise unknown_exc(path, extra=extra)
207
# TODO: jam 20060516 Consider re-raising the error wrapped in
208
# something like TransportError, but this loses the traceback
209
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
210
# to handle. Consider doing something like that here.
211
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
97
raise FtpTransportError(msg="Error setting up connection: %s"
98
% str(e), orig_error=e)
100
def should_cache(self):
101
"""Return True if the data pulled across should be cached locally.
105
def clone(self, offset=None):
106
"""Return a new FtpTransport with root at self.base + offset.
110
return FtpTransport(self.base, self._FTP_instance)
112
return FtpTransport(self.abspath(offset), self._FTP_instance)
114
def _abspath(self, relpath):
115
assert isinstance(relpath, basestring)
116
relpath = urllib.unquote(relpath)
117
if isinstance(relpath, basestring):
118
relpath_parts = relpath.split('/')
120
# TODO: Don't call this with an array - no magic interfaces
121
relpath_parts = relpath[:]
122
if len(relpath_parts) > 1:
123
if relpath_parts[0] == '':
124
raise ValueError("path %r within branch %r seems to be absolute"
125
% (relpath, self._path))
126
basepath = self._path.split('/')
127
if len(basepath) > 0 and basepath[-1] == '':
128
basepath = basepath[:-1]
129
for p in relpath_parts:
131
if len(basepath) == 0:
132
# In most filesystems, a request for the parent
133
# of root, just returns root.
136
elif p == '.' or p == '':
140
# Possibly, we could use urlparse.urljoin() here, but
141
# I'm concerned about when it chooses to strip the last
142
# portion of the path, and when it doesn't.
143
return '/'.join(basepath)
145
def abspath(self, relpath):
146
"""Return the full url to the given relative path.
147
This can be supplied with a string or a list
149
path = self._abspath(relpath)
150
return urlparse.urlunparse((self._proto,
151
self._host, path, '', '', ''))
214
153
def has(self, relpath):
215
"""Does the target location exist?"""
216
# FIXME jam 20060516 We *do* ask about directories in the test suite
217
# We don't seem to in the actual codebase
218
# XXX: I assume we're never asked has(dirname) and thus I use
219
# the FTP size command and assume that if it doesn't raise,
221
abspath = self._remote_path(relpath)
154
"""Does the target location exist?
156
XXX: I assume we're never asked has(dirname) and thus I use
157
the FTP size command and assume that if it doesn't raise,
223
161
f = self._get_FTP()
224
mutter('FTP has check: %s => %s', relpath, abspath)
226
mutter("FTP has: %s", abspath)
162
s = f.size(self._abspath(relpath))
163
mutter("FTP has: %s" % self._abspath(relpath))
228
except ftplib.error_perm, e:
229
if ('is a directory' in str(e).lower()):
230
mutter("FTP has dir: %s: %s", abspath, e)
232
mutter("FTP has not: %s: %s", abspath, e)
165
except ftplib.error_perm:
166
mutter("FTP has not: %s" % self._abspath(relpath))
235
def get(self, relpath, decode=False, retries=0):
169
def get(self, relpath, decode=False):
236
170
"""Get the file at the given relative path.
238
172
:param relpath: The relative path to the file
239
:param retries: Number of retries after temporary failures so far
242
174
We're meant to return a file-like object which bzr will
243
175
then read from. For now we do this via the magic of StringIO
245
# TODO: decode should be deprecated
247
mutter("FTP get: %s", self._remote_path(relpath))
178
mutter("FTP get: %s" % self._abspath(relpath))
248
179
f = self._get_FTP()
250
f.retrbinary('RETR '+self._remote_path(relpath), ret.write, 8192)
181
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
253
184
except ftplib.error_perm, e:
254
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
255
except ftplib.error_temp, e:
256
if retries > _number_of_retries:
257
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
258
% self.abspath(relpath),
261
warning("FTP temporary error: %s. Retrying.", str(e))
263
return self.get(relpath, decode, retries+1)
265
if retries > _number_of_retries:
266
raise errors.TransportError("FTP control connection closed during GET %s."
267
% self.abspath(relpath),
270
warning("FTP control connection closed. Trying to reopen.")
271
time.sleep(_sleep_between_retries)
273
return self.get(relpath, decode, retries+1)
185
raise NoSuchFile(msg="Error retrieving %s: %s"
186
% (self.abspath(relpath), str(e)),
275
def put_file(self, relpath, fp, mode=None, retries=0):
189
def put(self, relpath, fp):
276
190
"""Copy the file-like or string object into the location.
278
192
:param relpath: Location to put the contents, relative to base.
279
:param fp: File-like or string object.
280
:param retries: Number of retries after temporary failures so far
193
:param f: File-like or string object.
195
if not hasattr(fp, 'read'):
198
mutter("FTP put: %s" % self._abspath(relpath))
200
f.storbinary('STOR '+self._abspath(relpath), fp, 8192)
201
except ftplib.error_perm, e:
202
raise FtpTransportError(orig_error=e)
283
TODO: jam 20051215 ftp as a protocol seems to support chmod, but
286
abspath = self._remote_path(relpath)
287
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
288
os.getpid(), random.randint(0,0x7FFFFFFF))
290
if getattr(fp, 'read', None) is None:
291
# hand in a string IO
295
# capture the byte count; .read() may be read only so
297
class byte_counter(object):
298
def __init__(self, fp):
300
self.counted_bytes = 0
301
def read(self, count):
302
result = self.fp.read(count)
303
self.counted_bytes += len(result)
305
fp = byte_counter(fp)
204
def mkdir(self, relpath):
205
"""Create a directory at the given path."""
307
mutter("FTP put: %s", abspath)
207
mutter("FTP mkd: %s" % self._abspath(relpath))
308
208
f = self._get_FTP()
310
f.storbinary('STOR '+tmp_abspath, fp)
311
self._rename_and_overwrite(tmp_abspath, abspath, f)
312
self._setmode(relpath, mode)
313
if bytes is not None:
210
f.mkd(self._abspath(relpath))
211
except ftplib.error_perm, e:
213
if 'File exists' in s:
214
# Swallow attempts to mkdir something which is already
215
# present. Hopefully this will shush some errors.
316
return fp.counted_bytes
317
except (ftplib.error_temp,EOFError), e:
318
warning("Failure during ftp PUT. Deleting temporary file.")
320
f.delete(tmp_abspath)
322
warning("Failed to delete temporary file on the"
323
" server.\nFile: %s", tmp_abspath)
326
except ftplib.error_perm, e:
327
self._translate_ftp_error(e, abspath, extra='could not store',
328
unknown_exc=errors.NoSuchFile)
329
except ftplib.error_temp, e:
330
if retries > _number_of_retries:
331
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
332
% self.abspath(relpath), orig_error=e)
334
warning("FTP temporary error: %s. Retrying.", str(e))
336
self.put_file(relpath, fp, mode, retries+1)
338
if retries > _number_of_retries:
339
raise errors.TransportError("FTP control connection closed during PUT %s."
340
% self.abspath(relpath), orig_error=e)
342
warning("FTP control connection closed. Trying to reopen.")
343
time.sleep(_sleep_between_retries)
345
self.put_file(relpath, fp, mode, retries+1)
347
def mkdir(self, relpath, mode=None):
348
"""Create a directory at the given path."""
349
abspath = self._remote_path(relpath)
351
mutter("FTP mkd: %s", abspath)
354
self._setmode(relpath, mode)
355
except ftplib.error_perm, e:
356
self._translate_ftp_error(e, abspath,
357
unknown_exc=errors.FileExists)
359
def open_write_stream(self, relpath, mode=None):
360
"""See Transport.open_write_stream."""
361
self.put_bytes(relpath, "", mode)
362
result = AppendBasedFileStream(self, relpath)
363
_file_streams[self.abspath(relpath)] = result
366
def recommended_page_size(self):
367
"""See Transport.recommended_page_size().
369
For FTP we suggest a large page size to reduce the overhead
370
introduced by latency.
374
def rmdir(self, rel_path):
375
"""Delete the directory at rel_path"""
376
abspath = self._remote_path(rel_path)
378
mutter("FTP rmd: %s", abspath)
381
except ftplib.error_perm, e:
382
self._translate_ftp_error(e, abspath, unknown_exc=errors.PathError)
384
def append_file(self, relpath, f, mode=None):
219
except ftplib.error_perm, e:
220
raise FtpTransportError(orig_error=e)
222
def append(self, relpath, f):
385
223
"""Append the text in the file-like object into the final
389
abspath = self._remote_path(relpath)
390
if self.has(relpath):
391
ftp = self._get_FTP()
392
result = ftp.size(abspath)
397
mutter("FTP appe to %s", abspath)
398
self._try_append(relpath, text, mode)
400
self._fallback_append(relpath, text, mode)
404
def _try_append(self, relpath, text, mode=None, retries=0):
405
"""Try repeatedly to append the given text to the file at relpath.
407
This is a recursive function. On errors, it will be called until the
408
number of retries is exceeded.
411
abspath = self._remote_path(relpath)
412
mutter("FTP appe (try %d) to %s", retries, abspath)
413
ftp = self._get_FTP()
414
cmd = "APPE %s" % abspath
415
conn = ftp.transfercmd(cmd)
418
self._setmode(relpath, mode)
420
except ftplib.error_perm, e:
421
# Check whether the command is not supported (reply code 502)
422
if str(e).startswith('502 '):
423
warning("FTP server does not support file appending natively. "
424
"Performance may be severely degraded! (%s)", e)
425
self._has_append = False
426
self._fallback_append(relpath, text, mode)
428
self._translate_ftp_error(e, abspath, extra='error appending',
429
unknown_exc=errors.NoSuchFile)
430
except ftplib.error_temp, e:
431
if retries > _number_of_retries:
432
raise errors.TransportError(
433
"FTP temporary error during APPEND %s. Aborting."
434
% abspath, orig_error=e)
436
warning("FTP temporary error: %s. Retrying.", str(e))
438
self._try_append(relpath, text, mode, retries+1)
440
def _fallback_append(self, relpath, text, mode = None):
441
remote = self.get(relpath)
442
remote.seek(0, os.SEEK_END)
445
return self.put_file(relpath, remote, mode)
447
def _setmode(self, relpath, mode):
448
"""Set permissions on a path.
450
Only set permissions if the FTP server supports the 'SITE CHMOD'
455
mutter("FTP site chmod: setting permissions to %s on %s",
456
oct(mode), self._remote_path(relpath))
457
ftp = self._get_FTP()
458
cmd = "SITE CHMOD %s %s" % (oct(mode),
459
self._remote_path(relpath))
461
except ftplib.error_perm, e:
462
# Command probably not available on this server
463
warning("FTP Could not set permissions to %s on %s. %s",
464
oct(mode), self._remote_path(relpath), str(e))
466
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
467
# to copy something to another machine. And you may be able
468
# to give it its own address as the 'to' location.
469
# So implement a fancier 'copy()'
471
def rename(self, rel_from, rel_to):
472
abs_from = self._remote_path(rel_from)
473
abs_to = self._remote_path(rel_to)
474
mutter("FTP rename: %s => %s", abs_from, abs_to)
476
return self._rename(abs_from, abs_to, f)
478
def _rename(self, abs_from, abs_to, f):
480
f.rename(abs_from, abs_to)
481
except (ftplib.error_temp, ftplib.error_perm), e:
482
self._translate_ftp_error(e, abs_from,
483
': unable to rename to %r' % (abs_to))
226
raise TransportNotPossible('ftp does not support append()')
228
def copy(self, rel_from, rel_to):
229
"""Copy the item at rel_from to the location at rel_to"""
230
raise TransportNotPossible('ftp does not (yet) support copy()')
485
232
def move(self, rel_from, rel_to):
486
233
"""Move the item at rel_from to the location at rel_to"""
487
abs_from = self._remote_path(rel_from)
488
abs_to = self._remote_path(rel_to)
490
mutter("FTP mv: %s => %s", abs_from, abs_to)
235
mutter("FTP mv: %s => %s" % (self._abspath(rel_from),
236
self._abspath(rel_to)))
491
237
f = self._get_FTP()
492
self._rename_and_overwrite(abs_from, abs_to, f)
238
f.rename(self._abspath(rel_from), self._abspath(rel_to))
493
239
except ftplib.error_perm, e:
494
self._translate_ftp_error(e, abs_from,
495
extra='unable to rename to %r' % (rel_to,),
496
unknown_exc=errors.PathError)
498
def _rename_and_overwrite(self, abs_from, abs_to, f):
499
"""Do a fancy rename on the remote server.
501
Using the implementation provided by osutils.
503
osutils.fancy_rename(abs_from, abs_to,
504
rename_func=lambda p1, p2: self._rename(p1, p2, f),
505
unlink_func=lambda p: self._delete(p, f))
240
raise FtpTransportError(orig_error=e)
507
242
def delete(self, relpath):
508
243
"""Delete the item at relpath"""
509
abspath = self._remote_path(relpath)
511
self._delete(abspath, f)
513
def _delete(self, abspath, f):
515
mutter("FTP rm: %s", abspath)
245
mutter("FTP rm: %s" % self._abspath(relpath))
247
f.delete(self._abspath(relpath))
517
248
except ftplib.error_perm, e:
518
self._translate_ftp_error(e, abspath, 'error deleting',
519
unknown_exc=errors.NoSuchFile)
521
def external_url(self):
522
"""See bzrlib.transport.Transport.external_url."""
523
# FTP URL's are externally usable.
249
raise FtpTransportError(orig_error=e)
526
251
def listable(self):
527
252
"""See Transport.listable."""
added
removed
bzrlib/transport/ftp.py
