27
27
from cStringIO import StringIO
36
38
from warnings import warn
39
from bzrlib.transport import Transport
40
from bzrlib.errors import (TransportNotPossible, TransportError,
41
NoSuchFile, FileExists, DirectoryNotEmpty)
40
from bzrlib.transport import (
45
import bzrlib.errors as errors
42
46
from bzrlib.trace import mutter, warning
52
class FtpPathError(errors.PathError):
53
"""FTP failed for path: %(path)s%(extra)s"""
46
def _find_FTP(hostname, username, password, is_active):
57
def _find_FTP(hostname, port, username, password, is_active):
47
58
"""Find an ftplib.FTP instance attached to this triplet."""
48
key = "%s|%s|%s|%s" % (hostname, username, password, is_active)
59
key = (hostname, port, username, password, is_active)
60
alt_key = (hostname, port, username, '********', is_active)
49
61
if key not in _FTP_cache:
50
mutter("Constructing FTP instance against %r" % key)
51
_FTP_cache[key] = ftplib.FTP(hostname, username, password)
52
_FTP_cache[key].set_pasv(not is_active)
62
mutter("Constructing FTP instance against %r" % (alt_key,))
65
conn.connect(host=hostname, port=port)
66
if username and username != 'anonymous' and not password:
67
password = bzrlib.ui.ui_factory.get_password(
68
prompt='FTP %(user)s@%(host)s password',
69
user=username, host=hostname)
70
conn.login(user=username, passwd=password)
71
conn.set_pasv(not is_active)
73
_FTP_cache[key] = conn
53
75
return _FTP_cache[key]
56
class FtpTransportError(TransportError):
60
78
class FtpStatResult(object):
61
79
def __init__(self, f, relpath):
80
98
def __init__(self, base, _provided_instance=None):
81
99
"""Set the base path where files will be stored."""
82
100
assert base.startswith('ftp://') or base.startswith('aftp://')
83
super(FtpTransport, self).__init__(base)
84
102
self.is_active = base.startswith('aftp://')
85
103
if self.is_active:
104
# urlparse won't handle aftp://
87
(self._proto, self._host,
88
self._path, self._parameters,
89
self._query, self._fragment) = urlparse.urlparse(self.base)
106
if not base.endswith('/'):
108
(self._proto, self._username,
109
self._password, self._host,
110
self._port, self._path) = split_url(base)
111
base = self._unparse_url()
113
super(FtpTransport, self).__init__(base)
90
114
self._FTP_instance = _provided_instance
116
def _unparse_url(self, path=None):
119
path = urllib.quote(path)
120
netloc = urllib.quote(self._host)
121
if self._username is not None:
122
netloc = '%s@%s' % (urllib.quote(self._username), netloc)
123
if self._port is not None:
124
netloc = '%s:%d' % (netloc, self._port)
125
return urlparse.urlunparse(('ftp', netloc, path, '', '', ''))
92
127
def _get_FTP(self):
93
128
"""Return the ftplib.FTP instance for this object."""
94
129
if self._FTP_instance is not None:
95
130
return self._FTP_instance
100
hostname = self._host
102
username, hostname = hostname.split("@", 1)
104
username, password = username.split(":", 1)
106
self._FTP_instance = _find_FTP(hostname, username, password,
133
self._FTP_instance = _find_FTP(self._host, self._port,
134
self._username, self._password,
108
136
return self._FTP_instance
109
137
except ftplib.error_perm, e:
110
raise TransportError(msg="Error setting up connection: %s"
138
raise errors.TransportError(msg="Error setting up connection: %s"
111
139
% str(e), orig_error=e)
141
def _translate_perm_error(self, err, path, extra=None, unknown_exc=FtpPathError):
142
"""Try to translate an ftplib.error_perm exception.
144
:param err: The error to translate into a bzr error
145
:param path: The path which had problems
146
:param extra: Extra information which can be included
147
:param unknown_exc: If None, we will just raise the original exception
148
otherwise we raise unknown_exc(path, extra=extra)
154
extra += ': ' + str(err)
155
if ('no such file' in s
156
or 'could not open' in s
157
or 'no such dir' in s
159
raise errors.NoSuchFile(path, extra=extra)
160
if ('file exists' in s):
161
raise errors.FileExists(path, extra=extra)
162
if ('not a directory' in s):
163
raise errors.PathError(path, extra=extra)
165
mutter('unable to understand error for path: %s: %s', path, err)
168
raise unknown_exc(path, extra=extra)
169
# TODO: jam 20060516 Consider re-raising the error wrapped in
170
# something like TransportError, but this loses the traceback
171
# Also, 'sftp' has a generic 'Failure' mode, which we use failure_exc
172
# to handle. Consider doing something like that here.
173
#raise TransportError(msg='Error for path: %s' % (path,), orig_error=e)
113
176
def should_cache(self):
114
177
"""Return True if the data pulled across should be cached locally.
153
212
# Possibly, we could use urlparse.urljoin() here, but
154
213
# I'm concerned about when it chooses to strip the last
155
214
# portion of the path, and when it doesn't.
156
return '/'.join(basepath)
215
return '/'.join(basepath) or '/'
158
217
def abspath(self, relpath):
159
218
"""Return the full url to the given relative path.
160
219
This can be supplied with a string or a list
162
221
path = self._abspath(relpath)
163
return urlparse.urlunparse((self._proto,
164
self._host, path, '', '', ''))
222
return self._unparse_url(path)
166
224
def has(self, relpath):
167
"""Does the target location exist?
169
XXX: I assume we're never asked has(dirname) and thus I use
170
the FTP size command and assume that if it doesn't raise,
225
"""Does the target location exist?"""
226
# FIXME jam 20060516 We *do* ask about directories in the test suite
227
# We don't seem to in the actual codebase
228
# XXX: I assume we're never asked has(dirname) and thus I use
229
# the FTP size command and assume that if it doesn't raise,
231
abspath = self._abspath(relpath)
174
233
f = self._get_FTP()
175
s = f.size(self._abspath(relpath))
176
mutter("FTP has: %s" % self._abspath(relpath))
234
mutter('FTP has check: %s => %s', relpath, abspath)
236
mutter("FTP has: %s", abspath)
178
except ftplib.error_perm:
179
mutter("FTP has not: %s" % self._abspath(relpath))
238
except ftplib.error_perm, e:
239
if ('is a directory' in str(e).lower()):
240
mutter("FTP has dir: %s: %s", abspath, e)
242
mutter("FTP has not: %s: %s", abspath, e)
182
245
def get(self, relpath, decode=False, retries=0):
192
255
# TODO: decode should be deprecated
194
mutter("FTP get: %s" % self._abspath(relpath))
257
mutter("FTP get: %s", self._abspath(relpath))
195
258
f = self._get_FTP()
197
260
f.retrbinary('RETR '+self._abspath(relpath), ret.write, 8192)
200
263
except ftplib.error_perm, e:
201
raise NoSuchFile(self.abspath(relpath), extra=str(e))
264
raise errors.NoSuchFile(self.abspath(relpath), extra=str(e))
202
265
except ftplib.error_temp, e:
203
266
if retries > _number_of_retries:
204
raise TransportError(msg="FTP temporary error during GET %s. Aborting."
267
raise errors.TransportError(msg="FTP temporary error during GET %s. Aborting."
205
268
% self.abspath(relpath),
208
warning("FTP temporary error: %s. Retrying." % str(e))
271
warning("FTP temporary error: %s. Retrying.", str(e))
209
272
self._FTP_instance = None
210
273
return self.get(relpath, decode, retries+1)
211
274
except EOFError, e:
212
275
if retries > _number_of_retries:
213
raise TransportError("FTP control connection closed during GET %s."
276
raise errors.TransportError("FTP control connection closed during GET %s."
214
277
% self.abspath(relpath),
230
293
TODO: jam 20051215 ftp as a protocol seems to support chmod, but ftplib does not
232
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (self._abspath(relpath), time.time(),
295
abspath = self._abspath(relpath)
296
tmp_abspath = '%s.tmp.%.9f.%d.%d' % (abspath, time.time(),
233
297
os.getpid(), random.randint(0,0x7FFFFFFF))
234
298
if not hasattr(fp, 'read'):
235
299
fp = StringIO(fp)
237
mutter("FTP put: %s" % self._abspath(relpath))
301
mutter("FTP put: %s", abspath)
238
302
f = self._get_FTP()
240
304
f.storbinary('STOR '+tmp_abspath, fp)
241
f.rename(tmp_abspath, self._abspath(relpath))
305
f.rename(tmp_abspath, abspath)
242
306
except (ftplib.error_temp,EOFError), e:
243
307
warning("Failure during ftp PUT. Deleting temporary file.")
245
309
f.delete(tmp_abspath)
247
warning("Failed to delete temporary file on the server.\nFile: %s"
311
warning("Failed to delete temporary file on the"
312
" server.\nFile: %s", tmp_abspath)
251
315
except ftplib.error_perm, e:
252
if "no such file" in str(e).lower():
253
raise NoSuchFile("Error storing %s: %s"
254
% (self.abspath(relpath), str(e)), extra=e)
256
raise FtpTransportError(orig_error=e)
316
self._translate_perm_error(e, abspath, extra='could not store')
257
317
except ftplib.error_temp, e:
258
318
if retries > _number_of_retries:
259
raise TransportError("FTP temporary error during PUT %s. Aborting."
319
raise errors.TransportError("FTP temporary error during PUT %s. Aborting."
260
320
% self.abspath(relpath), orig_error=e)
262
warning("FTP temporary error: %s. Retrying." % str(e))
322
warning("FTP temporary error: %s. Retrying.", str(e))
263
323
self._FTP_instance = None
264
324
self.put(relpath, fp, mode, retries+1)
266
326
if retries > _number_of_retries:
267
raise TransportError("FTP control connection closed during PUT %s."
327
raise errors.TransportError("FTP control connection closed during PUT %s."
268
328
% self.abspath(relpath), orig_error=e)
270
330
warning("FTP control connection closed. Trying to reopen.")
272
332
self._FTP_instance = None
273
333
self.put(relpath, fp, mode, retries+1)
276
335
def mkdir(self, relpath, mode=None):
277
336
"""Create a directory at the given path."""
337
abspath = self._abspath(relpath)
279
mutter("FTP mkd: %s" % self._abspath(relpath))
339
mutter("FTP mkd: %s", abspath)
280
340
f = self._get_FTP()
282
f.mkd(self._abspath(relpath))
283
except ftplib.error_perm, e:
285
if 'File exists' in s:
286
raise FileExists(self.abspath(relpath), extra=s)
289
342
except ftplib.error_perm, e:
290
raise TransportError(orig_error=e)
343
self._translate_perm_error(e, abspath,
344
unknown_exc=errors.FileExists)
292
346
def rmdir(self, rel_path):
293
347
"""Delete the directory at rel_path"""
348
abspath = self._abspath(rel_path)
295
mutter("FTP rmd: %s" % self._abspath(rel_path))
350
mutter("FTP rmd: %s", abspath)
297
351
f = self._get_FTP()
298
f.rmd(self._abspath(rel_path))
299
353
except ftplib.error_perm, e:
300
if str(e).endswith("Directory not empty"):
301
raise DirectoryNotEmpty(self._abspath(rel_path), extra=str(e))
303
raise TransportError(msg="Cannot remove directory at %s" % \
304
self._abspath(rel_path), extra=str(e))
354
self._translate_perm_error(e, abspath, unknown_exc=errors.PathError)
306
def append(self, relpath, f):
356
def append(self, relpath, f, mode=None):
307
357
"""Append the text in the file-like object into the final
310
raise TransportNotPossible('ftp does not support append()')
312
def copy(self, rel_from, rel_to):
313
"""Copy the item at rel_from to the location at rel_to"""
314
raise TransportNotPossible('ftp does not (yet) support copy()')
360
abspath = self._abspath(relpath)
361
if self.has(relpath):
362
ftp = self._get_FTP()
363
result = ftp.size(abspath)
367
mutter("FTP appe to %s", abspath)
368
self._try_append(relpath, f.read(), mode)
372
def _try_append(self, relpath, text, mode=None, retries=0):
373
"""Try repeatedly to append the given text to the file at relpath.
375
This is a recursive function. On errors, it will be called until the
376
number of retries is exceeded.
379
abspath = self._abspath(relpath)
380
mutter("FTP appe (try %d) to %s", retries, abspath)
381
ftp = self._get_FTP()
382
ftp.voidcmd("TYPE I")
383
cmd = "APPE %s" % abspath
384
conn = ftp.transfercmd(cmd)
388
self._setmode(relpath, mode)
390
except ftplib.error_perm, e:
391
self._translate_perm_error(e, abspath, extra='error appending',
392
unknown_exc=errors.NoSuchFile)
393
except ftplib.error_temp, e:
394
if retries > _number_of_retries:
395
raise errors.TransportError("FTP temporary error during APPEND %s." \
396
"Aborting." % abspath, orig_error=e)
398
warning("FTP temporary error: %s. Retrying.", str(e))
399
self._FTP_instance = None
400
self._try_append(relpath, text, mode, retries+1)
402
def _setmode(self, relpath, mode):
403
"""Set permissions on a path.
405
Only set permissions if the FTP server supports the 'SITE CHMOD'
409
mutter("FTP site chmod: setting permissions to %s on %s",
410
str(mode), self._abspath(relpath))
411
ftp = self._get_FTP()
412
cmd = "SITE CHMOD %s %s" % (self._abspath(relpath), str(mode))
414
except ftplib.error_perm, e:
415
# Command probably not available on this server
416
warning("FTP Could not set permissions to %s on %s. %s",
417
str(mode), self._abspath(relpath), str(e))
419
# TODO: jam 20060516 I believe ftp allows you to tell an ftp server
420
# to copy something to another machine. And you may be able
421
# to give it its own address as the 'to' location.
422
# So implement a fancier 'copy()'
316
424
def move(self, rel_from, rel_to):
317
425
"""Move the item at rel_from to the location at rel_to"""
426
abs_from = self._abspath(rel_from)
427
abs_to = self._abspath(rel_to)
319
mutter("FTP mv: %s => %s" % (self._abspath(rel_from),
320
self._abspath(rel_to)))
429
mutter("FTP mv: %s => %s", abs_from, abs_to)
321
430
f = self._get_FTP()
322
f.rename(self._abspath(rel_from), self._abspath(rel_to))
431
f.rename(abs_from, abs_to)
323
432
except ftplib.error_perm, e:
324
raise TransportError(orig_error=e)
433
self._translate_perm_error(e, abs_from,
434
extra='unable to rename to %r' % (rel_to,),
435
unknown_exc=errors.PathError)
328
439
def delete(self, relpath):
329
440
"""Delete the item at relpath"""
441
abspath = self._abspath(relpath)
331
mutter("FTP rm: %s" % self._abspath(relpath))
443
mutter("FTP rm: %s", abspath)
332
444
f = self._get_FTP()
333
f.delete(self._abspath(relpath))
334
446
except ftplib.error_perm, e:
335
if str(e).endswith("No such file or directory"):
336
raise NoSuchFile(self._abspath(relpath), extra=str(e))
338
raise TransportError(orig_error=e)
447
self._translate_perm_error(e, abspath, 'error deleting',
448
unknown_exc=errors.NoSuchFile)
340
450
def listable(self):
341
451
"""See Transport.listable."""
344
454
def list_dir(self, relpath):
345
455
"""See Transport.list_dir."""
347
mutter("FTP nlst: %s" % self._abspath(relpath))
457
mutter("FTP nlst: %s", self._abspath(relpath))
348
458
f = self._get_FTP()
349
459
basepath = self._abspath(relpath)
350
# FTP.nlst returns paths prefixed by relpath, strip 'em
351
the_list = f.nlst(basepath)
352
stripped = [path[len(basepath)+1:] for path in the_list]
460
paths = f.nlst(basepath)
461
# If FTP.nlst returns paths prefixed by relpath, strip 'em
462
if paths and paths[0].startswith(basepath):
463
paths = [path[len(basepath)+1:] for path in paths]
353
464
# Remove . and .. if present, and return
354
return [path for path in stripped if path not in (".", "..")]
465
return [path for path in paths if path not in (".", "..")]
355
466
except ftplib.error_perm, e:
356
raise TransportError(orig_error=e)
467
self._translate_perm_error(e, relpath, extra='error with list_dir')
358
469
def iter_files_recursive(self):
359
470
"""See Transport.iter_files_recursive.
406
513
return self.lock_read(relpath)
516
class FtpServer(Server):
517
"""Common code for SFTP server facilities."""
521
self._ftp_server = None
523
self._async_thread = None
528
"""Calculate an ftp url to this server."""
529
return 'ftp://foo:bar@localhost:%d/' % (self._port)
531
# def get_bogus_url(self):
532
# """Return a URL which cannot be connected to."""
533
# return 'ftp://127.0.0.1:1'
535
def log(self, message):
536
"""This is used by medusa.ftp_server to log connections, etc."""
537
self.logs.append(message)
542
raise RuntimeError('Must have medusa to run the FtpServer')
544
self._root = os.getcwdu()
545
self._ftp_server = _ftp_server(
546
authorizer=_test_authorizer(root=self._root),
548
port=0, # bind to a random port
550
logger_object=self # Use FtpServer.log() for messages
552
self._port = self._ftp_server.getsockname()[1]
553
# Don't let it loop forever, or handle an infinite number of requests.
554
# In this case it will run for 100s, or 1000 requests
555
self._async_thread = threading.Thread(target=asyncore.loop,
556
kwargs={'timeout':0.1, 'count':1000})
557
self._async_thread.setDaemon(True)
558
self._async_thread.start()
561
"""See bzrlib.transport.Server.tearDown."""
562
# have asyncore release the channel
563
self._ftp_server.del_channel()
565
self._async_thread.join()
570
_test_authorizer = None
574
global _have_medusa, _ftp_channel, _ftp_server, _test_authorizer
577
import medusa.filesys
578
import medusa.ftp_server
584
class test_authorizer(object):
585
"""A custom Authorizer object for running the test suite.
587
The reason we cannot use dummy_authorizer, is because it sets the
588
channel to readonly, which we don't always want to do.
591
def __init__(self, root):
594
def authorize(self, channel, username, password):
595
"""Return (success, reply_string, filesystem)"""
597
return 0, 'No Medusa.', None
599
channel.persona = -1, -1
600
if username == 'anonymous':
601
channel.read_only = 1
603
channel.read_only = 0
605
return 1, 'OK.', medusa.filesys.os_filesystem(self.root)
608
class ftp_channel(medusa.ftp_server.ftp_channel):
609
"""Customized ftp channel"""
611
def log(self, message):
612
"""Redirect logging requests."""
613
mutter('_ftp_channel: %s', message)
615
def log_info(self, message, type='info'):
616
"""Redirect logging requests."""
617
mutter('_ftp_channel %s: %s', type, message)
619
def cmd_rnfr(self, line):
620
"""Prepare for renaming a file."""
621
self._renaming = line[1]
622
self.respond('350 Ready for RNTO')
623
# TODO: jam 20060516 in testing, the ftp server seems to
624
# check that the file already exists, or it sends
625
# 550 RNFR command failed
627
def cmd_rnto(self, line):
628
"""Rename a file based on the target given.
630
rnto must be called after calling rnfr.
632
if not self._renaming:
633
self.respond('503 RNFR required first.')
634
pfrom = self.filesystem.translate(self._renaming)
635
self._renaming = None
636
pto = self.filesystem.translate(line[1])
638
os.rename(pfrom, pto)
639
except (IOError, OSError), e:
640
# TODO: jam 20060516 return custom responses based on
641
# why the command failed
642
self.respond('550 RNTO failed: %s' % (e,))
644
self.respond('550 RNTO failed')
645
# For a test server, we will go ahead and just die
648
self.respond('250 Rename successful.')
650
def cmd_size(self, line):
651
"""Return the size of a file
653
This is overloaded to help the test suite determine if the
654
target is a directory.
657
if not self.filesystem.isfile(filename):
658
if self.filesystem.isdir(filename):
659
self.respond('550 "%s" is a directory' % (filename,))
661
self.respond('550 "%s" is not a file' % (filename,))
663
self.respond('213 %d'
664
% (self.filesystem.stat(filename)[stat.ST_SIZE]),)
666
def cmd_mkd(self, line):
667
"""Create a directory.
669
Overloaded because default implementation does not distinguish
670
*why* it cannot make a directory.
673
self.command_not_understood (string.join (line))
677
self.filesystem.mkdir (path)
678
self.respond ('257 MKD command successful.')
679
except (IOError, OSError), e:
680
self.respond ('550 error creating directory: %s' % (e,))
682
self.respond ('550 error creating directory.')
685
class ftp_server(medusa.ftp_server.ftp_server):
686
"""Customize the behavior of the Medusa ftp_server.
688
There are a few warts on the ftp_server, based on how it expects
692
ftp_channel_class = ftp_channel
694
def __init__(self, *args, **kwargs):
695
mutter('Initializing _ftp_server: %r, %r', args, kwargs)
696
medusa.ftp_server.ftp_server.__init__(self, *args, **kwargs)
698
def log(self, message):
699
"""Redirect logging requests."""
700
mutter('_ftp_server: %s', message)
702
def log_info(self, message, type='info'):
703
"""Override the asyncore.log_info so we don't stipple the screen."""
704
mutter('_ftp_server %s: %s', type, message)
706
_test_authorizer = test_authorizer
707
_ftp_channel = ftp_channel
708
_ftp_server = ftp_server
409
713
def get_test_permutations():
410
714
"""Return the permutations to be used in testing."""
411
warn("There are no FTP transport provider tests yet.")
715
if not _setup_medusa():
716
warn("You must install medusa (http://www.amk.ca/python/code/medusa.html) for FTP tests")
719
return [(FtpTransport, FtpServer)]