15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
from cStringIO import StringIO
30
from logging import debug, warning, error
28
32
import bzrlib.commands
30
33
import bzrlib.trace
34
import bzrlib.osutils as osutils
35
from bzrlib.selftest import TestUtil
36
from bzrlib.selftest.TestUtil import TestLoader, TestSuite
37
from bzrlib.selftest.treeshape import build_tree_contents
34
39
MODULES_TO_TEST = []
35
40
MODULES_TO_DOCTEST = []
37
from logging import debug, warning, error
44
class EarlyStoppingTestResultAdapter(object):
45
"""An adapter for TestResult to stop at the first first failure or error"""
47
def __init__(self, result):
50
def addError(self, test, err):
51
self._result.addError(test, err)
54
def addFailure(self, test, err):
55
self._result.addFailure(test, err)
58
def __getattr__(self, name):
59
return getattr(self._result, name)
61
def __setattr__(self, name, value):
63
object.__setattr__(self, name, value)
64
return setattr(self._result, name, value)
67
class _MyResult(unittest._TextTestResult):
71
No special behaviour for now.
74
def _elapsedTime(self):
75
return "(Took %.3fs)" % (time.time() - self._start_time)
77
def startTest(self, test):
78
unittest.TestResult.startTest(self, test)
79
# TODO: Maybe show test.shortDescription somewhere?
80
what = test.shortDescription() or test.id()
82
self.stream.write('%-70.70s' % what)
84
self._start_time = time.time()
86
def addError(self, test, err):
87
unittest.TestResult.addError(self, test, err)
89
self.stream.writeln("ERROR %s" % self._elapsedTime())
91
self.stream.write('E')
94
def addFailure(self, test, err):
95
unittest.TestResult.addFailure(self, test, err)
97
self.stream.writeln("FAIL %s" % self._elapsedTime())
99
self.stream.write('F')
102
def addSuccess(self, test):
104
self.stream.writeln('OK %s' % self._elapsedTime())
106
self.stream.write('~')
108
unittest.TestResult.addSuccess(self, test)
110
def printErrorList(self, flavour, errors):
111
for test, err in errors:
112
self.stream.writeln(self.separator1)
113
self.stream.writeln("%s: %s" % (flavour,self.getDescription(test)))
114
if hasattr(test, '_get_log'):
115
self.stream.writeln()
116
self.stream.writeln('log from this test:')
117
print >>self.stream, test._get_log()
118
self.stream.writeln(self.separator2)
119
self.stream.writeln("%s" % err)
122
class TextTestRunner(unittest.TextTestRunner):
123
stop_on_failure = False
125
def _makeResult(self):
126
result = _MyResult(self.stream, self.descriptions, self.verbosity)
127
if self.stop_on_failure:
128
result = EarlyStoppingTestResultAdapter(result)
132
def iter_suite_tests(suite):
133
"""Return all tests in a suite, recursing through nested suites"""
134
for item in suite._tests:
135
if isinstance(item, unittest.TestCase):
137
elif isinstance(item, unittest.TestSuite):
138
for r in iter_suite_tests(item):
141
raise Exception('unknown object %r inside test suite %r'
145
class TestSkipped(Exception):
146
"""Indicates that a test was intentionally skipped, rather than failing."""
39
150
class CommandFailed(Exception):
48
159
Error and debug log messages are redirected from their usual
49
160
location into a temporary file, the contents of which can be
50
retrieved by _get_log().
161
retrieved by _get_log(). We use a real OS file, not an in-memory object,
162
so that it can also capture file IO. When the test completes this file
163
is read into memory and removed from disk.
52
165
There are also convenience functions to invoke bzr's command-line
53
routine, and to build and check bzr trees."""
166
routine, and to build and check bzr trees.
168
In addition to the usual method of overriding tearDown(), this class also
169
allows subclasses to register functions into the _cleanups list, which is
170
run in order as the object is torn down. It's less likely this will be
171
accidentally overlooked.
175
_log_file_name = None
58
# this replaces the default testsweet.TestCase; we don't want logging changed
59
179
unittest.TestCase.setUp(self)
181
self._cleanEnvironment()
60
182
bzrlib.trace.disable_default_logging()
61
self._enable_file_logging()
64
def _enable_file_logging(self):
185
def _ndiff_strings(self, a, b):
186
"""Return ndiff between two strings containing lines.
188
A trailing newline is added if missing to make the strings
190
if b and b[-1] != '\n':
192
if a and a[-1] != '\n':
194
difflines = difflib.ndiff(a.splitlines(True),
196
linejunk=lambda x: False,
197
charjunk=lambda x: False)
198
return ''.join(difflines)
200
def assertEqualDiff(self, a, b):
201
"""Assert two texts are equal, if not raise an exception.
203
This is intended for use with multi-line strings where it can
204
be hard to find the differences by eye.
206
# TODO: perhaps override assertEquals to call this for strings?
209
raise AssertionError("texts not equal:\n" +
210
self._ndiff_strings(a, b))
212
def assertContainsRe(self, haystack, needle_re):
213
"""Assert that a contains something matching a regular expression."""
214
if not re.search(needle_re, haystack):
215
raise AssertionError('pattern "%s" not found in "%s"'
216
% (needle_re, haystack))
218
def _startLogFile(self):
219
"""Send bzr and test log messages to a temporary file.
221
The file is removed as the test is torn down.
65
223
fileno, name = tempfile.mkstemp(suffix='.log', prefix='testbzr')
67
224
self._log_file = os.fdopen(fileno, 'w+')
69
hdlr = logging.StreamHandler(self._log_file)
70
hdlr.setLevel(logging.DEBUG)
71
hdlr.setFormatter(logging.Formatter('%(levelname)8s %(message)s'))
72
logging.getLogger('').addHandler(hdlr)
73
logging.getLogger('').setLevel(logging.DEBUG)
225
bzrlib.trace.enable_test_log(self._log_file)
75
226
debug('opened log file %s', name)
77
227
self._log_file_name = name
81
logging.getLogger('').removeHandler(self._log_hdlr)
82
bzrlib.trace.enable_default_logging()
83
logging.debug('%s teardown', self.id())
228
self.addCleanup(self._finishLogFile)
230
def _finishLogFile(self):
231
"""Finished with the log file.
233
Read contents into memory, close, and delete.
235
bzrlib.trace.disable_test_log()
236
self._log_file.seek(0)
237
self._log_contents = self._log_file.read()
84
238
self._log_file.close()
239
os.remove(self._log_file_name)
240
self._log_file = self._log_file_name = None
242
def addCleanup(self, callable):
243
"""Arrange to run a callable when this case is torn down.
245
Callables are run in the reverse of the order they are registered,
246
ie last-in first-out.
248
if callable in self._cleanups:
249
raise ValueError("cleanup function %r already registered on %s"
251
self._cleanups.append(callable)
253
def _cleanEnvironment(self):
254
self.oldenv = os.environ.get('HOME', None)
255
os.environ['HOME'] = os.getcwd()
256
self.bzr_email = os.environ.get('BZREMAIL')
257
if self.bzr_email is not None:
258
del os.environ['BZREMAIL']
259
self.email = os.environ.get('EMAIL')
260
if self.email is not None:
261
del os.environ['EMAIL']
262
self.addCleanup(self._restoreEnvironment)
264
def _restoreEnvironment(self):
265
os.environ['HOME'] = self.oldenv
266
if os.environ.get('BZREMAIL') is not None:
267
del os.environ['BZREMAIL']
268
if self.bzr_email is not None:
269
os.environ['BZREMAIL'] = self.bzr_email
270
if os.environ.get('EMAIL') is not None:
271
del os.environ['EMAIL']
272
if self.email is not None:
273
os.environ['EMAIL'] = self.email
85
277
unittest.TestCase.tearDown(self)
279
def _runCleanups(self):
280
"""Run registered cleanup functions.
282
This should only be called from TestCase.tearDown.
284
for callable in reversed(self._cleanups):
88
287
def log(self, *args):
89
288
logging.debug(*args)
91
290
def _get_log(self):
92
291
"""Return as a string the log for this test"""
93
return open(self._log_file_name).read()
292
if self._log_file_name:
293
return open(self._log_file_name).read()
295
return self._log_contents
297
def capture(self, cmd, retcode=0):
298
"""Shortcut that splits cmd into words, runs, and returns stdout"""
299
return self.run_bzr_captured(cmd.split(), retcode=retcode)[0]
301
def run_bzr_captured(self, argv, retcode=0):
302
"""Invoke bzr and return (stdout, stderr).
304
Useful for code that wants to check the contents of the
305
output, the way error messages are presented, etc.
307
This should be the main method for tests that want to exercise the
308
overall behavior of the bzr application (rather than a unit test
309
or a functional test of the library.)
311
Much of the old code runs bzr by forking a new copy of Python, but
312
that is slower, harder to debug, and generally not necessary.
314
This runs bzr through the interface that catches and reports
315
errors, and with logging set to something approximating the
316
default, so that error reporting can be checked.
318
argv -- arguments to invoke bzr
319
retcode -- expected return code, or None for don't-care.
323
self.log('run bzr: %s', ' '.join(argv))
324
handler = logging.StreamHandler(stderr)
325
handler.setFormatter(bzrlib.trace.QuietFormatter())
326
handler.setLevel(logging.INFO)
327
logger = logging.getLogger('')
328
logger.addHandler(handler)
330
result = self.apply_redirected(None, stdout, stderr,
331
bzrlib.commands.run_bzr_catch_errors,
334
logger.removeHandler(handler)
335
out = stdout.getvalue()
336
err = stderr.getvalue()
338
self.log('output:\n%s', out)
340
self.log('errors:\n%s', err)
341
if retcode is not None:
342
self.assertEquals(result, retcode)
95
345
def run_bzr(self, *args, **kwargs):
96
346
"""Invoke bzr, as if it were run from the command line.
210
461
super(TestCaseInTempDir, self).setUp()
212
462
self._make_test_root()
213
self._currentdir = os.getcwdu()
463
_currentdir = os.getcwdu()
214
464
short_id = self.id().replace('bzrlib.selftest.', '') \
215
465
.replace('__main__.', '')
216
466
self.test_dir = os.path.join(self.TEST_ROOT, short_id)
217
467
os.mkdir(self.test_dir)
218
468
os.chdir(self.test_dir)
469
os.environ['HOME'] = self.test_dir
470
def _leaveDirectory():
471
os.chdir(_currentdir)
472
self.addCleanup(_leaveDirectory)
222
os.chdir(self._currentdir)
223
super(TestCaseInTempDir, self).tearDown()
225
def _formcmd(self, cmd):
226
if isinstance(cmd, basestring):
229
cmd[0] = self.BZRPATH
230
if self.OVERRIDE_PYTHON:
231
cmd.insert(0, self.OVERRIDE_PYTHON)
232
self.log('$ %r' % cmd)
235
def runcmd(self, cmd, retcode=0):
236
"""Run one command and check the return code.
238
Returns a tuple of (stdout,stderr) strings.
240
If a single string is based, it is split into words.
241
For commands that are not simple space-separated words, please
242
pass a list instead."""
243
cmd = self._formcmd(cmd)
244
self.log('$ ' + ' '.join(cmd))
245
actual_retcode = subprocess.call(cmd, stdout=self._log_file,
246
stderr=self._log_file)
247
if retcode != actual_retcode:
248
raise CommandFailed("test failed: %r returned %d, expected %d"
249
% (cmd, actual_retcode, retcode))
251
def backtick(self, cmd, retcode=0):
252
"""Run a command and return its output"""
253
cmd = self._formcmd(cmd)
254
child = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=self._log_file)
255
outd, errd = child.communicate()
257
actual_retcode = child.wait()
259
outd = outd.replace('\r', '')
261
if retcode != actual_retcode:
262
raise CommandFailed("test failed: %r returned %d, expected %d"
263
% (cmd, actual_retcode, retcode))
269
474
def build_tree(self, shape):
270
475
"""Build a test tree according to a pattern.
275
480
This doesn't add anything to a branch.
277
482
# XXX: It's OK to just create them using forward slashes on windows?
279
483
for name in shape:
280
assert isinstance(name, basestring)
484
self.assert_(isinstance(name, basestring))
281
485
if name[-1] == '/':
282
486
os.mkdir(name[:-1])
284
488
f = file(name, 'wt')
285
489
print >>f, "contents of", name
492
def build_tree_contents(self, shape):
493
bzrlib.selftest.build_tree_contents(shape)
495
def failUnlessExists(self, path):
496
"""Fail unless path, which may be abs or relative, exists."""
497
self.failUnless(osutils.lexists(path))
499
def assertFileEqual(self, content, path):
500
"""Fail if path does not contain 'content'."""
501
self.failUnless(osutils.lexists(path))
502
self.assertEqualDiff(content, open(path, 'r').read())
290
505
class MetaTestLog(TestCase):
291
506
def test_logging(self):
292
507
"""Test logs are captured when a test fails."""
293
logging.info('an info message')
294
warning('something looks dodgy...')
295
logging.debug('hello, test is running')
299
def selftest(verbose=False, pattern=".*"):
508
self.log('a test message')
509
self.assertContainsRe(self._get_log(), 'a test message\n')
512
def filter_suite_by_re(suite, pattern):
513
result = TestUtil.TestSuite()
514
filter_re = re.compile(pattern)
515
for test in iter_suite_tests(suite):
516
if filter_re.search(test.id()):
521
def run_suite(suite, name='test', verbose=False, pattern=".*",
522
stop_on_failure=False, keep_output=False):
523
TestCaseInTempDir._TEST_NAME = name
528
runner = TextTestRunner(stream=sys.stdout,
531
runner.stop_on_failure=stop_on_failure
533
suite = filter_suite_by_re(suite, pattern)
534
result = runner.run(suite)
535
# This is still a little bogus,
536
# but only a little. Folk not using our testrunner will
537
# have to delete their temp directories themselves.
538
if result.wasSuccessful() or not keep_output:
539
if TestCaseInTempDir.TEST_ROOT is not None:
540
shutil.rmtree(TestCaseInTempDir.TEST_ROOT)
542
print "Failed tests working directories are in '%s'\n" % TestCaseInTempDir.TEST_ROOT
543
return result.wasSuccessful()
546
def selftest(verbose=False, pattern=".*", stop_on_failure=True,
300
548
"""Run the whole test suite under the enhanced runner"""
301
return testsweet.run_suite(test_suite(), 'testbzr', verbose=verbose, pattern=pattern)
549
return run_suite(test_suite(), 'testbzr', verbose=verbose, pattern=pattern,
550
stop_on_failure=stop_on_failure, keep_output=keep_output)
304
553
def test_suite():
305
554
"""Build and return TestSuite for the whole program."""
306
from bzrlib.selftest.TestUtil import TestLoader, TestSuite
307
import bzrlib, bzrlib.store, bzrlib.inventory, bzrlib.branch
308
import bzrlib.osutils, bzrlib.commands, bzrlib.merge3, bzrlib.plugin
555
import bzrlib.store, bzrlib.inventory, bzrlib.branch
556
import bzrlib.osutils, bzrlib.merge3, bzrlib.plugin
309
557
from doctest import DocTestSuite
315
559
global MODULES_TO_TEST, MODULES_TO_DOCTEST
561
# FIXME: If these fail to load, e.g. because of a syntax error, the
562
# exception is hidden by unittest. Sucks. Should either fix that or
563
# perhaps import them and pass them to unittest as modules.
317
564
testmod_names = \
318
565
['bzrlib.selftest.MetaTestLog',
566
'bzrlib.selftest.testapi',
567
'bzrlib.selftest.testgpg',
568
'bzrlib.selftest.testidentitymap',
319
569
'bzrlib.selftest.testinv',
570
'bzrlib.selftest.test_ancestry',
571
'bzrlib.selftest.test_commit',
572
'bzrlib.selftest.test_command',
573
'bzrlib.selftest.test_commit_merge',
574
'bzrlib.selftest.testconfig',
320
575
'bzrlib.selftest.versioning',
321
576
'bzrlib.selftest.testmerge3',
577
'bzrlib.selftest.testmerge',
322
578
'bzrlib.selftest.testhashcache',
323
579
'bzrlib.selftest.teststatus',
324
580
'bzrlib.selftest.testlog',
325
581
'bzrlib.selftest.testrevisionnamespaces',
326
582
'bzrlib.selftest.testbranch',
327
# 'bzrlib.selftest.testrevision',
328
# 'bzrlib.selftest.test_merge_core',
583
'bzrlib.selftest.testrevision',
584
'bzrlib.selftest.test_revision_info',
585
'bzrlib.selftest.test_merge_core',
329
586
'bzrlib.selftest.test_smart_add',
587
'bzrlib.selftest.test_bad_files',
330
588
'bzrlib.selftest.testdiff',
331
# 'bzrlib.selftest.test_parent',
589
'bzrlib.selftest.test_parent',
332
590
'bzrlib.selftest.test_xml',
333
# 'bzrlib.selftest.testfetch',
334
# 'bzrlib.selftest.whitebox',
591
'bzrlib.selftest.test_weave',
592
'bzrlib.selftest.testfetch',
593
'bzrlib.selftest.whitebox',
335
594
'bzrlib.selftest.teststore',
336
# 'bzrlib.selftest.blackbox',
595
'bzrlib.selftest.blackbox',
596
'bzrlib.selftest.testsampler',
597
'bzrlib.selftest.testtransactions',
598
'bzrlib.selftest.testtransport',
599
'bzrlib.selftest.testsftp',
600
'bzrlib.selftest.testgraph',
601
'bzrlib.selftest.testworkingtree',
602
'bzrlib.selftest.test_upgrade',
603
'bzrlib.selftest.test_conflicts',
604
'bzrlib.selftest.testtestament',
605
'bzrlib.selftest.testannotate',
606
'bzrlib.selftest.testrevprops',
607
'bzrlib.selftest.testoptions',
608
'bzrlib.selftest.testhttp',
609
'bzrlib.selftest.testnonascii',
610
'bzrlib.selftest.testreweave',
611
'bzrlib.selftest.testtsort',
612
'bzrlib.selftest.testtrace',
339
615
for m in (bzrlib.store, bzrlib.inventory, bzrlib.branch,
340
bzrlib.osutils, bzrlib.commands, bzrlib.merge3):
616
bzrlib.osutils, bzrlib.commands, bzrlib.merge3,
341
619
if m not in MODULES_TO_DOCTEST:
342
620
MODULES_TO_DOCTEST.append(m)