← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_selftest.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2009-12-23 00:59:10 UTC
  • mfrom: (4794.1.21 integration)
  • Revision ID: pqm@pqm.ubuntu.com-20091223005910-zatr8ajlw8ul6d6s
(robertc) Add testtools 0.9.2 as a hard dependency to run the test
        suite, improving debug output on selftest --parallel,
        reducing code size and adding support for assertThat. (Robert Collins)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_selftest.py
17
17
"""Tests for the test framework."""
18
18
 
19
19
from cStringIO import StringIO
 
20
from doctest import ELLIPSIS
20
21
import os
21
22
import signal
22
23
import sys
24
25
import unittest
25
26
import warnings
26
27
 
 
28
from testtools import MultiTestResult
 
29
from testtools.content_type import ContentType
 
30
from testtools.matchers import (
 
31
    DocTestMatches,
 
32
    Equals,
 
33
    )
 
34
import testtools.tests.helpers
 
35
 
27
36
import bzrlib
28
37
from bzrlib import (
29
38
    branchbuilder,
78
87
                          TestUtil._load_module_by_name,
79
88
                          'bzrlib.no-name-yet')
80
89
 
 
90
 
81
91
class MetaTestLog(tests.TestCase):
82
92
 
83
93
    def test_logging(self):
84
94
        """Test logs are captured when a test fails."""
85
95
        self.log('a test message')
86
 
        self._log_file.flush()
87
 
        self.assertContainsRe(self._get_log(keep_log_file=True),
88
 
                              'a test message\n')
 
96
        details = self.getDetails()
 
97
        log = details['log']
 
98
        self.assertThat(log.content_type, Equals(ContentType(
 
99
            "text", "plain", {"charset": "utf8"})))
 
100
        self.assertThat(u"".join(log.iter_text()), Equals(self.get_log()))
 
101
        self.assertThat(self.get_log(),
 
102
            DocTestMatches(u"...a test message\n", ELLIPSIS))
89
103
 
90
104
 
91
105
class TestUnicodeFilename(tests.TestCase):
687
701
 
688
702
    def test_profiles_tests(self):
689
703
        self.requireFeature(test_lsprof.LSProfFeature)
690
 
        terminal = unittest.TestResult()
 
704
        terminal = testtools.tests.helpers.ExtendedTestResult()
691
705
        result = tests.ProfileResult(terminal)
692
706
        class Sample(tests.TestCase):
693
707
            def a(self):
695
709
            def sample_function(self):
696
710
                pass
697
711
        test = Sample("a")
698
 
        test.attrs_to_keep = test.attrs_to_keep + ('_benchcalls',)
699
712
        test.run(result)
700
 
        self.assertLength(1, test._benchcalls)
 
713
        case = terminal._events[0][1]
 
714
        self.assertLength(1, case._benchcalls)
701
715
        # We must be able to unpack it as the test reporting code wants
702
 
        (_, _, _), stats = test._benchcalls[0]
 
716
        (_, _, _), stats = case._benchcalls[0]
703
717
        self.assertTrue(callable(stats.pprint))
704
718
 
705
719
 
710
724
                descriptions=0,
711
725
                verbosity=1,
712
726
                )
713
 
        test_case.run(result)
714
 
        timed_string = result._testTimeString(test_case)
 
727
        capture = testtools.tests.helpers.ExtendedTestResult()
 
728
        test_case.run(MultiTestResult(result, capture))
 
729
        run_case = capture._events[0][1]
 
730
        timed_string = result._testTimeString(run_case)
715
731
        self.assertContainsRe(timed_string, expected_re)
716
732
 
717
733
    def test_test_reporting(self):
832
848
            def stopTestRun(self): pass
833
849
            def startTests(self): pass
834
850
            def report_test_start(self, test): pass
835
 
            def report_known_failure(self, test, err):
836
 
                self._call = test, err
 
851
            def report_known_failure(self, test, err=None, details=None):
 
852
                self._call = test, 'known failure'
837
853
        result = InstrumentedTestResult(None, None, None, None)
838
854
        class Test(tests.TestCase):
839
855
            def test_function(self):
842
858
        test.run(result)
843
859
        # it should invoke 'report_known_failure'.
844
860
        self.assertEqual(2, len(result._call))
845
 
        self.assertEqual(test, result._call[0])
846
 
        self.assertEqual(tests.KnownFailure, result._call[1][0])
847
 
        self.assertIsInstance(result._call[1][1], tests.KnownFailure)
 
861
        self.assertEqual(test.id(), result._call[0].id())
 
862
        self.assertEqual('known failure', result._call[1])
848
863
        # we dont introspec the traceback, if the rest is ok, it would be
849
864
        # exceptional for it not to be.
850
865
        # it should update the known_failure_count on the object.
944
959
        test.run(result)
945
960
        # it should invoke 'addNotSupported'.
946
961
        self.assertEqual(2, len(result._call))
947
 
        self.assertEqual(test, result._call[0])
 
962
        self.assertEqual(test.id(), result._call[0].id())
948
963
        self.assertEqual(feature, result._call[1])
949
964
        # and not count as an error
950
965
        self.assertEqual(0, result.error_count)
1028
1043
        # the final output when real failures occur.
1029
1044
        class Test(tests.TestCase):
1030
1045
            def known_failure_test(self):
1031
 
                raise tests.KnownFailure('failed')
 
1046
                self.expectFailure('failed', self.assertTrue, False)
1032
1047
        test = unittest.TestSuite()
1033
1048
        test.addTest(Test("known_failure_test"))
1034
1049
        def failing_test():
1035
 
            raise AssertionError('foo')
 
1050
            self.fail('foo')
1036
1051
        test.addTest(unittest.FunctionTestCase(failing_test))
1037
1052
        stream = StringIO()
1038
1053
        runner = tests.TextTestRunner(stream=stream)
1039
1054
        result = self.run_test_runner(runner, test)
1040
1055
        lines = stream.getvalue().splitlines()
1041
1056
        self.assertContainsRe(stream.getvalue(),
1042
 
            '(?sm)^testing.*$'
 
1057
            '(?sm)^bzr selftest.*$'
1043
1058
            '.*'
1044
1059
            '^======================================================================\n'
1045
1060
            '^FAIL: unittest.FunctionTestCase \\(failing_test\\)\n'
1046
1061
            '^----------------------------------------------------------------------\n'
1047
1062
            'Traceback \\(most recent call last\\):\n'
1048
1063
            '  .*' # File .*, line .*, in failing_test' - but maybe not from .pyc
1049
 
            '    raise AssertionError\\(\'foo\'\\)\n'
 
1064
            '    self.fail\\(\'foo\'\\)\n'
1050
1065
            '.*'
1051
1066
            '^----------------------------------------------------------------------\n'
1052
1067
            '.*'
1058
1073
        # the final output.
1059
1074
        class Test(tests.TestCase):
1060
1075
            def known_failure_test(self):
1061
 
                raise tests.KnownFailure('failed')
 
1076
                self.expectFailure('failed', self.assertTrue, False)
1062
1077
        test = Test("known_failure_test")
1063
1078
        stream = StringIO()
1064
1079
        runner = tests.TextTestRunner(stream=stream)
1209
1224
        self.assertContainsRe(output_string, "--date [0-9.]+")
1210
1225
        self.assertLength(1, self._get_source_tree_calls)
1211
1226
 
1212
 
    def assertLogDeleted(self, test):
1213
 
        log = test._get_log()
1214
 
        self.assertEqual("DELETED log file to reduce memory footprint", log)
1215
 
        self.assertEqual('', test._log_contents)
1216
 
        self.assertIs(None, test._log_file_name)
1217
 
 
1218
 
    def test_success_log_deleted(self):
1219
 
        """Successful tests have their log deleted"""
1220
 
 
1221
 
        class LogTester(tests.TestCase):
1222
 
 
1223
 
            def test_success(self):
1224
 
                self.log('this will be removed\n')
1225
 
 
1226
 
        sio = StringIO()
1227
 
        runner = tests.TextTestRunner(stream=sio)
1228
 
        test = LogTester('test_success')
1229
 
        result = self.run_test_runner(runner, test)
1230
 
 
1231
 
        self.assertLogDeleted(test)
1232
 
 
1233
 
    def test_skipped_log_deleted(self):
1234
 
        """Skipped tests have their log deleted"""
1235
 
 
1236
 
        class LogTester(tests.TestCase):
1237
 
 
1238
 
            def test_skipped(self):
1239
 
                self.log('this will be removed\n')
1240
 
                raise tests.TestSkipped()
1241
 
 
1242
 
        sio = StringIO()
1243
 
        runner = tests.TextTestRunner(stream=sio)
1244
 
        test = LogTester('test_skipped')
1245
 
        result = self.run_test_runner(runner, test)
1246
 
 
1247
 
        self.assertLogDeleted(test)
1248
 
 
1249
 
    def test_not_aplicable_log_deleted(self):
1250
 
        """Not applicable tests have their log deleted"""
1251
 
 
1252
 
        class LogTester(tests.TestCase):
1253
 
 
1254
 
            def test_not_applicable(self):
1255
 
                self.log('this will be removed\n')
1256
 
                raise tests.TestNotApplicable()
1257
 
 
1258
 
        sio = StringIO()
1259
 
        runner = tests.TextTestRunner(stream=sio)
1260
 
        test = LogTester('test_not_applicable')
1261
 
        result = self.run_test_runner(runner, test)
1262
 
 
1263
 
        self.assertLogDeleted(test)
1264
 
 
1265
 
    def test_known_failure_log_deleted(self):
1266
 
        """Know failure tests have their log deleted"""
1267
 
 
1268
 
        class LogTester(tests.TestCase):
1269
 
 
1270
 
            def test_known_failure(self):
1271
 
                self.log('this will be removed\n')
1272
 
                raise tests.KnownFailure()
1273
 
 
1274
 
        sio = StringIO()
1275
 
        runner = tests.TextTestRunner(stream=sio)
1276
 
        test = LogTester('test_known_failure')
1277
 
        result = self.run_test_runner(runner, test)
1278
 
 
1279
 
        self.assertLogDeleted(test)
1280
 
 
1281
 
    def test_fail_log_kept(self):
1282
 
        """Failed tests have their log kept"""
1283
 
 
1284
 
        class LogTester(tests.TestCase):
1285
 
 
1286
 
            def test_fail(self):
1287
 
                self.log('this will be kept\n')
1288
 
                self.fail('this test fails')
1289
 
 
1290
 
        sio = StringIO()
1291
 
        runner = tests.TextTestRunner(stream=sio)
1292
 
        test = LogTester('test_fail')
1293
 
        result = self.run_test_runner(runner, test)
1294
 
 
1295
 
        text = sio.getvalue()
1296
 
        self.assertContainsRe(text, 'this will be kept')
1297
 
        self.assertContainsRe(text, 'this test fails')
1298
 
 
1299
 
        log = test._get_log()
1300
 
        self.assertContainsRe(log, 'this will be kept')
1301
 
        self.assertEqual(log, test._log_contents)
1302
 
 
1303
 
    def test_error_log_kept(self):
1304
 
        """Tests with errors have their log kept"""
1305
 
 
1306
 
        class LogTester(tests.TestCase):
1307
 
 
1308
 
            def test_error(self):
1309
 
                self.log('this will be kept\n')
1310
 
                raise ValueError('random exception raised')
1311
 
 
1312
 
        sio = StringIO()
1313
 
        runner = tests.TextTestRunner(stream=sio)
1314
 
        test = LogTester('test_error')
1315
 
        result = self.run_test_runner(runner, test)
1316
 
 
1317
 
        text = sio.getvalue()
1318
 
        self.assertContainsRe(text, 'this will be kept')
1319
 
        self.assertContainsRe(text, 'random exception raised')
1320
 
 
1321
 
        log = test._get_log()
1322
 
        self.assertContainsRe(log, 'this will be kept')
1323
 
        self.assertEqual(log, test._log_contents)
1324
 
 
1325
1227
    def test_startTestRun(self):
1326
1228
        """run should call result.startTestRun()"""
1327
1229
        calls = []
1491
1393
        self.assertEqual(set(['original-state']), bzrlib.debug.debug_flags)
1492
1394
 
1493
1395
    def make_test_result(self):
 
1396
        """Get a test result that writes to the test log file."""
1494
1397
        return tests.TextTestResult(self._log_file, descriptions=0, verbosity=1)
1495
1398
 
1496
1399
    def inner_test(self):
1504
1407
        result = self.make_test_result()
1505
1408
        self.inner_test.run(result)
1506
1409
        note("outer finish")
 
1410
        self.addCleanup(osutils.delete_any, self._log_file_name)
1507
1411
 
1508
1412
    def test_trace_nesting(self):
1509
1413
        # this tests that each test case nests its trace facility correctly.
1521
1425
        outer_test = TestTestCase("outer_child")
1522
1426
        result = self.make_test_result()
1523
1427
        outer_test.run(result)
1524
 
        self.addCleanup(osutils.delete_any, outer_test._log_file_name)
1525
1428
        self.assertEqual(original_trace, bzrlib.trace._trace_file)
1526
1429
 
1527
1430
    def method_that_times_a_bit_twice(self):
1642
1545
        """Test disabled tests behaviour with support aware results."""
1643
1546
        test = SampleTestCase('_test_pass')
1644
1547
        class DisabledFeature(object):
 
1548
            def __eq__(self, other):
 
1549
                return isinstance(other, DisabledFeature)
1645
1550
            def available(self):
1646
1551
                return False
1647
1552
        the_feature = DisabledFeature()
1658
1563
                self.calls.append(('addNotSupported', test, feature))
1659
1564
        result = InstrumentedTestResult()
1660
1565
        test.run(result)
 
1566
        case = result.calls[0][1]
1661
1567
        self.assertEqual([
1662
 
            ('startTest', test),
1663
 
            ('addNotSupported', test, the_feature),
1664
 
            ('stopTest', test),
 
1568
            ('startTest', case),
 
1569
            ('addNotSupported', case, the_feature),
 
1570
            ('stopTest', case),
1665
1571
            ],
1666
1572
            result.calls)
1667
1573
 
2431
2337
        self.assertEqual('bzr: interrupted\n', result[1])
2432
2338
 
2433
2339
 
2434
 
class TestKnownFailure(tests.TestCase):
2435
 
 
2436
 
    def test_known_failure(self):
2437
 
        """Check that KnownFailure is defined appropriately."""
2438
 
        # a KnownFailure is an assertion error for compatability with unaware
2439
 
        # runners.
2440
 
        self.assertIsInstance(tests.KnownFailure(""), AssertionError)
2441
 
 
2442
 
    def test_expect_failure(self):
2443
 
        try:
2444
 
            self.expectFailure("Doomed to failure", self.assertTrue, False)
2445
 
        except tests.KnownFailure, e:
2446
 
            self.assertEqual('Doomed to failure', e.args[0])
2447
 
        try:
2448
 
            self.expectFailure("Doomed to failure", self.assertTrue, True)
2449
 
        except AssertionError, e:
2450
 
            self.assertEqual('Unexpected success.  Should have failed:'
2451
 
                             ' Doomed to failure', e.args[0])
2452
 
        else:
2453
 
            self.fail('Assertion not raised')
2454
 
 
2455
 
 
2456
2340
class TestFeature(tests.TestCase):
2457
2341
 
2458
2342
    def test_caching(self):
2709
2593
        # the test framework
2710
2594
        self.assertEquals('always fails', str(e))
2711
2595
        # check that there's no traceback in the test log
2712
 
        self.assertNotContainsRe(self._get_log(keep_log_file=True),
2713
 
            r'Traceback')
 
2596
        self.assertNotContainsRe(self.get_log(), r'Traceback')
2714
2597
 
2715
2598
    def test_run_bzr_user_error_caught(self):
2716
2599
        # Running bzr in blackbox mode, normal/expected/user errors should be
2969
2852
    def test_load_tests(self):
2970
2853
        test_list = ['bzrlib.tests.test_sampler.DemoTest.test_nothing']
2971
2854
        loader = self._create_loader(test_list)
2972
 
 
2973
2855
        suite = loader.loadTestsFromModuleName('bzrlib.tests.test_sampler')
2974
2856
        self.assertEquals(test_list, _test_ids(suite))
2975
2857
 
2976
2858
    def test_exclude_tests(self):
2977
2859
        test_list = ['bogus']
2978
2860
        loader = self._create_loader(test_list)
2979
 
 
2980
2861
        suite = loader.loadTestsFromModuleName('bzrlib.tests.test_sampler')
2981
2862
        self.assertEquals([], _test_ids(suite))
2982
2863
 
3027
2908
        tpr.register('bar', 'bbb.aaa.rrr')
3028
2909
        tpr.register('bar', 'bBB.aAA.rRR')
3029
2910
        self.assertEquals('bbb.aaa.rrr', tpr.get('bar'))
3030
 
        self.assertContainsRe(self._get_log(keep_log_file=True),
3031
 
                              r'.*bar.*bbb.aaa.rrr.*bBB.aAA.rRR')
 
2911
        self.assertThat(self.get_log(),
 
2912
            DocTestMatches("...bar...bbb.aaa.rrr...BB.aAA.rRR", ELLIPSIS))
3032
2913
 
3033
2914
    def test_get_unknown_prefix(self):
3034
2915
        tpr = self._get_registry()