~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_selftest.py

Merged bzr.dev into cmdline-parser.

Show diffs side-by-side

added added

removed removed

Lines of Context:
17
17
"""Tests for the test framework."""
18
18
 
19
19
from cStringIO import StringIO
 
20
from doctest import ELLIPSIS
20
21
import os
21
22
import signal
22
23
import sys
24
25
import unittest
25
26
import warnings
26
27
 
 
28
from testtools import MultiTestResult
 
29
from testtools.content_type import ContentType
 
30
from testtools.matchers import (
 
31
    DocTestMatches,
 
32
    Equals,
 
33
    )
 
34
import testtools.tests.helpers
 
35
 
27
36
import bzrlib
28
37
from bzrlib import (
29
38
    branchbuilder,
78
87
                          TestUtil._load_module_by_name,
79
88
                          'bzrlib.no-name-yet')
80
89
 
 
90
 
81
91
class MetaTestLog(tests.TestCase):
82
92
 
83
93
    def test_logging(self):
84
94
        """Test logs are captured when a test fails."""
85
95
        self.log('a test message')
86
 
        self._log_file.flush()
87
 
        self.assertContainsRe(self._get_log(keep_log_file=True),
88
 
                              'a test message\n')
 
96
        details = self.getDetails()
 
97
        log = details['log']
 
98
        self.assertThat(log.content_type, Equals(ContentType(
 
99
            "text", "plain", {"charset": "utf8"})))
 
100
        self.assertThat(u"".join(log.iter_text()), Equals(self.get_log()))
 
101
        self.assertThat(self.get_log(),
 
102
            DocTestMatches(u"...a test message\n", ELLIPSIS))
89
103
 
90
104
 
91
105
class TestUnicodeFilename(tests.TestCase):
687
701
 
688
702
    def test_profiles_tests(self):
689
703
        self.requireFeature(test_lsprof.LSProfFeature)
690
 
        terminal = unittest.TestResult()
 
704
        terminal = testtools.tests.helpers.ExtendedTestResult()
691
705
        result = tests.ProfileResult(terminal)
692
706
        class Sample(tests.TestCase):
693
707
            def a(self):
695
709
            def sample_function(self):
696
710
                pass
697
711
        test = Sample("a")
698
 
        test.attrs_to_keep = test.attrs_to_keep + ('_benchcalls',)
699
712
        test.run(result)
700
 
        self.assertLength(1, test._benchcalls)
 
713
        case = terminal._events[0][1]
 
714
        self.assertLength(1, case._benchcalls)
701
715
        # We must be able to unpack it as the test reporting code wants
702
 
        (_, _, _), stats = test._benchcalls[0]
 
716
        (_, _, _), stats = case._benchcalls[0]
703
717
        self.assertTrue(callable(stats.pprint))
704
718
 
705
719
 
710
724
                descriptions=0,
711
725
                verbosity=1,
712
726
                )
713
 
        test_case.run(result)
714
 
        timed_string = result._testTimeString(test_case)
 
727
        capture = testtools.tests.helpers.ExtendedTestResult()
 
728
        test_case.run(MultiTestResult(result, capture))
 
729
        run_case = capture._events[0][1]
 
730
        timed_string = result._testTimeString(run_case)
715
731
        self.assertContainsRe(timed_string, expected_re)
716
732
 
717
733
    def test_test_reporting(self):
738
754
    def _patch_get_bzr_source_tree(self):
739
755
        # Reading from the actual source tree breaks isolation, but we don't
740
756
        # want to assume that thats *all* that would happen.
741
 
        def _get_bzr_source_tree():
742
 
            return None
743
 
        orig_get_bzr_source_tree = bzrlib.version._get_bzr_source_tree
744
 
        bzrlib.version._get_bzr_source_tree = _get_bzr_source_tree
745
 
        def restore():
746
 
            bzrlib.version._get_bzr_source_tree = orig_get_bzr_source_tree
747
 
        self.addCleanup(restore)
 
757
        self.overrideAttr(bzrlib.version, '_get_bzr_source_tree', lambda: None)
748
758
 
749
759
    def test_assigned_benchmark_file_stores_date(self):
750
760
        self._patch_get_bzr_source_tree()
832
842
            def stopTestRun(self): pass
833
843
            def startTests(self): pass
834
844
            def report_test_start(self, test): pass
835
 
            def report_known_failure(self, test, err):
836
 
                self._call = test, err
 
845
            def report_known_failure(self, test, err=None, details=None):
 
846
                self._call = test, 'known failure'
837
847
        result = InstrumentedTestResult(None, None, None, None)
838
848
        class Test(tests.TestCase):
839
849
            def test_function(self):
842
852
        test.run(result)
843
853
        # it should invoke 'report_known_failure'.
844
854
        self.assertEqual(2, len(result._call))
845
 
        self.assertEqual(test, result._call[0])
846
 
        self.assertEqual(tests.KnownFailure, result._call[1][0])
847
 
        self.assertIsInstance(result._call[1][1], tests.KnownFailure)
 
855
        self.assertEqual(test.id(), result._call[0].id())
 
856
        self.assertEqual('known failure', result._call[1])
848
857
        # we dont introspec the traceback, if the rest is ok, it would be
849
858
        # exceptional for it not to be.
850
859
        # it should update the known_failure_count on the object.
944
953
        test.run(result)
945
954
        # it should invoke 'addNotSupported'.
946
955
        self.assertEqual(2, len(result._call))
947
 
        self.assertEqual(test, result._call[0])
 
956
        self.assertEqual(test.id(), result._call[0].id())
948
957
        self.assertEqual(feature, result._call[1])
949
958
        # and not count as an error
950
959
        self.assertEqual(0, result.error_count)
1028
1037
        # the final output when real failures occur.
1029
1038
        class Test(tests.TestCase):
1030
1039
            def known_failure_test(self):
1031
 
                raise tests.KnownFailure('failed')
 
1040
                self.expectFailure('failed', self.assertTrue, False)
1032
1041
        test = unittest.TestSuite()
1033
1042
        test.addTest(Test("known_failure_test"))
1034
1043
        def failing_test():
1035
 
            raise AssertionError('foo')
 
1044
            self.fail('foo')
1036
1045
        test.addTest(unittest.FunctionTestCase(failing_test))
1037
1046
        stream = StringIO()
1038
1047
        runner = tests.TextTestRunner(stream=stream)
1039
1048
        result = self.run_test_runner(runner, test)
1040
1049
        lines = stream.getvalue().splitlines()
1041
1050
        self.assertContainsRe(stream.getvalue(),
1042
 
            '(?sm)^testing.*$'
 
1051
            '(?sm)^bzr selftest.*$'
1043
1052
            '.*'
1044
1053
            '^======================================================================\n'
1045
 
            '^FAIL: unittest.FunctionTestCase \\(failing_test\\)\n'
 
1054
            '^FAIL: failing_test\n'
1046
1055
            '^----------------------------------------------------------------------\n'
1047
1056
            'Traceback \\(most recent call last\\):\n'
1048
1057
            '  .*' # File .*, line .*, in failing_test' - but maybe not from .pyc
1049
 
            '    raise AssertionError\\(\'foo\'\\)\n'
 
1058
            '    self.fail\\(\'foo\'\\)\n'
1050
1059
            '.*'
1051
1060
            '^----------------------------------------------------------------------\n'
1052
1061
            '.*'
1058
1067
        # the final output.
1059
1068
        class Test(tests.TestCase):
1060
1069
            def known_failure_test(self):
1061
 
                raise tests.KnownFailure('failed')
 
1070
                self.expectFailure('failed', self.assertTrue, False)
1062
1071
        test = Test("known_failure_test")
1063
1072
        stream = StringIO()
1064
1073
        runner = tests.TextTestRunner(stream=stream)
1186
1195
        # Reading from the actual source tree breaks isolation, but we don't
1187
1196
        # want to assume that thats *all* that would happen.
1188
1197
        self._get_source_tree_calls = []
1189
 
        def _get_bzr_source_tree():
 
1198
        def new_get():
1190
1199
            self._get_source_tree_calls.append("called")
1191
1200
            return None
1192
 
        orig_get_bzr_source_tree = bzrlib.version._get_bzr_source_tree
1193
 
        bzrlib.version._get_bzr_source_tree = _get_bzr_source_tree
1194
 
        def restore():
1195
 
            bzrlib.version._get_bzr_source_tree = orig_get_bzr_source_tree
1196
 
        self.addCleanup(restore)
 
1201
        self.overrideAttr(bzrlib.version, '_get_bzr_source_tree',  new_get)
1197
1202
 
1198
1203
    def test_bench_history(self):
1199
1204
        # tests that the running the benchmark passes bench_history into
1209
1214
        self.assertContainsRe(output_string, "--date [0-9.]+")
1210
1215
        self.assertLength(1, self._get_source_tree_calls)
1211
1216
 
1212
 
    def assertLogDeleted(self, test):
1213
 
        log = test._get_log()
1214
 
        self.assertEqual("DELETED log file to reduce memory footprint", log)
1215
 
        self.assertEqual('', test._log_contents)
1216
 
        self.assertIs(None, test._log_file_name)
1217
 
 
1218
 
    def test_success_log_deleted(self):
1219
 
        """Successful tests have their log deleted"""
1220
 
 
1221
 
        class LogTester(tests.TestCase):
1222
 
 
1223
 
            def test_success(self):
1224
 
                self.log('this will be removed\n')
1225
 
 
1226
 
        sio = StringIO()
1227
 
        runner = tests.TextTestRunner(stream=sio)
1228
 
        test = LogTester('test_success')
1229
 
        result = self.run_test_runner(runner, test)
1230
 
 
1231
 
        self.assertLogDeleted(test)
1232
 
 
1233
 
    def test_skipped_log_deleted(self):
1234
 
        """Skipped tests have their log deleted"""
1235
 
 
1236
 
        class LogTester(tests.TestCase):
1237
 
 
1238
 
            def test_skipped(self):
1239
 
                self.log('this will be removed\n')
1240
 
                raise tests.TestSkipped()
1241
 
 
1242
 
        sio = StringIO()
1243
 
        runner = tests.TextTestRunner(stream=sio)
1244
 
        test = LogTester('test_skipped')
1245
 
        result = self.run_test_runner(runner, test)
1246
 
 
1247
 
        self.assertLogDeleted(test)
1248
 
 
1249
 
    def test_not_aplicable_log_deleted(self):
1250
 
        """Not applicable tests have their log deleted"""
1251
 
 
1252
 
        class LogTester(tests.TestCase):
1253
 
 
1254
 
            def test_not_applicable(self):
1255
 
                self.log('this will be removed\n')
1256
 
                raise tests.TestNotApplicable()
1257
 
 
1258
 
        sio = StringIO()
1259
 
        runner = tests.TextTestRunner(stream=sio)
1260
 
        test = LogTester('test_not_applicable')
1261
 
        result = self.run_test_runner(runner, test)
1262
 
 
1263
 
        self.assertLogDeleted(test)
1264
 
 
1265
 
    def test_known_failure_log_deleted(self):
1266
 
        """Know failure tests have their log deleted"""
1267
 
 
1268
 
        class LogTester(tests.TestCase):
1269
 
 
1270
 
            def test_known_failure(self):
1271
 
                self.log('this will be removed\n')
1272
 
                raise tests.KnownFailure()
1273
 
 
1274
 
        sio = StringIO()
1275
 
        runner = tests.TextTestRunner(stream=sio)
1276
 
        test = LogTester('test_known_failure')
1277
 
        result = self.run_test_runner(runner, test)
1278
 
 
1279
 
        self.assertLogDeleted(test)
1280
 
 
1281
 
    def test_fail_log_kept(self):
1282
 
        """Failed tests have their log kept"""
1283
 
 
1284
 
        class LogTester(tests.TestCase):
1285
 
 
1286
 
            def test_fail(self):
1287
 
                self.log('this will be kept\n')
1288
 
                self.fail('this test fails')
1289
 
 
1290
 
        sio = StringIO()
1291
 
        runner = tests.TextTestRunner(stream=sio)
1292
 
        test = LogTester('test_fail')
1293
 
        result = self.run_test_runner(runner, test)
1294
 
 
1295
 
        text = sio.getvalue()
1296
 
        self.assertContainsRe(text, 'this will be kept')
1297
 
        self.assertContainsRe(text, 'this test fails')
1298
 
 
1299
 
        log = test._get_log()
1300
 
        self.assertContainsRe(log, 'this will be kept')
1301
 
        self.assertEqual(log, test._log_contents)
1302
 
 
1303
 
    def test_error_log_kept(self):
1304
 
        """Tests with errors have their log kept"""
1305
 
 
1306
 
        class LogTester(tests.TestCase):
1307
 
 
1308
 
            def test_error(self):
1309
 
                self.log('this will be kept\n')
1310
 
                raise ValueError('random exception raised')
1311
 
 
1312
 
        sio = StringIO()
1313
 
        runner = tests.TextTestRunner(stream=sio)
1314
 
        test = LogTester('test_error')
1315
 
        result = self.run_test_runner(runner, test)
1316
 
 
1317
 
        text = sio.getvalue()
1318
 
        self.assertContainsRe(text, 'this will be kept')
1319
 
        self.assertContainsRe(text, 'random exception raised')
1320
 
 
1321
 
        log = test._get_log()
1322
 
        self.assertContainsRe(log, 'this will be kept')
1323
 
        self.assertEqual(log, test._log_contents)
1324
 
 
1325
1217
    def test_startTestRun(self):
1326
1218
        """run should call result.startTestRun()"""
1327
1219
        calls = []
1420
1312
        self.assertEqual(flags, bzrlib.debug.debug_flags)
1421
1313
 
1422
1314
    def change_selftest_debug_flags(self, new_flags):
1423
 
        orig_selftest_flags = tests.selftest_debug_flags
1424
 
        self.addCleanup(self._restore_selftest_debug_flags, orig_selftest_flags)
1425
 
        tests.selftest_debug_flags = set(new_flags)
1426
 
 
1427
 
    def _restore_selftest_debug_flags(self, flags):
1428
 
        tests.selftest_debug_flags = flags
 
1315
        self.overrideAttr(tests, 'selftest_debug_flags', set(new_flags))
1429
1316
 
1430
1317
    def test_allow_debug_flag(self):
1431
1318
        """The -Eallow_debug flag prevents bzrlib.debug.debug_flags from being
1491
1378
        self.assertEqual(set(['original-state']), bzrlib.debug.debug_flags)
1492
1379
 
1493
1380
    def make_test_result(self):
 
1381
        """Get a test result that writes to the test log file."""
1494
1382
        return tests.TextTestResult(self._log_file, descriptions=0, verbosity=1)
1495
1383
 
1496
1384
    def inner_test(self):
1504
1392
        result = self.make_test_result()
1505
1393
        self.inner_test.run(result)
1506
1394
        note("outer finish")
 
1395
        self.addCleanup(osutils.delete_any, self._log_file_name)
1507
1396
 
1508
1397
    def test_trace_nesting(self):
1509
1398
        # this tests that each test case nests its trace facility correctly.
1521
1410
        outer_test = TestTestCase("outer_child")
1522
1411
        result = self.make_test_result()
1523
1412
        outer_test.run(result)
1524
 
        self.addCleanup(osutils.delete_any, outer_test._log_file_name)
1525
1413
        self.assertEqual(original_trace, bzrlib.trace._trace_file)
1526
1414
 
1527
1415
    def method_that_times_a_bit_twice(self):
1582
1470
        # Manually set one up (TestCase doesn't and shouldn't provide magic
1583
1471
        # machinery)
1584
1472
        transport_server = MemoryServer()
1585
 
        transport_server.setUp()
1586
 
        self.addCleanup(transport_server.tearDown)
 
1473
        transport_server.start_server()
 
1474
        self.addCleanup(transport_server.stop_server)
1587
1475
        t = transport.get_transport(transport_server.get_url())
1588
1476
        bzrdir.BzrDir.create(t.base)
1589
1477
        self.assertRaises(errors.BzrError,
1642
1530
        """Test disabled tests behaviour with support aware results."""
1643
1531
        test = SampleTestCase('_test_pass')
1644
1532
        class DisabledFeature(object):
 
1533
            def __eq__(self, other):
 
1534
                return isinstance(other, DisabledFeature)
1645
1535
            def available(self):
1646
1536
                return False
1647
1537
        the_feature = DisabledFeature()
1658
1548
                self.calls.append(('addNotSupported', test, feature))
1659
1549
        result = InstrumentedTestResult()
1660
1550
        test.run(result)
 
1551
        case = result.calls[0][1]
1661
1552
        self.assertEqual([
1662
 
            ('startTest', test),
1663
 
            ('addNotSupported', test, the_feature),
1664
 
            ('stopTest', test),
 
1553
            ('startTest', case),
 
1554
            ('addNotSupported', case, the_feature),
 
1555
            ('stopTest', case),
1665
1556
            ],
1666
1557
            result.calls)
1667
1558
 
1728
1619
        self.assertRaises(AssertionError,
1729
1620
            self.assertListRaises, _TestException, success_generator)
1730
1621
 
 
1622
    def test_overrideAttr_without_value(self):
 
1623
        self.test_attr = 'original' # Define a test attribute
 
1624
        obj = self # Make 'obj' visible to the embedded test
 
1625
        class Test(tests.TestCase):
 
1626
 
 
1627
            def setUp(self):
 
1628
                tests.TestCase.setUp(self)
 
1629
                self.orig = self.overrideAttr(obj, 'test_attr')
 
1630
 
 
1631
            def test_value(self):
 
1632
                self.assertEqual('original', self.orig)
 
1633
                self.assertEqual('original', obj.test_attr)
 
1634
                obj.test_attr = 'modified'
 
1635
                self.assertEqual('modified', obj.test_attr)
 
1636
 
 
1637
        test = Test('test_value')
 
1638
        test.run(unittest.TestResult())
 
1639
        self.assertEqual('original', obj.test_attr)
 
1640
 
 
1641
    def test_overrideAttr_with_value(self):
 
1642
        self.test_attr = 'original' # Define a test attribute
 
1643
        obj = self # Make 'obj' visible to the embedded test
 
1644
        class Test(tests.TestCase):
 
1645
 
 
1646
            def setUp(self):
 
1647
                tests.TestCase.setUp(self)
 
1648
                self.orig = self.overrideAttr(obj, 'test_attr', new='modified')
 
1649
 
 
1650
            def test_value(self):
 
1651
                self.assertEqual('original', self.orig)
 
1652
                self.assertEqual('modified', obj.test_attr)
 
1653
 
 
1654
        test = Test('test_value')
 
1655
        test.run(unittest.TestResult())
 
1656
        self.assertEqual('original', obj.test_attr)
 
1657
 
1731
1658
 
1732
1659
# NB: Don't delete this; it's not actually from 0.11!
1733
1660
@deprecated_function(deprecated_in((0, 11, 0)))
2431
2358
        self.assertEqual('bzr: interrupted\n', result[1])
2432
2359
 
2433
2360
 
2434
 
class TestKnownFailure(tests.TestCase):
2435
 
 
2436
 
    def test_known_failure(self):
2437
 
        """Check that KnownFailure is defined appropriately."""
2438
 
        # a KnownFailure is an assertion error for compatability with unaware
2439
 
        # runners.
2440
 
        self.assertIsInstance(tests.KnownFailure(""), AssertionError)
2441
 
 
2442
 
    def test_expect_failure(self):
2443
 
        try:
2444
 
            self.expectFailure("Doomed to failure", self.assertTrue, False)
2445
 
        except tests.KnownFailure, e:
2446
 
            self.assertEqual('Doomed to failure', e.args[0])
2447
 
        try:
2448
 
            self.expectFailure("Doomed to failure", self.assertTrue, True)
2449
 
        except AssertionError, e:
2450
 
            self.assertEqual('Unexpected success.  Should have failed:'
2451
 
                             ' Doomed to failure', e.args[0])
2452
 
        else:
2453
 
            self.fail('Assertion not raised')
2454
 
 
2455
 
 
2456
2361
class TestFeature(tests.TestCase):
2457
2362
 
2458
2363
    def test_caching(self):
2709
2614
        # the test framework
2710
2615
        self.assertEquals('always fails', str(e))
2711
2616
        # check that there's no traceback in the test log
2712
 
        self.assertNotContainsRe(self._get_log(keep_log_file=True),
2713
 
            r'Traceback')
 
2617
        self.assertNotContainsRe(self.get_log(), r'Traceback')
2714
2618
 
2715
2619
    def test_run_bzr_user_error_caught(self):
2716
2620
        # Running bzr in blackbox mode, normal/expected/user errors should be
2717
2621
        # caught in the regular way and turned into an error message plus exit
2718
2622
        # code.
2719
2623
        transport_server = MemoryServer()
2720
 
        transport_server.setUp()
2721
 
        self.addCleanup(transport_server.tearDown)
 
2624
        transport_server.start_server()
 
2625
        self.addCleanup(transport_server.stop_server)
2722
2626
        url = transport_server.get_url()
2723
2627
        self.permit_url(url)
2724
2628
        out, err = self.run_bzr(["log", "%s/nonexistantpath" % url], retcode=3)
2880
2784
        # test doubles that supply a few sample tests to load, and check they
2881
2785
        # are loaded.
2882
2786
        calls = []
2883
 
        def _test_suite_testmod_names():
 
2787
        def testmod_names():
2884
2788
            calls.append("testmod_names")
2885
2789
            return [
2886
2790
                'bzrlib.tests.blackbox.test_branch',
2887
2791
                'bzrlib.tests.per_transport',
2888
2792
                'bzrlib.tests.test_selftest',
2889
2793
                ]
2890
 
        original_testmod_names = tests._test_suite_testmod_names
2891
 
        def _test_suite_modules_to_doctest():
 
2794
        self.overrideAttr(tests, '_test_suite_testmod_names', testmod_names)
 
2795
        def doctests():
2892
2796
            calls.append("modules_to_doctest")
2893
2797
            return ['bzrlib.timestamp']
2894
 
        orig_modules_to_doctest = tests._test_suite_modules_to_doctest
2895
 
        def restore_names():
2896
 
            tests._test_suite_testmod_names = original_testmod_names
2897
 
            tests._test_suite_modules_to_doctest = orig_modules_to_doctest
2898
 
        self.addCleanup(restore_names)
2899
 
        tests._test_suite_testmod_names = _test_suite_testmod_names
2900
 
        tests._test_suite_modules_to_doctest = _test_suite_modules_to_doctest
 
2798
        self.overrideAttr(tests, '_test_suite_modules_to_doctest', doctests)
2901
2799
        expected_test_list = [
2902
2800
            # testmod_names
2903
2801
            'bzrlib.tests.blackbox.test_branch.TestBranch.test_branch',
2969
2867
    def test_load_tests(self):
2970
2868
        test_list = ['bzrlib.tests.test_sampler.DemoTest.test_nothing']
2971
2869
        loader = self._create_loader(test_list)
2972
 
 
2973
2870
        suite = loader.loadTestsFromModuleName('bzrlib.tests.test_sampler')
2974
2871
        self.assertEquals(test_list, _test_ids(suite))
2975
2872
 
2976
2873
    def test_exclude_tests(self):
2977
2874
        test_list = ['bogus']
2978
2875
        loader = self._create_loader(test_list)
2979
 
 
2980
2876
        suite = loader.loadTestsFromModuleName('bzrlib.tests.test_sampler')
2981
2877
        self.assertEquals([], _test_ids(suite))
2982
2878
 
3027
2923
        tpr.register('bar', 'bbb.aaa.rrr')
3028
2924
        tpr.register('bar', 'bBB.aAA.rRR')
3029
2925
        self.assertEquals('bbb.aaa.rrr', tpr.get('bar'))
3030
 
        self.assertContainsRe(self._get_log(keep_log_file=True),
3031
 
                              r'.*bar.*bbb.aaa.rrr.*bBB.aAA.rRR')
 
2926
        self.assertThat(self.get_log(),
 
2927
            DocTestMatches("...bar...bbb.aaa.rrr...BB.aAA.rRR", ELLIPSIS))
3032
2928
 
3033
2929
    def test_get_unknown_prefix(self):
3034
2930
        tpr = self._get_registry()