← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_selftest.py

Merge cleanup into description

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_selftest.py
17
17
"""Tests for the test framework."""
18
18
 
19
19
from cStringIO import StringIO
 
20
from doctest import ELLIPSIS
20
21
import os
21
22
import signal
22
23
import sys
24
25
import unittest
25
26
import warnings
26
27
 
 
28
from testtools import MultiTestResult
 
29
from testtools.content_type import ContentType
 
30
from testtools.matchers import (
 
31
    DocTestMatches,
 
32
    Equals,
 
33
    )
 
34
import testtools.tests.helpers
 
35
 
27
36
import bzrlib
28
37
from bzrlib import (
29
38
    branchbuilder,
52
61
    deprecated_method,
53
62
    )
54
63
from bzrlib.tests import (
55
 
    SubUnitFeature,
 
64
    features,
56
65
    test_lsprof,
57
66
    test_sftp_transport,
58
67
    TestUtil,
78
87
                          TestUtil._load_module_by_name,
79
88
                          'bzrlib.no-name-yet')
80
89
 
 
90
 
81
91
class MetaTestLog(tests.TestCase):
82
92
 
83
93
    def test_logging(self):
84
94
        """Test logs are captured when a test fails."""
85
95
        self.log('a test message')
86
 
        self._log_file.flush()
87
 
        self.assertContainsRe(self._get_log(keep_log_file=True),
88
 
                              'a test message\n')
 
96
        details = self.getDetails()
 
97
        log = details['log']
 
98
        self.assertThat(log.content_type, Equals(ContentType(
 
99
            "text", "plain", {"charset": "utf8"})))
 
100
        self.assertThat(u"".join(log.iter_text()), Equals(self.get_log()))
 
101
        self.assertThat(self.get_log(),
 
102
            DocTestMatches(u"...a test message\n", ELLIPSIS))
89
103
 
90
104
 
91
105
class TestUnicodeFilename(tests.TestCase):
687
701
 
688
702
    def test_profiles_tests(self):
689
703
        self.requireFeature(test_lsprof.LSProfFeature)
690
 
        terminal = unittest.TestResult()
 
704
        terminal = testtools.tests.helpers.ExtendedTestResult()
691
705
        result = tests.ProfileResult(terminal)
692
706
        class Sample(tests.TestCase):
693
707
            def a(self):
695
709
            def sample_function(self):
696
710
                pass
697
711
        test = Sample("a")
698
 
        test.attrs_to_keep = test.attrs_to_keep + ('_benchcalls',)
699
712
        test.run(result)
700
 
        self.assertLength(1, test._benchcalls)
 
713
        case = terminal._events[0][1]
 
714
        self.assertLength(1, case._benchcalls)
701
715
        # We must be able to unpack it as the test reporting code wants
702
 
        (_, _, _), stats = test._benchcalls[0]
 
716
        (_, _, _), stats = case._benchcalls[0]
703
717
        self.assertTrue(callable(stats.pprint))
704
718
 
705
719
 
710
724
                descriptions=0,
711
725
                verbosity=1,
712
726
                )
713
 
        test_case.run(result)
714
 
        timed_string = result._testTimeString(test_case)
 
727
        capture = testtools.tests.helpers.ExtendedTestResult()
 
728
        test_case.run(MultiTestResult(result, capture))
 
729
        run_case = capture._events[0][1]
 
730
        timed_string = result._testTimeString(run_case)
715
731
        self.assertContainsRe(timed_string, expected_re)
716
732
 
717
733
    def test_test_reporting(self):
832
848
            def stopTestRun(self): pass
833
849
            def startTests(self): pass
834
850
            def report_test_start(self, test): pass
835
 
            def report_known_failure(self, test, err):
836
 
                self._call = test, err
 
851
            def report_known_failure(self, test, err=None, details=None):
 
852
                self._call = test, 'known failure'
837
853
        result = InstrumentedTestResult(None, None, None, None)
838
854
        class Test(tests.TestCase):
839
855
            def test_function(self):
842
858
        test.run(result)
843
859
        # it should invoke 'report_known_failure'.
844
860
        self.assertEqual(2, len(result._call))
845
 
        self.assertEqual(test, result._call[0])
846
 
        self.assertEqual(tests.KnownFailure, result._call[1][0])
847
 
        self.assertIsInstance(result._call[1][1], tests.KnownFailure)
 
861
        self.assertEqual(test.id(), result._call[0].id())
 
862
        self.assertEqual('known failure', result._call[1])
848
863
        # we dont introspec the traceback, if the rest is ok, it would be
849
864
        # exceptional for it not to be.
850
865
        # it should update the known_failure_count on the object.
944
959
        test.run(result)
945
960
        # it should invoke 'addNotSupported'.
946
961
        self.assertEqual(2, len(result._call))
947
 
        self.assertEqual(test, result._call[0])
 
962
        self.assertEqual(test.id(), result._call[0].id())
948
963
        self.assertEqual(feature, result._call[1])
949
964
        # and not count as an error
950
965
        self.assertEqual(0, result.error_count)
1028
1043
        # the final output when real failures occur.
1029
1044
        class Test(tests.TestCase):
1030
1045
            def known_failure_test(self):
1031
 
                raise tests.KnownFailure('failed')
 
1046
                self.expectFailure('failed', self.assertTrue, False)
1032
1047
        test = unittest.TestSuite()
1033
1048
        test.addTest(Test("known_failure_test"))
1034
1049
        def failing_test():
1035
 
            raise AssertionError('foo')
 
1050
            self.fail('foo')
1036
1051
        test.addTest(unittest.FunctionTestCase(failing_test))
1037
1052
        stream = StringIO()
1038
1053
        runner = tests.TextTestRunner(stream=stream)
1039
1054
        result = self.run_test_runner(runner, test)
1040
1055
        lines = stream.getvalue().splitlines()
1041
1056
        self.assertContainsRe(stream.getvalue(),
1042
 
            '(?sm)^testing.*$'
 
1057
            '(?sm)^bzr selftest.*$'
1043
1058
            '.*'
1044
1059
            '^======================================================================\n'
1045
1060
            '^FAIL: unittest.FunctionTestCase \\(failing_test\\)\n'
1046
1061
            '^----------------------------------------------------------------------\n'
1047
1062
            'Traceback \\(most recent call last\\):\n'
1048
1063
            '  .*' # File .*, line .*, in failing_test' - but maybe not from .pyc
1049
 
            '    raise AssertionError\\(\'foo\'\\)\n'
 
1064
            '    self.fail\\(\'foo\'\\)\n'
1050
1065
            '.*'
1051
1066
            '^----------------------------------------------------------------------\n'
1052
1067
            '.*'
1058
1073
        # the final output.
1059
1074
        class Test(tests.TestCase):
1060
1075
            def known_failure_test(self):
1061
 
                raise tests.KnownFailure('failed')
 
1076
                self.expectFailure('failed', self.assertTrue, False)
1062
1077
        test = Test("known_failure_test")
1063
1078
        stream = StringIO()
1064
1079
        runner = tests.TextTestRunner(stream=stream)
1209
1224
        self.assertContainsRe(output_string, "--date [0-9.]+")
1210
1225
        self.assertLength(1, self._get_source_tree_calls)
1211
1226
 
1212
 
    def assertLogDeleted(self, test):
1213
 
        log = test._get_log()
1214
 
        self.assertEqual("DELETED log file to reduce memory footprint", log)
1215
 
        self.assertEqual('', test._log_contents)
1216
 
        self.assertIs(None, test._log_file_name)
1217
 
 
1218
 
    def test_success_log_deleted(self):
1219
 
        """Successful tests have their log deleted"""
1220
 
 
1221
 
        class LogTester(tests.TestCase):
1222
 
 
1223
 
            def test_success(self):
1224
 
                self.log('this will be removed\n')
1225
 
 
1226
 
        sio = StringIO()
1227
 
        runner = tests.TextTestRunner(stream=sio)
1228
 
        test = LogTester('test_success')
1229
 
        result = self.run_test_runner(runner, test)
1230
 
 
1231
 
        self.assertLogDeleted(test)
1232
 
 
1233
 
    def test_skipped_log_deleted(self):
1234
 
        """Skipped tests have their log deleted"""
1235
 
 
1236
 
        class LogTester(tests.TestCase):
1237
 
 
1238
 
            def test_skipped(self):
1239
 
                self.log('this will be removed\n')
1240
 
                raise tests.TestSkipped()
1241
 
 
1242
 
        sio = StringIO()
1243
 
        runner = tests.TextTestRunner(stream=sio)
1244
 
        test = LogTester('test_skipped')
1245
 
        result = self.run_test_runner(runner, test)
1246
 
 
1247
 
        self.assertLogDeleted(test)
1248
 
 
1249
 
    def test_not_aplicable_log_deleted(self):
1250
 
        """Not applicable tests have their log deleted"""
1251
 
 
1252
 
        class LogTester(tests.TestCase):
1253
 
 
1254
 
            def test_not_applicable(self):
1255
 
                self.log('this will be removed\n')
1256
 
                raise tests.TestNotApplicable()
1257
 
 
1258
 
        sio = StringIO()
1259
 
        runner = tests.TextTestRunner(stream=sio)
1260
 
        test = LogTester('test_not_applicable')
1261
 
        result = self.run_test_runner(runner, test)
1262
 
 
1263
 
        self.assertLogDeleted(test)
1264
 
 
1265
 
    def test_known_failure_log_deleted(self):
1266
 
        """Know failure tests have their log deleted"""
1267
 
 
1268
 
        class LogTester(tests.TestCase):
1269
 
 
1270
 
            def test_known_failure(self):
1271
 
                self.log('this will be removed\n')
1272
 
                raise tests.KnownFailure()
1273
 
 
1274
 
        sio = StringIO()
1275
 
        runner = tests.TextTestRunner(stream=sio)
1276
 
        test = LogTester('test_known_failure')
1277
 
        result = self.run_test_runner(runner, test)
1278
 
 
1279
 
        self.assertLogDeleted(test)
1280
 
 
1281
 
    def test_fail_log_kept(self):
1282
 
        """Failed tests have their log kept"""
1283
 
 
1284
 
        class LogTester(tests.TestCase):
1285
 
 
1286
 
            def test_fail(self):
1287
 
                self.log('this will be kept\n')
1288
 
                self.fail('this test fails')
1289
 
 
1290
 
        sio = StringIO()
1291
 
        runner = tests.TextTestRunner(stream=sio)
1292
 
        test = LogTester('test_fail')
1293
 
        result = self.run_test_runner(runner, test)
1294
 
 
1295
 
        text = sio.getvalue()
1296
 
        self.assertContainsRe(text, 'this will be kept')
1297
 
        self.assertContainsRe(text, 'this test fails')
1298
 
 
1299
 
        log = test._get_log()
1300
 
        self.assertContainsRe(log, 'this will be kept')
1301
 
        self.assertEqual(log, test._log_contents)
1302
 
 
1303
 
    def test_error_log_kept(self):
1304
 
        """Tests with errors have their log kept"""
1305
 
 
1306
 
        class LogTester(tests.TestCase):
1307
 
 
1308
 
            def test_error(self):
1309
 
                self.log('this will be kept\n')
1310
 
                raise ValueError('random exception raised')
1311
 
 
1312
 
        sio = StringIO()
1313
 
        runner = tests.TextTestRunner(stream=sio)
1314
 
        test = LogTester('test_error')
1315
 
        result = self.run_test_runner(runner, test)
1316
 
 
1317
 
        text = sio.getvalue()
1318
 
        self.assertContainsRe(text, 'this will be kept')
1319
 
        self.assertContainsRe(text, 'random exception raised')
1320
 
 
1321
 
        log = test._get_log()
1322
 
        self.assertContainsRe(log, 'this will be kept')
1323
 
        self.assertEqual(log, test._log_contents)
1324
 
 
1325
1227
    def test_startTestRun(self):
1326
1228
        """run should call result.startTestRun()"""
1327
1229
        calls = []
1491
1393
        self.assertEqual(set(['original-state']), bzrlib.debug.debug_flags)
1492
1394
 
1493
1395
    def make_test_result(self):
 
1396
        """Get a test result that writes to the test log file."""
1494
1397
        return tests.TextTestResult(self._log_file, descriptions=0, verbosity=1)
1495
1398
 
1496
1399
    def inner_test(self):
1504
1407
        result = self.make_test_result()
1505
1408
        self.inner_test.run(result)
1506
1409
        note("outer finish")
 
1410
        self.addCleanup(osutils.delete_any, self._log_file_name)
1507
1411
 
1508
1412
    def test_trace_nesting(self):
1509
1413
        # this tests that each test case nests its trace facility correctly.
1521
1425
        outer_test = TestTestCase("outer_child")
1522
1426
        result = self.make_test_result()
1523
1427
        outer_test.run(result)
1524
 
        self.addCleanup(osutils.delete_any, outer_test._log_file_name)
1525
1428
        self.assertEqual(original_trace, bzrlib.trace._trace_file)
1526
1429
 
1527
1430
    def method_that_times_a_bit_twice(self):
1642
1545
        """Test disabled tests behaviour with support aware results."""
1643
1546
        test = SampleTestCase('_test_pass')
1644
1547
        class DisabledFeature(object):
 
1548
            def __eq__(self, other):
 
1549
                return isinstance(other, DisabledFeature)
1645
1550
            def available(self):
1646
1551
                return False
1647
1552
        the_feature = DisabledFeature()
1658
1563
                self.calls.append(('addNotSupported', test, feature))
1659
1564
        result = InstrumentedTestResult()
1660
1565
        test.run(result)
 
1566
        case = result.calls[0][1]
1661
1567
        self.assertEqual([
1662
 
            ('startTest', test),
1663
 
            ('addNotSupported', test, the_feature),
1664
 
            ('stopTest', test),
 
1568
            ('startTest', case),
 
1569
            ('addNotSupported', case, the_feature),
 
1570
            ('stopTest', case),
1665
1571
            ],
1666
1572
            result.calls)
1667
1573
 
1982
1888
        self.assertEqual(expected.getvalue(), repeated.getvalue())
1983
1889
 
1984
1890
    def test_runner_class(self):
1985
 
        self.requireFeature(SubUnitFeature)
 
1891
        self.requireFeature(features.subunit)
1986
1892
        from subunit import ProtocolTestCase
1987
1893
        stream = self.run_selftest(runner_class=tests.SubUnitBzrRunner,
1988
1894
            test_suite_factory=self.factory)
2020
1926
        self.assertEqual(transport_server, captured_transport[0])
2021
1927
 
2022
1928
    def test_transport_sftp(self):
2023
 
        try:
2024
 
            import bzrlib.transport.sftp
2025
 
        except errors.ParamikoNotPresent:
2026
 
            raise tests.TestSkipped("Paramiko not present")
 
1929
        self.requireFeature(features.paramiko)
2027
1930
        self.check_transport_set(bzrlib.transport.sftp.SFTPAbsoluteServer)
2028
1931
 
2029
1932
    def test_transport_memory(self):
2434
2337
        self.assertEqual('bzr: interrupted\n', result[1])
2435
2338
 
2436
2339
 
2437
 
class TestKnownFailure(tests.TestCase):
2438
 
 
2439
 
    def test_known_failure(self):
2440
 
        """Check that KnownFailure is defined appropriately."""
2441
 
        # a KnownFailure is an assertion error for compatability with unaware
2442
 
        # runners.
2443
 
        self.assertIsInstance(tests.KnownFailure(""), AssertionError)
2444
 
 
2445
 
    def test_expect_failure(self):
2446
 
        try:
2447
 
            self.expectFailure("Doomed to failure", self.assertTrue, False)
2448
 
        except tests.KnownFailure, e:
2449
 
            self.assertEqual('Doomed to failure', e.args[0])
2450
 
        try:
2451
 
            self.expectFailure("Doomed to failure", self.assertTrue, True)
2452
 
        except AssertionError, e:
2453
 
            self.assertEqual('Unexpected success.  Should have failed:'
2454
 
                             ' Doomed to failure', e.args[0])
2455
 
        else:
2456
 
            self.fail('Assertion not raised')
2457
 
 
2458
 
 
2459
2340
class TestFeature(tests.TestCase):
2460
2341
 
2461
2342
    def test_caching(self):
2497
2378
        self.assertIs(feature, exception.args[0])
2498
2379
 
2499
2380
 
 
2381
simple_thunk_feature = tests._CompatabilityThunkFeature(
 
2382
    'bzrlib.tests', 'UnicodeFilename',
 
2383
    'bzrlib.tests.test_selftest.simple_thunk_feature',
 
2384
    deprecated_in((2,1,0)))
 
2385
 
 
2386
class Test_CompatibilityFeature(tests.TestCase):
 
2387
 
 
2388
    def test_does_thunk(self):
 
2389
        res = self.callDeprecated(
 
2390
            ['bzrlib.tests.test_selftest.simple_thunk_feature was deprecated'
 
2391
             ' in version 2.1.0. Use bzrlib.tests.UnicodeFilename instead.'],
 
2392
            simple_thunk_feature.available)
 
2393
        self.assertEqual(tests.UnicodeFilename.available(), res)
 
2394
 
 
2395
        
2500
2396
class TestModuleAvailableFeature(tests.TestCase):
2501
2397
 
2502
2398
    def test_available_module(self):
2697
2593
        # the test framework
2698
2594
        self.assertEquals('always fails', str(e))
2699
2595
        # check that there's no traceback in the test log
2700
 
        self.assertNotContainsRe(self._get_log(keep_log_file=True),
2701
 
            r'Traceback')
 
2596
        self.assertNotContainsRe(self.get_log(), r'Traceback')
2702
2597
 
2703
2598
    def test_run_bzr_user_error_caught(self):
2704
2599
        # Running bzr in blackbox mode, normal/expected/user errors should be
2957
2852
    def test_load_tests(self):
2958
2853
        test_list = ['bzrlib.tests.test_sampler.DemoTest.test_nothing']
2959
2854
        loader = self._create_loader(test_list)
2960
 
 
2961
2855
        suite = loader.loadTestsFromModuleName('bzrlib.tests.test_sampler')
2962
2856
        self.assertEquals(test_list, _test_ids(suite))
2963
2857
 
2964
2858
    def test_exclude_tests(self):
2965
2859
        test_list = ['bogus']
2966
2860
        loader = self._create_loader(test_list)
2967
 
 
2968
2861
        suite = loader.loadTestsFromModuleName('bzrlib.tests.test_sampler')
2969
2862
        self.assertEquals([], _test_ids(suite))
2970
2863
 
3015
2908
        tpr.register('bar', 'bbb.aaa.rrr')
3016
2909
        tpr.register('bar', 'bBB.aAA.rRR')
3017
2910
        self.assertEquals('bbb.aaa.rrr', tpr.get('bar'))
3018
 
        self.assertContainsRe(self._get_log(keep_log_file=True),
3019
 
                              r'.*bar.*bbb.aaa.rrr.*bBB.aAA.rRR')
 
2911
        self.assertThat(self.get_log(),
 
2912
            DocTestMatches("...bar...bbb.aaa.rrr...BB.aAA.rRR", ELLIPSIS))
3020
2913
 
3021
2914
    def test_get_unknown_prefix(self):
3022
2915
        tpr = self._get_registry()