← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_selftest.py

  • Committer: Robert Collins
  • Date: 2007-03-27 07:58:34 UTC
  • mfrom: (2367.1.9 benchmark-strace)
  • mto: This revision was merged to the branch mainline in revision 2379.
  • Revision ID: robertc@robertcollins.net-20070327075834-51wgidn6o63h8lqo
(robertc) Merge bzrlib.strace module which adds support for stracing individual callables from within bzrlib.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_selftest.py
38
38
from bzrlib.symbol_versioning import zero_ten, zero_eleven
39
39
from bzrlib.tests import (
40
40
                          ChrootedTestCase,
 
41
                          ExtendedTestResult,
 
42
                          Feature,
 
43
                          KnownFailure,
41
44
                          TestCase,
42
45
                          TestCaseInTempDir,
43
46
                          TestCaseWithMemoryTransport,
45
48
                          TestSkipped,
46
49
                          TestSuite,
47
50
                          TextTestRunner,
 
51
                          UnavailableFeature,
48
52
                          )
49
53
from bzrlib.tests.test_sftp_transport import TestCaseWithSFTPServer
50
54
from bzrlib.tests.TestUtil import _load_module_by_name
690
694
        self.assertContainsRe(output,
691
695
            r"LSProf output for <type 'unicode'>\(\('world',\), {'errors': 'replace'}\)\n")
692
696
 
 
697
    def test_known_failure(self):
 
698
        """A KnownFailure being raised should trigger several result actions."""
 
699
        class InstrumentedTestResult(ExtendedTestResult):
 
700
 
 
701
            def report_test_start(self, test): pass
 
702
            def report_known_failure(self, test, err):
 
703
                self._call = test, err
 
704
        result = InstrumentedTestResult(None, None, None, None)
 
705
        def test_function():
 
706
            raise KnownFailure('failed!')
 
707
        test = unittest.FunctionTestCase(test_function)
 
708
        test.run(result)
 
709
        # it should invoke 'report_known_failure'.
 
710
        self.assertEqual(2, len(result._call))
 
711
        self.assertEqual(test, result._call[0])
 
712
        self.assertEqual(KnownFailure, result._call[1][0])
 
713
        self.assertIsInstance(result._call[1][1], KnownFailure)
 
714
        # we dont introspec the traceback, if the rest is ok, it would be
 
715
        # exceptional for it not to be.
 
716
        # it should update the known_failure_count on the object.
 
717
        self.assertEqual(1, result.known_failure_count)
 
718
        # the result should be successful.
 
719
        self.assertTrue(result.wasSuccessful())
 
720
 
 
721
    def test_verbose_report_known_failure(self):
 
722
        # verbose test output formatting
 
723
        result_stream = StringIO()
 
724
        result = bzrlib.tests.VerboseTestResult(
 
725
            unittest._WritelnDecorator(result_stream),
 
726
            descriptions=0,
 
727
            verbosity=2,
 
728
            )
 
729
        test = self.get_passing_test()
 
730
        result.startTest(test)
 
731
        result.extractBenchmarkTime(test)
 
732
        prefix = len(result_stream.getvalue())
 
733
        # the err parameter has the shape:
 
734
        # (class, exception object, traceback)
 
735
        # KnownFailures dont get their tracebacks shown though, so we
 
736
        # can skip that.
 
737
        err = (KnownFailure, KnownFailure('foo'), None)
 
738
        result.report_known_failure(test, err)
 
739
        output = result_stream.getvalue()[prefix:]
 
740
        lines = output.splitlines()
 
741
        self.assertEqual(lines, ['XFAIL                   0ms', '    foo'])
 
742
    
 
743
    def test_text_report_known_failure(self):
 
744
        # text test output formatting
 
745
        pb = MockProgress()
 
746
        result = bzrlib.tests.TextTestResult(
 
747
            None,
 
748
            descriptions=0,
 
749
            verbosity=1,
 
750
            pb=pb,
 
751
            )
 
752
        test = self.get_passing_test()
 
753
        # this seeds the state to handle reporting the test.
 
754
        result.startTest(test)
 
755
        result.extractBenchmarkTime(test)
 
756
        # the err parameter has the shape:
 
757
        # (class, exception object, traceback)
 
758
        # KnownFailures dont get their tracebacks shown though, so we
 
759
        # can skip that.
 
760
        err = (KnownFailure, KnownFailure('foo'), None)
 
761
        result.report_known_failure(test, err)
 
762
        self.assertEqual(
 
763
            [
 
764
            ('update', '[1 in 0s] passing_test', None, None),
 
765
            ('note', 'XFAIL: %s\n%s\n', ('passing_test', err[1]))
 
766
            ],
 
767
            pb.calls)
 
768
        # known_failures should be printed in the summary, so if we run a test
 
769
        # after there are some known failures, the update prefix should match
 
770
        # this.
 
771
        result.known_failure_count = 3
 
772
        test.run(result)
 
773
        self.assertEqual(
 
774
            [
 
775
            ('update', '[2 in 0s, 3 known failures] passing_test', None, None),
 
776
            ],
 
777
            pb.calls[2:])
 
778
 
 
779
    def get_passing_test(self):
 
780
        """Return a test object that can't be run usefully."""
 
781
        def passing_test():
 
782
            pass
 
783
        return unittest.FunctionTestCase(passing_test)
 
784
 
 
785
    def test_add_not_supported(self):
 
786
        """Test the behaviour of invoking addNotSupported."""
 
787
        class InstrumentedTestResult(ExtendedTestResult):
 
788
            def report_test_start(self, test): pass
 
789
            def report_unsupported(self, test, feature):
 
790
                self._call = test, feature
 
791
        result = InstrumentedTestResult(None, None, None, None)
 
792
        test = SampleTestCase('_test_pass')
 
793
        feature = Feature()
 
794
        result.startTest(test)
 
795
        result.addNotSupported(test, feature)
 
796
        # it should invoke 'report_unsupported'.
 
797
        self.assertEqual(2, len(result._call))
 
798
        self.assertEqual(test, result._call[0])
 
799
        self.assertEqual(feature, result._call[1])
 
800
        # the result should be successful.
 
801
        self.assertTrue(result.wasSuccessful())
 
802
        # it should record the test against a count of tests not run due to
 
803
        # this feature.
 
804
        self.assertEqual(1, result.unsupported['Feature'])
 
805
        # and invoking it again should increment that counter
 
806
        result.addNotSupported(test, feature)
 
807
        self.assertEqual(2, result.unsupported['Feature'])
 
808
 
 
809
    def test_verbose_report_unsupported(self):
 
810
        # verbose test output formatting
 
811
        result_stream = StringIO()
 
812
        result = bzrlib.tests.VerboseTestResult(
 
813
            unittest._WritelnDecorator(result_stream),
 
814
            descriptions=0,
 
815
            verbosity=2,
 
816
            )
 
817
        test = self.get_passing_test()
 
818
        feature = Feature()
 
819
        result.startTest(test)
 
820
        result.extractBenchmarkTime(test)
 
821
        prefix = len(result_stream.getvalue())
 
822
        result.report_unsupported(test, feature)
 
823
        output = result_stream.getvalue()[prefix:]
 
824
        lines = output.splitlines()
 
825
        self.assertEqual(lines, ['NODEP                   0ms', "    The feature 'Feature' is not available."])
 
826
    
 
827
    def test_text_report_unsupported(self):
 
828
        # text test output formatting
 
829
        pb = MockProgress()
 
830
        result = bzrlib.tests.TextTestResult(
 
831
            None,
 
832
            descriptions=0,
 
833
            verbosity=1,
 
834
            pb=pb,
 
835
            )
 
836
        test = self.get_passing_test()
 
837
        feature = Feature()
 
838
        # this seeds the state to handle reporting the test.
 
839
        result.startTest(test)
 
840
        result.extractBenchmarkTime(test)
 
841
        result.report_unsupported(test, feature)
 
842
        # no output on unsupported features
 
843
        self.assertEqual(
 
844
            [('update', '[1 in 0s] passing_test', None, None)
 
845
            ],
 
846
            pb.calls)
 
847
        # the number of missing features should be printed in the progress
 
848
        # summary, so check for that.
 
849
        result.unsupported = {'foo':0, 'bar':0}
 
850
        test.run(result)
 
851
        self.assertEqual(
 
852
            [
 
853
            ('update', '[2 in 0s, 2 missing features] passing_test', None, None),
 
854
            ],
 
855
            pb.calls[1:])
 
856
    
 
857
    def test_unavailable_exception(self):
 
858
        """An UnavailableFeature being raised should invoke addNotSupported."""
 
859
        class InstrumentedTestResult(ExtendedTestResult):
 
860
 
 
861
            def report_test_start(self, test): pass
 
862
            def addNotSupported(self, test, feature):
 
863
                self._call = test, feature
 
864
        result = InstrumentedTestResult(None, None, None, None)
 
865
        feature = Feature()
 
866
        def test_function():
 
867
            raise UnavailableFeature(feature)
 
868
        test = unittest.FunctionTestCase(test_function)
 
869
        test.run(result)
 
870
        # it should invoke 'addNotSupported'.
 
871
        self.assertEqual(2, len(result._call))
 
872
        self.assertEqual(test, result._call[0])
 
873
        self.assertEqual(feature, result._call[1])
 
874
        # and not count as an error
 
875
        self.assertEqual(0, result.error_count)
 
876
 
693
877
 
694
878
class TestRunner(TestCase):
695
879
 
712
896
        finally:
713
897
            TestCaseInTempDir.TEST_ROOT = old_root
714
898
 
 
899
    def test_known_failure_failed_run(self):
 
900
        # run a test that generates a known failure which should be printed in
 
901
        # the final output when real failures occur.
 
902
        def known_failure_test():
 
903
            raise KnownFailure('failed')
 
904
        test = unittest.TestSuite()
 
905
        test.addTest(unittest.FunctionTestCase(known_failure_test))
 
906
        def failing_test():
 
907
            raise AssertionError('foo')
 
908
        test.addTest(unittest.FunctionTestCase(failing_test))
 
909
        stream = StringIO()
 
910
        runner = TextTestRunner(stream=stream)
 
911
        result = self.run_test_runner(runner, test)
 
912
        lines = stream.getvalue().splitlines()
 
913
        self.assertEqual([
 
914
            '',
 
915
            '======================================================================',
 
916
            'FAIL: unittest.FunctionTestCase (failing_test)',
 
917
            '----------------------------------------------------------------------',
 
918
            'Traceback (most recent call last):',
 
919
            '    raise AssertionError(\'foo\')',
 
920
            'AssertionError: foo',
 
921
            '',
 
922
            '----------------------------------------------------------------------',
 
923
            '',
 
924
            'FAILED (failures=1, known_failure_count=1)'],
 
925
            lines[0:5] + lines[6:10] + lines[11:])
 
926
 
 
927
    def test_known_failure_ok_run(self):
 
928
        # run a test that generates a known failure which should be printed in the final output.
 
929
        def known_failure_test():
 
930
            raise KnownFailure('failed')
 
931
        test = unittest.FunctionTestCase(known_failure_test)
 
932
        stream = StringIO()
 
933
        runner = TextTestRunner(stream=stream)
 
934
        result = self.run_test_runner(runner, test)
 
935
        self.assertEqual(
 
936
            '\n'
 
937
            '----------------------------------------------------------------------\n'
 
938
            'Ran 1 test in 0.000s\n'
 
939
            '\n'
 
940
            'OK (known_failures=1)\n',
 
941
            stream.getvalue())
 
942
 
715
943
    def test_skipped_test(self):
716
944
        # run a test that is skipped, and check the suite as a whole still
717
945
        # succeeds.
765
993
        # Check if cleanup was called the right number of times.
766
994
        self.assertEqual(0, test.counter)
767
995
 
 
996
    def test_unsupported_features_listed(self):
 
997
        """When unsupported features are encountered they are detailed."""
 
998
        class Feature1(Feature):
 
999
            def _probe(self): return False
 
1000
        class Feature2(Feature):
 
1001
            def _probe(self): return False
 
1002
        # create sample tests
 
1003
        test1 = SampleTestCase('_test_pass')
 
1004
        test1._test_needs_features = [Feature1()]
 
1005
        test2 = SampleTestCase('_test_pass')
 
1006
        test2._test_needs_features = [Feature2()]
 
1007
        test = unittest.TestSuite()
 
1008
        test.addTest(test1)
 
1009
        test.addTest(test2)
 
1010
        stream = StringIO()
 
1011
        runner = TextTestRunner(stream=stream)
 
1012
        result = self.run_test_runner(runner, test)
 
1013
        lines = stream.getvalue().splitlines()
 
1014
        self.assertEqual([
 
1015
            'OK',
 
1016
            "Missing feature 'Feature1' skipped 1 tests.",
 
1017
            "Missing feature 'Feature2' skipped 1 tests.",
 
1018
            ],
 
1019
            lines[-3:])
 
1020
 
768
1021
    def test_bench_history(self):
769
1022
        # tests that the running the benchmark produces a history file
770
1023
        # containing a timestamp and the revision id of the bzrlib source which
843
1096
        self.assertEqual(log, test._log_contents)
844
1097
 
845
1098
 
 
1099
class SampleTestCase(TestCase):
 
1100
 
 
1101
    def _test_pass(self):
 
1102
        pass
 
1103
 
 
1104
 
846
1105
class TestTestCase(TestCase):
847
1106
    """Tests that test the core bzrlib TestCase."""
848
1107
 
924
1183
        self.assertIsInstance(self._benchcalls[0][1], bzrlib.lsprof.Stats)
925
1184
        self.assertIsInstance(self._benchcalls[1][1], bzrlib.lsprof.Stats)
926
1185
 
 
1186
    def test_knownFailure(self):
 
1187
        """Self.knownFailure() should raise a KnownFailure exception."""
 
1188
        self.assertRaises(KnownFailure, self.knownFailure, "A Failure")
 
1189
 
 
1190
    def test_requireFeature_available(self):
 
1191
        """self.requireFeature(available) is a no-op."""
 
1192
        class Available(Feature):
 
1193
            def _probe(self):return True
 
1194
        feature = Available()
 
1195
        self.requireFeature(feature)
 
1196
 
 
1197
    def test_requireFeature_unavailable(self):
 
1198
        """self.requireFeature(unavailable) raises UnavailableFeature."""
 
1199
        class Unavailable(Feature):
 
1200
            def _probe(self):return False
 
1201
        feature = Unavailable()
 
1202
        self.assertRaises(UnavailableFeature, self.requireFeature, feature)
 
1203
 
 
1204
    def test_run_no_parameters(self):
 
1205
        test = SampleTestCase('_test_pass')
 
1206
        test.run()
 
1207
    
 
1208
    def test_run_enabled_unittest_result(self):
 
1209
        """Test we revert to regular behaviour when the test is enabled."""
 
1210
        test = SampleTestCase('_test_pass')
 
1211
        class EnabledFeature(object):
 
1212
            def available(self):
 
1213
                return True
 
1214
        test._test_needs_features = [EnabledFeature()]
 
1215
        result = unittest.TestResult()
 
1216
        test.run(result)
 
1217
        self.assertEqual(1, result.testsRun)
 
1218
        self.assertEqual([], result.errors)
 
1219
        self.assertEqual([], result.failures)
 
1220
 
 
1221
    def test_run_disabled_unittest_result(self):
 
1222
        """Test our compatability for disabled tests with unittest results."""
 
1223
        test = SampleTestCase('_test_pass')
 
1224
        class DisabledFeature(object):
 
1225
            def available(self):
 
1226
                return False
 
1227
        test._test_needs_features = [DisabledFeature()]
 
1228
        result = unittest.TestResult()
 
1229
        test.run(result)
 
1230
        self.assertEqual(1, result.testsRun)
 
1231
        self.assertEqual([], result.errors)
 
1232
        self.assertEqual([], result.failures)
 
1233
 
 
1234
    def test_run_disabled_supporting_result(self):
 
1235
        """Test disabled tests behaviour with support aware results."""
 
1236
        test = SampleTestCase('_test_pass')
 
1237
        class DisabledFeature(object):
 
1238
            def available(self):
 
1239
                return False
 
1240
        the_feature = DisabledFeature()
 
1241
        test._test_needs_features = [the_feature]
 
1242
        class InstrumentedTestResult(unittest.TestResult):
 
1243
            def __init__(self):
 
1244
                unittest.TestResult.__init__(self)
 
1245
                self.calls = []
 
1246
            def startTest(self, test):
 
1247
                self.calls.append(('startTest', test))
 
1248
            def stopTest(self, test):
 
1249
                self.calls.append(('stopTest', test))
 
1250
            def addNotSupported(self, test, feature):
 
1251
                self.calls.append(('addNotSupported', test, feature))
 
1252
        result = InstrumentedTestResult()
 
1253
        test.run(result)
 
1254
        self.assertEqual([
 
1255
            ('startTest', test),
 
1256
            ('addNotSupported', test, the_feature),
 
1257
            ('stopTest', test),
 
1258
            ],
 
1259
            result.calls)
 
1260
 
927
1261
 
928
1262
@symbol_versioning.deprecated_function(zero_eleven)
929
1263
def sample_deprecated_function():
1085
1419
        self.assertEquals(['bzr','bzrlib','setup.py',
1086
1420
                           'test9999.tmp','tests'],
1087
1421
                           after)
 
1422
 
 
1423
 
 
1424
class TestKnownFailure(TestCase):
 
1425
 
 
1426
    def test_known_failure(self):
 
1427
        """Check that KnownFailure is defined appropriately."""
 
1428
        # a KnownFailure is an assertion error for compatability with unaware
 
1429
        # runners.
 
1430
        self.assertIsInstance(KnownFailure(""), AssertionError)
 
1431
 
 
1432
 
 
1433
class TestFeature(TestCase):
 
1434
 
 
1435
    def test_caching(self):
 
1436
        """Feature._probe is called by the feature at most once."""
 
1437
        class InstrumentedFeature(Feature):
 
1438
            def __init__(self):
 
1439
                Feature.__init__(self)
 
1440
                self.calls = []
 
1441
            def _probe(self):
 
1442
                self.calls.append('_probe')
 
1443
                return False
 
1444
        feature = InstrumentedFeature()
 
1445
        feature.available()
 
1446
        self.assertEqual(['_probe'], feature.calls)
 
1447
        feature.available()
 
1448
        self.assertEqual(['_probe'], feature.calls)
 
1449
 
 
1450
    def test_named_str(self):
 
1451
        """Feature.__str__ should thunk to feature_name()."""
 
1452
        class NamedFeature(Feature):
 
1453
            def feature_name(self):
 
1454
                return 'symlinks'
 
1455
        feature = NamedFeature()
 
1456
        self.assertEqual('symlinks', str(feature))
 
1457
 
 
1458
    def test_default_str(self):
 
1459
        """Feature.__str__ should default to __class__.__name__."""
 
1460
        class NamedFeature(Feature):
 
1461
            pass
 
1462
        feature = NamedFeature()
 
1463
        self.assertEqual('NamedFeature', str(feature))
 
1464
 
 
1465
 
 
1466
class TestUnavailableFeature(TestCase):
 
1467
 
 
1468
    def test_access_feature(self):
 
1469
        feature = Feature()
 
1470
        exception = UnavailableFeature(feature)
 
1471
        self.assertIs(feature, exception.args[0])