836
836
self.assertContainsRe(output,
837
837
r"LSProf output for <type 'unicode'>\(\('world',\), {'errors': 'replace'}\)\n")
839
def test_uses_time_from_testtools(self):
840
"""Test case timings in verbose results should use testtools times"""
842
class TimeAddedVerboseTestResult(tests.VerboseTestResult):
843
def startTest(self, test):
844
self.time(datetime.datetime.utcfromtimestamp(1.145))
845
super(TimeAddedVerboseTestResult, self).startTest(test)
846
def addSuccess(self, test):
847
self.time(datetime.datetime.utcfromtimestamp(51.147))
848
super(TimeAddedVerboseTestResult, self).addSuccess(test)
849
def report_tests_starting(self): pass
851
self.get_passing_test().run(TimeAddedVerboseTestResult(sio, 0, 2))
852
self.assertEndsWith(sio.getvalue(), "OK 50002ms\n")
854
839
def test_known_failure(self):
855
840
"""A KnownFailure being raised should trigger several result actions."""
856
841
class InstrumentedTestResult(tests.ExtendedTestResult):
857
842
def stopTestRun(self): pass
858
def report_tests_starting(self): pass
843
def startTests(self): pass
844
def report_test_start(self, test): pass
859
845
def report_known_failure(self, test, err=None, details=None):
860
846
self._call = test, 'known failure'
861
847
result = InstrumentedTestResult(None, None, None, None)
1238
1214
self.assertContainsRe(output_string, "--date [0-9.]+")
1239
1215
self.assertLength(1, self._get_source_tree_calls)
1241
def test_verbose_test_count(self):
1242
"""A verbose test run reports the right test count at the start"""
1243
suite = TestUtil.TestSuite([
1244
unittest.FunctionTestCase(lambda:None),
1245
unittest.FunctionTestCase(lambda:None)])
1246
self.assertEqual(suite.countTestCases(), 2)
1248
runner = tests.TextTestRunner(stream=stream, verbosity=2)
1249
# Need to use the CountingDecorator as that's what sets num_tests
1250
result = self.run_test_runner(runner, tests.CountingDecorator(suite))
1251
self.assertStartsWith(stream.getvalue(), "running 2 tests")
1253
1217
def test_startTestRun(self):
1254
1218
"""run should call result.startTestRun()"""
1256
class LoggingDecorator(ExtendedToOriginalDecorator):
1220
class LoggingDecorator(tests.ForwardingResult):
1257
1221
def startTestRun(self):
1258
ExtendedToOriginalDecorator.startTestRun(self)
1222
tests.ForwardingResult.startTestRun(self)
1259
1223
calls.append('startTestRun')
1260
1224
test = unittest.FunctionTestCase(lambda:None)
1261
1225
stream = StringIO()
1709
1656
self.assertEqual('original', obj.test_attr)
1712
class _MissingFeature(tests.Feature):
1715
missing_feature = _MissingFeature()
1718
def _get_test(name):
1719
"""Get an instance of a specific example test.
1721
We protect this in a function so that they don't auto-run in the test
1725
class ExampleTests(tests.TestCase):
1727
def test_fail(self):
1728
mutter('this was a failing test')
1729
self.fail('this test will fail')
1731
def test_error(self):
1732
mutter('this test errored')
1733
raise RuntimeError('gotcha')
1735
def test_missing_feature(self):
1736
mutter('missing the feature')
1737
self.requireFeature(missing_feature)
1739
def test_skip(self):
1740
mutter('this test will be skipped')
1741
raise tests.TestSkipped('reason')
1743
def test_success(self):
1744
mutter('this test succeeds')
1746
def test_xfail(self):
1747
mutter('test with expected failure')
1748
self.knownFailure('this_fails')
1750
def test_unexpected_success(self):
1751
mutter('test with unexpected success')
1752
self.expectFailure('should_fail', lambda: None)
1754
return ExampleTests(name)
1757
class TestTestCaseLogDetails(tests.TestCase):
1759
def _run_test(self, test_name):
1760
test = _get_test(test_name)
1761
result = testtools.TestResult()
1765
def test_fail_has_log(self):
1766
result = self._run_test('test_fail')
1767
self.assertEqual(1, len(result.failures))
1768
result_content = result.failures[0][1]
1769
self.assertContainsRe(result_content, 'Text attachment: log')
1770
self.assertContainsRe(result_content, 'this was a failing test')
1772
def test_error_has_log(self):
1773
result = self._run_test('test_error')
1774
self.assertEqual(1, len(result.errors))
1775
result_content = result.errors[0][1]
1776
self.assertContainsRe(result_content, 'Text attachment: log')
1777
self.assertContainsRe(result_content, 'this test errored')
1779
def test_skip_has_no_log(self):
1780
result = self._run_test('test_skip')
1781
self.assertEqual(['reason'], result.skip_reasons.keys())
1782
skips = result.skip_reasons['reason']
1783
self.assertEqual(1, len(skips))
1785
self.assertFalse('log' in test.getDetails())
1787
def test_missing_feature_has_no_log(self):
1788
# testtools doesn't know about addNotSupported, so it just gets
1789
# considered as a skip
1790
result = self._run_test('test_missing_feature')
1791
self.assertEqual([missing_feature], result.skip_reasons.keys())
1792
skips = result.skip_reasons[missing_feature]
1793
self.assertEqual(1, len(skips))
1795
self.assertFalse('log' in test.getDetails())
1797
def test_xfail_has_no_log(self):
1798
result = self._run_test('test_xfail')
1799
self.assertEqual(1, len(result.expectedFailures))
1800
result_content = result.expectedFailures[0][1]
1801
self.assertNotContainsRe(result_content, 'Text attachment: log')
1802
self.assertNotContainsRe(result_content, 'test with expected failure')
1804
def test_unexpected_success_has_log(self):
1805
result = self._run_test('test_unexpected_success')
1806
self.assertEqual(1, len(result.unexpectedSuccesses))
1807
# Inconsistency, unexpectedSuccesses is a list of tests,
1808
# expectedFailures is a list of reasons?
1809
test = result.unexpectedSuccesses[0]
1810
details = test.getDetails()
1811
self.assertTrue('log' in details)
1814
class TestTestCloning(tests.TestCase):
1815
"""Tests that test cloning of TestCases (as used by multiply_tests)."""
1817
def test_cloned_testcase_does_not_share_details(self):
1818
"""A TestCase cloned with clone_test does not share mutable attributes
1819
such as details or cleanups.
1821
class Test(tests.TestCase):
1823
self.addDetail('foo', Content('text/plain', lambda: 'foo'))
1824
orig_test = Test('test_foo')
1825
cloned_test = tests.clone_test(orig_test, orig_test.id() + '(cloned)')
1826
orig_test.run(unittest.TestResult())
1827
self.assertEqual('foo', orig_test.getDetails()['foo'].iter_bytes())
1828
self.assertEqual(None, cloned_test.getDetails().get('foo'))
1830
def test_double_apply_scenario_preserves_first_scenario(self):
1831
"""Applying two levels of scenarios to a test preserves the attributes
1832
added by both scenarios.
1834
class Test(tests.TestCase):
1837
test = Test('test_foo')
1838
scenarios_x = [('x=1', {'x': 1}), ('x=2', {'x': 2})]
1839
scenarios_y = [('y=1', {'y': 1}), ('y=2', {'y': 2})]
1840
suite = tests.multiply_tests(test, scenarios_x, unittest.TestSuite())
1841
suite = tests.multiply_tests(suite, scenarios_y, unittest.TestSuite())
1842
all_tests = list(tests.iter_suite_tests(suite))
1843
self.assertLength(4, all_tests)
1844
all_xys = sorted((t.x, t.y) for t in all_tests)
1845
self.assertEqual([(1, 1), (1, 2), (2, 1), (2, 2)], all_xys)
1848
1659
# NB: Don't delete this; it's not actually from 0.11!
1849
1660
@deprecated_function(deprecated_in((0, 11, 0)))
1850
1661
def sample_deprecated_function():
2161
1972
load_list='missing file name', list_only=True)
2164
class TestSubunitLogDetails(tests.TestCase, SelfTestHelper):
2166
_test_needs_features = [features.subunit]
2168
def run_subunit_stream(self, test_name):
2169
from subunit import ProtocolTestCase
2171
return TestUtil.TestSuite([_get_test(test_name)])
2172
stream = self.run_selftest(runner_class=tests.SubUnitBzrRunner,
2173
test_suite_factory=factory)
2174
test = ProtocolTestCase(stream)
2175
result = testtools.TestResult()
2177
content = stream.getvalue()
2178
return content, result
2180
def test_fail_has_log(self):
2181
content, result = self.run_subunit_stream('test_fail')
2182
self.assertEqual(1, len(result.failures))
2183
self.assertContainsRe(content, '(?m)^log$')
2184
self.assertContainsRe(content, 'this test will fail')
2186
def test_error_has_log(self):
2187
content, result = self.run_subunit_stream('test_error')
2188
self.assertContainsRe(content, '(?m)^log$')
2189
self.assertContainsRe(content, 'this test errored')
2191
def test_skip_has_no_log(self):
2192
content, result = self.run_subunit_stream('test_skip')
2193
self.assertNotContainsRe(content, '(?m)^log$')
2194
self.assertNotContainsRe(content, 'this test will be skipped')
2195
self.assertEqual(['reason'], result.skip_reasons.keys())
2196
skips = result.skip_reasons['reason']
2197
self.assertEqual(1, len(skips))
2199
# RemotedTestCase doesn't preserve the "details"
2200
## self.assertFalse('log' in test.getDetails())
2202
def test_missing_feature_has_no_log(self):
2203
content, result = self.run_subunit_stream('test_missing_feature')
2204
self.assertNotContainsRe(content, '(?m)^log$')
2205
self.assertNotContainsRe(content, 'missing the feature')
2206
self.assertEqual(['_MissingFeature\n'], result.skip_reasons.keys())
2207
skips = result.skip_reasons['_MissingFeature\n']
2208
self.assertEqual(1, len(skips))
2210
# RemotedTestCase doesn't preserve the "details"
2211
## self.assertFalse('log' in test.getDetails())
2213
def test_xfail_has_no_log(self):
2214
content, result = self.run_subunit_stream('test_xfail')
2215
self.assertNotContainsRe(content, '(?m)^log$')
2216
self.assertNotContainsRe(content, 'test with expected failure')
2217
self.assertEqual(1, len(result.expectedFailures))
2218
result_content = result.expectedFailures[0][1]
2219
self.assertNotContainsRe(result_content, 'Text attachment: log')
2220
self.assertNotContainsRe(result_content, 'test with expected failure')
2222
def test_unexpected_success_has_log(self):
2223
content, result = self.run_subunit_stream('test_unexpected_success')
2224
self.assertContainsRe(content, '(?m)^log$')
2225
self.assertContainsRe(content, 'test with unexpected success')
2226
self.expectFailure('subunit treats "unexpectedSuccess"'
2227
' as a plain success',
2228
self.assertEqual, 1, len(result.unexpectedSuccesses))
2229
self.assertEqual(1, len(result.unexpectedSuccesses))
2230
test = result.unexpectedSuccesses[0]
2231
# RemotedTestCase doesn't preserve the "details"
2232
## self.assertTrue('log' in test.getDetails())
2234
def test_success_has_no_log(self):
2235
content, result = self.run_subunit_stream('test_success')
2236
self.assertEqual(1, result.testsRun)
2237
self.assertNotContainsRe(content, '(?m)^log$')
2238
self.assertNotContainsRe(content, 'this test succeeds')
2241
1975
class TestRunBzr(tests.TestCase):
3233
2961
self.assertEquals('bzrlib.plugins', tpr.resolve_alias('bp'))
3236
class TestThreadLeakDetection(tests.TestCase):
3237
"""Ensure when tests leak threads we detect and report it"""
3239
class LeakRecordingResult(tests.ExtendedTestResult):
3241
tests.ExtendedTestResult.__init__(self, StringIO(), 0, 1)
3243
def _report_thread_leak(self, test, leaks, alive):
3244
self.leaks.append((test, leaks))
3246
def test_testcase_without_addCleanups(self):
3247
"""Check old TestCase instances don't break with leak detection"""
3248
class Test(unittest.TestCase):
3251
result = self.LeakRecordingResult()
3253
result.startTestRun()
3255
result.stopTestRun()
3256
self.assertEqual(result._tests_leaking_threads_count, 0)
3257
self.assertEqual(result.leaks, [])
3259
def test_thread_leak(self):
3260
"""Ensure a thread that outlives the running of a test is reported
3262
Uses a thread that blocks on an event, and is started by the inner
3263
test case. As the thread outlives the inner case's run, it should be
3264
detected as a leak, but the event is then set so that the thread can
3265
be safely joined in cleanup so it's not leaked for real.
3267
event = threading.Event()
3268
thread = threading.Thread(name="Leaker", target=event.wait)
3269
class Test(tests.TestCase):
3270
def test_leak(self):
3272
result = self.LeakRecordingResult()
3273
test = Test("test_leak")
3274
self.addCleanup(thread.join)
3275
self.addCleanup(event.set)
3276
result.startTestRun()
3278
result.stopTestRun()
3279
self.assertEqual(result._tests_leaking_threads_count, 1)
3280
self.assertEqual(result._first_thread_leaker_id, test.id())
3281
self.assertEqual(result.leaks, [(test, set([thread]))])
3282
self.assertContainsString(result.stream.getvalue(), "leaking threads")
3284
def test_multiple_leaks(self):
3285
"""Check multiple leaks are blamed on the test cases at fault
3287
Same concept as the previous test, but has one inner test method that
3288
leaks two threads, and one that doesn't leak at all.
3290
event = threading.Event()
3291
thread_a = threading.Thread(name="LeakerA", target=event.wait)
3292
thread_b = threading.Thread(name="LeakerB", target=event.wait)
3293
thread_c = threading.Thread(name="LeakerC", target=event.wait)
3294
class Test(tests.TestCase):
3295
def test_first_leak(self):
3297
def test_second_no_leak(self):
3299
def test_third_leak(self):
3302
result = self.LeakRecordingResult()
3303
first_test = Test("test_first_leak")
3304
third_test = Test("test_third_leak")
3305
self.addCleanup(thread_a.join)
3306
self.addCleanup(thread_b.join)
3307
self.addCleanup(thread_c.join)
3308
self.addCleanup(event.set)
3309
result.startTestRun()
3311
[first_test, Test("test_second_no_leak"), third_test]
3313
result.stopTestRun()
3314
self.assertEqual(result._tests_leaking_threads_count, 2)
3315
self.assertEqual(result._first_thread_leaker_id, first_test.id())
3316
self.assertEqual(result.leaks, [
3317
(first_test, set([thread_b])),
3318
(third_test, set([thread_a, thread_c]))])
3319
self.assertContainsString(result.stream.getvalue(), "leaking threads")
3322
class TestPostMortemDebugging(tests.TestCase):
3323
"""Check post mortem debugging works when tests fail or error"""
3325
class TracebackRecordingResult(tests.ExtendedTestResult):
3327
tests.ExtendedTestResult.__init__(self, StringIO(), 0, 1)
3328
self.postcode = None
3329
def _post_mortem(self, tb=None):
3330
"""Record the code object at the end of the current traceback"""
3331
tb = tb or sys.exc_info()[2]
3334
while next is not None:
3337
self.postcode = tb.tb_frame.f_code
3338
def report_error(self, test, err):
3340
def report_failure(self, test, err):
3343
def test_location_unittest_error(self):
3344
"""Needs right post mortem traceback with erroring unittest case"""
3345
class Test(unittest.TestCase):
3348
result = self.TracebackRecordingResult()
3350
self.assertEqual(result.postcode, Test.runTest.func_code)
3352
def test_location_unittest_failure(self):
3353
"""Needs right post mortem traceback with failing unittest case"""
3354
class Test(unittest.TestCase):
3356
raise self.failureException
3357
result = self.TracebackRecordingResult()
3359
self.assertEqual(result.postcode, Test.runTest.func_code)
3361
def test_location_bt_error(self):
3362
"""Needs right post mortem traceback with erroring bzrlib.tests case"""
3363
class Test(tests.TestCase):
3364
def test_error(self):
3366
result = self.TracebackRecordingResult()
3367
Test("test_error").run(result)
3368
self.assertEqual(result.postcode, Test.test_error.func_code)
3370
def test_location_bt_failure(self):
3371
"""Needs right post mortem traceback with failing bzrlib.tests case"""
3372
class Test(tests.TestCase):
3373
def test_failure(self):
3374
raise self.failureException
3375
result = self.TracebackRecordingResult()
3376
Test("test_failure").run(result)
3377
self.assertEqual(result.postcode, Test.test_failure.func_code)
3379
def test_env_var_triggers_post_mortem(self):
3380
"""Check pdb.post_mortem is called iff BZR_TEST_PDB is set"""
3382
result = tests.ExtendedTestResult(StringIO(), 0, 1)
3383
post_mortem_calls = []
3384
self.overrideAttr(pdb, "post_mortem", post_mortem_calls.append)
3385
self.overrideEnv('BZR_TEST_PDB', None)
3386
result._post_mortem(1)
3387
self.overrideEnv('BZR_TEST_PDB', 'on')
3388
result._post_mortem(2)
3389
self.assertEqual([2], post_mortem_calls)
3392
2964
class TestRunSuite(tests.TestCase):
3394
2966
def test_runner_class(self):
3405
2977
self.verbosity)
3406
2978
tests.run_suite(suite, runner_class=MyRunner, stream=StringIO())
3407
2979
self.assertLength(1, calls)
3410
class TestEnvironHandling(tests.TestCase):
3412
def test_overrideEnv_None_called_twice_doesnt_leak(self):
3413
self.failIf('MYVAR' in os.environ)
3414
self.overrideEnv('MYVAR', '42')
3415
# We use an embedded test to make sure we fix the _captureVar bug
3416
class Test(tests.TestCase):
3418
# The first call save the 42 value
3419
self.overrideEnv('MYVAR', None)
3420
self.assertEquals(None, os.environ.get('MYVAR'))
3421
# Make sure we can call it twice
3422
self.overrideEnv('MYVAR', None)
3423
self.assertEquals(None, os.environ.get('MYVAR'))
3425
result = tests.TextTestResult(output, 0, 1)
3426
Test('test_me').run(result)
3427
if not result.wasStrictlySuccessful():
3428
self.fail(output.getvalue())
3429
# We get our value back
3430
self.assertEquals('42', os.environ.get('MYVAR'))
3433
class TestIsolatedEnv(tests.TestCase):
3434
"""Test isolating tests from os.environ.
3436
Since we use tests that are already isolated from os.environ a bit of care
3437
should be taken when designing the tests to avoid bootstrap side-effects.
3438
The tests start an already clean os.environ which allow doing valid
3439
assertions about which variables are present or not and design tests around
3443
class ScratchMonkey(tests.TestCase):
3448
def test_basics(self):
3449
# Make sure we know the definition of BZR_HOME: not part of os.environ
3450
# for tests.TestCase.
3451
self.assertTrue('BZR_HOME' in tests.isolated_environ)
3452
self.assertEquals(None, tests.isolated_environ['BZR_HOME'])
3453
# Being part of isolated_environ, BZR_HOME should not appear here
3454
self.assertFalse('BZR_HOME' in os.environ)
3455
# Make sure we know the definition of LINES: part of os.environ for
3457
self.assertTrue('LINES' in tests.isolated_environ)
3458
self.assertEquals('25', tests.isolated_environ['LINES'])
3459
self.assertEquals('25', os.environ['LINES'])
3461
def test_injecting_unknown_variable(self):
3462
# BZR_HOME is known to be absent from os.environ
3463
test = self.ScratchMonkey('test_me')
3464
tests.override_os_environ(test, {'BZR_HOME': 'foo'})
3465
self.assertEquals('foo', os.environ['BZR_HOME'])
3466
tests.restore_os_environ(test)
3467
self.assertFalse('BZR_HOME' in os.environ)
3469
def test_injecting_known_variable(self):
3470
test = self.ScratchMonkey('test_me')
3471
# LINES is known to be present in os.environ
3472
tests.override_os_environ(test, {'LINES': '42'})
3473
self.assertEquals('42', os.environ['LINES'])
3474
tests.restore_os_environ(test)
3475
self.assertEquals('25', os.environ['LINES'])
3477
def test_deleting_variable(self):
3478
test = self.ScratchMonkey('test_me')
3479
# LINES is known to be present in os.environ
3480
tests.override_os_environ(test, {'LINES': None})
3481
self.assertTrue('LINES' not in os.environ)
3482
tests.restore_os_environ(test)
3483
self.assertEquals('25', os.environ['LINES'])
3486
class TestDocTestSuiteIsolation(tests.TestCase):
3487
"""Test that `tests.DocTestSuite` isolates doc tests from os.environ.
3489
Since tests.TestCase alreay provides an isolation from os.environ, we use
3490
the clean environment as a base for testing. To precisely capture the
3491
isolation provided by tests.DocTestSuite, we use doctest.DocTestSuite to
3494
We want to make sure `tests.DocTestSuite` respect `tests.isolated_environ`,
3495
not `os.environ` so each test overrides it to suit its needs.
3499
def get_doctest_suite_for_string(self, klass, string):
3500
class Finder(doctest.DocTestFinder):
3502
def find(*args, **kwargs):
3503
test = doctest.DocTestParser().get_doctest(
3504
string, {}, 'foo', 'foo.py', 0)
3507
suite = klass(test_finder=Finder())
3510
def run_doctest_suite_for_string(self, klass, string):
3511
suite = self.get_doctest_suite_for_string(klass, string)
3513
result = tests.TextTestResult(output, 0, 1)
3515
return result, output
3517
def assertDocTestStringSucceds(self, klass, string):
3518
result, output = self.run_doctest_suite_for_string(klass, string)
3519
if not result.wasStrictlySuccessful():
3520
self.fail(output.getvalue())
3522
def assertDocTestStringFails(self, klass, string):
3523
result, output = self.run_doctest_suite_for_string(klass, string)
3524
if result.wasStrictlySuccessful():
3525
self.fail(output.getvalue())
3527
def test_injected_variable(self):
3528
self.overrideAttr(tests, 'isolated_environ', {'LINES': '42'})
3531
>>> os.environ['LINES']
3534
# doctest.DocTestSuite fails as it sees '25'
3535
self.assertDocTestStringFails(doctest.DocTestSuite, test)
3536
# tests.DocTestSuite sees '42'
3537
self.assertDocTestStringSucceds(tests.IsolatedDocTestSuite, test)
3539
def test_deleted_variable(self):
3540
self.overrideAttr(tests, 'isolated_environ', {'LINES': None})
3543
>>> os.environ.get('LINES')
3545
# doctest.DocTestSuite fails as it sees '25'
3546
self.assertDocTestStringFails(doctest.DocTestSuite, test)
3547
# tests.DocTestSuite sees None
3548
self.assertDocTestStringSucceds(tests.IsolatedDocTestSuite, test)
added
removed
bzrlib/tests/test_selftest.py
