833
837
self.assertContainsRe(output,
834
838
r"LSProf output for <type 'unicode'>\(\('world',\), {'errors': 'replace'}\)\n")
840
def test_uses_time_from_testtools(self):
841
"""Test case timings in verbose results should use testtools times"""
843
class TimeAddedVerboseTestResult(tests.VerboseTestResult):
844
def startTest(self, test):
845
self.time(datetime.datetime.utcfromtimestamp(1.145))
846
super(TimeAddedVerboseTestResult, self).startTest(test)
847
def addSuccess(self, test):
848
self.time(datetime.datetime.utcfromtimestamp(51.147))
849
super(TimeAddedVerboseTestResult, self).addSuccess(test)
850
def report_tests_starting(self): pass
852
self.get_passing_test().run(TimeAddedVerboseTestResult(sio, 0, 2))
853
self.assertEndsWith(sio.getvalue(), "OK 50002ms\n")
836
855
def test_known_failure(self):
837
856
"""A KnownFailure being raised should trigger several result actions."""
838
857
class InstrumentedTestResult(tests.ExtendedTestResult):
839
858
def stopTestRun(self): pass
840
def startTests(self): pass
841
def report_test_start(self, test): pass
859
def report_tests_starting(self): pass
842
860
def report_known_failure(self, test, err=None, details=None):
843
861
self._call = test, 'known failure'
844
862
result = InstrumentedTestResult(None, None, None, None)
1214
1239
self.assertContainsRe(output_string, "--date [0-9.]+")
1215
1240
self.assertLength(1, self._get_source_tree_calls)
1242
def test_verbose_test_count(self):
1243
"""A verbose test run reports the right test count at the start"""
1244
suite = TestUtil.TestSuite([
1245
unittest.FunctionTestCase(lambda:None),
1246
unittest.FunctionTestCase(lambda:None)])
1247
self.assertEqual(suite.countTestCases(), 2)
1249
runner = tests.TextTestRunner(stream=stream, verbosity=2)
1250
# Need to use the CountingDecorator as that's what sets num_tests
1251
result = self.run_test_runner(runner, tests.CountingDecorator(suite))
1252
self.assertStartsWith(stream.getvalue(), "running 2 tests")
1217
1254
def test_startTestRun(self):
1218
1255
"""run should call result.startTestRun()"""
1220
class LoggingDecorator(tests.ForwardingResult):
1257
class LoggingDecorator(ExtendedToOriginalDecorator):
1221
1258
def startTestRun(self):
1222
tests.ForwardingResult.startTestRun(self)
1259
ExtendedToOriginalDecorator.startTestRun(self)
1223
1260
calls.append('startTestRun')
1224
1261
test = unittest.FunctionTestCase(lambda:None)
1225
1262
stream = StringIO()
1656
1710
self.assertEqual('original', obj.test_attr)
1713
class _MissingFeature(tests.Feature):
1716
missing_feature = _MissingFeature()
1719
def _get_test(name):
1720
"""Get an instance of a specific example test.
1722
We protect this in a function so that they don't auto-run in the test
1726
class ExampleTests(tests.TestCase):
1728
def test_fail(self):
1729
mutter('this was a failing test')
1730
self.fail('this test will fail')
1732
def test_error(self):
1733
mutter('this test errored')
1734
raise RuntimeError('gotcha')
1736
def test_missing_feature(self):
1737
mutter('missing the feature')
1738
self.requireFeature(missing_feature)
1740
def test_skip(self):
1741
mutter('this test will be skipped')
1742
raise tests.TestSkipped('reason')
1744
def test_success(self):
1745
mutter('this test succeeds')
1747
def test_xfail(self):
1748
mutter('test with expected failure')
1749
self.knownFailure('this_fails')
1751
def test_unexpected_success(self):
1752
mutter('test with unexpected success')
1753
self.expectFailure('should_fail', lambda: None)
1755
return ExampleTests(name)
1758
class TestTestCaseLogDetails(tests.TestCase):
1760
def _run_test(self, test_name):
1761
test = _get_test(test_name)
1762
result = testtools.TestResult()
1766
def test_fail_has_log(self):
1767
result = self._run_test('test_fail')
1768
self.assertEqual(1, len(result.failures))
1769
result_content = result.failures[0][1]
1770
self.assertContainsRe(result_content, 'Text attachment: log')
1771
self.assertContainsRe(result_content, 'this was a failing test')
1773
def test_error_has_log(self):
1774
result = self._run_test('test_error')
1775
self.assertEqual(1, len(result.errors))
1776
result_content = result.errors[0][1]
1777
self.assertContainsRe(result_content, 'Text attachment: log')
1778
self.assertContainsRe(result_content, 'this test errored')
1780
def test_skip_has_no_log(self):
1781
result = self._run_test('test_skip')
1782
self.assertEqual(['reason'], result.skip_reasons.keys())
1783
skips = result.skip_reasons['reason']
1784
self.assertEqual(1, len(skips))
1786
self.assertFalse('log' in test.getDetails())
1788
def test_missing_feature_has_no_log(self):
1789
# testtools doesn't know about addNotSupported, so it just gets
1790
# considered as a skip
1791
result = self._run_test('test_missing_feature')
1792
self.assertEqual([missing_feature], result.skip_reasons.keys())
1793
skips = result.skip_reasons[missing_feature]
1794
self.assertEqual(1, len(skips))
1796
self.assertFalse('log' in test.getDetails())
1798
def test_xfail_has_no_log(self):
1799
result = self._run_test('test_xfail')
1800
self.assertEqual(1, len(result.expectedFailures))
1801
result_content = result.expectedFailures[0][1]
1802
self.assertNotContainsRe(result_content, 'Text attachment: log')
1803
self.assertNotContainsRe(result_content, 'test with expected failure')
1805
def test_unexpected_success_has_log(self):
1806
result = self._run_test('test_unexpected_success')
1807
self.assertEqual(1, len(result.unexpectedSuccesses))
1808
# Inconsistency, unexpectedSuccesses is a list of tests,
1809
# expectedFailures is a list of reasons?
1810
test = result.unexpectedSuccesses[0]
1811
details = test.getDetails()
1812
self.assertTrue('log' in details)
1815
class TestTestCloning(tests.TestCase):
1816
"""Tests that test cloning of TestCases (as used by multiply_tests)."""
1818
def test_cloned_testcase_does_not_share_details(self):
1819
"""A TestCase cloned with clone_test does not share mutable attributes
1820
such as details or cleanups.
1822
class Test(tests.TestCase):
1824
self.addDetail('foo', Content('text/plain', lambda: 'foo'))
1825
orig_test = Test('test_foo')
1826
cloned_test = tests.clone_test(orig_test, orig_test.id() + '(cloned)')
1827
orig_test.run(unittest.TestResult())
1828
self.assertEqual('foo', orig_test.getDetails()['foo'].iter_bytes())
1829
self.assertEqual(None, cloned_test.getDetails().get('foo'))
1831
def test_double_apply_scenario_preserves_first_scenario(self):
1832
"""Applying two levels of scenarios to a test preserves the attributes
1833
added by both scenarios.
1835
class Test(tests.TestCase):
1838
test = Test('test_foo')
1839
scenarios_x = [('x=1', {'x': 1}), ('x=2', {'x': 2})]
1840
scenarios_y = [('y=1', {'y': 1}), ('y=2', {'y': 2})]
1841
suite = tests.multiply_tests(test, scenarios_x, unittest.TestSuite())
1842
suite = tests.multiply_tests(suite, scenarios_y, unittest.TestSuite())
1843
all_tests = list(tests.iter_suite_tests(suite))
1844
self.assertLength(4, all_tests)
1845
all_xys = sorted((t.x, t.y) for t in all_tests)
1846
self.assertEqual([(1, 1), (1, 2), (2, 1), (2, 2)], all_xys)
1659
1849
# NB: Don't delete this; it's not actually from 0.11!
1660
1850
@deprecated_function(deprecated_in((0, 11, 0)))
1661
1851
def sample_deprecated_function():
1972
2163
load_list='missing file name', list_only=True)
2166
class TestSubunitLogDetails(tests.TestCase, SelfTestHelper):
2168
_test_needs_features = [features.subunit]
2170
def run_subunit_stream(self, test_name):
2171
from subunit import ProtocolTestCase
2173
return TestUtil.TestSuite([_get_test(test_name)])
2174
stream = self.run_selftest(runner_class=tests.SubUnitBzrRunner,
2175
test_suite_factory=factory)
2176
test = ProtocolTestCase(stream)
2177
result = testtools.TestResult()
2179
content = stream.getvalue()
2180
return content, result
2182
def test_fail_has_log(self):
2183
content, result = self.run_subunit_stream('test_fail')
2184
self.assertEqual(1, len(result.failures))
2185
self.assertContainsRe(content, '(?m)^log$')
2186
self.assertContainsRe(content, 'this test will fail')
2188
def test_error_has_log(self):
2189
content, result = self.run_subunit_stream('test_error')
2190
self.assertContainsRe(content, '(?m)^log$')
2191
self.assertContainsRe(content, 'this test errored')
2193
def test_skip_has_no_log(self):
2194
content, result = self.run_subunit_stream('test_skip')
2195
self.assertNotContainsRe(content, '(?m)^log$')
2196
self.assertNotContainsRe(content, 'this test will be skipped')
2197
self.assertEqual(['reason'], result.skip_reasons.keys())
2198
skips = result.skip_reasons['reason']
2199
self.assertEqual(1, len(skips))
2201
# RemotedTestCase doesn't preserve the "details"
2202
## self.assertFalse('log' in test.getDetails())
2204
def test_missing_feature_has_no_log(self):
2205
content, result = self.run_subunit_stream('test_missing_feature')
2206
self.assertNotContainsRe(content, '(?m)^log$')
2207
self.assertNotContainsRe(content, 'missing the feature')
2208
self.assertEqual(['_MissingFeature\n'], result.skip_reasons.keys())
2209
skips = result.skip_reasons['_MissingFeature\n']
2210
self.assertEqual(1, len(skips))
2212
# RemotedTestCase doesn't preserve the "details"
2213
## self.assertFalse('log' in test.getDetails())
2215
def test_xfail_has_no_log(self):
2216
content, result = self.run_subunit_stream('test_xfail')
2217
self.assertNotContainsRe(content, '(?m)^log$')
2218
self.assertNotContainsRe(content, 'test with expected failure')
2219
self.assertEqual(1, len(result.expectedFailures))
2220
result_content = result.expectedFailures[0][1]
2221
self.assertNotContainsRe(result_content, 'Text attachment: log')
2222
self.assertNotContainsRe(result_content, 'test with expected failure')
2224
def test_unexpected_success_has_log(self):
2225
content, result = self.run_subunit_stream('test_unexpected_success')
2226
self.assertContainsRe(content, '(?m)^log$')
2227
self.assertContainsRe(content, 'test with unexpected success')
2228
self.expectFailure('subunit treats "unexpectedSuccess"'
2229
' as a plain success',
2230
self.assertEqual, 1, len(result.unexpectedSuccesses))
2231
self.assertEqual(1, len(result.unexpectedSuccesses))
2232
test = result.unexpectedSuccesses[0]
2233
# RemotedTestCase doesn't preserve the "details"
2234
## self.assertTrue('log' in test.getDetails())
2236
def test_success_has_no_log(self):
2237
content, result = self.run_subunit_stream('test_success')
2238
self.assertEqual(1, result.testsRun)
2239
self.assertNotContainsRe(content, '(?m)^log$')
2240
self.assertNotContainsRe(content, 'this test succeeds')
1975
2243
class TestRunBzr(tests.TestCase):
2966
3235
self.assertEquals('bzrlib.plugins', tpr.resolve_alias('bp'))
3238
class TestThreadLeakDetection(tests.TestCase):
3239
"""Ensure when tests leak threads we detect and report it"""
3241
class LeakRecordingResult(tests.ExtendedTestResult):
3243
tests.ExtendedTestResult.__init__(self, StringIO(), 0, 1)
3245
def _report_thread_leak(self, test, leaks, alive):
3246
self.leaks.append((test, leaks))
3248
def test_testcase_without_addCleanups(self):
3249
"""Check old TestCase instances don't break with leak detection"""
3250
class Test(unittest.TestCase):
3253
result = self.LeakRecordingResult()
3255
result.startTestRun()
3257
result.stopTestRun()
3258
self.assertEqual(result._tests_leaking_threads_count, 0)
3259
self.assertEqual(result.leaks, [])
3261
def test_thread_leak(self):
3262
"""Ensure a thread that outlives the running of a test is reported
3264
Uses a thread that blocks on an event, and is started by the inner
3265
test case. As the thread outlives the inner case's run, it should be
3266
detected as a leak, but the event is then set so that the thread can
3267
be safely joined in cleanup so it's not leaked for real.
3269
event = threading.Event()
3270
thread = threading.Thread(name="Leaker", target=event.wait)
3271
class Test(tests.TestCase):
3272
def test_leak(self):
3274
result = self.LeakRecordingResult()
3275
test = Test("test_leak")
3276
self.addCleanup(thread.join)
3277
self.addCleanup(event.set)
3278
result.startTestRun()
3280
result.stopTestRun()
3281
self.assertEqual(result._tests_leaking_threads_count, 1)
3282
self.assertEqual(result._first_thread_leaker_id, test.id())
3283
self.assertEqual(result.leaks, [(test, set([thread]))])
3284
self.assertContainsString(result.stream.getvalue(), "leaking threads")
3286
def test_multiple_leaks(self):
3287
"""Check multiple leaks are blamed on the test cases at fault
3289
Same concept as the previous test, but has one inner test method that
3290
leaks two threads, and one that doesn't leak at all.
3292
event = threading.Event()
3293
thread_a = threading.Thread(name="LeakerA", target=event.wait)
3294
thread_b = threading.Thread(name="LeakerB", target=event.wait)
3295
thread_c = threading.Thread(name="LeakerC", target=event.wait)
3296
class Test(tests.TestCase):
3297
def test_first_leak(self):
3299
def test_second_no_leak(self):
3301
def test_third_leak(self):
3304
result = self.LeakRecordingResult()
3305
first_test = Test("test_first_leak")
3306
third_test = Test("test_third_leak")
3307
self.addCleanup(thread_a.join)
3308
self.addCleanup(thread_b.join)
3309
self.addCleanup(thread_c.join)
3310
self.addCleanup(event.set)
3311
result.startTestRun()
3313
[first_test, Test("test_second_no_leak"), third_test]
3315
result.stopTestRun()
3316
self.assertEqual(result._tests_leaking_threads_count, 2)
3317
self.assertEqual(result._first_thread_leaker_id, first_test.id())
3318
self.assertEqual(result.leaks, [
3319
(first_test, set([thread_b])),
3320
(third_test, set([thread_a, thread_c]))])
3321
self.assertContainsString(result.stream.getvalue(), "leaking threads")
3324
class TestPostMortemDebugging(tests.TestCase):
3325
"""Check post mortem debugging works when tests fail or error"""
3327
class TracebackRecordingResult(tests.ExtendedTestResult):
3329
tests.ExtendedTestResult.__init__(self, StringIO(), 0, 1)
3330
self.postcode = None
3331
def _post_mortem(self, tb=None):
3332
"""Record the code object at the end of the current traceback"""
3333
tb = tb or sys.exc_info()[2]
3336
while next is not None:
3339
self.postcode = tb.tb_frame.f_code
3340
def report_error(self, test, err):
3342
def report_failure(self, test, err):
3345
def test_location_unittest_error(self):
3346
"""Needs right post mortem traceback with erroring unittest case"""
3347
class Test(unittest.TestCase):
3350
result = self.TracebackRecordingResult()
3352
self.assertEqual(result.postcode, Test.runTest.func_code)
3354
def test_location_unittest_failure(self):
3355
"""Needs right post mortem traceback with failing unittest case"""
3356
class Test(unittest.TestCase):
3358
raise self.failureException
3359
result = self.TracebackRecordingResult()
3361
self.assertEqual(result.postcode, Test.runTest.func_code)
3363
def test_location_bt_error(self):
3364
"""Needs right post mortem traceback with erroring bzrlib.tests case"""
3365
class Test(tests.TestCase):
3366
def test_error(self):
3368
result = self.TracebackRecordingResult()
3369
Test("test_error").run(result)
3370
self.assertEqual(result.postcode, Test.test_error.func_code)
3372
def test_location_bt_failure(self):
3373
"""Needs right post mortem traceback with failing bzrlib.tests case"""
3374
class Test(tests.TestCase):
3375
def test_failure(self):
3376
raise self.failureException
3377
result = self.TracebackRecordingResult()
3378
Test("test_failure").run(result)
3379
self.assertEqual(result.postcode, Test.test_failure.func_code)
3381
def test_env_var_triggers_post_mortem(self):
3382
"""Check pdb.post_mortem is called iff BZR_TEST_PDB is set"""
3384
result = tests.ExtendedTestResult(StringIO(), 0, 1)
3385
post_mortem_calls = []
3386
self.overrideAttr(pdb, "post_mortem", post_mortem_calls.append)
3387
self.overrideEnv('BZR_TEST_PDB', None)
3388
result._post_mortem(1)
3389
self.overrideEnv('BZR_TEST_PDB', 'on')
3390
result._post_mortem(2)
3391
self.assertEqual([2], post_mortem_calls)
2969
3394
class TestRunSuite(tests.TestCase):
2971
3396
def test_runner_class(self):
2982
3407
self.verbosity)
2983
3408
tests.run_suite(suite, runner_class=MyRunner, stream=StringIO())
2984
3409
self.assertLength(1, calls)
3412
class TestEnvironHandling(tests.TestCase):
3414
def test_overrideEnv_None_called_twice_doesnt_leak(self):
3415
self.failIf('MYVAR' in os.environ)
3416
self.overrideEnv('MYVAR', '42')
3417
# We use an embedded test to make sure we fix the _captureVar bug
3418
class Test(tests.TestCase):
3420
# The first call save the 42 value
3421
self.overrideEnv('MYVAR', None)
3422
self.assertEquals(None, os.environ.get('MYVAR'))
3423
# Make sure we can call it twice
3424
self.overrideEnv('MYVAR', None)
3425
self.assertEquals(None, os.environ.get('MYVAR'))
3427
result = tests.TextTestResult(output, 0, 1)
3428
Test('test_me').run(result)
3429
if not result.wasStrictlySuccessful():
3430
self.fail(output.getvalue())
3431
# We get our value back
3432
self.assertEquals('42', os.environ.get('MYVAR'))
3435
class TestIsolatedEnv(tests.TestCase):
3436
"""Test isolating tests from os.environ.
3438
Since we use tests that are already isolated from os.environ a bit of care
3439
should be taken when designing the tests to avoid bootstrap side-effects.
3440
The tests start an already clean os.environ which allow doing valid
3441
assertions about which variables are present or not and design tests around
3445
class ScratchMonkey(tests.TestCase):
3450
def test_basics(self):
3451
# Make sure we know the definition of BZR_HOME: not part of os.environ
3452
# for tests.TestCase.
3453
self.assertTrue('BZR_HOME' in tests.isolated_environ)
3454
self.assertEquals(None, tests.isolated_environ['BZR_HOME'])
3455
# Being part of isolated_environ, BZR_HOME should not appear here
3456
self.assertFalse('BZR_HOME' in os.environ)
3457
# Make sure we know the definition of LINES: part of os.environ for
3459
self.assertTrue('LINES' in tests.isolated_environ)
3460
self.assertEquals('25', tests.isolated_environ['LINES'])
3461
self.assertEquals('25', os.environ['LINES'])
3463
def test_injecting_unknown_variable(self):
3464
# BZR_HOME is known to be absent from os.environ
3465
test = self.ScratchMonkey('test_me')
3466
tests.override_os_environ(test, {'BZR_HOME': 'foo'})
3467
self.assertEquals('foo', os.environ['BZR_HOME'])
3468
tests.restore_os_environ(test)
3469
self.assertFalse('BZR_HOME' in os.environ)
3471
def test_injecting_known_variable(self):
3472
test = self.ScratchMonkey('test_me')
3473
# LINES is known to be present in os.environ
3474
tests.override_os_environ(test, {'LINES': '42'})
3475
self.assertEquals('42', os.environ['LINES'])
3476
tests.restore_os_environ(test)
3477
self.assertEquals('25', os.environ['LINES'])
3479
def test_deleting_variable(self):
3480
test = self.ScratchMonkey('test_me')
3481
# LINES is known to be present in os.environ
3482
tests.override_os_environ(test, {'LINES': None})
3483
self.assertTrue('LINES' not in os.environ)
3484
tests.restore_os_environ(test)
3485
self.assertEquals('25', os.environ['LINES'])
3488
class TestDocTestSuiteIsolation(tests.TestCase):
3489
"""Test that `tests.DocTestSuite` isolates doc tests from os.environ.
3491
Since tests.TestCase alreay provides an isolation from os.environ, we use
3492
the clean environment as a base for testing. To precisely capture the
3493
isolation provided by tests.DocTestSuite, we use doctest.DocTestSuite to
3496
We want to make sure `tests.DocTestSuite` respect `tests.isolated_environ`,
3497
not `os.environ` so each test overrides it to suit its needs.
3501
def get_doctest_suite_for_string(self, klass, string):
3502
class Finder(doctest.DocTestFinder):
3504
def find(*args, **kwargs):
3505
test = doctest.DocTestParser().get_doctest(
3506
string, {}, 'foo', 'foo.py', 0)
3509
suite = klass(test_finder=Finder())
3512
def run_doctest_suite_for_string(self, klass, string):
3513
suite = self.get_doctest_suite_for_string(klass, string)
3515
result = tests.TextTestResult(output, 0, 1)
3517
return result, output
3519
def assertDocTestStringSucceds(self, klass, string):
3520
result, output = self.run_doctest_suite_for_string(klass, string)
3521
if not result.wasStrictlySuccessful():
3522
self.fail(output.getvalue())
3524
def assertDocTestStringFails(self, klass, string):
3525
result, output = self.run_doctest_suite_for_string(klass, string)
3526
if result.wasStrictlySuccessful():
3527
self.fail(output.getvalue())
3529
def test_injected_variable(self):
3530
self.overrideAttr(tests, 'isolated_environ', {'LINES': '42'})
3533
>>> os.environ['LINES']
3536
# doctest.DocTestSuite fails as it sees '25'
3537
self.assertDocTestStringFails(doctest.DocTestSuite, test)
3538
# tests.DocTestSuite sees '42'
3539
self.assertDocTestStringSucceds(tests.IsolatedDocTestSuite, test)
3541
def test_deleted_variable(self):
3542
self.overrideAttr(tests, 'isolated_environ', {'LINES': None})
3545
>>> os.environ.get('LINES')
3547
# doctest.DocTestSuite fails as it sees '25'
3548
self.assertDocTestStringFails(doctest.DocTestSuite, test)
3549
# tests.DocTestSuite sees None
3550
self.assertDocTestStringSucceds(tests.IsolatedDocTestSuite, test)
added
removed
bzrlib/tests/test_selftest.py
