← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2010-09-01 08:02:42 UTC
  • mfrom: (5390.3.3 faster-revert-593560)
  • Revision ID: pqm@pqm.ubuntu.com-20100901080242-esg62ody4frwmy66
(spiv) Avoid repeatedly calling self.target.all_file_ids() in
 InterTree.iter_changes. (Andrew Bennetts)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
 
1
# Copyright (C) 2006-2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
20
 
import difflib
21
 
import gzip
22
20
import sys
23
21
 
24
22
from bzrlib import (
25
23
    errors,
26
 
    generate_ids,
27
24
    knit,
28
25
    multiparent,
29
26
    osutils,
30
27
    pack,
 
28
    tests,
 
29
    transport,
 
30
    tuned_gzip,
31
31
    )
32
32
from bzrlib.errors import (
33
 
    RevisionAlreadyPresent,
34
33
    KnitHeaderError,
35
 
    RevisionNotPresent,
36
34
    NoSuchFile,
37
35
    )
38
36
from bzrlib.index import *
39
37
from bzrlib.knit import (
40
38
    AnnotatedKnitContent,
41
39
    KnitContent,
42
 
    KnitSequenceMatcher,
43
40
    KnitVersionedFiles,
44
41
    PlainKnitContent,
45
42
    _VFContentMapGenerator,
49
46
    _KnitKeyAccess,
50
47
    make_file_factory,
51
48
    )
 
49
from bzrlib.patiencediff import PatienceSequenceMatcher
52
50
from bzrlib.repofmt import pack_repo
53
51
from bzrlib.tests import (
54
 
    Feature,
55
 
    KnownFailure,
56
52
    TestCase,
57
53
    TestCaseWithMemoryTransport,
58
54
    TestCaseWithTransport,
59
55
    TestNotApplicable,
60
56
    )
61
 
from bzrlib.transport import get_transport
62
 
from bzrlib.transport.memory import MemoryTransport
63
 
from bzrlib.tuned_gzip import GzipFile
64
57
from bzrlib.versionedfile import (
65
58
    AbsentContentFactory,
66
59
    ConstantMapper,
69
62
    )
70
63
 
71
64
 
72
 
class _CompiledKnitFeature(Feature):
73
 
 
74
 
    def _probe(self):
75
 
        try:
76
 
            import bzrlib._knit_load_data_c
77
 
        except ImportError:
78
 
            return False
79
 
        return True
80
 
 
81
 
    def feature_name(self):
82
 
        return 'bzrlib._knit_load_data_c'
83
 
 
84
 
CompiledKnitFeature = _CompiledKnitFeature()
 
65
compiled_knit_feature = tests.ModuleAvailableFeature(
 
66
                            'bzrlib._knit_load_data_pyx')
85
67
 
86
68
 
87
69
class KnitContentTestsMixin(object):
116
98
        line_delta = source_content.line_delta(target_content)
117
99
        delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
118
100
            source_lines, target_lines))
119
 
        matcher = KnitSequenceMatcher(None, source_lines, target_lines)
120
 
        matcher_blocks = list(list(matcher.get_matching_blocks()))
 
101
        matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
 
102
        matcher_blocks = list(matcher.get_matching_blocks())
121
103
        self.assertEqual(matcher_blocks, delta_blocks)
122
104
 
123
105
    def test_get_line_delta_blocks(self):
366
348
        :return: (versioned_file, reload_counter)
367
349
            versioned_file  a KnitVersionedFiles using the packs for access
368
350
        """
369
 
        tree = self.make_branch_and_memory_tree('tree')
370
 
        tree.lock_write()
371
 
        self.addCleanup(tree.branch.repository.unlock)
372
 
        tree.add([''], ['root-id'])
373
 
        tree.commit('one', rev_id='rev-1')
374
 
        tree.commit('two', rev_id='rev-2')
375
 
        tree.commit('three', rev_id='rev-3')
 
351
        builder = self.make_branch_builder('.', format="1.9")
 
352
        builder.start_series()
 
353
        builder.build_snapshot('rev-1', None, [
 
354
            ('add', ('', 'root-id', 'directory', None)),
 
355
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
 
356
            ])
 
357
        builder.build_snapshot('rev-2', ['rev-1'], [
 
358
            ('modify', ('file-id', 'content\nrev 2\n')),
 
359
            ])
 
360
        builder.build_snapshot('rev-3', ['rev-2'], [
 
361
            ('modify', ('file-id', 'content\nrev 3\n')),
 
362
            ])
 
363
        builder.finish_series()
 
364
        b = builder.get_branch()
 
365
        b.lock_write()
 
366
        self.addCleanup(b.unlock)
376
367
        # Pack these three revisions into another pack file, but don't remove
377
368
        # the originals
378
 
        repo = tree.branch.repository
 
369
        repo = b.repository
379
370
        collection = repo._pack_collection
380
371
        collection.ensure_loaded()
381
372
        orig_packs = collection.packs
384
375
        # forget about the new pack
385
376
        collection.reset()
386
377
        repo.refresh_data()
387
 
        vf = tree.branch.repository.revisions
388
 
        del tree
 
378
        vf = repo.revisions
389
379
        # Set up a reload() function that switches to using the new pack file
390
380
        new_index = new_pack.revision_index
391
381
        access_tuple = new_pack.access_tuple()
702
692
 
703
693
    def create_gz_content(self, text):
704
694
        sio = StringIO()
705
 
        gz_file = gzip.GzipFile(mode='wb', fileobj=sio)
 
695
        gz_file = tuned_gzip.GzipFile(mode='wb', fileobj=sio)
706
696
        gz_file.write(text)
707
697
        gz_file.close()
708
698
        return sio.getvalue()
864
854
 
865
855
    def get_knit_index(self, transport, name, mode):
866
856
        mapper = ConstantMapper(name)
867
 
        orig = knit._load_data
868
 
        def reset():
869
 
            knit._load_data = orig
870
 
        self.addCleanup(reset)
871
857
        from bzrlib._knit_load_data_py import _load_data_py
872
 
        knit._load_data = _load_data_py
 
858
        self.overrideAttr(knit, '_load_data', _load_data_py)
873
859
        allow_writes = lambda: 'w' in mode
874
860
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
875
861
 
1300
1286
 
1301
1287
class LowLevelKnitIndexTests_c(LowLevelKnitIndexTests):
1302
1288
 
1303
 
    _test_needs_features = [CompiledKnitFeature]
 
1289
    _test_needs_features = [compiled_knit_feature]
1304
1290
 
1305
1291
    def get_knit_index(self, transport, name, mode):
1306
1292
        mapper = ConstantMapper(name)
1307
 
        orig = knit._load_data
1308
 
        def reset():
1309
 
            knit._load_data = orig
1310
 
        self.addCleanup(reset)
1311
 
        from bzrlib._knit_load_data_c import _load_data_c
1312
 
        knit._load_data = _load_data_c
 
1293
        from bzrlib._knit_load_data_pyx import _load_data_c
 
1294
        self.overrideAttr(knit, '_load_data', _load_data_c)
1313
1295
        allow_writes = lambda: mode == 'w'
1314
 
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
 
1296
        return _KndxIndex(transport, mapper, lambda:None,
 
1297
                          allow_writes, lambda:True)
 
1298
 
 
1299
 
 
1300
class Test_KnitAnnotator(TestCaseWithMemoryTransport):
 
1301
 
 
1302
    def make_annotator(self):
 
1303
        factory = knit.make_pack_factory(True, True, 1)
 
1304
        vf = factory(self.get_transport())
 
1305
        return knit._KnitAnnotator(vf)
 
1306
 
 
1307
    def test__expand_fulltext(self):
 
1308
        ann = self.make_annotator()
 
1309
        rev_key = ('rev-id',)
 
1310
        ann._num_compression_children[rev_key] = 1
 
1311
        res = ann._expand_record(rev_key, (('parent-id',),), None,
 
1312
                           ['line1\n', 'line2\n'], ('fulltext', True))
 
1313
        # The content object and text lines should be cached appropriately
 
1314
        self.assertEqual(['line1\n', 'line2'], res)
 
1315
        content_obj = ann._content_objects[rev_key]
 
1316
        self.assertEqual(['line1\n', 'line2\n'], content_obj._lines)
 
1317
        self.assertEqual(res, content_obj.text())
 
1318
        self.assertEqual(res, ann._text_cache[rev_key])
 
1319
 
 
1320
    def test__expand_delta_comp_parent_not_available(self):
 
1321
        # Parent isn't available yet, so we return nothing, but queue up this
 
1322
        # node for later processing
 
1323
        ann = self.make_annotator()
 
1324
        rev_key = ('rev-id',)
 
1325
        parent_key = ('parent-id',)
 
1326
        record = ['0,1,1\n', 'new-line\n']
 
1327
        details = ('line-delta', False)
 
1328
        res = ann._expand_record(rev_key, (parent_key,), parent_key,
 
1329
                                 record, details)
 
1330
        self.assertEqual(None, res)
 
1331
        self.assertTrue(parent_key in ann._pending_deltas)
 
1332
        pending = ann._pending_deltas[parent_key]
 
1333
        self.assertEqual(1, len(pending))
 
1334
        self.assertEqual((rev_key, (parent_key,), record, details), pending[0])
 
1335
 
 
1336
    def test__expand_record_tracks_num_children(self):
 
1337
        ann = self.make_annotator()
 
1338
        rev_key = ('rev-id',)
 
1339
        rev2_key = ('rev2-id',)
 
1340
        parent_key = ('parent-id',)
 
1341
        record = ['0,1,1\n', 'new-line\n']
 
1342
        details = ('line-delta', False)
 
1343
        ann._num_compression_children[parent_key] = 2
 
1344
        ann._expand_record(parent_key, (), None, ['line1\n', 'line2\n'],
 
1345
                           ('fulltext', False))
 
1346
        res = ann._expand_record(rev_key, (parent_key,), parent_key,
 
1347
                                 record, details)
 
1348
        self.assertEqual({parent_key: 1}, ann._num_compression_children)
 
1349
        # Expanding the second child should remove the content object, and the
 
1350
        # num_compression_children entry
 
1351
        res = ann._expand_record(rev2_key, (parent_key,), parent_key,
 
1352
                                 record, details)
 
1353
        self.assertFalse(parent_key in ann._content_objects)
 
1354
        self.assertEqual({}, ann._num_compression_children)
 
1355
        # We should not cache the content_objects for rev2 and rev, because
 
1356
        # they do not have compression children of their own.
 
1357
        self.assertEqual({}, ann._content_objects)
 
1358
 
 
1359
    def test__expand_delta_records_blocks(self):
 
1360
        ann = self.make_annotator()
 
1361
        rev_key = ('rev-id',)
 
1362
        parent_key = ('parent-id',)
 
1363
        record = ['0,1,1\n', 'new-line\n']
 
1364
        details = ('line-delta', True)
 
1365
        ann._num_compression_children[parent_key] = 2
 
1366
        ann._expand_record(parent_key, (), None,
 
1367
                           ['line1\n', 'line2\n', 'line3\n'],
 
1368
                           ('fulltext', False))
 
1369
        ann._expand_record(rev_key, (parent_key,), parent_key, record, details)
 
1370
        self.assertEqual({(rev_key, parent_key): [(1, 1, 1), (3, 3, 0)]},
 
1371
                         ann._matching_blocks)
 
1372
        rev2_key = ('rev2-id',)
 
1373
        record = ['0,1,1\n', 'new-line\n']
 
1374
        details = ('line-delta', False)
 
1375
        ann._expand_record(rev2_key, (parent_key,), parent_key, record, details)
 
1376
        self.assertEqual([(1, 1, 2), (3, 3, 0)],
 
1377
                         ann._matching_blocks[(rev2_key, parent_key)])
 
1378
 
 
1379
    def test__get_parent_ann_uses_matching_blocks(self):
 
1380
        ann = self.make_annotator()
 
1381
        rev_key = ('rev-id',)
 
1382
        parent_key = ('parent-id',)
 
1383
        parent_ann = [(parent_key,)]*3
 
1384
        block_key = (rev_key, parent_key)
 
1385
        ann._annotations_cache[parent_key] = parent_ann
 
1386
        ann._matching_blocks[block_key] = [(0, 1, 1), (3, 3, 0)]
 
1387
        # We should not try to access any parent_lines content, because we know
 
1388
        # we already have the matching blocks
 
1389
        par_ann, blocks = ann._get_parent_annotations_and_matches(rev_key,
 
1390
                                        ['1\n', '2\n', '3\n'], parent_key)
 
1391
        self.assertEqual(parent_ann, par_ann)
 
1392
        self.assertEqual([(0, 1, 1), (3, 3, 0)], blocks)
 
1393
        self.assertEqual({}, ann._matching_blocks)
 
1394
 
 
1395
    def test__process_pending(self):
 
1396
        ann = self.make_annotator()
 
1397
        rev_key = ('rev-id',)
 
1398
        p1_key = ('p1-id',)
 
1399
        p2_key = ('p2-id',)
 
1400
        record = ['0,1,1\n', 'new-line\n']
 
1401
        details = ('line-delta', False)
 
1402
        p1_record = ['line1\n', 'line2\n']
 
1403
        ann._num_compression_children[p1_key] = 1
 
1404
        res = ann._expand_record(rev_key, (p1_key,p2_key), p1_key,
 
1405
                                 record, details)
 
1406
        self.assertEqual(None, res)
 
1407
        # self.assertTrue(p1_key in ann._pending_deltas)
 
1408
        self.assertEqual({}, ann._pending_annotation)
 
1409
        # Now insert p1, and we should be able to expand the delta
 
1410
        res = ann._expand_record(p1_key, (), None, p1_record,
 
1411
                                 ('fulltext', False))
 
1412
        self.assertEqual(p1_record, res)
 
1413
        ann._annotations_cache[p1_key] = [(p1_key,)]*2
 
1414
        res = ann._process_pending(p1_key)
 
1415
        self.assertEqual([], res)
 
1416
        self.assertFalse(p1_key in ann._pending_deltas)
 
1417
        self.assertTrue(p2_key in ann._pending_annotation)
 
1418
        self.assertEqual({p2_key: [(rev_key, (p1_key, p2_key))]},
 
1419
                         ann._pending_annotation)
 
1420
        # Now fill in parent 2, and pending annotation should be satisfied
 
1421
        res = ann._expand_record(p2_key, (), None, [], ('fulltext', False))
 
1422
        ann._annotations_cache[p2_key] = []
 
1423
        res = ann._process_pending(p2_key)
 
1424
        self.assertEqual([rev_key], res)
 
1425
        self.assertEqual({}, ann._pending_annotation)
 
1426
        self.assertEqual({}, ann._pending_deltas)
 
1427
 
 
1428
    def test_record_delta_removes_basis(self):
 
1429
        ann = self.make_annotator()
 
1430
        ann._expand_record(('parent-id',), (), None,
 
1431
                           ['line1\n', 'line2\n'], ('fulltext', False))
 
1432
        ann._num_compression_children['parent-id'] = 2
 
1433
 
 
1434
    def test_annotate_special_text(self):
 
1435
        ann = self.make_annotator()
 
1436
        vf = ann._vf
 
1437
        rev1_key = ('rev-1',)
 
1438
        rev2_key = ('rev-2',)
 
1439
        rev3_key = ('rev-3',)
 
1440
        spec_key = ('special:',)
 
1441
        vf.add_lines(rev1_key, [], ['initial content\n'])
 
1442
        vf.add_lines(rev2_key, [rev1_key], ['initial content\n',
 
1443
                                            'common content\n',
 
1444
                                            'content in 2\n'])
 
1445
        vf.add_lines(rev3_key, [rev1_key], ['initial content\n',
 
1446
                                            'common content\n',
 
1447
                                            'content in 3\n'])
 
1448
        spec_text = ('initial content\n'
 
1449
                     'common content\n'
 
1450
                     'content in 2\n'
 
1451
                     'content in 3\n')
 
1452
        ann.add_special_text(spec_key, [rev2_key, rev3_key], spec_text)
 
1453
        anns, lines = ann.annotate(spec_key)
 
1454
        self.assertEqual([(rev1_key,),
 
1455
                          (rev2_key, rev3_key),
 
1456
                          (rev2_key,),
 
1457
                          (rev3_key,),
 
1458
                         ], anns)
 
1459
        self.assertEqualDiff(spec_text, ''.join(lines))
1315
1460
 
1316
1461
 
1317
1462
class KnitTests(TestCaseWithTransport):
1426
1571
        # could leave an empty .kndx file, which bzr would later claim was a
1427
1572
        # corrupted file since the header was not present. In reality, the file
1428
1573
        # just wasn't created, so it should be ignored.
1429
 
        t = get_transport('.')
 
1574
        t = transport.get_transport('.')
1430
1575
        t.put_bytes('test.kndx', '')
1431
1576
 
1432
1577
        knit = self.make_test_knit()
1433
1578
 
1434
1579
    def test_knit_index_checks_header(self):
1435
 
        t = get_transport('.')
 
1580
        t = transport.get_transport('.')
1436
1581
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1437
1582
        k = self.make_test_knit()
1438
1583
        self.assertRaises(KnitHeaderError, k.keys)
1637
1782
              ([('missing-parent', ), ('ghost', )], [('missing-parent', )]))])
1638
1783
        return graph_index
1639
1784
 
 
1785
    def make_g_index_missing_parent(self):
 
1786
        graph_index = self.make_g_index('missing_parent', 2,
 
1787
            [(('parent', ), ' 100 78', ([], [])),
 
1788
             (('tip', ), ' 100 78',
 
1789
              ([('parent', ), ('missing-parent', )], [('parent', )])),
 
1790
              ])
 
1791
        return graph_index
 
1792
 
1640
1793
    def make_g_index_no_external_refs(self):
1641
1794
        graph_index = self.make_g_index('no_external_refs', 2,
1642
1795
            [(('rev', ), ' 100 78',
1650
1803
        index.scan_unvalidated_index(unvalidated)
1651
1804
        self.assertEqual(frozenset(), index.get_missing_compression_parents())
1652
1805
 
1653
 
    def test_add_incomplete_unvalidated_index(self):
 
1806
    def test_add_missing_compression_parent_unvalidated_index(self):
1654
1807
        unvalidated = self.make_g_index_missing_compression_parent()
1655
1808
        combined = CombinedGraphIndex([unvalidated])
1656
1809
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
1662
1815
            frozenset([('missing-parent',)]),
1663
1816
            index.get_missing_compression_parents())
1664
1817
 
 
1818
    def test_add_missing_noncompression_parent_unvalidated_index(self):
 
1819
        unvalidated = self.make_g_index_missing_parent()
 
1820
        combined = CombinedGraphIndex([unvalidated])
 
1821
        index = _KnitGraphIndex(combined, lambda: True, deltas=True,
 
1822
            track_external_parent_refs=True)
 
1823
        index.scan_unvalidated_index(unvalidated)
 
1824
        self.assertEqual(
 
1825
            frozenset([('missing-parent',)]), index.get_missing_parents())
 
1826
 
 
1827
    def test_track_external_parent_refs(self):
 
1828
        g_index = self.make_g_index('empty', 2, [])
 
1829
        combined = CombinedGraphIndex([g_index])
 
1830
        index = _KnitGraphIndex(combined, lambda: True, deltas=True,
 
1831
            add_callback=self.catch_add, track_external_parent_refs=True)
 
1832
        self.caught_entries = []
 
1833
        index.add_records([
 
1834
            (('new-key',), 'fulltext,no-eol', (None, 50, 60),
 
1835
             [('parent-1',), ('parent-2',)])])
 
1836
        self.assertEqual(
 
1837
            frozenset([('parent-1',), ('parent-2',)]),
 
1838
            index.get_missing_parents())
 
1839
 
1665
1840
    def test_add_unvalidated_index_with_present_external_references(self):
1666
1841
        index = self.two_graph_index(deltas=True)
1667
1842
        # Ugly hack to get at one of the underlying GraphIndex objects that
2030
2205
        # self.assertEqual([("annotate", key_basis)], basis.calls)
2031
2206
        self.assertEqual([('get_parent_map', set([key_basis])),
2032
2207
            ('get_parent_map', set([key_basis])),
2033
 
            ('get_parent_map', set([key_basis])),
2034
 
            ('get_record_stream', [key_basis], 'unordered', True)],
 
2208
            ('get_record_stream', [key_basis], 'topological', True)],
2035
2209
            basis.calls)
2036
2210
 
2037
2211
    def test_check(self):
2143
2317
        # ask which fallbacks have which parents.
2144
2318
        self.assertEqual([
2145
2319
            ("get_parent_map", set([key_basis, key_basis_2, key_missing])),
2146
 
            # unordered is asked for by the underlying worker as it still
2147
 
            # buffers everything while answering - which is a problem!
2148
 
            ("get_record_stream", [key_basis_2, key_basis], 'unordered', True)],
 
2320
            # topological is requested from the fallback, because that is what
 
2321
            # was requested at the top level.
 
2322
            ("get_record_stream", [key_basis_2, key_basis], 'topological', True)],
2149
2323
            calls)
2150
2324
 
2151
2325
    def test_get_record_stream_unordered_deltas(self):
2372
2546
        last_call = basis.calls[-1]
2373
2547
        self.assertEqual('get_record_stream', last_call[0])
2374
2548
        self.assertEqual(set([key_left, key_right]), set(last_call[1]))
2375
 
        self.assertEqual('unordered', last_call[2])
 
2549
        self.assertEqual('topological', last_call[2])
2376
2550
        self.assertEqual(True, last_call[3])
2377
2551
 
2378
2552