← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: John Arbash Meinel
  • Date: 2009-06-19 17:53:37 UTC
  • mto: This revision was merged to the branch mainline in revision 4466.
  • Revision ID: john@arbash-meinel.com-20090619175337-uozt3bntdd48lh4z
Update time_graph to use X:1 ratios rather than 0.xxx ratios.
It is just easier to track now that the new code is much faster.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2006-2010 Canonical Ltd
 
1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
 
20
import difflib
 
21
import gzip
20
22
import sys
21
23
 
22
24
from bzrlib import (
23
25
    errors,
 
26
    generate_ids,
24
27
    knit,
25
28
    multiparent,
26
29
    osutils,
27
30
    pack,
28
 
    tests,
29
 
    transport,
30
 
    tuned_gzip,
31
31
    )
32
32
from bzrlib.errors import (
 
33
    RevisionAlreadyPresent,
33
34
    KnitHeaderError,
 
35
    RevisionNotPresent,
34
36
    NoSuchFile,
35
37
    )
36
38
from bzrlib.index import *
37
39
from bzrlib.knit import (
38
40
    AnnotatedKnitContent,
39
41
    KnitContent,
 
42
    KnitSequenceMatcher,
40
43
    KnitVersionedFiles,
41
44
    PlainKnitContent,
42
45
    _VFContentMapGenerator,
46
49
    _KnitKeyAccess,
47
50
    make_file_factory,
48
51
    )
49
 
from bzrlib.patiencediff import PatienceSequenceMatcher
50
52
from bzrlib.repofmt import pack_repo
51
53
from bzrlib.tests import (
 
54
    Feature,
 
55
    KnownFailure,
52
56
    TestCase,
53
57
    TestCaseWithMemoryTransport,
54
58
    TestCaseWithTransport,
55
59
    TestNotApplicable,
56
60
    )
 
61
from bzrlib.transport import get_transport
 
62
from bzrlib.transport.memory import MemoryTransport
 
63
from bzrlib.tuned_gzip import GzipFile
57
64
from bzrlib.versionedfile import (
58
65
    AbsentContentFactory,
59
66
    ConstantMapper,
62
69
    )
63
70
 
64
71
 
65
 
compiled_knit_feature = tests.ModuleAvailableFeature(
66
 
                            'bzrlib._knit_load_data_pyx')
 
72
class _CompiledKnitFeature(Feature):
 
73
 
 
74
    def _probe(self):
 
75
        try:
 
76
            import bzrlib._knit_load_data_c
 
77
        except ImportError:
 
78
            return False
 
79
        return True
 
80
 
 
81
    def feature_name(self):
 
82
        return 'bzrlib._knit_load_data_c'
 
83
 
 
84
CompiledKnitFeature = _CompiledKnitFeature()
67
85
 
68
86
 
69
87
class KnitContentTestsMixin(object):
98
116
        line_delta = source_content.line_delta(target_content)
99
117
        delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
100
118
            source_lines, target_lines))
101
 
        matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
102
 
        matcher_blocks = list(matcher.get_matching_blocks())
 
119
        matcher = KnitSequenceMatcher(None, source_lines, target_lines)
 
120
        matcher_blocks = list(list(matcher.get_matching_blocks()))
103
121
        self.assertEqual(matcher_blocks, delta_blocks)
104
122
 
105
123
    def test_get_line_delta_blocks(self):
348
366
        :return: (versioned_file, reload_counter)
349
367
            versioned_file  a KnitVersionedFiles using the packs for access
350
368
        """
351
 
        builder = self.make_branch_builder('.', format="1.9")
352
 
        builder.start_series()
353
 
        builder.build_snapshot('rev-1', None, [
354
 
            ('add', ('', 'root-id', 'directory', None)),
355
 
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
356
 
            ])
357
 
        builder.build_snapshot('rev-2', ['rev-1'], [
358
 
            ('modify', ('file-id', 'content\nrev 2\n')),
359
 
            ])
360
 
        builder.build_snapshot('rev-3', ['rev-2'], [
361
 
            ('modify', ('file-id', 'content\nrev 3\n')),
362
 
            ])
363
 
        builder.finish_series()
364
 
        b = builder.get_branch()
365
 
        b.lock_write()
366
 
        self.addCleanup(b.unlock)
 
369
        tree = self.make_branch_and_memory_tree('tree')
 
370
        tree.lock_write()
 
371
        self.addCleanup(tree.unlock)
 
372
        tree.add([''], ['root-id'])
 
373
        tree.commit('one', rev_id='rev-1')
 
374
        tree.commit('two', rev_id='rev-2')
 
375
        tree.commit('three', rev_id='rev-3')
367
376
        # Pack these three revisions into another pack file, but don't remove
368
377
        # the originals
369
 
        repo = b.repository
 
378
        repo = tree.branch.repository
370
379
        collection = repo._pack_collection
371
380
        collection.ensure_loaded()
372
381
        orig_packs = collection.packs
375
384
        # forget about the new pack
376
385
        collection.reset()
377
386
        repo.refresh_data()
378
 
        vf = repo.revisions
 
387
        vf = tree.branch.repository.revisions
379
388
        # Set up a reload() function that switches to using the new pack file
380
389
        new_index = new_pack.revision_index
381
390
        access_tuple = new_pack.access_tuple()
692
701
 
693
702
    def create_gz_content(self, text):
694
703
        sio = StringIO()
695
 
        gz_file = tuned_gzip.GzipFile(mode='wb', fileobj=sio)
 
704
        gz_file = gzip.GzipFile(mode='wb', fileobj=sio)
696
705
        gz_file.write(text)
697
706
        gz_file.close()
698
707
        return sio.getvalue()
854
863
 
855
864
    def get_knit_index(self, transport, name, mode):
856
865
        mapper = ConstantMapper(name)
 
866
        orig = knit._load_data
 
867
        def reset():
 
868
            knit._load_data = orig
 
869
        self.addCleanup(reset)
857
870
        from bzrlib._knit_load_data_py import _load_data_py
858
 
        self.overrideAttr(knit, '_load_data', _load_data_py)
 
871
        knit._load_data = _load_data_py
859
872
        allow_writes = lambda: 'w' in mode
860
873
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
861
874
 
1286
1299
 
1287
1300
class LowLevelKnitIndexTests_c(LowLevelKnitIndexTests):
1288
1301
 
1289
 
    _test_needs_features = [compiled_knit_feature]
 
1302
    _test_needs_features = [CompiledKnitFeature]
1290
1303
 
1291
1304
    def get_knit_index(self, transport, name, mode):
1292
1305
        mapper = ConstantMapper(name)
1293
 
        from bzrlib._knit_load_data_pyx import _load_data_c
1294
 
        self.overrideAttr(knit, '_load_data', _load_data_c)
 
1306
        orig = knit._load_data
 
1307
        def reset():
 
1308
            knit._load_data = orig
 
1309
        self.addCleanup(reset)
 
1310
        from bzrlib._knit_load_data_c import _load_data_c
 
1311
        knit._load_data = _load_data_c
1295
1312
        allow_writes = lambda: mode == 'w'
1296
 
        return _KndxIndex(transport, mapper, lambda:None,
1297
 
                          allow_writes, lambda:True)
1298
 
 
1299
 
 
1300
 
class Test_KnitAnnotator(TestCaseWithMemoryTransport):
1301
 
 
1302
 
    def make_annotator(self):
1303
 
        factory = knit.make_pack_factory(True, True, 1)
1304
 
        vf = factory(self.get_transport())
1305
 
        return knit._KnitAnnotator(vf)
1306
 
 
1307
 
    def test__expand_fulltext(self):
1308
 
        ann = self.make_annotator()
1309
 
        rev_key = ('rev-id',)
1310
 
        ann._num_compression_children[rev_key] = 1
1311
 
        res = ann._expand_record(rev_key, (('parent-id',),), None,
1312
 
                           ['line1\n', 'line2\n'], ('fulltext', True))
1313
 
        # The content object and text lines should be cached appropriately
1314
 
        self.assertEqual(['line1\n', 'line2'], res)
1315
 
        content_obj = ann._content_objects[rev_key]
1316
 
        self.assertEqual(['line1\n', 'line2\n'], content_obj._lines)
1317
 
        self.assertEqual(res, content_obj.text())
1318
 
        self.assertEqual(res, ann._text_cache[rev_key])
1319
 
 
1320
 
    def test__expand_delta_comp_parent_not_available(self):
1321
 
        # Parent isn't available yet, so we return nothing, but queue up this
1322
 
        # node for later processing
1323
 
        ann = self.make_annotator()
1324
 
        rev_key = ('rev-id',)
1325
 
        parent_key = ('parent-id',)
1326
 
        record = ['0,1,1\n', 'new-line\n']
1327
 
        details = ('line-delta', False)
1328
 
        res = ann._expand_record(rev_key, (parent_key,), parent_key,
1329
 
                                 record, details)
1330
 
        self.assertEqual(None, res)
1331
 
        self.assertTrue(parent_key in ann._pending_deltas)
1332
 
        pending = ann._pending_deltas[parent_key]
1333
 
        self.assertEqual(1, len(pending))
1334
 
        self.assertEqual((rev_key, (parent_key,), record, details), pending[0])
1335
 
 
1336
 
    def test__expand_record_tracks_num_children(self):
1337
 
        ann = self.make_annotator()
1338
 
        rev_key = ('rev-id',)
1339
 
        rev2_key = ('rev2-id',)
1340
 
        parent_key = ('parent-id',)
1341
 
        record = ['0,1,1\n', 'new-line\n']
1342
 
        details = ('line-delta', False)
1343
 
        ann._num_compression_children[parent_key] = 2
1344
 
        ann._expand_record(parent_key, (), None, ['line1\n', 'line2\n'],
1345
 
                           ('fulltext', False))
1346
 
        res = ann._expand_record(rev_key, (parent_key,), parent_key,
1347
 
                                 record, details)
1348
 
        self.assertEqual({parent_key: 1}, ann._num_compression_children)
1349
 
        # Expanding the second child should remove the content object, and the
1350
 
        # num_compression_children entry
1351
 
        res = ann._expand_record(rev2_key, (parent_key,), parent_key,
1352
 
                                 record, details)
1353
 
        self.assertFalse(parent_key in ann._content_objects)
1354
 
        self.assertEqual({}, ann._num_compression_children)
1355
 
        # We should not cache the content_objects for rev2 and rev, because
1356
 
        # they do not have compression children of their own.
1357
 
        self.assertEqual({}, ann._content_objects)
1358
 
 
1359
 
    def test__expand_delta_records_blocks(self):
1360
 
        ann = self.make_annotator()
1361
 
        rev_key = ('rev-id',)
1362
 
        parent_key = ('parent-id',)
1363
 
        record = ['0,1,1\n', 'new-line\n']
1364
 
        details = ('line-delta', True)
1365
 
        ann._num_compression_children[parent_key] = 2
1366
 
        ann._expand_record(parent_key, (), None,
1367
 
                           ['line1\n', 'line2\n', 'line3\n'],
1368
 
                           ('fulltext', False))
1369
 
        ann._expand_record(rev_key, (parent_key,), parent_key, record, details)
1370
 
        self.assertEqual({(rev_key, parent_key): [(1, 1, 1), (3, 3, 0)]},
1371
 
                         ann._matching_blocks)
1372
 
        rev2_key = ('rev2-id',)
1373
 
        record = ['0,1,1\n', 'new-line\n']
1374
 
        details = ('line-delta', False)
1375
 
        ann._expand_record(rev2_key, (parent_key,), parent_key, record, details)
1376
 
        self.assertEqual([(1, 1, 2), (3, 3, 0)],
1377
 
                         ann._matching_blocks[(rev2_key, parent_key)])
1378
 
 
1379
 
    def test__get_parent_ann_uses_matching_blocks(self):
1380
 
        ann = self.make_annotator()
1381
 
        rev_key = ('rev-id',)
1382
 
        parent_key = ('parent-id',)
1383
 
        parent_ann = [(parent_key,)]*3
1384
 
        block_key = (rev_key, parent_key)
1385
 
        ann._annotations_cache[parent_key] = parent_ann
1386
 
        ann._matching_blocks[block_key] = [(0, 1, 1), (3, 3, 0)]
1387
 
        # We should not try to access any parent_lines content, because we know
1388
 
        # we already have the matching blocks
1389
 
        par_ann, blocks = ann._get_parent_annotations_and_matches(rev_key,
1390
 
                                        ['1\n', '2\n', '3\n'], parent_key)
1391
 
        self.assertEqual(parent_ann, par_ann)
1392
 
        self.assertEqual([(0, 1, 1), (3, 3, 0)], blocks)
1393
 
        self.assertEqual({}, ann._matching_blocks)
1394
 
 
1395
 
    def test__process_pending(self):
1396
 
        ann = self.make_annotator()
1397
 
        rev_key = ('rev-id',)
1398
 
        p1_key = ('p1-id',)
1399
 
        p2_key = ('p2-id',)
1400
 
        record = ['0,1,1\n', 'new-line\n']
1401
 
        details = ('line-delta', False)
1402
 
        p1_record = ['line1\n', 'line2\n']
1403
 
        ann._num_compression_children[p1_key] = 1
1404
 
        res = ann._expand_record(rev_key, (p1_key,p2_key), p1_key,
1405
 
                                 record, details)
1406
 
        self.assertEqual(None, res)
1407
 
        # self.assertTrue(p1_key in ann._pending_deltas)
1408
 
        self.assertEqual({}, ann._pending_annotation)
1409
 
        # Now insert p1, and we should be able to expand the delta
1410
 
        res = ann._expand_record(p1_key, (), None, p1_record,
1411
 
                                 ('fulltext', False))
1412
 
        self.assertEqual(p1_record, res)
1413
 
        ann._annotations_cache[p1_key] = [(p1_key,)]*2
1414
 
        res = ann._process_pending(p1_key)
1415
 
        self.assertEqual([], res)
1416
 
        self.assertFalse(p1_key in ann._pending_deltas)
1417
 
        self.assertTrue(p2_key in ann._pending_annotation)
1418
 
        self.assertEqual({p2_key: [(rev_key, (p1_key, p2_key))]},
1419
 
                         ann._pending_annotation)
1420
 
        # Now fill in parent 2, and pending annotation should be satisfied
1421
 
        res = ann._expand_record(p2_key, (), None, [], ('fulltext', False))
1422
 
        ann._annotations_cache[p2_key] = []
1423
 
        res = ann._process_pending(p2_key)
1424
 
        self.assertEqual([rev_key], res)
1425
 
        self.assertEqual({}, ann._pending_annotation)
1426
 
        self.assertEqual({}, ann._pending_deltas)
1427
 
 
1428
 
    def test_record_delta_removes_basis(self):
1429
 
        ann = self.make_annotator()
1430
 
        ann._expand_record(('parent-id',), (), None,
1431
 
                           ['line1\n', 'line2\n'], ('fulltext', False))
1432
 
        ann._num_compression_children['parent-id'] = 2
1433
 
 
1434
 
    def test_annotate_special_text(self):
1435
 
        ann = self.make_annotator()
1436
 
        vf = ann._vf
1437
 
        rev1_key = ('rev-1',)
1438
 
        rev2_key = ('rev-2',)
1439
 
        rev3_key = ('rev-3',)
1440
 
        spec_key = ('special:',)
1441
 
        vf.add_lines(rev1_key, [], ['initial content\n'])
1442
 
        vf.add_lines(rev2_key, [rev1_key], ['initial content\n',
1443
 
                                            'common content\n',
1444
 
                                            'content in 2\n'])
1445
 
        vf.add_lines(rev3_key, [rev1_key], ['initial content\n',
1446
 
                                            'common content\n',
1447
 
                                            'content in 3\n'])
1448
 
        spec_text = ('initial content\n'
1449
 
                     'common content\n'
1450
 
                     'content in 2\n'
1451
 
                     'content in 3\n')
1452
 
        ann.add_special_text(spec_key, [rev2_key, rev3_key], spec_text)
1453
 
        anns, lines = ann.annotate(spec_key)
1454
 
        self.assertEqual([(rev1_key,),
1455
 
                          (rev2_key, rev3_key),
1456
 
                          (rev2_key,),
1457
 
                          (rev3_key,),
1458
 
                         ], anns)
1459
 
        self.assertEqualDiff(spec_text, ''.join(lines))
 
1313
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
1460
1314
 
1461
1315
 
1462
1316
class KnitTests(TestCaseWithTransport):
1571
1425
        # could leave an empty .kndx file, which bzr would later claim was a
1572
1426
        # corrupted file since the header was not present. In reality, the file
1573
1427
        # just wasn't created, so it should be ignored.
1574
 
        t = transport.get_transport('.')
 
1428
        t = get_transport('.')
1575
1429
        t.put_bytes('test.kndx', '')
1576
1430
 
1577
1431
        knit = self.make_test_knit()
1578
1432
 
1579
1433
    def test_knit_index_checks_header(self):
1580
 
        t = transport.get_transport('.')
 
1434
        t = get_transport('.')
1581
1435
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1582
1436
        k = self.make_test_knit()
1583
1437
        self.assertRaises(KnitHeaderError, k.keys)
2205
2059
        # self.assertEqual([("annotate", key_basis)], basis.calls)
2206
2060
        self.assertEqual([('get_parent_map', set([key_basis])),
2207
2061
            ('get_parent_map', set([key_basis])),
2208
 
            ('get_record_stream', [key_basis], 'topological', True)],
 
2062
            ('get_record_stream', [key_basis], 'unordered', True)],
2209
2063
            basis.calls)
2210
2064
 
2211
2065
    def test_check(self):
2317
2171
        # ask which fallbacks have which parents.
2318
2172
        self.assertEqual([
2319
2173
            ("get_parent_map", set([key_basis, key_basis_2, key_missing])),
2320
 
            # topological is requested from the fallback, because that is what
2321
 
            # was requested at the top level.
2322
 
            ("get_record_stream", [key_basis_2, key_basis], 'topological', True)],
 
2174
            # unordered is asked for by the underlying worker as it still
 
2175
            # buffers everything while answering - which is a problem!
 
2176
            ("get_record_stream", [key_basis_2, key_basis], 'unordered', True)],
2323
2177
            calls)
2324
2178
 
2325
2179
    def test_get_record_stream_unordered_deltas(self):
2546
2400
        last_call = basis.calls[-1]
2547
2401
        self.assertEqual('get_record_stream', last_call[0])
2548
2402
        self.assertEqual(set([key_left, key_right]), set(last_call[1]))
2549
 
        self.assertEqual('topological', last_call[2])
 
2403
        self.assertEqual('unordered', last_call[2])
2550
2404
        self.assertEqual(True, last_call[3])
2551
2405
 
2552
2406