← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2010-07-07 04:03:32 UTC
  • mfrom: (5335.2.3 doc)
  • Revision ID: pqm@pqm.ubuntu.com-20100707040332-we60v2hsd39rumlr
(mbp) developer docs about testing (Martin Pool)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
 
1
# Copyright (C) 2006-2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
 
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
20
 
import difflib
21
 
import gzip
22
20
import sys
23
21
 
24
22
from bzrlib import (
25
23
    errors,
26
 
    generate_ids,
27
24
    knit,
28
25
    multiparent,
29
26
    osutils,
30
27
    pack,
 
28
    tests,
 
29
    transport,
 
30
    tuned_gzip,
31
31
    )
32
32
from bzrlib.errors import (
33
 
    RevisionAlreadyPresent,
34
33
    KnitHeaderError,
35
 
    RevisionNotPresent,
36
34
    NoSuchFile,
37
35
    )
38
36
from bzrlib.index import *
39
37
from bzrlib.knit import (
40
38
    AnnotatedKnitContent,
41
39
    KnitContent,
42
 
    KnitSequenceMatcher,
43
40
    KnitVersionedFiles,
44
41
    PlainKnitContent,
 
42
    _VFContentMapGenerator,
45
43
    _DirectPackAccess,
46
44
    _KndxIndex,
47
45
    _KnitGraphIndex,
48
46
    _KnitKeyAccess,
49
47
    make_file_factory,
50
48
    )
 
49
from bzrlib.patiencediff import PatienceSequenceMatcher
51
50
from bzrlib.repofmt import pack_repo
52
51
from bzrlib.tests import (
53
 
    Feature,
54
 
    KnownFailure,
55
52
    TestCase,
56
53
    TestCaseWithMemoryTransport,
57
54
    TestCaseWithTransport,
58
55
    TestNotApplicable,
59
56
    )
60
 
from bzrlib.transport import get_transport
61
 
from bzrlib.transport.memory import MemoryTransport
62
 
from bzrlib.tuned_gzip import GzipFile
63
57
from bzrlib.versionedfile import (
64
58
    AbsentContentFactory,
65
59
    ConstantMapper,
 
60
    network_bytes_to_kind_and_offset,
66
61
    RecordingVersionedFilesDecorator,
67
62
    )
68
63
 
69
64
 
70
 
class _CompiledKnitFeature(Feature):
71
 
 
72
 
    def _probe(self):
73
 
        try:
74
 
            import bzrlib._knit_load_data_c
75
 
        except ImportError:
76
 
            return False
77
 
        return True
78
 
 
79
 
    def feature_name(self):
80
 
        return 'bzrlib._knit_load_data_c'
81
 
 
82
 
CompiledKnitFeature = _CompiledKnitFeature()
 
65
compiled_knit_feature = tests.ModuleAvailableFeature(
 
66
                            'bzrlib._knit_load_data_pyx')
83
67
 
84
68
 
85
69
class KnitContentTestsMixin(object):
114
98
        line_delta = source_content.line_delta(target_content)
115
99
        delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
116
100
            source_lines, target_lines))
117
 
        matcher = KnitSequenceMatcher(None, source_lines, target_lines)
118
 
        matcher_blocks = list(list(matcher.get_matching_blocks()))
 
101
        matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
 
102
        matcher_blocks = list(matcher.get_matching_blocks())
119
103
        self.assertEqual(matcher_blocks, delta_blocks)
120
104
 
121
105
    def test_get_line_delta_blocks(self):
297
281
        access = self.get_access()
298
282
        memos = access.add_raw_records([('key', 10)], '1234567890')
299
283
        self.assertEqual(['1234567890'], list(access.get_raw_records(memos)))
300
 
 
 
284
 
301
285
    def test_add_several_raw_records(self):
302
286
        """add_raw_records with many records and read some back."""
303
287
        access = self.get_access()
364
348
        :return: (versioned_file, reload_counter)
365
349
            versioned_file  a KnitVersionedFiles using the packs for access
366
350
        """
367
 
        tree = self.make_branch_and_memory_tree('tree')
368
 
        tree.lock_write()
369
 
        try:
370
 
            tree.add([''], ['root-id'])
371
 
            tree.commit('one', rev_id='rev-1')
372
 
            tree.commit('two', rev_id='rev-2')
373
 
            tree.commit('three', rev_id='rev-3')
374
 
            # Pack these two revisions into another pack file, but don't remove
375
 
            # the originials
376
 
            repo = tree.branch.repository
377
 
            collection = repo._pack_collection
378
 
            collection.ensure_loaded()
379
 
            orig_packs = collection.packs
380
 
            packer = pack_repo.Packer(collection, orig_packs, '.testpack')
381
 
            new_pack = packer.pack()
382
 
 
383
 
            vf = tree.branch.repository.revisions
384
 
        finally:
385
 
            tree.unlock()
386
 
        tree.branch.repository.lock_read()
387
 
        self.addCleanup(tree.branch.repository.unlock)
388
 
        del tree
 
351
        builder = self.make_branch_builder('.', format="1.9")
 
352
        builder.start_series()
 
353
        builder.build_snapshot('rev-1', None, [
 
354
            ('add', ('', 'root-id', 'directory', None)),
 
355
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
 
356
            ])
 
357
        builder.build_snapshot('rev-2', ['rev-1'], [
 
358
            ('modify', ('file-id', 'content\nrev 2\n')),
 
359
            ])
 
360
        builder.build_snapshot('rev-3', ['rev-2'], [
 
361
            ('modify', ('file-id', 'content\nrev 3\n')),
 
362
            ])
 
363
        builder.finish_series()
 
364
        b = builder.get_branch()
 
365
        b.lock_write()
 
366
        self.addCleanup(b.unlock)
 
367
        # Pack these three revisions into another pack file, but don't remove
 
368
        # the originals
 
369
        repo = b.repository
 
370
        collection = repo._pack_collection
 
371
        collection.ensure_loaded()
 
372
        orig_packs = collection.packs
 
373
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
 
374
        new_pack = packer.pack()
 
375
        # forget about the new pack
 
376
        collection.reset()
 
377
        repo.refresh_data()
 
378
        vf = repo.revisions
389
379
        # Set up a reload() function that switches to using the new pack file
390
380
        new_index = new_pack.revision_index
391
381
        access_tuple = new_pack.access_tuple()
702
692
 
703
693
    def create_gz_content(self, text):
704
694
        sio = StringIO()
705
 
        gz_file = gzip.GzipFile(mode='wb', fileobj=sio)
 
695
        gz_file = tuned_gzip.GzipFile(mode='wb', fileobj=sio)
706
696
        gz_file.write(text)
707
697
        gz_file.close()
708
698
        return sio.getvalue()
864
854
 
865
855
    def get_knit_index(self, transport, name, mode):
866
856
        mapper = ConstantMapper(name)
867
 
        orig = knit._load_data
868
 
        def reset():
869
 
            knit._load_data = orig
870
 
        self.addCleanup(reset)
871
857
        from bzrlib._knit_load_data_py import _load_data_py
872
 
        knit._load_data = _load_data_py
 
858
        self.overrideAttr(knit, '_load_data', _load_data_py)
873
859
        allow_writes = lambda: 'w' in mode
874
860
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
875
861
 
1091
1077
            call[1][1].getvalue())
1092
1078
        self.assertEqual({'create_parent_dir': True}, call[2])
1093
1079
 
 
1080
    def assertTotalBuildSize(self, size, keys, positions):
 
1081
        self.assertEqual(size,
 
1082
                         knit._get_total_build_size(None, keys, positions))
 
1083
 
 
1084
    def test__get_total_build_size(self):
 
1085
        positions = {
 
1086
            ('a',): (('fulltext', False), (('a',), 0, 100), None),
 
1087
            ('b',): (('line-delta', False), (('b',), 100, 21), ('a',)),
 
1088
            ('c',): (('line-delta', False), (('c',), 121, 35), ('b',)),
 
1089
            ('d',): (('line-delta', False), (('d',), 156, 12), ('b',)),
 
1090
            }
 
1091
        self.assertTotalBuildSize(100, [('a',)], positions)
 
1092
        self.assertTotalBuildSize(121, [('b',)], positions)
 
1093
        # c needs both a & b
 
1094
        self.assertTotalBuildSize(156, [('c',)], positions)
 
1095
        # we shouldn't count 'b' twice
 
1096
        self.assertTotalBuildSize(156, [('b',), ('c',)], positions)
 
1097
        self.assertTotalBuildSize(133, [('d',)], positions)
 
1098
        self.assertTotalBuildSize(168, [('c',), ('d',)], positions)
 
1099
 
1094
1100
    def test_get_position(self):
1095
1101
        transport = MockTransport([
1096
1102
            _KndxIndex.HEADER,
1237
1243
            else:
1238
1244
                raise
1239
1245
 
 
1246
    def test_scan_unvalidated_index_not_implemented(self):
 
1247
        transport = MockTransport()
 
1248
        index = self.get_knit_index(transport, 'filename', 'r')
 
1249
        self.assertRaises(
 
1250
            NotImplementedError, index.scan_unvalidated_index,
 
1251
            'dummy graph_index')
 
1252
        self.assertRaises(
 
1253
            NotImplementedError, index.get_missing_compression_parents)
 
1254
 
1240
1255
    def test_short_line(self):
1241
1256
        transport = MockTransport([
1242
1257
            _KndxIndex.HEADER,
1271
1286
 
1272
1287
class LowLevelKnitIndexTests_c(LowLevelKnitIndexTests):
1273
1288
 
1274
 
    _test_needs_features = [CompiledKnitFeature]
 
1289
    _test_needs_features = [compiled_knit_feature]
1275
1290
 
1276
1291
    def get_knit_index(self, transport, name, mode):
1277
1292
        mapper = ConstantMapper(name)
1278
 
        orig = knit._load_data
1279
 
        def reset():
1280
 
            knit._load_data = orig
1281
 
        self.addCleanup(reset)
1282
 
        from bzrlib._knit_load_data_c import _load_data_c
1283
 
        knit._load_data = _load_data_c
 
1293
        from bzrlib._knit_load_data_pyx import _load_data_c
 
1294
        self.overrideAttr(knit, '_load_data', _load_data_c)
1284
1295
        allow_writes = lambda: mode == 'w'
1285
 
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
 
1296
        return _KndxIndex(transport, mapper, lambda:None,
 
1297
                          allow_writes, lambda:True)
 
1298
 
 
1299
 
 
1300
class Test_KnitAnnotator(TestCaseWithMemoryTransport):
 
1301
 
 
1302
    def make_annotator(self):
 
1303
        factory = knit.make_pack_factory(True, True, 1)
 
1304
        vf = factory(self.get_transport())
 
1305
        return knit._KnitAnnotator(vf)
 
1306
 
 
1307
    def test__expand_fulltext(self):
 
1308
        ann = self.make_annotator()
 
1309
        rev_key = ('rev-id',)
 
1310
        ann._num_compression_children[rev_key] = 1
 
1311
        res = ann._expand_record(rev_key, (('parent-id',),), None,
 
1312
                           ['line1\n', 'line2\n'], ('fulltext', True))
 
1313
        # The content object and text lines should be cached appropriately
 
1314
        self.assertEqual(['line1\n', 'line2'], res)
 
1315
        content_obj = ann._content_objects[rev_key]
 
1316
        self.assertEqual(['line1\n', 'line2\n'], content_obj._lines)
 
1317
        self.assertEqual(res, content_obj.text())
 
1318
        self.assertEqual(res, ann._text_cache[rev_key])
 
1319
 
 
1320
    def test__expand_delta_comp_parent_not_available(self):
 
1321
        # Parent isn't available yet, so we return nothing, but queue up this
 
1322
        # node for later processing
 
1323
        ann = self.make_annotator()
 
1324
        rev_key = ('rev-id',)
 
1325
        parent_key = ('parent-id',)
 
1326
        record = ['0,1,1\n', 'new-line\n']
 
1327
        details = ('line-delta', False)
 
1328
        res = ann._expand_record(rev_key, (parent_key,), parent_key,
 
1329
                                 record, details)
 
1330
        self.assertEqual(None, res)
 
1331
        self.assertTrue(parent_key in ann._pending_deltas)
 
1332
        pending = ann._pending_deltas[parent_key]
 
1333
        self.assertEqual(1, len(pending))
 
1334
        self.assertEqual((rev_key, (parent_key,), record, details), pending[0])
 
1335
 
 
1336
    def test__expand_record_tracks_num_children(self):
 
1337
        ann = self.make_annotator()
 
1338
        rev_key = ('rev-id',)
 
1339
        rev2_key = ('rev2-id',)
 
1340
        parent_key = ('parent-id',)
 
1341
        record = ['0,1,1\n', 'new-line\n']
 
1342
        details = ('line-delta', False)
 
1343
        ann._num_compression_children[parent_key] = 2
 
1344
        ann._expand_record(parent_key, (), None, ['line1\n', 'line2\n'],
 
1345
                           ('fulltext', False))
 
1346
        res = ann._expand_record(rev_key, (parent_key,), parent_key,
 
1347
                                 record, details)
 
1348
        self.assertEqual({parent_key: 1}, ann._num_compression_children)
 
1349
        # Expanding the second child should remove the content object, and the
 
1350
        # num_compression_children entry
 
1351
        res = ann._expand_record(rev2_key, (parent_key,), parent_key,
 
1352
                                 record, details)
 
1353
        self.assertFalse(parent_key in ann._content_objects)
 
1354
        self.assertEqual({}, ann._num_compression_children)
 
1355
        # We should not cache the content_objects for rev2 and rev, because
 
1356
        # they do not have compression children of their own.
 
1357
        self.assertEqual({}, ann._content_objects)
 
1358
 
 
1359
    def test__expand_delta_records_blocks(self):
 
1360
        ann = self.make_annotator()
 
1361
        rev_key = ('rev-id',)
 
1362
        parent_key = ('parent-id',)
 
1363
        record = ['0,1,1\n', 'new-line\n']
 
1364
        details = ('line-delta', True)
 
1365
        ann._num_compression_children[parent_key] = 2
 
1366
        ann._expand_record(parent_key, (), None,
 
1367
                           ['line1\n', 'line2\n', 'line3\n'],
 
1368
                           ('fulltext', False))
 
1369
        ann._expand_record(rev_key, (parent_key,), parent_key, record, details)
 
1370
        self.assertEqual({(rev_key, parent_key): [(1, 1, 1), (3, 3, 0)]},
 
1371
                         ann._matching_blocks)
 
1372
        rev2_key = ('rev2-id',)
 
1373
        record = ['0,1,1\n', 'new-line\n']
 
1374
        details = ('line-delta', False)
 
1375
        ann._expand_record(rev2_key, (parent_key,), parent_key, record, details)
 
1376
        self.assertEqual([(1, 1, 2), (3, 3, 0)],
 
1377
                         ann._matching_blocks[(rev2_key, parent_key)])
 
1378
 
 
1379
    def test__get_parent_ann_uses_matching_blocks(self):
 
1380
        ann = self.make_annotator()
 
1381
        rev_key = ('rev-id',)
 
1382
        parent_key = ('parent-id',)
 
1383
        parent_ann = [(parent_key,)]*3
 
1384
        block_key = (rev_key, parent_key)
 
1385
        ann._annotations_cache[parent_key] = parent_ann
 
1386
        ann._matching_blocks[block_key] = [(0, 1, 1), (3, 3, 0)]
 
1387
        # We should not try to access any parent_lines content, because we know
 
1388
        # we already have the matching blocks
 
1389
        par_ann, blocks = ann._get_parent_annotations_and_matches(rev_key,
 
1390
                                        ['1\n', '2\n', '3\n'], parent_key)
 
1391
        self.assertEqual(parent_ann, par_ann)
 
1392
        self.assertEqual([(0, 1, 1), (3, 3, 0)], blocks)
 
1393
        self.assertEqual({}, ann._matching_blocks)
 
1394
 
 
1395
    def test__process_pending(self):
 
1396
        ann = self.make_annotator()
 
1397
        rev_key = ('rev-id',)
 
1398
        p1_key = ('p1-id',)
 
1399
        p2_key = ('p2-id',)
 
1400
        record = ['0,1,1\n', 'new-line\n']
 
1401
        details = ('line-delta', False)
 
1402
        p1_record = ['line1\n', 'line2\n']
 
1403
        ann._num_compression_children[p1_key] = 1
 
1404
        res = ann._expand_record(rev_key, (p1_key,p2_key), p1_key,
 
1405
                                 record, details)
 
1406
        self.assertEqual(None, res)
 
1407
        # self.assertTrue(p1_key in ann._pending_deltas)
 
1408
        self.assertEqual({}, ann._pending_annotation)
 
1409
        # Now insert p1, and we should be able to expand the delta
 
1410
        res = ann._expand_record(p1_key, (), None, p1_record,
 
1411
                                 ('fulltext', False))
 
1412
        self.assertEqual(p1_record, res)
 
1413
        ann._annotations_cache[p1_key] = [(p1_key,)]*2
 
1414
        res = ann._process_pending(p1_key)
 
1415
        self.assertEqual([], res)
 
1416
        self.assertFalse(p1_key in ann._pending_deltas)
 
1417
        self.assertTrue(p2_key in ann._pending_annotation)
 
1418
        self.assertEqual({p2_key: [(rev_key, (p1_key, p2_key))]},
 
1419
                         ann._pending_annotation)
 
1420
        # Now fill in parent 2, and pending annotation should be satisfied
 
1421
        res = ann._expand_record(p2_key, (), None, [], ('fulltext', False))
 
1422
        ann._annotations_cache[p2_key] = []
 
1423
        res = ann._process_pending(p2_key)
 
1424
        self.assertEqual([rev_key], res)
 
1425
        self.assertEqual({}, ann._pending_annotation)
 
1426
        self.assertEqual({}, ann._pending_deltas)
 
1427
 
 
1428
    def test_record_delta_removes_basis(self):
 
1429
        ann = self.make_annotator()
 
1430
        ann._expand_record(('parent-id',), (), None,
 
1431
                           ['line1\n', 'line2\n'], ('fulltext', False))
 
1432
        ann._num_compression_children['parent-id'] = 2
 
1433
 
 
1434
    def test_annotate_special_text(self):
 
1435
        ann = self.make_annotator()
 
1436
        vf = ann._vf
 
1437
        rev1_key = ('rev-1',)
 
1438
        rev2_key = ('rev-2',)
 
1439
        rev3_key = ('rev-3',)
 
1440
        spec_key = ('special:',)
 
1441
        vf.add_lines(rev1_key, [], ['initial content\n'])
 
1442
        vf.add_lines(rev2_key, [rev1_key], ['initial content\n',
 
1443
                                            'common content\n',
 
1444
                                            'content in 2\n'])
 
1445
        vf.add_lines(rev3_key, [rev1_key], ['initial content\n',
 
1446
                                            'common content\n',
 
1447
                                            'content in 3\n'])
 
1448
        spec_text = ('initial content\n'
 
1449
                     'common content\n'
 
1450
                     'content in 2\n'
 
1451
                     'content in 3\n')
 
1452
        ann.add_special_text(spec_key, [rev2_key, rev3_key], spec_text)
 
1453
        anns, lines = ann.annotate(spec_key)
 
1454
        self.assertEqual([(rev1_key,),
 
1455
                          (rev2_key, rev3_key),
 
1456
                          (rev2_key,),
 
1457
                          (rev3_key,),
 
1458
                         ], anns)
 
1459
        self.assertEqualDiff(spec_text, ''.join(lines))
1286
1460
 
1287
1461
 
1288
1462
class KnitTests(TestCaseWithTransport):
1296
1470
class TestBadShaError(KnitTests):
1297
1471
    """Tests for handling of sha errors."""
1298
1472
 
1299
 
    def test_exception_has_text(self):
 
1473
    def test_sha_exception_has_text(self):
1300
1474
        # having the failed text included in the error allows for recovery.
1301
1475
        source = self.make_test_knit()
1302
1476
        target = self.make_test_knit(name="target")
1313
1487
        target.insert_record_stream(
1314
1488
            source.get_record_stream([broken], 'unordered', False))
1315
1489
        err = self.assertRaises(errors.KnitCorrupt,
1316
 
            target.get_record_stream([broken], 'unordered', True).next)
 
1490
            target.get_record_stream([broken], 'unordered', True
 
1491
            ).next().get_bytes_as, 'chunked')
1317
1492
        self.assertEqual(['gam\n', 'bar\n'], err.content)
1318
1493
        # Test for formatting with live data
1319
1494
        self.assertStartsWith(str(err), "Knit ")
1396
1571
        # could leave an empty .kndx file, which bzr would later claim was a
1397
1572
        # corrupted file since the header was not present. In reality, the file
1398
1573
        # just wasn't created, so it should be ignored.
1399
 
        t = get_transport('.')
 
1574
        t = transport.get_transport('.')
1400
1575
        t.put_bytes('test.kndx', '')
1401
1576
 
1402
1577
        knit = self.make_test_knit()
1403
1578
 
1404
1579
    def test_knit_index_checks_header(self):
1405
 
        t = get_transport('.')
 
1580
        t = transport.get_transport('.')
1406
1581
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1407
1582
        k = self.make_test_knit()
1408
1583
        self.assertRaises(KnitHeaderError, k.keys)
1524
1699
            [('parent',)])])
1525
1700
        # but neither should have added data:
1526
1701
        self.assertEqual([[], [], [], []], self.caught_entries)
1527
 
        
 
1702
 
1528
1703
    def test_add_version_different_dup(self):
1529
1704
        index = self.two_graph_index(deltas=True, catch_adds=True)
1530
1705
        # change options
1536
1711
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1537
1712
            [(('tip',), 'fulltext,no-eol', (None, 0, 100), [])])
1538
1713
        self.assertEqual([], self.caught_entries)
1539
 
        
 
1714
 
1540
1715
    def test_add_versions_nodeltas(self):
1541
1716
        index = self.two_graph_index(catch_adds=True)
1542
1717
        index.add_records([
1584
1759
            [('parent',)])])
1585
1760
        # but neither should have added data.
1586
1761
        self.assertEqual([[], [], [], []], self.caught_entries)
1587
 
        
 
1762
 
1588
1763
    def test_add_versions_different_dup(self):
1589
1764
        index = self.two_graph_index(deltas=True, catch_adds=True)
1590
1765
        # change options
1601
1776
             (('tip',), 'line-delta', (None, 0, 100), [('parent',)])])
1602
1777
        self.assertEqual([], self.caught_entries)
1603
1778
 
 
1779
    def make_g_index_missing_compression_parent(self):
 
1780
        graph_index = self.make_g_index('missing_comp', 2,
 
1781
            [(('tip', ), ' 100 78',
 
1782
              ([('missing-parent', ), ('ghost', )], [('missing-parent', )]))])
 
1783
        return graph_index
 
1784
 
 
1785
    def make_g_index_missing_parent(self):
 
1786
        graph_index = self.make_g_index('missing_parent', 2,
 
1787
            [(('parent', ), ' 100 78', ([], [])),
 
1788
             (('tip', ), ' 100 78',
 
1789
              ([('parent', ), ('missing-parent', )], [('parent', )])),
 
1790
              ])
 
1791
        return graph_index
 
1792
 
 
1793
    def make_g_index_no_external_refs(self):
 
1794
        graph_index = self.make_g_index('no_external_refs', 2,
 
1795
            [(('rev', ), ' 100 78',
 
1796
              ([('parent', ), ('ghost', )], []))])
 
1797
        return graph_index
 
1798
 
 
1799
    def test_add_good_unvalidated_index(self):
 
1800
        unvalidated = self.make_g_index_no_external_refs()
 
1801
        combined = CombinedGraphIndex([unvalidated])
 
1802
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1803
        index.scan_unvalidated_index(unvalidated)
 
1804
        self.assertEqual(frozenset(), index.get_missing_compression_parents())
 
1805
 
 
1806
    def test_add_missing_compression_parent_unvalidated_index(self):
 
1807
        unvalidated = self.make_g_index_missing_compression_parent()
 
1808
        combined = CombinedGraphIndex([unvalidated])
 
1809
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1810
        index.scan_unvalidated_index(unvalidated)
 
1811
        # This also checks that its only the compression parent that is
 
1812
        # examined, otherwise 'ghost' would also be reported as a missing
 
1813
        # parent.
 
1814
        self.assertEqual(
 
1815
            frozenset([('missing-parent',)]),
 
1816
            index.get_missing_compression_parents())
 
1817
 
 
1818
    def test_add_missing_noncompression_parent_unvalidated_index(self):
 
1819
        unvalidated = self.make_g_index_missing_parent()
 
1820
        combined = CombinedGraphIndex([unvalidated])
 
1821
        index = _KnitGraphIndex(combined, lambda: True, deltas=True,
 
1822
            track_external_parent_refs=True)
 
1823
        index.scan_unvalidated_index(unvalidated)
 
1824
        self.assertEqual(
 
1825
            frozenset([('missing-parent',)]), index.get_missing_parents())
 
1826
 
 
1827
    def test_track_external_parent_refs(self):
 
1828
        g_index = self.make_g_index('empty', 2, [])
 
1829
        combined = CombinedGraphIndex([g_index])
 
1830
        index = _KnitGraphIndex(combined, lambda: True, deltas=True,
 
1831
            add_callback=self.catch_add, track_external_parent_refs=True)
 
1832
        self.caught_entries = []
 
1833
        index.add_records([
 
1834
            (('new-key',), 'fulltext,no-eol', (None, 50, 60),
 
1835
             [('parent-1',), ('parent-2',)])])
 
1836
        self.assertEqual(
 
1837
            frozenset([('parent-1',), ('parent-2',)]),
 
1838
            index.get_missing_parents())
 
1839
 
 
1840
    def test_add_unvalidated_index_with_present_external_references(self):
 
1841
        index = self.two_graph_index(deltas=True)
 
1842
        # Ugly hack to get at one of the underlying GraphIndex objects that
 
1843
        # two_graph_index built.
 
1844
        unvalidated = index._graph_index._indices[1]
 
1845
        # 'parent' is an external ref of _indices[1] (unvalidated), but is
 
1846
        # present in _indices[0].
 
1847
        index.scan_unvalidated_index(unvalidated)
 
1848
        self.assertEqual(frozenset(), index.get_missing_compression_parents())
 
1849
 
 
1850
    def make_new_missing_parent_g_index(self, name):
 
1851
        missing_parent = name + '-missing-parent'
 
1852
        graph_index = self.make_g_index(name, 2,
 
1853
            [((name + 'tip', ), ' 100 78',
 
1854
              ([(missing_parent, ), ('ghost', )], [(missing_parent, )]))])
 
1855
        return graph_index
 
1856
 
 
1857
    def test_add_mulitiple_unvalidated_indices_with_missing_parents(self):
 
1858
        g_index_1 = self.make_new_missing_parent_g_index('one')
 
1859
        g_index_2 = self.make_new_missing_parent_g_index('two')
 
1860
        combined = CombinedGraphIndex([g_index_1, g_index_2])
 
1861
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1862
        index.scan_unvalidated_index(g_index_1)
 
1863
        index.scan_unvalidated_index(g_index_2)
 
1864
        self.assertEqual(
 
1865
            frozenset([('one-missing-parent',), ('two-missing-parent',)]),
 
1866
            index.get_missing_compression_parents())
 
1867
 
 
1868
    def test_add_mulitiple_unvalidated_indices_with_mutual_dependencies(self):
 
1869
        graph_index_a = self.make_g_index('one', 2,
 
1870
            [(('parent-one', ), ' 100 78', ([('non-compression-parent',)], [])),
 
1871
             (('child-of-two', ), ' 100 78',
 
1872
              ([('parent-two',)], [('parent-two',)]))])
 
1873
        graph_index_b = self.make_g_index('two', 2,
 
1874
            [(('parent-two', ), ' 100 78', ([('non-compression-parent',)], [])),
 
1875
             (('child-of-one', ), ' 100 78',
 
1876
              ([('parent-one',)], [('parent-one',)]))])
 
1877
        combined = CombinedGraphIndex([graph_index_a, graph_index_b])
 
1878
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1879
        index.scan_unvalidated_index(graph_index_a)
 
1880
        index.scan_unvalidated_index(graph_index_b)
 
1881
        self.assertEqual(
 
1882
            frozenset([]), index.get_missing_compression_parents())
 
1883
 
1604
1884
 
1605
1885
class TestNoParentsGraphIndexKnit(KnitTests):
1606
1886
    """Tests for knits using _KnitGraphIndex with no parents."""
1614
1894
        size = trans.put_file(name, stream)
1615
1895
        return GraphIndex(trans, name, size)
1616
1896
 
 
1897
    def test_add_good_unvalidated_index(self):
 
1898
        unvalidated = self.make_g_index('unvalidated')
 
1899
        combined = CombinedGraphIndex([unvalidated])
 
1900
        index = _KnitGraphIndex(combined, lambda: True, parents=False)
 
1901
        index.scan_unvalidated_index(unvalidated)
 
1902
        self.assertEqual(frozenset(),
 
1903
            index.get_missing_compression_parents())
 
1904
 
1617
1905
    def test_parents_deltas_incompatible(self):
1618
1906
        index = CombinedGraphIndex([])
1619
1907
        self.assertRaises(errors.KnitError, _KnitGraphIndex, lambda:True,
1700
1988
        index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000), [])])
1701
1989
        # but neither should have added data.
1702
1990
        self.assertEqual([[], [], [], []], self.caught_entries)
1703
 
        
 
1991
 
1704
1992
    def test_add_version_different_dup(self):
1705
1993
        index = self.two_graph_index(catch_adds=True)
1706
1994
        # change options
1714
2002
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1715
2003
            [(('tip',), 'fulltext,no-eol', (None, 0, 100), [('parent',)])])
1716
2004
        self.assertEqual([], self.caught_entries)
1717
 
        
 
2005
 
1718
2006
    def test_add_versions(self):
1719
2007
        index = self.two_graph_index(catch_adds=True)
1720
2008
        index.add_records([
1752
2040
        index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000), [])])
1753
2041
        # but neither should have added data.
1754
2042
        self.assertEqual([[], [], [], []], self.caught_entries)
1755
 
        
 
2043
 
1756
2044
    def test_add_versions_different_dup(self):
1757
2045
        index = self.two_graph_index(catch_adds=True)
1758
2046
        # change options
1772
2060
        self.assertEqual([], self.caught_entries)
1773
2061
 
1774
2062
 
 
2063
class TestKnitVersionedFiles(KnitTests):
 
2064
 
 
2065
    def assertGroupKeysForIo(self, exp_groups, keys, non_local_keys,
 
2066
                             positions, _min_buffer_size=None):
 
2067
        kvf = self.make_test_knit()
 
2068
        if _min_buffer_size is None:
 
2069
            _min_buffer_size = knit._STREAM_MIN_BUFFER_SIZE
 
2070
        self.assertEqual(exp_groups, kvf._group_keys_for_io(keys,
 
2071
                                        non_local_keys, positions,
 
2072
                                        _min_buffer_size=_min_buffer_size))
 
2073
 
 
2074
    def assertSplitByPrefix(self, expected_map, expected_prefix_order,
 
2075
                            keys):
 
2076
        split, prefix_order = KnitVersionedFiles._split_by_prefix(keys)
 
2077
        self.assertEqual(expected_map, split)
 
2078
        self.assertEqual(expected_prefix_order, prefix_order)
 
2079
 
 
2080
    def test__group_keys_for_io(self):
 
2081
        ft_detail = ('fulltext', False)
 
2082
        ld_detail = ('line-delta', False)
 
2083
        f_a = ('f', 'a')
 
2084
        f_b = ('f', 'b')
 
2085
        f_c = ('f', 'c')
 
2086
        g_a = ('g', 'a')
 
2087
        g_b = ('g', 'b')
 
2088
        g_c = ('g', 'c')
 
2089
        positions = {
 
2090
            f_a: (ft_detail, (f_a, 0, 100), None),
 
2091
            f_b: (ld_detail, (f_b, 100, 21), f_a),
 
2092
            f_c: (ld_detail, (f_c, 180, 15), f_b),
 
2093
            g_a: (ft_detail, (g_a, 121, 35), None),
 
2094
            g_b: (ld_detail, (g_b, 156, 12), g_a),
 
2095
            g_c: (ld_detail, (g_c, 195, 13), g_a),
 
2096
            }
 
2097
        self.assertGroupKeysForIo([([f_a], set())],
 
2098
                                  [f_a], [], positions)
 
2099
        self.assertGroupKeysForIo([([f_a], set([f_a]))],
 
2100
                                  [f_a], [f_a], positions)
 
2101
        self.assertGroupKeysForIo([([f_a, f_b], set([]))],
 
2102
                                  [f_a, f_b], [], positions)
 
2103
        self.assertGroupKeysForIo([([f_a, f_b], set([f_b]))],
 
2104
                                  [f_a, f_b], [f_b], positions)
 
2105
        self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
 
2106
                                  [f_a, g_a, f_b, g_b], [], positions)
 
2107
        self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
 
2108
                                  [f_a, g_a, f_b, g_b], [], positions,
 
2109
                                  _min_buffer_size=150)
 
2110
        self.assertGroupKeysForIo([([f_a, f_b], set()), ([g_a, g_b], set())],
 
2111
                                  [f_a, g_a, f_b, g_b], [], positions,
 
2112
                                  _min_buffer_size=100)
 
2113
        self.assertGroupKeysForIo([([f_c], set()), ([g_b], set())],
 
2114
                                  [f_c, g_b], [], positions,
 
2115
                                  _min_buffer_size=125)
 
2116
        self.assertGroupKeysForIo([([g_b, f_c], set())],
 
2117
                                  [g_b, f_c], [], positions,
 
2118
                                  _min_buffer_size=125)
 
2119
 
 
2120
    def test__split_by_prefix(self):
 
2121
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2122
                                  'g': [('g', 'b'), ('g', 'a')],
 
2123
                                 }, ['f', 'g'],
 
2124
                                 [('f', 'a'), ('g', 'b'),
 
2125
                                  ('g', 'a'), ('f', 'b')])
 
2126
 
 
2127
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2128
                                  'g': [('g', 'b'), ('g', 'a')],
 
2129
                                 }, ['f', 'g'],
 
2130
                                 [('f', 'a'), ('f', 'b'),
 
2131
                                  ('g', 'b'), ('g', 'a')])
 
2132
 
 
2133
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2134
                                  'g': [('g', 'b'), ('g', 'a')],
 
2135
                                 }, ['f', 'g'],
 
2136
                                 [('f', 'a'), ('f', 'b'),
 
2137
                                  ('g', 'b'), ('g', 'a')])
 
2138
 
 
2139
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2140
                                  'g': [('g', 'b'), ('g', 'a')],
 
2141
                                  '': [('a',), ('b',)]
 
2142
                                 }, ['f', 'g', ''],
 
2143
                                 [('f', 'a'), ('g', 'b'),
 
2144
                                  ('a',), ('b',),
 
2145
                                  ('g', 'a'), ('f', 'b')])
 
2146
 
 
2147
 
1775
2148
class TestStacking(KnitTests):
1776
2149
 
1777
2150
    def get_basis_and_test_knit(self):
1832
2205
        # self.assertEqual([("annotate", key_basis)], basis.calls)
1833
2206
        self.assertEqual([('get_parent_map', set([key_basis])),
1834
2207
            ('get_parent_map', set([key_basis])),
1835
 
            ('get_parent_map', set([key_basis])),
1836
 
            ('get_record_stream', [key_basis], 'unordered', True)],
 
2208
            ('get_record_stream', [key_basis], 'topological', True)],
1837
2209
            basis.calls)
1838
2210
 
1839
2211
    def test_check(self):
1840
2212
        # At the moment checking a stacked knit does implicitly check the
1841
 
        # fallback files.  
 
2213
        # fallback files.
1842
2214
        basis, test = self.get_basis_and_test_knit()
1843
2215
        test.check()
1844
2216
 
1936
2308
                True).next()
1937
2309
            self.assertEqual(record.key, result[0])
1938
2310
            self.assertEqual(record.sha1, result[1])
1939
 
            self.assertEqual(record.storage_kind, result[2])
 
2311
            # We used to check that the storage kind matched, but actually it
 
2312
            # depends on whether it was sourced from the basis, or in a single
 
2313
            # group, because asking for full texts returns proxy objects to a
 
2314
            # _ContentMapGenerator object; so checking the kind is unneeded.
1940
2315
            self.assertEqual(record.get_bytes_as('fulltext'), result[3])
1941
2316
        # It's not strictly minimal, but it seems reasonable for now for it to
1942
2317
        # ask which fallbacks have which parents.
1943
2318
        self.assertEqual([
1944
2319
            ("get_parent_map", set([key_basis, key_basis_2, key_missing])),
1945
 
            # unordered is asked for by the underlying worker as it still
1946
 
            # buffers everything while answering - which is a problem!
1947
 
            ("get_record_stream", [key_basis_2, key_basis], 'unordered', True)],
 
2320
            # topological is requested from the fallback, because that is what
 
2321
            # was requested at the top level.
 
2322
            ("get_record_stream", [key_basis_2, key_basis], 'topological', True)],
1948
2323
            calls)
1949
2324
 
1950
2325
    def test_get_record_stream_unordered_deltas(self):
2079
2454
 
2080
2455
    def test_iter_lines_added_or_present_in_keys(self):
2081
2456
        # Lines from the basis are returned, and lines for a given key are only
2082
 
        # returned once. 
 
2457
        # returned once.
2083
2458
        key1 = ('foo1',)
2084
2459
        key2 = ('foo2',)
2085
2460
        # all sources are asked for keys:
2171
2546
        last_call = basis.calls[-1]
2172
2547
        self.assertEqual('get_record_stream', last_call[0])
2173
2548
        self.assertEqual(set([key_left, key_right]), set(last_call[1]))
2174
 
        self.assertEqual('unordered', last_call[2])
 
2549
        self.assertEqual('topological', last_call[2])
2175
2550
        self.assertEqual(True, last_call[3])
 
2551
 
 
2552
 
 
2553
class TestNetworkBehaviour(KnitTests):
 
2554
    """Tests for getting data out of/into knits over the network."""
 
2555
 
 
2556
    def test_include_delta_closure_generates_a_knit_delta_closure(self):
 
2557
        vf = self.make_test_knit(name='test')
 
2558
        # put in three texts, giving ft, delta, delta
 
2559
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2560
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2561
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2562
        # But heuristics could interfere, so check what happened:
 
2563
        self.assertEqual(['knit-ft-gz', 'knit-delta-gz', 'knit-delta-gz'],
 
2564
            [record.storage_kind for record in
 
2565
             vf.get_record_stream([('base',), ('d1',), ('d2',)],
 
2566
                'topological', False)])
 
2567
        # generate a stream of just the deltas include_delta_closure=True,
 
2568
        # serialise to the network, and check that we get a delta closure on the wire.
 
2569
        stream = vf.get_record_stream([('d1',), ('d2',)], 'topological', True)
 
2570
        netb = [record.get_bytes_as(record.storage_kind) for record in stream]
 
2571
        # The first bytes should be a memo from _ContentMapGenerator, and the
 
2572
        # second bytes should be empty (because its a API proxy not something
 
2573
        # for wire serialisation.
 
2574
        self.assertEqual('', netb[1])
 
2575
        bytes = netb[0]
 
2576
        kind, line_end = network_bytes_to_kind_and_offset(bytes)
 
2577
        self.assertEqual('knit-delta-closure', kind)
 
2578
 
 
2579
 
 
2580
class TestContentMapGenerator(KnitTests):
 
2581
    """Tests for ContentMapGenerator"""
 
2582
 
 
2583
    def test_get_record_stream_gives_records(self):
 
2584
        vf = self.make_test_knit(name='test')
 
2585
        # put in three texts, giving ft, delta, delta
 
2586
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2587
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2588
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2589
        keys = [('d1',), ('d2',)]
 
2590
        generator = _VFContentMapGenerator(vf, keys,
 
2591
            global_map=vf.get_parent_map(keys))
 
2592
        for record in generator.get_record_stream():
 
2593
            if record.key == ('d1',):
 
2594
                self.assertEqual('d1\n', record.get_bytes_as('fulltext'))
 
2595
            else:
 
2596
                self.assertEqual('d2\n', record.get_bytes_as('fulltext'))
 
2597
 
 
2598
    def test_get_record_stream_kinds_are_raw(self):
 
2599
        vf = self.make_test_knit(name='test')
 
2600
        # put in three texts, giving ft, delta, delta
 
2601
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2602
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2603
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2604
        keys = [('base',), ('d1',), ('d2',)]
 
2605
        generator = _VFContentMapGenerator(vf, keys,
 
2606
            global_map=vf.get_parent_map(keys))
 
2607
        kinds = {('base',): 'knit-delta-closure',
 
2608
            ('d1',): 'knit-delta-closure-ref',
 
2609
            ('d2',): 'knit-delta-closure-ref',
 
2610
            }
 
2611
        for record in generator.get_record_stream():
 
2612
            self.assertEqual(kinds[record.key], record.storage_kind)