← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

(jameinel) Allow 'bzr serve' to interpret SIGHUP as a graceful shutdown.
 (bug #795025) (John A Meinel)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
 
1
# Copyright (C) 2006-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
 
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
20
 
import difflib
21
20
import gzip
22
21
import sys
23
22
 
24
23
from bzrlib import (
25
24
    errors,
26
 
    generate_ids,
27
25
    knit,
28
26
    multiparent,
29
27
    osutils,
30
28
    pack,
 
29
    tests,
 
30
    transport,
31
31
    )
32
32
from bzrlib.errors import (
33
 
    RevisionAlreadyPresent,
34
33
    KnitHeaderError,
35
 
    RevisionNotPresent,
36
34
    NoSuchFile,
37
35
    )
38
36
from bzrlib.index import *
39
37
from bzrlib.knit import (
40
38
    AnnotatedKnitContent,
41
39
    KnitContent,
42
 
    KnitSequenceMatcher,
43
40
    KnitVersionedFiles,
44
41
    PlainKnitContent,
45
 
    _DirectPackAccess,
 
42
    _VFContentMapGenerator,
46
43
    _KndxIndex,
47
44
    _KnitGraphIndex,
48
45
    _KnitKeyAccess,
49
46
    make_file_factory,
50
47
    )
51
 
from bzrlib.repofmt import pack_repo
 
48
from bzrlib.patiencediff import PatienceSequenceMatcher
 
49
from bzrlib.repofmt import (
 
50
    knitpack_repo,
 
51
    pack_repo,
 
52
    )
52
53
from bzrlib.tests import (
53
 
    Feature,
54
 
    KnownFailure,
55
54
    TestCase,
56
55
    TestCaseWithMemoryTransport,
57
56
    TestCaseWithTransport,
58
57
    TestNotApplicable,
59
58
    )
60
 
from bzrlib.transport import get_transport
61
 
from bzrlib.transport.memory import MemoryTransport
62
 
from bzrlib.tuned_gzip import GzipFile
63
59
from bzrlib.versionedfile import (
64
60
    AbsentContentFactory,
65
61
    ConstantMapper,
 
62
    network_bytes_to_kind_and_offset,
66
63
    RecordingVersionedFilesDecorator,
67
64
    )
68
 
 
69
 
 
70
 
class _CompiledKnitFeature(Feature):
71
 
 
72
 
    def _probe(self):
73
 
        try:
74
 
            import bzrlib._knit_load_data_c
75
 
        except ImportError:
76
 
            return False
77
 
        return True
78
 
 
79
 
    def feature_name(self):
80
 
        return 'bzrlib._knit_load_data_c'
81
 
 
82
 
CompiledKnitFeature = _CompiledKnitFeature()
 
65
from bzrlib.tests import (
 
66
    features,
 
67
    )
 
68
 
 
69
 
 
70
compiled_knit_feature = features.ModuleAvailableFeature(
 
71
    'bzrlib._knit_load_data_pyx')
83
72
 
84
73
 
85
74
class KnitContentTestsMixin(object):
114
103
        line_delta = source_content.line_delta(target_content)
115
104
        delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
116
105
            source_lines, target_lines))
117
 
        matcher = KnitSequenceMatcher(None, source_lines, target_lines)
118
 
        matcher_blocks = list(list(matcher.get_matching_blocks()))
 
106
        matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
 
107
        matcher_blocks = list(matcher.get_matching_blocks())
119
108
        self.assertEqual(matcher_blocks, delta_blocks)
120
109
 
121
110
    def test_get_line_delta_blocks(self):
297
286
        access = self.get_access()
298
287
        memos = access.add_raw_records([('key', 10)], '1234567890')
299
288
        self.assertEqual(['1234567890'], list(access.get_raw_records(memos)))
300
 
 
 
289
 
301
290
    def test_add_several_raw_records(self):
302
291
        """add_raw_records with many records and read some back."""
303
292
        access = self.get_access()
341
330
            transport.append_bytes(packname, bytes)
342
331
        writer = pack.ContainerWriter(write_data)
343
332
        writer.begin()
344
 
        access = _DirectPackAccess({})
 
333
        access = pack_repo._DirectPackAccess({})
345
334
        access.set_writer(writer, index, (transport, packname))
346
335
        return access, writer
347
336
 
354
343
        writer.end()
355
344
        return memos
356
345
 
 
346
    def test_pack_collection_pack_retries(self):
 
347
        """An explicit pack of a pack collection succeeds even when a
 
348
        concurrent pack happens.
 
349
        """
 
350
        builder = self.make_branch_builder('.')
 
351
        builder.start_series()
 
352
        builder.build_snapshot('rev-1', None, [
 
353
            ('add', ('', 'root-id', 'directory', None)),
 
354
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
 
355
            ])
 
356
        builder.build_snapshot('rev-2', ['rev-1'], [
 
357
            ('modify', ('file-id', 'content\nrev 2\n')),
 
358
            ])
 
359
        builder.build_snapshot('rev-3', ['rev-2'], [
 
360
            ('modify', ('file-id', 'content\nrev 3\n')),
 
361
            ])
 
362
        self.addCleanup(builder.finish_series)
 
363
        b = builder.get_branch()
 
364
        self.addCleanup(b.lock_write().unlock)
 
365
        repo = b.repository
 
366
        collection = repo._pack_collection
 
367
        # Concurrently repack the repo.
 
368
        reopened_repo = repo.bzrdir.open_repository()
 
369
        reopened_repo.pack()
 
370
        # Pack the new pack.
 
371
        collection.pack()
 
372
 
357
373
    def make_vf_for_retrying(self):
358
374
        """Create 3 packs and a reload function.
359
375
 
364
380
        :return: (versioned_file, reload_counter)
365
381
            versioned_file  a KnitVersionedFiles using the packs for access
366
382
        """
367
 
        tree = self.make_branch_and_memory_tree('tree')
368
 
        tree.lock_write()
369
 
        try:
370
 
            tree.add([''], ['root-id'])
371
 
            tree.commit('one', rev_id='rev-1')
372
 
            tree.commit('two', rev_id='rev-2')
373
 
            tree.commit('three', rev_id='rev-3')
374
 
            # Pack these two revisions into another pack file, but don't remove
375
 
            # the originials
376
 
            repo = tree.branch.repository
377
 
            collection = repo._pack_collection
378
 
            collection.ensure_loaded()
379
 
            orig_packs = collection.packs
380
 
            packer = pack_repo.Packer(collection, orig_packs, '.testpack')
381
 
            new_pack = packer.pack()
382
 
 
383
 
            vf = tree.branch.repository.revisions
384
 
        finally:
385
 
            tree.unlock()
386
 
        tree.branch.repository.lock_read()
387
 
        self.addCleanup(tree.branch.repository.unlock)
388
 
        del tree
 
383
        builder = self.make_branch_builder('.', format="1.9")
 
384
        builder.start_series()
 
385
        builder.build_snapshot('rev-1', None, [
 
386
            ('add', ('', 'root-id', 'directory', None)),
 
387
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
 
388
            ])
 
389
        builder.build_snapshot('rev-2', ['rev-1'], [
 
390
            ('modify', ('file-id', 'content\nrev 2\n')),
 
391
            ])
 
392
        builder.build_snapshot('rev-3', ['rev-2'], [
 
393
            ('modify', ('file-id', 'content\nrev 3\n')),
 
394
            ])
 
395
        builder.finish_series()
 
396
        b = builder.get_branch()
 
397
        b.lock_write()
 
398
        self.addCleanup(b.unlock)
 
399
        # Pack these three revisions into another pack file, but don't remove
 
400
        # the originals
 
401
        repo = b.repository
 
402
        collection = repo._pack_collection
 
403
        collection.ensure_loaded()
 
404
        orig_packs = collection.packs
 
405
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
 
406
        new_pack = packer.pack()
 
407
        # forget about the new pack
 
408
        collection.reset()
 
409
        repo.refresh_data()
 
410
        vf = repo.revisions
389
411
        # Set up a reload() function that switches to using the new pack file
390
412
        new_index = new_pack.revision_index
391
413
        access_tuple = new_pack.access_tuple()
425
447
        except _TestException, e:
426
448
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
427
449
                                                 exc_info=sys.exc_info())
 
450
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
428
451
        return retry_exc
429
452
 
430
453
    def test_read_from_several_packs(self):
439
462
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
440
463
        writer.end()
441
464
        transport = self.get_transport()
442
 
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
 
465
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
443
466
            "FOOBAR":(transport, 'pack2'),
444
467
            "BAZ":(transport, 'pack3')})
445
468
        self.assertEqual(['1234567890', '12345', 'alpha'],
455
478
 
456
479
    def test_set_writer(self):
457
480
        """The writer should be settable post construction."""
458
 
        access = _DirectPackAccess({})
 
481
        access = pack_repo._DirectPackAccess({})
459
482
        transport = self.get_transport()
460
483
        packname = 'packfile'
461
484
        index = 'foo'
473
496
        transport = self.get_transport()
474
497
        reload_called, reload_func = self.make_reload_func()
475
498
        # Note that the index key has changed from 'foo' to 'bar'
476
 
        access = _DirectPackAccess({'bar':(transport, 'packname')},
 
499
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
477
500
                                   reload_func=reload_func)
478
501
        e = self.assertListRaises(errors.RetryWithNewPacks,
479
502
                                  access.get_raw_records, memos)
488
511
        memos = self.make_pack_file()
489
512
        transport = self.get_transport()
490
513
        # Note that the index key has changed from 'foo' to 'bar'
491
 
        access = _DirectPackAccess({'bar':(transport, 'packname')})
 
514
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
492
515
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
493
516
 
494
517
    def test_missing_file_raises_retry(self):
496
519
        transport = self.get_transport()
497
520
        reload_called, reload_func = self.make_reload_func()
498
521
        # Note that the 'filename' has been changed to 'different-packname'
499
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
500
 
                                   reload_func=reload_func)
 
522
        access = pack_repo._DirectPackAccess(
 
523
            {'foo':(transport, 'different-packname')},
 
524
            reload_func=reload_func)
501
525
        e = self.assertListRaises(errors.RetryWithNewPacks,
502
526
                                  access.get_raw_records, memos)
503
527
        # The file has gone missing, so we assume we need to reload
511
535
        memos = self.make_pack_file()
512
536
        transport = self.get_transport()
513
537
        # Note that the 'filename' has been changed to 'different-packname'
514
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
 
538
        access = pack_repo._DirectPackAccess(
 
539
            {'foo': (transport, 'different-packname')})
515
540
        e = self.assertListRaises(errors.NoSuchFile,
516
541
                                  access.get_raw_records, memos)
517
542
 
521
546
        failing_transport = MockReadvFailingTransport(
522
547
                                [transport.get_bytes('packname')])
523
548
        reload_called, reload_func = self.make_reload_func()
524
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
525
 
                                   reload_func=reload_func)
 
549
        access = pack_repo._DirectPackAccess(
 
550
            {'foo': (failing_transport, 'packname')},
 
551
            reload_func=reload_func)
526
552
        # Asking for a single record will not trigger the Mock failure
527
553
        self.assertEqual(['1234567890'],
528
554
            list(access.get_raw_records(memos[:1])))
544
570
        failing_transport = MockReadvFailingTransport(
545
571
                                [transport.get_bytes('packname')])
546
572
        reload_called, reload_func = self.make_reload_func()
547
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
 
573
        access = pack_repo._DirectPackAccess(
 
574
            {'foo':(failing_transport, 'packname')})
548
575
        # Asking for a single record will not trigger the Mock failure
549
576
        self.assertEqual(['1234567890'],
550
577
            list(access.get_raw_records(memos[:1])))
555
582
                                  access.get_raw_records, memos)
556
583
 
557
584
    def test_reload_or_raise_no_reload(self):
558
 
        access = _DirectPackAccess({}, reload_func=None)
 
585
        access = pack_repo._DirectPackAccess({}, reload_func=None)
559
586
        retry_exc = self.make_retry_exception()
560
587
        # Without a reload_func, we will just re-raise the original exception
561
588
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
562
589
 
563
590
    def test_reload_or_raise_reload_changed(self):
564
591
        reload_called, reload_func = self.make_reload_func(return_val=True)
565
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
592
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
566
593
        retry_exc = self.make_retry_exception()
567
594
        access.reload_or_raise(retry_exc)
568
595
        self.assertEqual([1], reload_called)
572
599
 
573
600
    def test_reload_or_raise_reload_no_change(self):
574
601
        reload_called, reload_func = self.make_reload_func(return_val=False)
575
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
602
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
576
603
        retry_exc = self.make_retry_exception()
577
604
        # If reload_occurred is False, then we consider it an error to have
578
605
        # reload_func() return False (no changes).
709
736
 
710
737
    def make_multiple_records(self):
711
738
        """Create the content for multiple records."""
712
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
739
        sha1sum = osutils.sha_string('foo\nbar\n')
713
740
        total_txt = []
714
741
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
715
742
                                        'foo\n'
718
745
                                        % (sha1sum,))
719
746
        record_1 = (0, len(gz_txt), sha1sum)
720
747
        total_txt.append(gz_txt)
721
 
        sha1sum = osutils.sha('baz\n').hexdigest()
 
748
        sha1sum = osutils.sha_string('baz\n')
722
749
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
723
750
                                        'baz\n'
724
751
                                        'end rev-id-2\n'
728
755
        return total_txt, record_1, record_2
729
756
 
730
757
    def test_valid_knit_data(self):
731
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
758
        sha1sum = osutils.sha_string('foo\nbar\n')
732
759
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
733
760
                                        'foo\n'
734
761
                                        'bar\n'
765
792
                         raw_contents)
766
793
 
767
794
    def test_not_enough_lines(self):
768
 
        sha1sum = osutils.sha('foo\n').hexdigest()
 
795
        sha1sum = osutils.sha_string('foo\n')
769
796
        # record says 2 lines data says 1
770
797
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
771
798
                                        'foo\n'
783
810
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
784
811
 
785
812
    def test_too_many_lines(self):
786
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
813
        sha1sum = osutils.sha_string('foo\nbar\n')
787
814
        # record says 1 lines data says 2
788
815
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
789
816
                                        'foo\n'
802
829
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
803
830
 
804
831
    def test_mismatched_version_id(self):
805
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
832
        sha1sum = osutils.sha_string('foo\nbar\n')
806
833
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
807
834
                                        'foo\n'
808
835
                                        'bar\n'
821
848
            knit._read_records_iter_raw(records))
822
849
 
823
850
    def test_uncompressed_data(self):
824
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
851
        sha1sum = osutils.sha_string('foo\nbar\n')
825
852
        txt = ('version rev-id-1 2 %s\n'
826
853
               'foo\n'
827
854
               'bar\n'
841
868
            knit._read_records_iter_raw(records))
842
869
 
843
870
    def test_corrupted_data(self):
844
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
871
        sha1sum = osutils.sha_string('foo\nbar\n')
845
872
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
846
873
                                        'foo\n'
847
874
                                        'bar\n'
864
891
 
865
892
    def get_knit_index(self, transport, name, mode):
866
893
        mapper = ConstantMapper(name)
867
 
        orig = knit._load_data
868
 
        def reset():
869
 
            knit._load_data = orig
870
 
        self.addCleanup(reset)
871
894
        from bzrlib._knit_load_data_py import _load_data_py
872
 
        knit._load_data = _load_data_py
 
895
        self.overrideAttr(knit, '_load_data', _load_data_py)
873
896
        allow_writes = lambda: 'w' in mode
874
897
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
875
898
 
1091
1114
            call[1][1].getvalue())
1092
1115
        self.assertEqual({'create_parent_dir': True}, call[2])
1093
1116
 
 
1117
    def assertTotalBuildSize(self, size, keys, positions):
 
1118
        self.assertEqual(size,
 
1119
                         knit._get_total_build_size(None, keys, positions))
 
1120
 
 
1121
    def test__get_total_build_size(self):
 
1122
        positions = {
 
1123
            ('a',): (('fulltext', False), (('a',), 0, 100), None),
 
1124
            ('b',): (('line-delta', False), (('b',), 100, 21), ('a',)),
 
1125
            ('c',): (('line-delta', False), (('c',), 121, 35), ('b',)),
 
1126
            ('d',): (('line-delta', False), (('d',), 156, 12), ('b',)),
 
1127
            }
 
1128
        self.assertTotalBuildSize(100, [('a',)], positions)
 
1129
        self.assertTotalBuildSize(121, [('b',)], positions)
 
1130
        # c needs both a & b
 
1131
        self.assertTotalBuildSize(156, [('c',)], positions)
 
1132
        # we shouldn't count 'b' twice
 
1133
        self.assertTotalBuildSize(156, [('b',), ('c',)], positions)
 
1134
        self.assertTotalBuildSize(133, [('d',)], positions)
 
1135
        self.assertTotalBuildSize(168, [('c',), ('d',)], positions)
 
1136
 
1094
1137
    def test_get_position(self):
1095
1138
        transport = MockTransport([
1096
1139
            _KndxIndex.HEADER,
1153
1196
            self.assertRaises(errors.KnitCorrupt, index.keys)
1154
1197
        except TypeError, e:
1155
1198
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1156
 
                           ' not exceptions.IndexError')
1157
 
                and sys.version_info[0:2] >= (2,5)):
 
1199
                           ' not exceptions.IndexError')):
1158
1200
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1159
1201
                                  ' raising new style exceptions with python'
1160
1202
                                  ' >=2.5')
1173
1215
            self.assertRaises(errors.KnitCorrupt, index.keys)
1174
1216
        except TypeError, e:
1175
1217
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1176
 
                           ' not exceptions.ValueError')
1177
 
                and sys.version_info[0:2] >= (2,5)):
 
1218
                           ' not exceptions.ValueError')):
1178
1219
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1179
1220
                                  ' raising new style exceptions with python'
1180
1221
                                  ' >=2.5')
1193
1234
            self.assertRaises(errors.KnitCorrupt, index.keys)
1194
1235
        except TypeError, e:
1195
1236
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1196
 
                           ' not exceptions.ValueError')
1197
 
                and sys.version_info[0:2] >= (2,5)):
 
1237
                           ' not exceptions.ValueError')):
1198
1238
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1199
1239
                                  ' raising new style exceptions with python'
1200
1240
                                  ' >=2.5')
1211
1251
            self.assertRaises(errors.KnitCorrupt, index.keys)
1212
1252
        except TypeError, e:
1213
1253
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1214
 
                           ' not exceptions.ValueError')
1215
 
                and sys.version_info[0:2] >= (2,5)):
 
1254
                           ' not exceptions.ValueError')):
1216
1255
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1217
1256
                                  ' raising new style exceptions with python'
1218
1257
                                  ' >=2.5')
1229
1268
            self.assertRaises(errors.KnitCorrupt, index.keys)
1230
1269
        except TypeError, e:
1231
1270
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1232
 
                           ' not exceptions.ValueError')
1233
 
                and sys.version_info[0:2] >= (2,5)):
 
1271
                           ' not exceptions.ValueError')):
1234
1272
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1235
1273
                                  ' raising new style exceptions with python'
1236
1274
                                  ' >=2.5')
1237
1275
            else:
1238
1276
                raise
1239
1277
 
 
1278
    def test_scan_unvalidated_index_not_implemented(self):
 
1279
        transport = MockTransport()
 
1280
        index = self.get_knit_index(transport, 'filename', 'r')
 
1281
        self.assertRaises(
 
1282
            NotImplementedError, index.scan_unvalidated_index,
 
1283
            'dummy graph_index')
 
1284
        self.assertRaises(
 
1285
            NotImplementedError, index.get_missing_compression_parents)
 
1286
 
1240
1287
    def test_short_line(self):
1241
1288
        transport = MockTransport([
1242
1289
            _KndxIndex.HEADER,
1271
1318
 
1272
1319
class LowLevelKnitIndexTests_c(LowLevelKnitIndexTests):
1273
1320
 
1274
 
    _test_needs_features = [CompiledKnitFeature]
 
1321
    _test_needs_features = [compiled_knit_feature]
1275
1322
 
1276
1323
    def get_knit_index(self, transport, name, mode):
1277
1324
        mapper = ConstantMapper(name)
1278
 
        orig = knit._load_data
1279
 
        def reset():
1280
 
            knit._load_data = orig
1281
 
        self.addCleanup(reset)
1282
 
        from bzrlib._knit_load_data_c import _load_data_c
1283
 
        knit._load_data = _load_data_c
 
1325
        from bzrlib._knit_load_data_pyx import _load_data_c
 
1326
        self.overrideAttr(knit, '_load_data', _load_data_c)
1284
1327
        allow_writes = lambda: mode == 'w'
1285
 
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
 
1328
        return _KndxIndex(transport, mapper, lambda:None,
 
1329
                          allow_writes, lambda:True)
 
1330
 
 
1331
 
 
1332
class Test_KnitAnnotator(TestCaseWithMemoryTransport):
 
1333
 
 
1334
    def make_annotator(self):
 
1335
        factory = knit.make_pack_factory(True, True, 1)
 
1336
        vf = factory(self.get_transport())
 
1337
        return knit._KnitAnnotator(vf)
 
1338
 
 
1339
    def test__expand_fulltext(self):
 
1340
        ann = self.make_annotator()
 
1341
        rev_key = ('rev-id',)
 
1342
        ann._num_compression_children[rev_key] = 1
 
1343
        res = ann._expand_record(rev_key, (('parent-id',),), None,
 
1344
                           ['line1\n', 'line2\n'], ('fulltext', True))
 
1345
        # The content object and text lines should be cached appropriately
 
1346
        self.assertEqual(['line1\n', 'line2'], res)
 
1347
        content_obj = ann._content_objects[rev_key]
 
1348
        self.assertEqual(['line1\n', 'line2\n'], content_obj._lines)
 
1349
        self.assertEqual(res, content_obj.text())
 
1350
        self.assertEqual(res, ann._text_cache[rev_key])
 
1351
 
 
1352
    def test__expand_delta_comp_parent_not_available(self):
 
1353
        # Parent isn't available yet, so we return nothing, but queue up this
 
1354
        # node for later processing
 
1355
        ann = self.make_annotator()
 
1356
        rev_key = ('rev-id',)
 
1357
        parent_key = ('parent-id',)
 
1358
        record = ['0,1,1\n', 'new-line\n']
 
1359
        details = ('line-delta', False)
 
1360
        res = ann._expand_record(rev_key, (parent_key,), parent_key,
 
1361
                                 record, details)
 
1362
        self.assertEqual(None, res)
 
1363
        self.assertTrue(parent_key in ann._pending_deltas)
 
1364
        pending = ann._pending_deltas[parent_key]
 
1365
        self.assertEqual(1, len(pending))
 
1366
        self.assertEqual((rev_key, (parent_key,), record, details), pending[0])
 
1367
 
 
1368
    def test__expand_record_tracks_num_children(self):
 
1369
        ann = self.make_annotator()
 
1370
        rev_key = ('rev-id',)
 
1371
        rev2_key = ('rev2-id',)
 
1372
        parent_key = ('parent-id',)
 
1373
        record = ['0,1,1\n', 'new-line\n']
 
1374
        details = ('line-delta', False)
 
1375
        ann._num_compression_children[parent_key] = 2
 
1376
        ann._expand_record(parent_key, (), None, ['line1\n', 'line2\n'],
 
1377
                           ('fulltext', False))
 
1378
        res = ann._expand_record(rev_key, (parent_key,), parent_key,
 
1379
                                 record, details)
 
1380
        self.assertEqual({parent_key: 1}, ann._num_compression_children)
 
1381
        # Expanding the second child should remove the content object, and the
 
1382
        # num_compression_children entry
 
1383
        res = ann._expand_record(rev2_key, (parent_key,), parent_key,
 
1384
                                 record, details)
 
1385
        self.assertFalse(parent_key in ann._content_objects)
 
1386
        self.assertEqual({}, ann._num_compression_children)
 
1387
        # We should not cache the content_objects for rev2 and rev, because
 
1388
        # they do not have compression children of their own.
 
1389
        self.assertEqual({}, ann._content_objects)
 
1390
 
 
1391
    def test__expand_delta_records_blocks(self):
 
1392
        ann = self.make_annotator()
 
1393
        rev_key = ('rev-id',)
 
1394
        parent_key = ('parent-id',)
 
1395
        record = ['0,1,1\n', 'new-line\n']
 
1396
        details = ('line-delta', True)
 
1397
        ann._num_compression_children[parent_key] = 2
 
1398
        ann._expand_record(parent_key, (), None,
 
1399
                           ['line1\n', 'line2\n', 'line3\n'],
 
1400
                           ('fulltext', False))
 
1401
        ann._expand_record(rev_key, (parent_key,), parent_key, record, details)
 
1402
        self.assertEqual({(rev_key, parent_key): [(1, 1, 1), (3, 3, 0)]},
 
1403
                         ann._matching_blocks)
 
1404
        rev2_key = ('rev2-id',)
 
1405
        record = ['0,1,1\n', 'new-line\n']
 
1406
        details = ('line-delta', False)
 
1407
        ann._expand_record(rev2_key, (parent_key,), parent_key, record, details)
 
1408
        self.assertEqual([(1, 1, 2), (3, 3, 0)],
 
1409
                         ann._matching_blocks[(rev2_key, parent_key)])
 
1410
 
 
1411
    def test__get_parent_ann_uses_matching_blocks(self):
 
1412
        ann = self.make_annotator()
 
1413
        rev_key = ('rev-id',)
 
1414
        parent_key = ('parent-id',)
 
1415
        parent_ann = [(parent_key,)]*3
 
1416
        block_key = (rev_key, parent_key)
 
1417
        ann._annotations_cache[parent_key] = parent_ann
 
1418
        ann._matching_blocks[block_key] = [(0, 1, 1), (3, 3, 0)]
 
1419
        # We should not try to access any parent_lines content, because we know
 
1420
        # we already have the matching blocks
 
1421
        par_ann, blocks = ann._get_parent_annotations_and_matches(rev_key,
 
1422
                                        ['1\n', '2\n', '3\n'], parent_key)
 
1423
        self.assertEqual(parent_ann, par_ann)
 
1424
        self.assertEqual([(0, 1, 1), (3, 3, 0)], blocks)
 
1425
        self.assertEqual({}, ann._matching_blocks)
 
1426
 
 
1427
    def test__process_pending(self):
 
1428
        ann = self.make_annotator()
 
1429
        rev_key = ('rev-id',)
 
1430
        p1_key = ('p1-id',)
 
1431
        p2_key = ('p2-id',)
 
1432
        record = ['0,1,1\n', 'new-line\n']
 
1433
        details = ('line-delta', False)
 
1434
        p1_record = ['line1\n', 'line2\n']
 
1435
        ann._num_compression_children[p1_key] = 1
 
1436
        res = ann._expand_record(rev_key, (p1_key,p2_key), p1_key,
 
1437
                                 record, details)
 
1438
        self.assertEqual(None, res)
 
1439
        # self.assertTrue(p1_key in ann._pending_deltas)
 
1440
        self.assertEqual({}, ann._pending_annotation)
 
1441
        # Now insert p1, and we should be able to expand the delta
 
1442
        res = ann._expand_record(p1_key, (), None, p1_record,
 
1443
                                 ('fulltext', False))
 
1444
        self.assertEqual(p1_record, res)
 
1445
        ann._annotations_cache[p1_key] = [(p1_key,)]*2
 
1446
        res = ann._process_pending(p1_key)
 
1447
        self.assertEqual([], res)
 
1448
        self.assertFalse(p1_key in ann._pending_deltas)
 
1449
        self.assertTrue(p2_key in ann._pending_annotation)
 
1450
        self.assertEqual({p2_key: [(rev_key, (p1_key, p2_key))]},
 
1451
                         ann._pending_annotation)
 
1452
        # Now fill in parent 2, and pending annotation should be satisfied
 
1453
        res = ann._expand_record(p2_key, (), None, [], ('fulltext', False))
 
1454
        ann._annotations_cache[p2_key] = []
 
1455
        res = ann._process_pending(p2_key)
 
1456
        self.assertEqual([rev_key], res)
 
1457
        self.assertEqual({}, ann._pending_annotation)
 
1458
        self.assertEqual({}, ann._pending_deltas)
 
1459
 
 
1460
    def test_record_delta_removes_basis(self):
 
1461
        ann = self.make_annotator()
 
1462
        ann._expand_record(('parent-id',), (), None,
 
1463
                           ['line1\n', 'line2\n'], ('fulltext', False))
 
1464
        ann._num_compression_children['parent-id'] = 2
 
1465
 
 
1466
    def test_annotate_special_text(self):
 
1467
        ann = self.make_annotator()
 
1468
        vf = ann._vf
 
1469
        rev1_key = ('rev-1',)
 
1470
        rev2_key = ('rev-2',)
 
1471
        rev3_key = ('rev-3',)
 
1472
        spec_key = ('special:',)
 
1473
        vf.add_lines(rev1_key, [], ['initial content\n'])
 
1474
        vf.add_lines(rev2_key, [rev1_key], ['initial content\n',
 
1475
                                            'common content\n',
 
1476
                                            'content in 2\n'])
 
1477
        vf.add_lines(rev3_key, [rev1_key], ['initial content\n',
 
1478
                                            'common content\n',
 
1479
                                            'content in 3\n'])
 
1480
        spec_text = ('initial content\n'
 
1481
                     'common content\n'
 
1482
                     'content in 2\n'
 
1483
                     'content in 3\n')
 
1484
        ann.add_special_text(spec_key, [rev2_key, rev3_key], spec_text)
 
1485
        anns, lines = ann.annotate(spec_key)
 
1486
        self.assertEqual([(rev1_key,),
 
1487
                          (rev2_key, rev3_key),
 
1488
                          (rev2_key,),
 
1489
                          (rev3_key,),
 
1490
                         ], anns)
 
1491
        self.assertEqualDiff(spec_text, ''.join(lines))
1286
1492
 
1287
1493
 
1288
1494
class KnitTests(TestCaseWithTransport):
1296
1502
class TestBadShaError(KnitTests):
1297
1503
    """Tests for handling of sha errors."""
1298
1504
 
1299
 
    def test_exception_has_text(self):
 
1505
    def test_sha_exception_has_text(self):
1300
1506
        # having the failed text included in the error allows for recovery.
1301
1507
        source = self.make_test_knit()
1302
1508
        target = self.make_test_knit(name="target")
1313
1519
        target.insert_record_stream(
1314
1520
            source.get_record_stream([broken], 'unordered', False))
1315
1521
        err = self.assertRaises(errors.KnitCorrupt,
1316
 
            target.get_record_stream([broken], 'unordered', True).next)
 
1522
            target.get_record_stream([broken], 'unordered', True
 
1523
            ).next().get_bytes_as, 'chunked')
1317
1524
        self.assertEqual(['gam\n', 'bar\n'], err.content)
1318
1525
        # Test for formatting with live data
1319
1526
        self.assertStartsWith(str(err), "Knit ")
1396
1603
        # could leave an empty .kndx file, which bzr would later claim was a
1397
1604
        # corrupted file since the header was not present. In reality, the file
1398
1605
        # just wasn't created, so it should be ignored.
1399
 
        t = get_transport('.')
 
1606
        t = transport.get_transport_from_path('.')
1400
1607
        t.put_bytes('test.kndx', '')
1401
1608
 
1402
1609
        knit = self.make_test_knit()
1403
1610
 
1404
1611
    def test_knit_index_checks_header(self):
1405
 
        t = get_transport('.')
 
1612
        t = transport.get_transport_from_path('.')
1406
1613
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1407
1614
        k = self.make_test_knit()
1408
1615
        self.assertRaises(KnitHeaderError, k.keys)
1524
1731
            [('parent',)])])
1525
1732
        # but neither should have added data:
1526
1733
        self.assertEqual([[], [], [], []], self.caught_entries)
1527
 
        
 
1734
 
1528
1735
    def test_add_version_different_dup(self):
1529
1736
        index = self.two_graph_index(deltas=True, catch_adds=True)
1530
1737
        # change options
1536
1743
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1537
1744
            [(('tip',), 'fulltext,no-eol', (None, 0, 100), [])])
1538
1745
        self.assertEqual([], self.caught_entries)
1539
 
        
 
1746
 
1540
1747
    def test_add_versions_nodeltas(self):
1541
1748
        index = self.two_graph_index(catch_adds=True)
1542
1749
        index.add_records([
1584
1791
            [('parent',)])])
1585
1792
        # but neither should have added data.
1586
1793
        self.assertEqual([[], [], [], []], self.caught_entries)
1587
 
        
 
1794
 
1588
1795
    def test_add_versions_different_dup(self):
1589
1796
        index = self.two_graph_index(deltas=True, catch_adds=True)
1590
1797
        # change options
1601
1808
             (('tip',), 'line-delta', (None, 0, 100), [('parent',)])])
1602
1809
        self.assertEqual([], self.caught_entries)
1603
1810
 
 
1811
    def make_g_index_missing_compression_parent(self):
 
1812
        graph_index = self.make_g_index('missing_comp', 2,
 
1813
            [(('tip', ), ' 100 78',
 
1814
              ([('missing-parent', ), ('ghost', )], [('missing-parent', )]))])
 
1815
        return graph_index
 
1816
 
 
1817
    def make_g_index_missing_parent(self):
 
1818
        graph_index = self.make_g_index('missing_parent', 2,
 
1819
            [(('parent', ), ' 100 78', ([], [])),
 
1820
             (('tip', ), ' 100 78',
 
1821
              ([('parent', ), ('missing-parent', )], [('parent', )])),
 
1822
              ])
 
1823
        return graph_index
 
1824
 
 
1825
    def make_g_index_no_external_refs(self):
 
1826
        graph_index = self.make_g_index('no_external_refs', 2,
 
1827
            [(('rev', ), ' 100 78',
 
1828
              ([('parent', ), ('ghost', )], []))])
 
1829
        return graph_index
 
1830
 
 
1831
    def test_add_good_unvalidated_index(self):
 
1832
        unvalidated = self.make_g_index_no_external_refs()
 
1833
        combined = CombinedGraphIndex([unvalidated])
 
1834
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1835
        index.scan_unvalidated_index(unvalidated)
 
1836
        self.assertEqual(frozenset(), index.get_missing_compression_parents())
 
1837
 
 
1838
    def test_add_missing_compression_parent_unvalidated_index(self):
 
1839
        unvalidated = self.make_g_index_missing_compression_parent()
 
1840
        combined = CombinedGraphIndex([unvalidated])
 
1841
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1842
        index.scan_unvalidated_index(unvalidated)
 
1843
        # This also checks that its only the compression parent that is
 
1844
        # examined, otherwise 'ghost' would also be reported as a missing
 
1845
        # parent.
 
1846
        self.assertEqual(
 
1847
            frozenset([('missing-parent',)]),
 
1848
            index.get_missing_compression_parents())
 
1849
 
 
1850
    def test_add_missing_noncompression_parent_unvalidated_index(self):
 
1851
        unvalidated = self.make_g_index_missing_parent()
 
1852
        combined = CombinedGraphIndex([unvalidated])
 
1853
        index = _KnitGraphIndex(combined, lambda: True, deltas=True,
 
1854
            track_external_parent_refs=True)
 
1855
        index.scan_unvalidated_index(unvalidated)
 
1856
        self.assertEqual(
 
1857
            frozenset([('missing-parent',)]), index.get_missing_parents())
 
1858
 
 
1859
    def test_track_external_parent_refs(self):
 
1860
        g_index = self.make_g_index('empty', 2, [])
 
1861
        combined = CombinedGraphIndex([g_index])
 
1862
        index = _KnitGraphIndex(combined, lambda: True, deltas=True,
 
1863
            add_callback=self.catch_add, track_external_parent_refs=True)
 
1864
        self.caught_entries = []
 
1865
        index.add_records([
 
1866
            (('new-key',), 'fulltext,no-eol', (None, 50, 60),
 
1867
             [('parent-1',), ('parent-2',)])])
 
1868
        self.assertEqual(
 
1869
            frozenset([('parent-1',), ('parent-2',)]),
 
1870
            index.get_missing_parents())
 
1871
 
 
1872
    def test_add_unvalidated_index_with_present_external_references(self):
 
1873
        index = self.two_graph_index(deltas=True)
 
1874
        # Ugly hack to get at one of the underlying GraphIndex objects that
 
1875
        # two_graph_index built.
 
1876
        unvalidated = index._graph_index._indices[1]
 
1877
        # 'parent' is an external ref of _indices[1] (unvalidated), but is
 
1878
        # present in _indices[0].
 
1879
        index.scan_unvalidated_index(unvalidated)
 
1880
        self.assertEqual(frozenset(), index.get_missing_compression_parents())
 
1881
 
 
1882
    def make_new_missing_parent_g_index(self, name):
 
1883
        missing_parent = name + '-missing-parent'
 
1884
        graph_index = self.make_g_index(name, 2,
 
1885
            [((name + 'tip', ), ' 100 78',
 
1886
              ([(missing_parent, ), ('ghost', )], [(missing_parent, )]))])
 
1887
        return graph_index
 
1888
 
 
1889
    def test_add_mulitiple_unvalidated_indices_with_missing_parents(self):
 
1890
        g_index_1 = self.make_new_missing_parent_g_index('one')
 
1891
        g_index_2 = self.make_new_missing_parent_g_index('two')
 
1892
        combined = CombinedGraphIndex([g_index_1, g_index_2])
 
1893
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1894
        index.scan_unvalidated_index(g_index_1)
 
1895
        index.scan_unvalidated_index(g_index_2)
 
1896
        self.assertEqual(
 
1897
            frozenset([('one-missing-parent',), ('two-missing-parent',)]),
 
1898
            index.get_missing_compression_parents())
 
1899
 
 
1900
    def test_add_mulitiple_unvalidated_indices_with_mutual_dependencies(self):
 
1901
        graph_index_a = self.make_g_index('one', 2,
 
1902
            [(('parent-one', ), ' 100 78', ([('non-compression-parent',)], [])),
 
1903
             (('child-of-two', ), ' 100 78',
 
1904
              ([('parent-two',)], [('parent-two',)]))])
 
1905
        graph_index_b = self.make_g_index('two', 2,
 
1906
            [(('parent-two', ), ' 100 78', ([('non-compression-parent',)], [])),
 
1907
             (('child-of-one', ), ' 100 78',
 
1908
              ([('parent-one',)], [('parent-one',)]))])
 
1909
        combined = CombinedGraphIndex([graph_index_a, graph_index_b])
 
1910
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1911
        index.scan_unvalidated_index(graph_index_a)
 
1912
        index.scan_unvalidated_index(graph_index_b)
 
1913
        self.assertEqual(
 
1914
            frozenset([]), index.get_missing_compression_parents())
 
1915
 
1604
1916
 
1605
1917
class TestNoParentsGraphIndexKnit(KnitTests):
1606
1918
    """Tests for knits using _KnitGraphIndex with no parents."""
1614
1926
        size = trans.put_file(name, stream)
1615
1927
        return GraphIndex(trans, name, size)
1616
1928
 
 
1929
    def test_add_good_unvalidated_index(self):
 
1930
        unvalidated = self.make_g_index('unvalidated')
 
1931
        combined = CombinedGraphIndex([unvalidated])
 
1932
        index = _KnitGraphIndex(combined, lambda: True, parents=False)
 
1933
        index.scan_unvalidated_index(unvalidated)
 
1934
        self.assertEqual(frozenset(),
 
1935
            index.get_missing_compression_parents())
 
1936
 
1617
1937
    def test_parents_deltas_incompatible(self):
1618
1938
        index = CombinedGraphIndex([])
1619
1939
        self.assertRaises(errors.KnitError, _KnitGraphIndex, lambda:True,
1700
2020
        index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000), [])])
1701
2021
        # but neither should have added data.
1702
2022
        self.assertEqual([[], [], [], []], self.caught_entries)
1703
 
        
 
2023
 
1704
2024
    def test_add_version_different_dup(self):
1705
2025
        index = self.two_graph_index(catch_adds=True)
1706
2026
        # change options
1714
2034
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1715
2035
            [(('tip',), 'fulltext,no-eol', (None, 0, 100), [('parent',)])])
1716
2036
        self.assertEqual([], self.caught_entries)
1717
 
        
 
2037
 
1718
2038
    def test_add_versions(self):
1719
2039
        index = self.two_graph_index(catch_adds=True)
1720
2040
        index.add_records([
1752
2072
        index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000), [])])
1753
2073
        # but neither should have added data.
1754
2074
        self.assertEqual([[], [], [], []], self.caught_entries)
1755
 
        
 
2075
 
1756
2076
    def test_add_versions_different_dup(self):
1757
2077
        index = self.two_graph_index(catch_adds=True)
1758
2078
        # change options
1772
2092
        self.assertEqual([], self.caught_entries)
1773
2093
 
1774
2094
 
 
2095
class TestKnitVersionedFiles(KnitTests):
 
2096
 
 
2097
    def assertGroupKeysForIo(self, exp_groups, keys, non_local_keys,
 
2098
                             positions, _min_buffer_size=None):
 
2099
        kvf = self.make_test_knit()
 
2100
        if _min_buffer_size is None:
 
2101
            _min_buffer_size = knit._STREAM_MIN_BUFFER_SIZE
 
2102
        self.assertEqual(exp_groups, kvf._group_keys_for_io(keys,
 
2103
                                        non_local_keys, positions,
 
2104
                                        _min_buffer_size=_min_buffer_size))
 
2105
 
 
2106
    def assertSplitByPrefix(self, expected_map, expected_prefix_order,
 
2107
                            keys):
 
2108
        split, prefix_order = KnitVersionedFiles._split_by_prefix(keys)
 
2109
        self.assertEqual(expected_map, split)
 
2110
        self.assertEqual(expected_prefix_order, prefix_order)
 
2111
 
 
2112
    def test__group_keys_for_io(self):
 
2113
        ft_detail = ('fulltext', False)
 
2114
        ld_detail = ('line-delta', False)
 
2115
        f_a = ('f', 'a')
 
2116
        f_b = ('f', 'b')
 
2117
        f_c = ('f', 'c')
 
2118
        g_a = ('g', 'a')
 
2119
        g_b = ('g', 'b')
 
2120
        g_c = ('g', 'c')
 
2121
        positions = {
 
2122
            f_a: (ft_detail, (f_a, 0, 100), None),
 
2123
            f_b: (ld_detail, (f_b, 100, 21), f_a),
 
2124
            f_c: (ld_detail, (f_c, 180, 15), f_b),
 
2125
            g_a: (ft_detail, (g_a, 121, 35), None),
 
2126
            g_b: (ld_detail, (g_b, 156, 12), g_a),
 
2127
            g_c: (ld_detail, (g_c, 195, 13), g_a),
 
2128
            }
 
2129
        self.assertGroupKeysForIo([([f_a], set())],
 
2130
                                  [f_a], [], positions)
 
2131
        self.assertGroupKeysForIo([([f_a], set([f_a]))],
 
2132
                                  [f_a], [f_a], positions)
 
2133
        self.assertGroupKeysForIo([([f_a, f_b], set([]))],
 
2134
                                  [f_a, f_b], [], positions)
 
2135
        self.assertGroupKeysForIo([([f_a, f_b], set([f_b]))],
 
2136
                                  [f_a, f_b], [f_b], positions)
 
2137
        self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
 
2138
                                  [f_a, g_a, f_b, g_b], [], positions)
 
2139
        self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
 
2140
                                  [f_a, g_a, f_b, g_b], [], positions,
 
2141
                                  _min_buffer_size=150)
 
2142
        self.assertGroupKeysForIo([([f_a, f_b], set()), ([g_a, g_b], set())],
 
2143
                                  [f_a, g_a, f_b, g_b], [], positions,
 
2144
                                  _min_buffer_size=100)
 
2145
        self.assertGroupKeysForIo([([f_c], set()), ([g_b], set())],
 
2146
                                  [f_c, g_b], [], positions,
 
2147
                                  _min_buffer_size=125)
 
2148
        self.assertGroupKeysForIo([([g_b, f_c], set())],
 
2149
                                  [g_b, f_c], [], positions,
 
2150
                                  _min_buffer_size=125)
 
2151
 
 
2152
    def test__split_by_prefix(self):
 
2153
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2154
                                  'g': [('g', 'b'), ('g', 'a')],
 
2155
                                 }, ['f', 'g'],
 
2156
                                 [('f', 'a'), ('g', 'b'),
 
2157
                                  ('g', 'a'), ('f', 'b')])
 
2158
 
 
2159
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2160
                                  'g': [('g', 'b'), ('g', 'a')],
 
2161
                                 }, ['f', 'g'],
 
2162
                                 [('f', 'a'), ('f', 'b'),
 
2163
                                  ('g', 'b'), ('g', 'a')])
 
2164
 
 
2165
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2166
                                  'g': [('g', 'b'), ('g', 'a')],
 
2167
                                 }, ['f', 'g'],
 
2168
                                 [('f', 'a'), ('f', 'b'),
 
2169
                                  ('g', 'b'), ('g', 'a')])
 
2170
 
 
2171
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2172
                                  'g': [('g', 'b'), ('g', 'a')],
 
2173
                                  '': [('a',), ('b',)]
 
2174
                                 }, ['f', 'g', ''],
 
2175
                                 [('f', 'a'), ('g', 'b'),
 
2176
                                  ('a',), ('b',),
 
2177
                                  ('g', 'a'), ('f', 'b')])
 
2178
 
 
2179
 
1775
2180
class TestStacking(KnitTests):
1776
2181
 
1777
2182
    def get_basis_and_test_knit(self):
1832
2237
        # self.assertEqual([("annotate", key_basis)], basis.calls)
1833
2238
        self.assertEqual([('get_parent_map', set([key_basis])),
1834
2239
            ('get_parent_map', set([key_basis])),
1835
 
            ('get_parent_map', set([key_basis])),
1836
 
            ('get_record_stream', [key_basis], 'unordered', True)],
 
2240
            ('get_record_stream', [key_basis], 'topological', True)],
1837
2241
            basis.calls)
1838
2242
 
1839
2243
    def test_check(self):
1840
2244
        # At the moment checking a stacked knit does implicitly check the
1841
 
        # fallback files.  
 
2245
        # fallback files.
1842
2246
        basis, test = self.get_basis_and_test_knit()
1843
2247
        test.check()
1844
2248
 
1936
2340
                True).next()
1937
2341
            self.assertEqual(record.key, result[0])
1938
2342
            self.assertEqual(record.sha1, result[1])
1939
 
            self.assertEqual(record.storage_kind, result[2])
 
2343
            # We used to check that the storage kind matched, but actually it
 
2344
            # depends on whether it was sourced from the basis, or in a single
 
2345
            # group, because asking for full texts returns proxy objects to a
 
2346
            # _ContentMapGenerator object; so checking the kind is unneeded.
1940
2347
            self.assertEqual(record.get_bytes_as('fulltext'), result[3])
1941
2348
        # It's not strictly minimal, but it seems reasonable for now for it to
1942
2349
        # ask which fallbacks have which parents.
1943
2350
        self.assertEqual([
1944
2351
            ("get_parent_map", set([key_basis, key_basis_2, key_missing])),
1945
 
            # unordered is asked for by the underlying worker as it still
1946
 
            # buffers everything while answering - which is a problem!
1947
 
            ("get_record_stream", [key_basis_2, key_basis], 'unordered', True)],
 
2352
            # topological is requested from the fallback, because that is what
 
2353
            # was requested at the top level.
 
2354
            ("get_record_stream", [key_basis_2, key_basis], 'topological', True)],
1948
2355
            calls)
1949
2356
 
1950
2357
    def test_get_record_stream_unordered_deltas(self):
2036
2443
        key_basis = ('bar',)
2037
2444
        key_missing = ('missing',)
2038
2445
        test.add_lines(key, (), ['foo\n'])
2039
 
        key_sha1sum = osutils.sha('foo\n').hexdigest()
 
2446
        key_sha1sum = osutils.sha_string('foo\n')
2040
2447
        sha1s = test.get_sha1s([key])
2041
2448
        self.assertEqual({key: key_sha1sum}, sha1s)
2042
2449
        self.assertEqual([], basis.calls)
2044
2451
        # directly (rather than via text reconstruction) so that remote servers
2045
2452
        # etc don't have to answer with full content.
2046
2453
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2047
 
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
2454
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
2048
2455
        basis.calls = []
2049
2456
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2050
2457
        self.assertEqual({key: key_sha1sum,
2079
2486
 
2080
2487
    def test_iter_lines_added_or_present_in_keys(self):
2081
2488
        # Lines from the basis are returned, and lines for a given key are only
2082
 
        # returned once. 
 
2489
        # returned once.
2083
2490
        key1 = ('foo1',)
2084
2491
        key2 = ('foo2',)
2085
2492
        # all sources are asked for keys:
2171
2578
        last_call = basis.calls[-1]
2172
2579
        self.assertEqual('get_record_stream', last_call[0])
2173
2580
        self.assertEqual(set([key_left, key_right]), set(last_call[1]))
2174
 
        self.assertEqual('unordered', last_call[2])
 
2581
        self.assertEqual('topological', last_call[2])
2175
2582
        self.assertEqual(True, last_call[3])
 
2583
 
 
2584
 
 
2585
class TestNetworkBehaviour(KnitTests):
 
2586
    """Tests for getting data out of/into knits over the network."""
 
2587
 
 
2588
    def test_include_delta_closure_generates_a_knit_delta_closure(self):
 
2589
        vf = self.make_test_knit(name='test')
 
2590
        # put in three texts, giving ft, delta, delta
 
2591
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2592
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2593
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2594
        # But heuristics could interfere, so check what happened:
 
2595
        self.assertEqual(['knit-ft-gz', 'knit-delta-gz', 'knit-delta-gz'],
 
2596
            [record.storage_kind for record in
 
2597
             vf.get_record_stream([('base',), ('d1',), ('d2',)],
 
2598
                'topological', False)])
 
2599
        # generate a stream of just the deltas include_delta_closure=True,
 
2600
        # serialise to the network, and check that we get a delta closure on the wire.
 
2601
        stream = vf.get_record_stream([('d1',), ('d2',)], 'topological', True)
 
2602
        netb = [record.get_bytes_as(record.storage_kind) for record in stream]
 
2603
        # The first bytes should be a memo from _ContentMapGenerator, and the
 
2604
        # second bytes should be empty (because its a API proxy not something
 
2605
        # for wire serialisation.
 
2606
        self.assertEqual('', netb[1])
 
2607
        bytes = netb[0]
 
2608
        kind, line_end = network_bytes_to_kind_and_offset(bytes)
 
2609
        self.assertEqual('knit-delta-closure', kind)
 
2610
 
 
2611
 
 
2612
class TestContentMapGenerator(KnitTests):
 
2613
    """Tests for ContentMapGenerator"""
 
2614
 
 
2615
    def test_get_record_stream_gives_records(self):
 
2616
        vf = self.make_test_knit(name='test')
 
2617
        # put in three texts, giving ft, delta, delta
 
2618
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2619
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2620
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2621
        keys = [('d1',), ('d2',)]
 
2622
        generator = _VFContentMapGenerator(vf, keys,
 
2623
            global_map=vf.get_parent_map(keys))
 
2624
        for record in generator.get_record_stream():
 
2625
            if record.key == ('d1',):
 
2626
                self.assertEqual('d1\n', record.get_bytes_as('fulltext'))
 
2627
            else:
 
2628
                self.assertEqual('d2\n', record.get_bytes_as('fulltext'))
 
2629
 
 
2630
    def test_get_record_stream_kinds_are_raw(self):
 
2631
        vf = self.make_test_knit(name='test')
 
2632
        # put in three texts, giving ft, delta, delta
 
2633
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2634
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2635
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2636
        keys = [('base',), ('d1',), ('d2',)]
 
2637
        generator = _VFContentMapGenerator(vf, keys,
 
2638
            global_map=vf.get_parent_map(keys))
 
2639
        kinds = {('base',): 'knit-delta-closure',
 
2640
            ('d1',): 'knit-delta-closure-ref',
 
2641
            ('d2',): 'knit-delta-closure-ref',
 
2642
            }
 
2643
        for record in generator.get_record_stream():
 
2644
            self.assertEqual(kinds[record.key], record.storage_kind)