← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

(vila) Forbid more operations on ReadonlyTransportDecorator (Vincent Ladeuil)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
 
1
# Copyright (C) 2006-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
 
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
20
 
import difflib
21
20
import gzip
22
 
import sha
23
21
import sys
24
22
 
25
23
from bzrlib import (
26
24
    errors,
27
 
    generate_ids,
28
25
    knit,
29
26
    multiparent,
 
27
    osutils,
30
28
    pack,
 
29
    tests,
 
30
    transport,
31
31
    )
32
32
from bzrlib.errors import (
33
 
    RevisionAlreadyPresent,
34
33
    KnitHeaderError,
35
 
    RevisionNotPresent,
36
34
    NoSuchFile,
37
35
    )
38
36
from bzrlib.index import *
39
37
from bzrlib.knit import (
40
38
    AnnotatedKnitContent,
41
39
    KnitContent,
42
 
    KnitSequenceMatcher,
43
40
    KnitVersionedFiles,
44
41
    PlainKnitContent,
45
 
    _DirectPackAccess,
 
42
    _VFContentMapGenerator,
46
43
    _KndxIndex,
47
44
    _KnitGraphIndex,
48
45
    _KnitKeyAccess,
49
46
    make_file_factory,
50
47
    )
51
 
from bzrlib.osutils import split_lines
52
 
from bzrlib.symbol_versioning import one_four
 
48
from bzrlib.patiencediff import PatienceSequenceMatcher
 
49
from bzrlib.repofmt import (
 
50
    knitpack_repo,
 
51
    pack_repo,
 
52
    )
53
53
from bzrlib.tests import (
54
 
    Feature,
55
 
    KnownFailure,
56
54
    TestCase,
57
55
    TestCaseWithMemoryTransport,
58
56
    TestCaseWithTransport,
 
57
    TestNotApplicable,
59
58
    )
60
 
from bzrlib.transport import get_transport
61
 
from bzrlib.transport.memory import MemoryTransport
62
 
from bzrlib.tuned_gzip import GzipFile
63
59
from bzrlib.versionedfile import (
64
60
    AbsentContentFactory,
65
61
    ConstantMapper,
 
62
    network_bytes_to_kind_and_offset,
66
63
    RecordingVersionedFilesDecorator,
67
64
    )
68
 
 
69
 
 
70
 
class _CompiledKnitFeature(Feature):
71
 
 
72
 
    def _probe(self):
73
 
        try:
74
 
            import bzrlib._knit_load_data_c
75
 
        except ImportError:
76
 
            return False
77
 
        return True
78
 
 
79
 
    def feature_name(self):
80
 
        return 'bzrlib._knit_load_data_c'
81
 
 
82
 
CompiledKnitFeature = _CompiledKnitFeature()
 
65
from bzrlib.tests import (
 
66
    features,
 
67
    )
 
68
 
 
69
 
 
70
compiled_knit_feature = features.ModuleAvailableFeature(
 
71
    'bzrlib._knit_load_data_pyx')
83
72
 
84
73
 
85
74
class KnitContentTestsMixin(object):
114
103
        line_delta = source_content.line_delta(target_content)
115
104
        delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
116
105
            source_lines, target_lines))
117
 
        matcher = KnitSequenceMatcher(None, source_lines, target_lines)
118
 
        matcher_blocks = list(list(matcher.get_matching_blocks()))
 
106
        matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
 
107
        matcher_blocks = list(matcher.get_matching_blocks())
119
108
        self.assertEqual(matcher_blocks, delta_blocks)
120
109
 
121
110
    def test_get_line_delta_blocks(self):
271
260
        return queue_call
272
261
 
273
262
 
 
263
class MockReadvFailingTransport(MockTransport):
 
264
    """Fail in the middle of a readv() result.
 
265
 
 
266
    This Transport will successfully yield the first two requested hunks, but
 
267
    raise NoSuchFile for the rest.
 
268
    """
 
269
 
 
270
    def readv(self, relpath, offsets):
 
271
        count = 0
 
272
        for result in MockTransport.readv(self, relpath, offsets):
 
273
            count += 1
 
274
            # we use 2 because the first offset is the pack header, the second
 
275
            # is the first actual content requset
 
276
            if count > 2:
 
277
                raise errors.NoSuchFile(relpath)
 
278
            yield result
 
279
 
 
280
 
274
281
class KnitRecordAccessTestsMixin(object):
275
282
    """Tests for getting and putting knit records."""
276
283
 
279
286
        access = self.get_access()
280
287
        memos = access.add_raw_records([('key', 10)], '1234567890')
281
288
        self.assertEqual(['1234567890'], list(access.get_raw_records(memos)))
282
 
 
 
289
 
283
290
    def test_add_several_raw_records(self):
284
291
        """add_raw_records with many records and read some back."""
285
292
        access = self.get_access()
305
312
        mapper = ConstantMapper("foo")
306
313
        access = _KnitKeyAccess(self.get_transport(), mapper)
307
314
        return access
308
 
    
 
315
 
 
316
 
 
317
class _TestException(Exception):
 
318
    """Just an exception for local tests to use."""
 
319
 
309
320
 
310
321
class TestPackKnitAccess(TestCaseWithMemoryTransport, KnitRecordAccessTestsMixin):
311
322
    """Tests for the pack based access."""
319
330
            transport.append_bytes(packname, bytes)
320
331
        writer = pack.ContainerWriter(write_data)
321
332
        writer.begin()
322
 
        access = _DirectPackAccess({})
 
333
        access = pack_repo._DirectPackAccess({})
323
334
        access.set_writer(writer, index, (transport, packname))
324
335
        return access, writer
325
336
 
 
337
    def make_pack_file(self):
 
338
        """Create a pack file with 2 records."""
 
339
        access, writer = self._get_access(packname='packname', index='foo')
 
340
        memos = []
 
341
        memos.extend(access.add_raw_records([('key1', 10)], '1234567890'))
 
342
        memos.extend(access.add_raw_records([('key2', 5)], '12345'))
 
343
        writer.end()
 
344
        return memos
 
345
 
 
346
    def test_pack_collection_pack_retries(self):
 
347
        """An explicit pack of a pack collection succeeds even when a
 
348
        concurrent pack happens.
 
349
        """
 
350
        builder = self.make_branch_builder('.')
 
351
        builder.start_series()
 
352
        builder.build_snapshot('rev-1', None, [
 
353
            ('add', ('', 'root-id', 'directory', None)),
 
354
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
 
355
            ])
 
356
        builder.build_snapshot('rev-2', ['rev-1'], [
 
357
            ('modify', ('file-id', 'content\nrev 2\n')),
 
358
            ])
 
359
        builder.build_snapshot('rev-3', ['rev-2'], [
 
360
            ('modify', ('file-id', 'content\nrev 3\n')),
 
361
            ])
 
362
        self.addCleanup(builder.finish_series)
 
363
        b = builder.get_branch()
 
364
        self.addCleanup(b.lock_write().unlock)
 
365
        repo = b.repository
 
366
        collection = repo._pack_collection
 
367
        # Concurrently repack the repo.
 
368
        reopened_repo = repo.bzrdir.open_repository()
 
369
        reopened_repo.pack()
 
370
        # Pack the new pack.
 
371
        collection.pack()
 
372
 
 
373
    def make_vf_for_retrying(self):
 
374
        """Create 3 packs and a reload function.
 
375
 
 
376
        Originally, 2 pack files will have the data, but one will be missing.
 
377
        And then the third will be used in place of the first two if reload()
 
378
        is called.
 
379
 
 
380
        :return: (versioned_file, reload_counter)
 
381
            versioned_file  a KnitVersionedFiles using the packs for access
 
382
        """
 
383
        builder = self.make_branch_builder('.', format="1.9")
 
384
        builder.start_series()
 
385
        builder.build_snapshot('rev-1', None, [
 
386
            ('add', ('', 'root-id', 'directory', None)),
 
387
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
 
388
            ])
 
389
        builder.build_snapshot('rev-2', ['rev-1'], [
 
390
            ('modify', ('file-id', 'content\nrev 2\n')),
 
391
            ])
 
392
        builder.build_snapshot('rev-3', ['rev-2'], [
 
393
            ('modify', ('file-id', 'content\nrev 3\n')),
 
394
            ])
 
395
        builder.finish_series()
 
396
        b = builder.get_branch()
 
397
        b.lock_write()
 
398
        self.addCleanup(b.unlock)
 
399
        # Pack these three revisions into another pack file, but don't remove
 
400
        # the originals
 
401
        repo = b.repository
 
402
        collection = repo._pack_collection
 
403
        collection.ensure_loaded()
 
404
        orig_packs = collection.packs
 
405
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
 
406
        new_pack = packer.pack()
 
407
        # forget about the new pack
 
408
        collection.reset()
 
409
        repo.refresh_data()
 
410
        vf = repo.revisions
 
411
        # Set up a reload() function that switches to using the new pack file
 
412
        new_index = new_pack.revision_index
 
413
        access_tuple = new_pack.access_tuple()
 
414
        reload_counter = [0, 0, 0]
 
415
        def reload():
 
416
            reload_counter[0] += 1
 
417
            if reload_counter[1] > 0:
 
418
                # We already reloaded, nothing more to do
 
419
                reload_counter[2] += 1
 
420
                return False
 
421
            reload_counter[1] += 1
 
422
            vf._index._graph_index._indices[:] = [new_index]
 
423
            vf._access._indices.clear()
 
424
            vf._access._indices[new_index] = access_tuple
 
425
            return True
 
426
        # Delete one of the pack files so the data will need to be reloaded. We
 
427
        # will delete the file with 'rev-2' in it
 
428
        trans, name = orig_packs[1].access_tuple()
 
429
        trans.delete(name)
 
430
        # We don't have the index trigger reloading because we want to test
 
431
        # that we reload when the .pack disappears
 
432
        vf._access._reload_func = reload
 
433
        return vf, reload_counter
 
434
 
 
435
    def make_reload_func(self, return_val=True):
 
436
        reload_called = [0]
 
437
        def reload():
 
438
            reload_called[0] += 1
 
439
            return return_val
 
440
        return reload_called, reload
 
441
 
 
442
    def make_retry_exception(self):
 
443
        # We raise a real exception so that sys.exc_info() is properly
 
444
        # populated
 
445
        try:
 
446
            raise _TestException('foobar')
 
447
        except _TestException, e:
 
448
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
 
449
                                                 exc_info=sys.exc_info())
 
450
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
 
451
        return retry_exc
 
452
 
326
453
    def test_read_from_several_packs(self):
327
454
        access, writer = self._get_access()
328
455
        memos = []
335
462
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
336
463
        writer.end()
337
464
        transport = self.get_transport()
338
 
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
 
465
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
339
466
            "FOOBAR":(transport, 'pack2'),
340
467
            "BAZ":(transport, 'pack3')})
341
468
        self.assertEqual(['1234567890', '12345', 'alpha'],
351
478
 
352
479
    def test_set_writer(self):
353
480
        """The writer should be settable post construction."""
354
 
        access = _DirectPackAccess({})
 
481
        access = pack_repo._DirectPackAccess({})
355
482
        transport = self.get_transport()
356
483
        packname = 'packfile'
357
484
        index = 'foo'
364
491
        writer.end()
365
492
        self.assertEqual(['1234567890'], list(access.get_raw_records(memos)))
366
493
 
 
494
    def test_missing_index_raises_retry(self):
 
495
        memos = self.make_pack_file()
 
496
        transport = self.get_transport()
 
497
        reload_called, reload_func = self.make_reload_func()
 
498
        # Note that the index key has changed from 'foo' to 'bar'
 
499
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
 
500
                                   reload_func=reload_func)
 
501
        e = self.assertListRaises(errors.RetryWithNewPacks,
 
502
                                  access.get_raw_records, memos)
 
503
        # Because a key was passed in which does not match our index list, we
 
504
        # assume that the listing was already reloaded
 
505
        self.assertTrue(e.reload_occurred)
 
506
        self.assertIsInstance(e.exc_info, tuple)
 
507
        self.assertIs(e.exc_info[0], KeyError)
 
508
        self.assertIsInstance(e.exc_info[1], KeyError)
 
509
 
 
510
    def test_missing_index_raises_key_error_with_no_reload(self):
 
511
        memos = self.make_pack_file()
 
512
        transport = self.get_transport()
 
513
        # Note that the index key has changed from 'foo' to 'bar'
 
514
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
 
515
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
 
516
 
 
517
    def test_missing_file_raises_retry(self):
 
518
        memos = self.make_pack_file()
 
519
        transport = self.get_transport()
 
520
        reload_called, reload_func = self.make_reload_func()
 
521
        # Note that the 'filename' has been changed to 'different-packname'
 
522
        access = pack_repo._DirectPackAccess(
 
523
            {'foo':(transport, 'different-packname')},
 
524
            reload_func=reload_func)
 
525
        e = self.assertListRaises(errors.RetryWithNewPacks,
 
526
                                  access.get_raw_records, memos)
 
527
        # The file has gone missing, so we assume we need to reload
 
528
        self.assertFalse(e.reload_occurred)
 
529
        self.assertIsInstance(e.exc_info, tuple)
 
530
        self.assertIs(e.exc_info[0], errors.NoSuchFile)
 
531
        self.assertIsInstance(e.exc_info[1], errors.NoSuchFile)
 
532
        self.assertEqual('different-packname', e.exc_info[1].path)
 
533
 
 
534
    def test_missing_file_raises_no_such_file_with_no_reload(self):
 
535
        memos = self.make_pack_file()
 
536
        transport = self.get_transport()
 
537
        # Note that the 'filename' has been changed to 'different-packname'
 
538
        access = pack_repo._DirectPackAccess(
 
539
            {'foo': (transport, 'different-packname')})
 
540
        e = self.assertListRaises(errors.NoSuchFile,
 
541
                                  access.get_raw_records, memos)
 
542
 
 
543
    def test_failing_readv_raises_retry(self):
 
544
        memos = self.make_pack_file()
 
545
        transport = self.get_transport()
 
546
        failing_transport = MockReadvFailingTransport(
 
547
                                [transport.get_bytes('packname')])
 
548
        reload_called, reload_func = self.make_reload_func()
 
549
        access = pack_repo._DirectPackAccess(
 
550
            {'foo': (failing_transport, 'packname')},
 
551
            reload_func=reload_func)
 
552
        # Asking for a single record will not trigger the Mock failure
 
553
        self.assertEqual(['1234567890'],
 
554
            list(access.get_raw_records(memos[:1])))
 
555
        self.assertEqual(['12345'],
 
556
            list(access.get_raw_records(memos[1:2])))
 
557
        # A multiple offset readv() will fail mid-way through
 
558
        e = self.assertListRaises(errors.RetryWithNewPacks,
 
559
                                  access.get_raw_records, memos)
 
560
        # The file has gone missing, so we assume we need to reload
 
561
        self.assertFalse(e.reload_occurred)
 
562
        self.assertIsInstance(e.exc_info, tuple)
 
563
        self.assertIs(e.exc_info[0], errors.NoSuchFile)
 
564
        self.assertIsInstance(e.exc_info[1], errors.NoSuchFile)
 
565
        self.assertEqual('packname', e.exc_info[1].path)
 
566
 
 
567
    def test_failing_readv_raises_no_such_file_with_no_reload(self):
 
568
        memos = self.make_pack_file()
 
569
        transport = self.get_transport()
 
570
        failing_transport = MockReadvFailingTransport(
 
571
                                [transport.get_bytes('packname')])
 
572
        reload_called, reload_func = self.make_reload_func()
 
573
        access = pack_repo._DirectPackAccess(
 
574
            {'foo':(failing_transport, 'packname')})
 
575
        # Asking for a single record will not trigger the Mock failure
 
576
        self.assertEqual(['1234567890'],
 
577
            list(access.get_raw_records(memos[:1])))
 
578
        self.assertEqual(['12345'],
 
579
            list(access.get_raw_records(memos[1:2])))
 
580
        # A multiple offset readv() will fail mid-way through
 
581
        e = self.assertListRaises(errors.NoSuchFile,
 
582
                                  access.get_raw_records, memos)
 
583
 
 
584
    def test_reload_or_raise_no_reload(self):
 
585
        access = pack_repo._DirectPackAccess({}, reload_func=None)
 
586
        retry_exc = self.make_retry_exception()
 
587
        # Without a reload_func, we will just re-raise the original exception
 
588
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
 
589
 
 
590
    def test_reload_or_raise_reload_changed(self):
 
591
        reload_called, reload_func = self.make_reload_func(return_val=True)
 
592
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
593
        retry_exc = self.make_retry_exception()
 
594
        access.reload_or_raise(retry_exc)
 
595
        self.assertEqual([1], reload_called)
 
596
        retry_exc.reload_occurred=True
 
597
        access.reload_or_raise(retry_exc)
 
598
        self.assertEqual([2], reload_called)
 
599
 
 
600
    def test_reload_or_raise_reload_no_change(self):
 
601
        reload_called, reload_func = self.make_reload_func(return_val=False)
 
602
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
603
        retry_exc = self.make_retry_exception()
 
604
        # If reload_occurred is False, then we consider it an error to have
 
605
        # reload_func() return False (no changes).
 
606
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
 
607
        self.assertEqual([1], reload_called)
 
608
        retry_exc.reload_occurred=True
 
609
        # If reload_occurred is True, then we assume nothing changed because
 
610
        # it had changed earlier, but didn't change again
 
611
        access.reload_or_raise(retry_exc)
 
612
        self.assertEqual([2], reload_called)
 
613
 
 
614
    def test_annotate_retries(self):
 
615
        vf, reload_counter = self.make_vf_for_retrying()
 
616
        # It is a little bit bogus to annotate the Revision VF, but it works,
 
617
        # as we have ancestry stored there
 
618
        key = ('rev-3',)
 
619
        reload_lines = vf.annotate(key)
 
620
        self.assertEqual([1, 1, 0], reload_counter)
 
621
        plain_lines = vf.annotate(key)
 
622
        self.assertEqual([1, 1, 0], reload_counter) # No extra reloading
 
623
        if reload_lines != plain_lines:
 
624
            self.fail('Annotation was not identical with reloading.')
 
625
        # Now delete the packs-in-use, which should trigger another reload, but
 
626
        # this time we just raise an exception because we can't recover
 
627
        for trans, name in vf._access._indices.itervalues():
 
628
            trans.delete(name)
 
629
        self.assertRaises(errors.NoSuchFile, vf.annotate, key)
 
630
        self.assertEqual([2, 1, 1], reload_counter)
 
631
 
 
632
    def test__get_record_map_retries(self):
 
633
        vf, reload_counter = self.make_vf_for_retrying()
 
634
        keys = [('rev-1',), ('rev-2',), ('rev-3',)]
 
635
        records = vf._get_record_map(keys)
 
636
        self.assertEqual(keys, sorted(records.keys()))
 
637
        self.assertEqual([1, 1, 0], reload_counter)
 
638
        # Now delete the packs-in-use, which should trigger another reload, but
 
639
        # this time we just raise an exception because we can't recover
 
640
        for trans, name in vf._access._indices.itervalues():
 
641
            trans.delete(name)
 
642
        self.assertRaises(errors.NoSuchFile, vf._get_record_map, keys)
 
643
        self.assertEqual([2, 1, 1], reload_counter)
 
644
 
 
645
    def test_get_record_stream_retries(self):
 
646
        vf, reload_counter = self.make_vf_for_retrying()
 
647
        keys = [('rev-1',), ('rev-2',), ('rev-3',)]
 
648
        record_stream = vf.get_record_stream(keys, 'topological', False)
 
649
        record = record_stream.next()
 
650
        self.assertEqual(('rev-1',), record.key)
 
651
        self.assertEqual([0, 0, 0], reload_counter)
 
652
        record = record_stream.next()
 
653
        self.assertEqual(('rev-2',), record.key)
 
654
        self.assertEqual([1, 1, 0], reload_counter)
 
655
        record = record_stream.next()
 
656
        self.assertEqual(('rev-3',), record.key)
 
657
        self.assertEqual([1, 1, 0], reload_counter)
 
658
        # Now delete all pack files, and see that we raise the right error
 
659
        for trans, name in vf._access._indices.itervalues():
 
660
            trans.delete(name)
 
661
        self.assertListRaises(errors.NoSuchFile,
 
662
            vf.get_record_stream, keys, 'topological', False)
 
663
 
 
664
    def test_iter_lines_added_or_present_in_keys_retries(self):
 
665
        vf, reload_counter = self.make_vf_for_retrying()
 
666
        keys = [('rev-1',), ('rev-2',), ('rev-3',)]
 
667
        # Unfortunately, iter_lines_added_or_present_in_keys iterates the
 
668
        # result in random order (determined by the iteration order from a
 
669
        # set()), so we don't have any solid way to trigger whether data is
 
670
        # read before or after. However we tried to delete the middle node to
 
671
        # exercise the code well.
 
672
        # What we care about is that all lines are always yielded, but not
 
673
        # duplicated
 
674
        count = 0
 
675
        reload_lines = sorted(vf.iter_lines_added_or_present_in_keys(keys))
 
676
        self.assertEqual([1, 1, 0], reload_counter)
 
677
        # Now do it again, to make sure the result is equivalent
 
678
        plain_lines = sorted(vf.iter_lines_added_or_present_in_keys(keys))
 
679
        self.assertEqual([1, 1, 0], reload_counter) # No extra reloading
 
680
        self.assertEqual(plain_lines, reload_lines)
 
681
        self.assertEqual(21, len(plain_lines))
 
682
        # Now delete all pack files, and see that we raise the right error
 
683
        for trans, name in vf._access._indices.itervalues():
 
684
            trans.delete(name)
 
685
        self.assertListRaises(errors.NoSuchFile,
 
686
            vf.iter_lines_added_or_present_in_keys, keys)
 
687
        self.assertEqual([2, 1, 1], reload_counter)
 
688
 
 
689
    def test_get_record_stream_yields_disk_sorted_order(self):
 
690
        # if we get 'unordered' pick a semi-optimal order for reading. The
 
691
        # order should be grouped by pack file, and then by position in file
 
692
        repo = self.make_repository('test', format='pack-0.92')
 
693
        repo.lock_write()
 
694
        self.addCleanup(repo.unlock)
 
695
        repo.start_write_group()
 
696
        vf = repo.texts
 
697
        vf.add_lines(('f-id', 'rev-5'), [('f-id', 'rev-4')], ['lines\n'])
 
698
        vf.add_lines(('f-id', 'rev-1'), [], ['lines\n'])
 
699
        vf.add_lines(('f-id', 'rev-2'), [('f-id', 'rev-1')], ['lines\n'])
 
700
        repo.commit_write_group()
 
701
        # We inserted them as rev-5, rev-1, rev-2, we should get them back in
 
702
        # the same order
 
703
        stream = vf.get_record_stream([('f-id', 'rev-1'), ('f-id', 'rev-5'),
 
704
                                       ('f-id', 'rev-2')], 'unordered', False)
 
705
        keys = [r.key for r in stream]
 
706
        self.assertEqual([('f-id', 'rev-5'), ('f-id', 'rev-1'),
 
707
                          ('f-id', 'rev-2')], keys)
 
708
        repo.start_write_group()
 
709
        vf.add_lines(('f-id', 'rev-4'), [('f-id', 'rev-3')], ['lines\n'])
 
710
        vf.add_lines(('f-id', 'rev-3'), [('f-id', 'rev-2')], ['lines\n'])
 
711
        vf.add_lines(('f-id', 'rev-6'), [('f-id', 'rev-5')], ['lines\n'])
 
712
        repo.commit_write_group()
 
713
        # Request in random order, to make sure the output order isn't based on
 
714
        # the request
 
715
        request_keys = set(('f-id', 'rev-%d' % i) for i in range(1, 7))
 
716
        stream = vf.get_record_stream(request_keys, 'unordered', False)
 
717
        keys = [r.key for r in stream]
 
718
        # We want to get the keys back in disk order, but it doesn't matter
 
719
        # which pack we read from first. So this can come back in 2 orders
 
720
        alt1 = [('f-id', 'rev-%d' % i) for i in [4, 3, 6, 5, 1, 2]]
 
721
        alt2 = [('f-id', 'rev-%d' % i) for i in [5, 1, 2, 4, 3, 6]]
 
722
        if keys != alt1 and keys != alt2:
 
723
            self.fail('Returned key order did not match either expected order.'
 
724
                      ' expected %s or %s, not %s'
 
725
                      % (alt1, alt2, keys))
 
726
 
367
727
 
368
728
class LowLevelKnitDataTests(TestCase):
369
729
 
374
734
        gz_file.close()
375
735
        return sio.getvalue()
376
736
 
 
737
    def make_multiple_records(self):
 
738
        """Create the content for multiple records."""
 
739
        sha1sum = osutils.sha_string('foo\nbar\n')
 
740
        total_txt = []
 
741
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
 
742
                                        'foo\n'
 
743
                                        'bar\n'
 
744
                                        'end rev-id-1\n'
 
745
                                        % (sha1sum,))
 
746
        record_1 = (0, len(gz_txt), sha1sum)
 
747
        total_txt.append(gz_txt)
 
748
        sha1sum = osutils.sha_string('baz\n')
 
749
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
 
750
                                        'baz\n'
 
751
                                        'end rev-id-2\n'
 
752
                                        % (sha1sum,))
 
753
        record_2 = (record_1[1], len(gz_txt), sha1sum)
 
754
        total_txt.append(gz_txt)
 
755
        return total_txt, record_1, record_2
 
756
 
377
757
    def test_valid_knit_data(self):
378
 
        sha1sum = sha.new('foo\nbar\n').hexdigest()
 
758
        sha1sum = osutils.sha_string('foo\nbar\n')
379
759
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
380
760
                                        'foo\n'
381
761
                                        'bar\n'
393
773
        raw_contents = list(knit._read_records_iter_raw(records))
394
774
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
395
775
 
 
776
    def test_multiple_records_valid(self):
 
777
        total_txt, record_1, record_2 = self.make_multiple_records()
 
778
        transport = MockTransport([''.join(total_txt)])
 
779
        access = _KnitKeyAccess(transport, ConstantMapper('filename'))
 
780
        knit = KnitVersionedFiles(None, access)
 
781
        records = [(('rev-id-1',), (('rev-id-1',), record_1[0], record_1[1])),
 
782
                   (('rev-id-2',), (('rev-id-2',), record_2[0], record_2[1]))]
 
783
 
 
784
        contents = list(knit._read_records_iter(records))
 
785
        self.assertEqual([(('rev-id-1',), ['foo\n', 'bar\n'], record_1[2]),
 
786
                          (('rev-id-2',), ['baz\n'], record_2[2])],
 
787
                         contents)
 
788
 
 
789
        raw_contents = list(knit._read_records_iter_raw(records))
 
790
        self.assertEqual([(('rev-id-1',), total_txt[0], record_1[2]),
 
791
                          (('rev-id-2',), total_txt[1], record_2[2])],
 
792
                         raw_contents)
 
793
 
396
794
    def test_not_enough_lines(self):
397
 
        sha1sum = sha.new('foo\n').hexdigest()
 
795
        sha1sum = osutils.sha_string('foo\n')
398
796
        # record says 2 lines data says 1
399
797
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
400
798
                                        'foo\n'
412
810
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
413
811
 
414
812
    def test_too_many_lines(self):
415
 
        sha1sum = sha.new('foo\nbar\n').hexdigest()
 
813
        sha1sum = osutils.sha_string('foo\nbar\n')
416
814
        # record says 1 lines data says 2
417
815
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
418
816
                                        'foo\n'
431
829
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
432
830
 
433
831
    def test_mismatched_version_id(self):
434
 
        sha1sum = sha.new('foo\nbar\n').hexdigest()
 
832
        sha1sum = osutils.sha_string('foo\nbar\n')
435
833
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
436
834
                                        'foo\n'
437
835
                                        'bar\n'
450
848
            knit._read_records_iter_raw(records))
451
849
 
452
850
    def test_uncompressed_data(self):
453
 
        sha1sum = sha.new('foo\nbar\n').hexdigest()
 
851
        sha1sum = osutils.sha_string('foo\nbar\n')
454
852
        txt = ('version rev-id-1 2 %s\n'
455
853
               'foo\n'
456
854
               'bar\n'
470
868
            knit._read_records_iter_raw(records))
471
869
 
472
870
    def test_corrupted_data(self):
473
 
        sha1sum = sha.new('foo\nbar\n').hexdigest()
 
871
        sha1sum = osutils.sha_string('foo\nbar\n')
474
872
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
475
873
                                        'foo\n'
476
874
                                        'bar\n'
493
891
 
494
892
    def get_knit_index(self, transport, name, mode):
495
893
        mapper = ConstantMapper(name)
496
 
        orig = knit._load_data
497
 
        def reset():
498
 
            knit._load_data = orig
499
 
        self.addCleanup(reset)
500
894
        from bzrlib._knit_load_data_py import _load_data_py
501
 
        knit._load_data = _load_data_py
 
895
        self.overrideAttr(knit, '_load_data', _load_data_py)
502
896
        allow_writes = lambda: 'w' in mode
503
897
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
504
898
 
720
1114
            call[1][1].getvalue())
721
1115
        self.assertEqual({'create_parent_dir': True}, call[2])
722
1116
 
 
1117
    def assertTotalBuildSize(self, size, keys, positions):
 
1118
        self.assertEqual(size,
 
1119
                         knit._get_total_build_size(None, keys, positions))
 
1120
 
 
1121
    def test__get_total_build_size(self):
 
1122
        positions = {
 
1123
            ('a',): (('fulltext', False), (('a',), 0, 100), None),
 
1124
            ('b',): (('line-delta', False), (('b',), 100, 21), ('a',)),
 
1125
            ('c',): (('line-delta', False), (('c',), 121, 35), ('b',)),
 
1126
            ('d',): (('line-delta', False), (('d',), 156, 12), ('b',)),
 
1127
            }
 
1128
        self.assertTotalBuildSize(100, [('a',)], positions)
 
1129
        self.assertTotalBuildSize(121, [('b',)], positions)
 
1130
        # c needs both a & b
 
1131
        self.assertTotalBuildSize(156, [('c',)], positions)
 
1132
        # we shouldn't count 'b' twice
 
1133
        self.assertTotalBuildSize(156, [('b',), ('c',)], positions)
 
1134
        self.assertTotalBuildSize(133, [('d',)], positions)
 
1135
        self.assertTotalBuildSize(168, [('c',), ('d',)], positions)
 
1136
 
723
1137
    def test_get_position(self):
724
1138
        transport = MockTransport([
725
1139
            _KndxIndex.HEADER,
782
1196
            self.assertRaises(errors.KnitCorrupt, index.keys)
783
1197
        except TypeError, e:
784
1198
            if (str(e) == ('exceptions must be strings, classes, or instances,'
785
 
                           ' not exceptions.IndexError')
786
 
                and sys.version_info[0:2] >= (2,5)):
 
1199
                           ' not exceptions.IndexError')):
787
1200
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
788
1201
                                  ' raising new style exceptions with python'
789
1202
                                  ' >=2.5')
802
1215
            self.assertRaises(errors.KnitCorrupt, index.keys)
803
1216
        except TypeError, e:
804
1217
            if (str(e) == ('exceptions must be strings, classes, or instances,'
805
 
                           ' not exceptions.ValueError')
806
 
                and sys.version_info[0:2] >= (2,5)):
 
1218
                           ' not exceptions.ValueError')):
807
1219
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
808
1220
                                  ' raising new style exceptions with python'
809
1221
                                  ' >=2.5')
822
1234
            self.assertRaises(errors.KnitCorrupt, index.keys)
823
1235
        except TypeError, e:
824
1236
            if (str(e) == ('exceptions must be strings, classes, or instances,'
825
 
                           ' not exceptions.ValueError')
826
 
                and sys.version_info[0:2] >= (2,5)):
 
1237
                           ' not exceptions.ValueError')):
827
1238
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
828
1239
                                  ' raising new style exceptions with python'
829
1240
                                  ' >=2.5')
840
1251
            self.assertRaises(errors.KnitCorrupt, index.keys)
841
1252
        except TypeError, e:
842
1253
            if (str(e) == ('exceptions must be strings, classes, or instances,'
843
 
                           ' not exceptions.ValueError')
844
 
                and sys.version_info[0:2] >= (2,5)):
 
1254
                           ' not exceptions.ValueError')):
845
1255
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
846
1256
                                  ' raising new style exceptions with python'
847
1257
                                  ' >=2.5')
858
1268
            self.assertRaises(errors.KnitCorrupt, index.keys)
859
1269
        except TypeError, e:
860
1270
            if (str(e) == ('exceptions must be strings, classes, or instances,'
861
 
                           ' not exceptions.ValueError')
862
 
                and sys.version_info[0:2] >= (2,5)):
 
1271
                           ' not exceptions.ValueError')):
863
1272
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
864
1273
                                  ' raising new style exceptions with python'
865
1274
                                  ' >=2.5')
866
1275
            else:
867
1276
                raise
868
1277
 
 
1278
    def test_scan_unvalidated_index_not_implemented(self):
 
1279
        transport = MockTransport()
 
1280
        index = self.get_knit_index(transport, 'filename', 'r')
 
1281
        self.assertRaises(
 
1282
            NotImplementedError, index.scan_unvalidated_index,
 
1283
            'dummy graph_index')
 
1284
        self.assertRaises(
 
1285
            NotImplementedError, index.get_missing_compression_parents)
 
1286
 
869
1287
    def test_short_line(self):
870
1288
        transport = MockTransport([
871
1289
            _KndxIndex.HEADER,
900
1318
 
901
1319
class LowLevelKnitIndexTests_c(LowLevelKnitIndexTests):
902
1320
 
903
 
    _test_needs_features = [CompiledKnitFeature]
 
1321
    _test_needs_features = [compiled_knit_feature]
904
1322
 
905
1323
    def get_knit_index(self, transport, name, mode):
906
1324
        mapper = ConstantMapper(name)
907
 
        orig = knit._load_data
908
 
        def reset():
909
 
            knit._load_data = orig
910
 
        self.addCleanup(reset)
911
 
        from bzrlib._knit_load_data_c import _load_data_c
912
 
        knit._load_data = _load_data_c
 
1325
        from bzrlib._knit_load_data_pyx import _load_data_c
 
1326
        self.overrideAttr(knit, '_load_data', _load_data_c)
913
1327
        allow_writes = lambda: mode == 'w'
914
 
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
 
1328
        return _KndxIndex(transport, mapper, lambda:None,
 
1329
                          allow_writes, lambda:True)
 
1330
 
 
1331
 
 
1332
class Test_KnitAnnotator(TestCaseWithMemoryTransport):
 
1333
 
 
1334
    def make_annotator(self):
 
1335
        factory = knit.make_pack_factory(True, True, 1)
 
1336
        vf = factory(self.get_transport())
 
1337
        return knit._KnitAnnotator(vf)
 
1338
 
 
1339
    def test__expand_fulltext(self):
 
1340
        ann = self.make_annotator()
 
1341
        rev_key = ('rev-id',)
 
1342
        ann._num_compression_children[rev_key] = 1
 
1343
        res = ann._expand_record(rev_key, (('parent-id',),), None,
 
1344
                           ['line1\n', 'line2\n'], ('fulltext', True))
 
1345
        # The content object and text lines should be cached appropriately
 
1346
        self.assertEqual(['line1\n', 'line2'], res)
 
1347
        content_obj = ann._content_objects[rev_key]
 
1348
        self.assertEqual(['line1\n', 'line2\n'], content_obj._lines)
 
1349
        self.assertEqual(res, content_obj.text())
 
1350
        self.assertEqual(res, ann._text_cache[rev_key])
 
1351
 
 
1352
    def test__expand_delta_comp_parent_not_available(self):
 
1353
        # Parent isn't available yet, so we return nothing, but queue up this
 
1354
        # node for later processing
 
1355
        ann = self.make_annotator()
 
1356
        rev_key = ('rev-id',)
 
1357
        parent_key = ('parent-id',)
 
1358
        record = ['0,1,1\n', 'new-line\n']
 
1359
        details = ('line-delta', False)
 
1360
        res = ann._expand_record(rev_key, (parent_key,), parent_key,
 
1361
                                 record, details)
 
1362
        self.assertEqual(None, res)
 
1363
        self.assertTrue(parent_key in ann._pending_deltas)
 
1364
        pending = ann._pending_deltas[parent_key]
 
1365
        self.assertEqual(1, len(pending))
 
1366
        self.assertEqual((rev_key, (parent_key,), record, details), pending[0])
 
1367
 
 
1368
    def test__expand_record_tracks_num_children(self):
 
1369
        ann = self.make_annotator()
 
1370
        rev_key = ('rev-id',)
 
1371
        rev2_key = ('rev2-id',)
 
1372
        parent_key = ('parent-id',)
 
1373
        record = ['0,1,1\n', 'new-line\n']
 
1374
        details = ('line-delta', False)
 
1375
        ann._num_compression_children[parent_key] = 2
 
1376
        ann._expand_record(parent_key, (), None, ['line1\n', 'line2\n'],
 
1377
                           ('fulltext', False))
 
1378
        res = ann._expand_record(rev_key, (parent_key,), parent_key,
 
1379
                                 record, details)
 
1380
        self.assertEqual({parent_key: 1}, ann._num_compression_children)
 
1381
        # Expanding the second child should remove the content object, and the
 
1382
        # num_compression_children entry
 
1383
        res = ann._expand_record(rev2_key, (parent_key,), parent_key,
 
1384
                                 record, details)
 
1385
        self.assertFalse(parent_key in ann._content_objects)
 
1386
        self.assertEqual({}, ann._num_compression_children)
 
1387
        # We should not cache the content_objects for rev2 and rev, because
 
1388
        # they do not have compression children of their own.
 
1389
        self.assertEqual({}, ann._content_objects)
 
1390
 
 
1391
    def test__expand_delta_records_blocks(self):
 
1392
        ann = self.make_annotator()
 
1393
        rev_key = ('rev-id',)
 
1394
        parent_key = ('parent-id',)
 
1395
        record = ['0,1,1\n', 'new-line\n']
 
1396
        details = ('line-delta', True)
 
1397
        ann._num_compression_children[parent_key] = 2
 
1398
        ann._expand_record(parent_key, (), None,
 
1399
                           ['line1\n', 'line2\n', 'line3\n'],
 
1400
                           ('fulltext', False))
 
1401
        ann._expand_record(rev_key, (parent_key,), parent_key, record, details)
 
1402
        self.assertEqual({(rev_key, parent_key): [(1, 1, 1), (3, 3, 0)]},
 
1403
                         ann._matching_blocks)
 
1404
        rev2_key = ('rev2-id',)
 
1405
        record = ['0,1,1\n', 'new-line\n']
 
1406
        details = ('line-delta', False)
 
1407
        ann._expand_record(rev2_key, (parent_key,), parent_key, record, details)
 
1408
        self.assertEqual([(1, 1, 2), (3, 3, 0)],
 
1409
                         ann._matching_blocks[(rev2_key, parent_key)])
 
1410
 
 
1411
    def test__get_parent_ann_uses_matching_blocks(self):
 
1412
        ann = self.make_annotator()
 
1413
        rev_key = ('rev-id',)
 
1414
        parent_key = ('parent-id',)
 
1415
        parent_ann = [(parent_key,)]*3
 
1416
        block_key = (rev_key, parent_key)
 
1417
        ann._annotations_cache[parent_key] = parent_ann
 
1418
        ann._matching_blocks[block_key] = [(0, 1, 1), (3, 3, 0)]
 
1419
        # We should not try to access any parent_lines content, because we know
 
1420
        # we already have the matching blocks
 
1421
        par_ann, blocks = ann._get_parent_annotations_and_matches(rev_key,
 
1422
                                        ['1\n', '2\n', '3\n'], parent_key)
 
1423
        self.assertEqual(parent_ann, par_ann)
 
1424
        self.assertEqual([(0, 1, 1), (3, 3, 0)], blocks)
 
1425
        self.assertEqual({}, ann._matching_blocks)
 
1426
 
 
1427
    def test__process_pending(self):
 
1428
        ann = self.make_annotator()
 
1429
        rev_key = ('rev-id',)
 
1430
        p1_key = ('p1-id',)
 
1431
        p2_key = ('p2-id',)
 
1432
        record = ['0,1,1\n', 'new-line\n']
 
1433
        details = ('line-delta', False)
 
1434
        p1_record = ['line1\n', 'line2\n']
 
1435
        ann._num_compression_children[p1_key] = 1
 
1436
        res = ann._expand_record(rev_key, (p1_key,p2_key), p1_key,
 
1437
                                 record, details)
 
1438
        self.assertEqual(None, res)
 
1439
        # self.assertTrue(p1_key in ann._pending_deltas)
 
1440
        self.assertEqual({}, ann._pending_annotation)
 
1441
        # Now insert p1, and we should be able to expand the delta
 
1442
        res = ann._expand_record(p1_key, (), None, p1_record,
 
1443
                                 ('fulltext', False))
 
1444
        self.assertEqual(p1_record, res)
 
1445
        ann._annotations_cache[p1_key] = [(p1_key,)]*2
 
1446
        res = ann._process_pending(p1_key)
 
1447
        self.assertEqual([], res)
 
1448
        self.assertFalse(p1_key in ann._pending_deltas)
 
1449
        self.assertTrue(p2_key in ann._pending_annotation)
 
1450
        self.assertEqual({p2_key: [(rev_key, (p1_key, p2_key))]},
 
1451
                         ann._pending_annotation)
 
1452
        # Now fill in parent 2, and pending annotation should be satisfied
 
1453
        res = ann._expand_record(p2_key, (), None, [], ('fulltext', False))
 
1454
        ann._annotations_cache[p2_key] = []
 
1455
        res = ann._process_pending(p2_key)
 
1456
        self.assertEqual([rev_key], res)
 
1457
        self.assertEqual({}, ann._pending_annotation)
 
1458
        self.assertEqual({}, ann._pending_deltas)
 
1459
 
 
1460
    def test_record_delta_removes_basis(self):
 
1461
        ann = self.make_annotator()
 
1462
        ann._expand_record(('parent-id',), (), None,
 
1463
                           ['line1\n', 'line2\n'], ('fulltext', False))
 
1464
        ann._num_compression_children['parent-id'] = 2
 
1465
 
 
1466
    def test_annotate_special_text(self):
 
1467
        ann = self.make_annotator()
 
1468
        vf = ann._vf
 
1469
        rev1_key = ('rev-1',)
 
1470
        rev2_key = ('rev-2',)
 
1471
        rev3_key = ('rev-3',)
 
1472
        spec_key = ('special:',)
 
1473
        vf.add_lines(rev1_key, [], ['initial content\n'])
 
1474
        vf.add_lines(rev2_key, [rev1_key], ['initial content\n',
 
1475
                                            'common content\n',
 
1476
                                            'content in 2\n'])
 
1477
        vf.add_lines(rev3_key, [rev1_key], ['initial content\n',
 
1478
                                            'common content\n',
 
1479
                                            'content in 3\n'])
 
1480
        spec_text = ('initial content\n'
 
1481
                     'common content\n'
 
1482
                     'content in 2\n'
 
1483
                     'content in 3\n')
 
1484
        ann.add_special_text(spec_key, [rev2_key, rev3_key], spec_text)
 
1485
        anns, lines = ann.annotate(spec_key)
 
1486
        self.assertEqual([(rev1_key,),
 
1487
                          (rev2_key, rev3_key),
 
1488
                          (rev2_key,),
 
1489
                          (rev3_key,),
 
1490
                         ], anns)
 
1491
        self.assertEqualDiff(spec_text, ''.join(lines))
915
1492
 
916
1493
 
917
1494
class KnitTests(TestCaseWithTransport):
922
1499
        return make_file_factory(annotate, mapper)(self.get_transport())
923
1500
 
924
1501
 
 
1502
class TestBadShaError(KnitTests):
 
1503
    """Tests for handling of sha errors."""
 
1504
 
 
1505
    def test_sha_exception_has_text(self):
 
1506
        # having the failed text included in the error allows for recovery.
 
1507
        source = self.make_test_knit()
 
1508
        target = self.make_test_knit(name="target")
 
1509
        if not source._max_delta_chain:
 
1510
            raise TestNotApplicable(
 
1511
                "cannot get delta-caused sha failures without deltas.")
 
1512
        # create a basis
 
1513
        basis = ('basis',)
 
1514
        broken = ('broken',)
 
1515
        source.add_lines(basis, (), ['foo\n'])
 
1516
        source.add_lines(broken, (basis,), ['foo\n', 'bar\n'])
 
1517
        # Seed target with a bad basis text
 
1518
        target.add_lines(basis, (), ['gam\n'])
 
1519
        target.insert_record_stream(
 
1520
            source.get_record_stream([broken], 'unordered', False))
 
1521
        err = self.assertRaises(errors.KnitCorrupt,
 
1522
            target.get_record_stream([broken], 'unordered', True
 
1523
            ).next().get_bytes_as, 'chunked')
 
1524
        self.assertEqual(['gam\n', 'bar\n'], err.content)
 
1525
        # Test for formatting with live data
 
1526
        self.assertStartsWith(str(err), "Knit ")
 
1527
 
 
1528
 
925
1529
class TestKnitIndex(KnitTests):
926
1530
 
927
1531
    def test_add_versions_dictionary_compresses(self):
999
1603
        # could leave an empty .kndx file, which bzr would later claim was a
1000
1604
        # corrupted file since the header was not present. In reality, the file
1001
1605
        # just wasn't created, so it should be ignored.
1002
 
        t = get_transport('.')
 
1606
        t = transport.get_transport_from_path('.')
1003
1607
        t.put_bytes('test.kndx', '')
1004
1608
 
1005
1609
        knit = self.make_test_knit()
1006
1610
 
1007
1611
    def test_knit_index_checks_header(self):
1008
 
        t = get_transport('.')
 
1612
        t = transport.get_transport_from_path('.')
1009
1613
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1010
1614
        k = self.make_test_knit()
1011
1615
        self.assertRaises(KnitHeaderError, k.keys)
1127
1731
            [('parent',)])])
1128
1732
        # but neither should have added data:
1129
1733
        self.assertEqual([[], [], [], []], self.caught_entries)
1130
 
        
 
1734
 
1131
1735
    def test_add_version_different_dup(self):
1132
1736
        index = self.two_graph_index(deltas=True, catch_adds=True)
1133
1737
        # change options
1134
1738
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1135
 
            [(('tip',), 'no-eol,line-delta', (None, 0, 100), [('parent',)])])
1136
 
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1137
 
            [(('tip',), 'line-delta,no-eol', (None, 0, 100), [('parent',)])])
 
1739
            [(('tip',), 'line-delta', (None, 0, 100), [('parent',)])])
1138
1740
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1139
1741
            [(('tip',), 'fulltext', (None, 0, 100), [('parent',)])])
1140
1742
        # parents
1141
1743
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1142
1744
            [(('tip',), 'fulltext,no-eol', (None, 0, 100), [])])
1143
1745
        self.assertEqual([], self.caught_entries)
1144
 
        
 
1746
 
1145
1747
    def test_add_versions_nodeltas(self):
1146
1748
        index = self.two_graph_index(catch_adds=True)
1147
1749
        index.add_records([
1189
1791
            [('parent',)])])
1190
1792
        # but neither should have added data.
1191
1793
        self.assertEqual([[], [], [], []], self.caught_entries)
1192
 
        
 
1794
 
1193
1795
    def test_add_versions_different_dup(self):
1194
1796
        index = self.two_graph_index(deltas=True, catch_adds=True)
1195
1797
        # change options
1196
1798
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1197
 
            [(('tip',), 'no-eol,line-delta', (None, 0, 100), [('parent',)])])
1198
 
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1199
 
            [(('tip',), 'line-delta,no-eol', (None, 0, 100), [('parent',)])])
 
1799
            [(('tip',), 'line-delta', (None, 0, 100), [('parent',)])])
1200
1800
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1201
1801
            [(('tip',), 'fulltext', (None, 0, 100), [('parent',)])])
1202
1802
        # parents
1205
1805
        # change options in the second record
1206
1806
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1207
1807
            [(('tip',), 'fulltext,no-eol', (None, 0, 100), [('parent',)]),
1208
 
             (('tip',), 'no-eol,line-delta', (None, 0, 100), [('parent',)])])
 
1808
             (('tip',), 'line-delta', (None, 0, 100), [('parent',)])])
1209
1809
        self.assertEqual([], self.caught_entries)
1210
1810
 
 
1811
    def make_g_index_missing_compression_parent(self):
 
1812
        graph_index = self.make_g_index('missing_comp', 2,
 
1813
            [(('tip', ), ' 100 78',
 
1814
              ([('missing-parent', ), ('ghost', )], [('missing-parent', )]))])
 
1815
        return graph_index
 
1816
 
 
1817
    def make_g_index_missing_parent(self):
 
1818
        graph_index = self.make_g_index('missing_parent', 2,
 
1819
            [(('parent', ), ' 100 78', ([], [])),
 
1820
             (('tip', ), ' 100 78',
 
1821
              ([('parent', ), ('missing-parent', )], [('parent', )])),
 
1822
              ])
 
1823
        return graph_index
 
1824
 
 
1825
    def make_g_index_no_external_refs(self):
 
1826
        graph_index = self.make_g_index('no_external_refs', 2,
 
1827
            [(('rev', ), ' 100 78',
 
1828
              ([('parent', ), ('ghost', )], []))])
 
1829
        return graph_index
 
1830
 
 
1831
    def test_add_good_unvalidated_index(self):
 
1832
        unvalidated = self.make_g_index_no_external_refs()
 
1833
        combined = CombinedGraphIndex([unvalidated])
 
1834
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1835
        index.scan_unvalidated_index(unvalidated)
 
1836
        self.assertEqual(frozenset(), index.get_missing_compression_parents())
 
1837
 
 
1838
    def test_add_missing_compression_parent_unvalidated_index(self):
 
1839
        unvalidated = self.make_g_index_missing_compression_parent()
 
1840
        combined = CombinedGraphIndex([unvalidated])
 
1841
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1842
        index.scan_unvalidated_index(unvalidated)
 
1843
        # This also checks that its only the compression parent that is
 
1844
        # examined, otherwise 'ghost' would also be reported as a missing
 
1845
        # parent.
 
1846
        self.assertEqual(
 
1847
            frozenset([('missing-parent',)]),
 
1848
            index.get_missing_compression_parents())
 
1849
 
 
1850
    def test_add_missing_noncompression_parent_unvalidated_index(self):
 
1851
        unvalidated = self.make_g_index_missing_parent()
 
1852
        combined = CombinedGraphIndex([unvalidated])
 
1853
        index = _KnitGraphIndex(combined, lambda: True, deltas=True,
 
1854
            track_external_parent_refs=True)
 
1855
        index.scan_unvalidated_index(unvalidated)
 
1856
        self.assertEqual(
 
1857
            frozenset([('missing-parent',)]), index.get_missing_parents())
 
1858
 
 
1859
    def test_track_external_parent_refs(self):
 
1860
        g_index = self.make_g_index('empty', 2, [])
 
1861
        combined = CombinedGraphIndex([g_index])
 
1862
        index = _KnitGraphIndex(combined, lambda: True, deltas=True,
 
1863
            add_callback=self.catch_add, track_external_parent_refs=True)
 
1864
        self.caught_entries = []
 
1865
        index.add_records([
 
1866
            (('new-key',), 'fulltext,no-eol', (None, 50, 60),
 
1867
             [('parent-1',), ('parent-2',)])])
 
1868
        self.assertEqual(
 
1869
            frozenset([('parent-1',), ('parent-2',)]),
 
1870
            index.get_missing_parents())
 
1871
 
 
1872
    def test_add_unvalidated_index_with_present_external_references(self):
 
1873
        index = self.two_graph_index(deltas=True)
 
1874
        # Ugly hack to get at one of the underlying GraphIndex objects that
 
1875
        # two_graph_index built.
 
1876
        unvalidated = index._graph_index._indices[1]
 
1877
        # 'parent' is an external ref of _indices[1] (unvalidated), but is
 
1878
        # present in _indices[0].
 
1879
        index.scan_unvalidated_index(unvalidated)
 
1880
        self.assertEqual(frozenset(), index.get_missing_compression_parents())
 
1881
 
 
1882
    def make_new_missing_parent_g_index(self, name):
 
1883
        missing_parent = name + '-missing-parent'
 
1884
        graph_index = self.make_g_index(name, 2,
 
1885
            [((name + 'tip', ), ' 100 78',
 
1886
              ([(missing_parent, ), ('ghost', )], [(missing_parent, )]))])
 
1887
        return graph_index
 
1888
 
 
1889
    def test_add_mulitiple_unvalidated_indices_with_missing_parents(self):
 
1890
        g_index_1 = self.make_new_missing_parent_g_index('one')
 
1891
        g_index_2 = self.make_new_missing_parent_g_index('two')
 
1892
        combined = CombinedGraphIndex([g_index_1, g_index_2])
 
1893
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1894
        index.scan_unvalidated_index(g_index_1)
 
1895
        index.scan_unvalidated_index(g_index_2)
 
1896
        self.assertEqual(
 
1897
            frozenset([('one-missing-parent',), ('two-missing-parent',)]),
 
1898
            index.get_missing_compression_parents())
 
1899
 
 
1900
    def test_add_mulitiple_unvalidated_indices_with_mutual_dependencies(self):
 
1901
        graph_index_a = self.make_g_index('one', 2,
 
1902
            [(('parent-one', ), ' 100 78', ([('non-compression-parent',)], [])),
 
1903
             (('child-of-two', ), ' 100 78',
 
1904
              ([('parent-two',)], [('parent-two',)]))])
 
1905
        graph_index_b = self.make_g_index('two', 2,
 
1906
            [(('parent-two', ), ' 100 78', ([('non-compression-parent',)], [])),
 
1907
             (('child-of-one', ), ' 100 78',
 
1908
              ([('parent-one',)], [('parent-one',)]))])
 
1909
        combined = CombinedGraphIndex([graph_index_a, graph_index_b])
 
1910
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1911
        index.scan_unvalidated_index(graph_index_a)
 
1912
        index.scan_unvalidated_index(graph_index_b)
 
1913
        self.assertEqual(
 
1914
            frozenset([]), index.get_missing_compression_parents())
 
1915
 
1211
1916
 
1212
1917
class TestNoParentsGraphIndexKnit(KnitTests):
1213
1918
    """Tests for knits using _KnitGraphIndex with no parents."""
1221
1926
        size = trans.put_file(name, stream)
1222
1927
        return GraphIndex(trans, name, size)
1223
1928
 
 
1929
    def test_add_good_unvalidated_index(self):
 
1930
        unvalidated = self.make_g_index('unvalidated')
 
1931
        combined = CombinedGraphIndex([unvalidated])
 
1932
        index = _KnitGraphIndex(combined, lambda: True, parents=False)
 
1933
        index.scan_unvalidated_index(unvalidated)
 
1934
        self.assertEqual(frozenset(),
 
1935
            index.get_missing_compression_parents())
 
1936
 
1224
1937
    def test_parents_deltas_incompatible(self):
1225
1938
        index = CombinedGraphIndex([])
1226
1939
        self.assertRaises(errors.KnitError, _KnitGraphIndex, lambda:True,
1307
2020
        index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000), [])])
1308
2021
        # but neither should have added data.
1309
2022
        self.assertEqual([[], [], [], []], self.caught_entries)
1310
 
        
 
2023
 
1311
2024
    def test_add_version_different_dup(self):
1312
2025
        index = self.two_graph_index(catch_adds=True)
1313
2026
        # change options
1321
2034
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1322
2035
            [(('tip',), 'fulltext,no-eol', (None, 0, 100), [('parent',)])])
1323
2036
        self.assertEqual([], self.caught_entries)
1324
 
        
 
2037
 
1325
2038
    def test_add_versions(self):
1326
2039
        index = self.two_graph_index(catch_adds=True)
1327
2040
        index.add_records([
1359
2072
        index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000), [])])
1360
2073
        # but neither should have added data.
1361
2074
        self.assertEqual([[], [], [], []], self.caught_entries)
1362
 
        
 
2075
 
1363
2076
    def test_add_versions_different_dup(self):
1364
2077
        index = self.two_graph_index(catch_adds=True)
1365
2078
        # change options
1379
2092
        self.assertEqual([], self.caught_entries)
1380
2093
 
1381
2094
 
 
2095
class TestKnitVersionedFiles(KnitTests):
 
2096
 
 
2097
    def assertGroupKeysForIo(self, exp_groups, keys, non_local_keys,
 
2098
                             positions, _min_buffer_size=None):
 
2099
        kvf = self.make_test_knit()
 
2100
        if _min_buffer_size is None:
 
2101
            _min_buffer_size = knit._STREAM_MIN_BUFFER_SIZE
 
2102
        self.assertEqual(exp_groups, kvf._group_keys_for_io(keys,
 
2103
                                        non_local_keys, positions,
 
2104
                                        _min_buffer_size=_min_buffer_size))
 
2105
 
 
2106
    def assertSplitByPrefix(self, expected_map, expected_prefix_order,
 
2107
                            keys):
 
2108
        split, prefix_order = KnitVersionedFiles._split_by_prefix(keys)
 
2109
        self.assertEqual(expected_map, split)
 
2110
        self.assertEqual(expected_prefix_order, prefix_order)
 
2111
 
 
2112
    def test__group_keys_for_io(self):
 
2113
        ft_detail = ('fulltext', False)
 
2114
        ld_detail = ('line-delta', False)
 
2115
        f_a = ('f', 'a')
 
2116
        f_b = ('f', 'b')
 
2117
        f_c = ('f', 'c')
 
2118
        g_a = ('g', 'a')
 
2119
        g_b = ('g', 'b')
 
2120
        g_c = ('g', 'c')
 
2121
        positions = {
 
2122
            f_a: (ft_detail, (f_a, 0, 100), None),
 
2123
            f_b: (ld_detail, (f_b, 100, 21), f_a),
 
2124
            f_c: (ld_detail, (f_c, 180, 15), f_b),
 
2125
            g_a: (ft_detail, (g_a, 121, 35), None),
 
2126
            g_b: (ld_detail, (g_b, 156, 12), g_a),
 
2127
            g_c: (ld_detail, (g_c, 195, 13), g_a),
 
2128
            }
 
2129
        self.assertGroupKeysForIo([([f_a], set())],
 
2130
                                  [f_a], [], positions)
 
2131
        self.assertGroupKeysForIo([([f_a], set([f_a]))],
 
2132
                                  [f_a], [f_a], positions)
 
2133
        self.assertGroupKeysForIo([([f_a, f_b], set([]))],
 
2134
                                  [f_a, f_b], [], positions)
 
2135
        self.assertGroupKeysForIo([([f_a, f_b], set([f_b]))],
 
2136
                                  [f_a, f_b], [f_b], positions)
 
2137
        self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
 
2138
                                  [f_a, g_a, f_b, g_b], [], positions)
 
2139
        self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
 
2140
                                  [f_a, g_a, f_b, g_b], [], positions,
 
2141
                                  _min_buffer_size=150)
 
2142
        self.assertGroupKeysForIo([([f_a, f_b], set()), ([g_a, g_b], set())],
 
2143
                                  [f_a, g_a, f_b, g_b], [], positions,
 
2144
                                  _min_buffer_size=100)
 
2145
        self.assertGroupKeysForIo([([f_c], set()), ([g_b], set())],
 
2146
                                  [f_c, g_b], [], positions,
 
2147
                                  _min_buffer_size=125)
 
2148
        self.assertGroupKeysForIo([([g_b, f_c], set())],
 
2149
                                  [g_b, f_c], [], positions,
 
2150
                                  _min_buffer_size=125)
 
2151
 
 
2152
    def test__split_by_prefix(self):
 
2153
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2154
                                  'g': [('g', 'b'), ('g', 'a')],
 
2155
                                 }, ['f', 'g'],
 
2156
                                 [('f', 'a'), ('g', 'b'),
 
2157
                                  ('g', 'a'), ('f', 'b')])
 
2158
 
 
2159
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2160
                                  'g': [('g', 'b'), ('g', 'a')],
 
2161
                                 }, ['f', 'g'],
 
2162
                                 [('f', 'a'), ('f', 'b'),
 
2163
                                  ('g', 'b'), ('g', 'a')])
 
2164
 
 
2165
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2166
                                  'g': [('g', 'b'), ('g', 'a')],
 
2167
                                 }, ['f', 'g'],
 
2168
                                 [('f', 'a'), ('f', 'b'),
 
2169
                                  ('g', 'b'), ('g', 'a')])
 
2170
 
 
2171
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
2172
                                  'g': [('g', 'b'), ('g', 'a')],
 
2173
                                  '': [('a',), ('b',)]
 
2174
                                 }, ['f', 'g', ''],
 
2175
                                 [('f', 'a'), ('g', 'b'),
 
2176
                                  ('a',), ('b',),
 
2177
                                  ('g', 'a'), ('f', 'b')])
 
2178
 
 
2179
 
1382
2180
class TestStacking(KnitTests):
1383
2181
 
1384
2182
    def get_basis_and_test_knit(self):
1410
2208
        basis.calls = []
1411
2209
        test.add_lines(key_cross_border, (key_basis,), ['foo\n'])
1412
2210
        self.assertEqual('fulltext', test._index.get_method(key_cross_border))
1413
 
        self.assertEqual([("get_parent_map", set([key_basis]))], basis.calls)
 
2211
        # we don't even need to look at the basis to see that this should be
 
2212
        # stored as a fulltext
 
2213
        self.assertEqual([], basis.calls)
1414
2214
        # Subsequent adds do delta.
1415
2215
        basis.calls = []
1416
2216
        test.add_lines(key_delta, (key_cross_border,), ['foo\n'])
1437
2237
        # self.assertEqual([("annotate", key_basis)], basis.calls)
1438
2238
        self.assertEqual([('get_parent_map', set([key_basis])),
1439
2239
            ('get_parent_map', set([key_basis])),
1440
 
            ('get_parent_map', set([key_basis])),
1441
 
            ('get_record_stream', [key_basis], 'unordered', True)],
 
2240
            ('get_record_stream', [key_basis], 'topological', True)],
1442
2241
            basis.calls)
1443
2242
 
1444
2243
    def test_check(self):
1445
2244
        # At the moment checking a stacked knit does implicitly check the
1446
 
        # fallback files.  
 
2245
        # fallback files.
1447
2246
        basis, test = self.get_basis_and_test_knit()
1448
2247
        test.check()
1449
2248
 
1541
2340
                True).next()
1542
2341
            self.assertEqual(record.key, result[0])
1543
2342
            self.assertEqual(record.sha1, result[1])
1544
 
            self.assertEqual(record.storage_kind, result[2])
 
2343
            # We used to check that the storage kind matched, but actually it
 
2344
            # depends on whether it was sourced from the basis, or in a single
 
2345
            # group, because asking for full texts returns proxy objects to a
 
2346
            # _ContentMapGenerator object; so checking the kind is unneeded.
1545
2347
            self.assertEqual(record.get_bytes_as('fulltext'), result[3])
1546
2348
        # It's not strictly minimal, but it seems reasonable for now for it to
1547
2349
        # ask which fallbacks have which parents.
1548
2350
        self.assertEqual([
1549
2351
            ("get_parent_map", set([key_basis, key_basis_2, key_missing])),
1550
 
            # unordered is asked for by the underlying worker as it still
1551
 
            # buffers everything while answering - which is a problem!
1552
 
            ("get_record_stream", [key_basis_2, key_basis], 'unordered', True)],
 
2352
            # topological is requested from the fallback, because that is what
 
2353
            # was requested at the top level.
 
2354
            ("get_record_stream", [key_basis_2, key_basis], 'topological', True)],
1553
2355
            calls)
1554
2356
 
1555
2357
    def test_get_record_stream_unordered_deltas(self):
1641
2443
        key_basis = ('bar',)
1642
2444
        key_missing = ('missing',)
1643
2445
        test.add_lines(key, (), ['foo\n'])
1644
 
        key_sha1sum = sha.new('foo\n').hexdigest()
 
2446
        key_sha1sum = osutils.sha_string('foo\n')
1645
2447
        sha1s = test.get_sha1s([key])
1646
2448
        self.assertEqual({key: key_sha1sum}, sha1s)
1647
2449
        self.assertEqual([], basis.calls)
1649
2451
        # directly (rather than via text reconstruction) so that remote servers
1650
2452
        # etc don't have to answer with full content.
1651
2453
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
1652
 
        basis_sha1sum = sha.new('foo\nbar\n').hexdigest()
 
2454
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
1653
2455
        basis.calls = []
1654
2456
        sha1s = test.get_sha1s([key, key_missing, key_basis])
1655
2457
        self.assertEqual({key: key_sha1sum,
1671
2473
        source.add_lines(key_delta, (key_basis,), ['bar\n'])
1672
2474
        stream = source.get_record_stream([key_delta], 'unordered', False)
1673
2475
        test.insert_record_stream(stream)
1674
 
        self.assertEqual([("get_parent_map", set([key_basis]))],
 
2476
        # XXX: this does somewhat too many calls in making sure of whether it
 
2477
        # has to recreate the full text.
 
2478
        self.assertEqual([("get_parent_map", set([key_basis])),
 
2479
             ('get_parent_map', set([key_basis])),
 
2480
             ('get_record_stream', [key_basis], 'unordered', True)],
1675
2481
            basis.calls)
1676
2482
        self.assertEqual({key_delta:(key_basis,)},
1677
2483
            test.get_parent_map([key_delta]))
1680
2486
 
1681
2487
    def test_iter_lines_added_or_present_in_keys(self):
1682
2488
        # Lines from the basis are returned, and lines for a given key are only
1683
 
        # returned once. 
 
2489
        # returned once.
1684
2490
        key1 = ('foo1',)
1685
2491
        key2 = ('foo2',)
1686
2492
        # all sources are asked for keys:
1738
2544
        test.add_mpdiffs([(key_delta, (key_basis,),
1739
2545
            source.get_sha1s([key_delta])[key_delta], diffs[0])])
1740
2546
        self.assertEqual([("get_parent_map", set([key_basis])),
1741
 
            ('get_record_stream', [key_basis], 'unordered', True),
1742
 
            ('get_parent_map', set([key_basis]))],
 
2547
            ('get_record_stream', [key_basis], 'unordered', True),],
1743
2548
            basis.calls)
1744
2549
        self.assertEqual({key_delta:(key_basis,)},
1745
2550
            test.get_parent_map([key_delta]))
1764
2569
                multiparent.NewText(['foo\n']),
1765
2570
                multiparent.ParentText(1, 0, 2, 1)])],
1766
2571
            diffs)
1767
 
        self.assertEqual(4, len(basis.calls))
 
2572
        self.assertEqual(3, len(basis.calls))
1768
2573
        self.assertEqual([
1769
2574
            ("get_parent_map", set([key_left, key_right])),
1770
2575
            ("get_parent_map", set([key_left, key_right])),
1771
 
            ("get_parent_map", set([key_left, key_right])),
1772
2576
            ],
1773
 
            basis.calls[:3])
1774
 
        last_call = basis.calls[3]
 
2577
            basis.calls[:-1])
 
2578
        last_call = basis.calls[-1]
1775
2579
        self.assertEqual('get_record_stream', last_call[0])
1776
2580
        self.assertEqual(set([key_left, key_right]), set(last_call[1]))
1777
 
        self.assertEqual('unordered', last_call[2])
 
2581
        self.assertEqual('topological', last_call[2])
1778
2582
        self.assertEqual(True, last_call[3])
 
2583
 
 
2584
 
 
2585
class TestNetworkBehaviour(KnitTests):
 
2586
    """Tests for getting data out of/into knits over the network."""
 
2587
 
 
2588
    def test_include_delta_closure_generates_a_knit_delta_closure(self):
 
2589
        vf = self.make_test_knit(name='test')
 
2590
        # put in three texts, giving ft, delta, delta
 
2591
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2592
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2593
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2594
        # But heuristics could interfere, so check what happened:
 
2595
        self.assertEqual(['knit-ft-gz', 'knit-delta-gz', 'knit-delta-gz'],
 
2596
            [record.storage_kind for record in
 
2597
             vf.get_record_stream([('base',), ('d1',), ('d2',)],
 
2598
                'topological', False)])
 
2599
        # generate a stream of just the deltas include_delta_closure=True,
 
2600
        # serialise to the network, and check that we get a delta closure on the wire.
 
2601
        stream = vf.get_record_stream([('d1',), ('d2',)], 'topological', True)
 
2602
        netb = [record.get_bytes_as(record.storage_kind) for record in stream]
 
2603
        # The first bytes should be a memo from _ContentMapGenerator, and the
 
2604
        # second bytes should be empty (because its a API proxy not something
 
2605
        # for wire serialisation.
 
2606
        self.assertEqual('', netb[1])
 
2607
        bytes = netb[0]
 
2608
        kind, line_end = network_bytes_to_kind_and_offset(bytes)
 
2609
        self.assertEqual('knit-delta-closure', kind)
 
2610
 
 
2611
 
 
2612
class TestContentMapGenerator(KnitTests):
 
2613
    """Tests for ContentMapGenerator"""
 
2614
 
 
2615
    def test_get_record_stream_gives_records(self):
 
2616
        vf = self.make_test_knit(name='test')
 
2617
        # put in three texts, giving ft, delta, delta
 
2618
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2619
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2620
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2621
        keys = [('d1',), ('d2',)]
 
2622
        generator = _VFContentMapGenerator(vf, keys,
 
2623
            global_map=vf.get_parent_map(keys))
 
2624
        for record in generator.get_record_stream():
 
2625
            if record.key == ('d1',):
 
2626
                self.assertEqual('d1\n', record.get_bytes_as('fulltext'))
 
2627
            else:
 
2628
                self.assertEqual('d2\n', record.get_bytes_as('fulltext'))
 
2629
 
 
2630
    def test_get_record_stream_kinds_are_raw(self):
 
2631
        vf = self.make_test_knit(name='test')
 
2632
        # put in three texts, giving ft, delta, delta
 
2633
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2634
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2635
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2636
        keys = [('base',), ('d1',), ('d2',)]
 
2637
        generator = _VFContentMapGenerator(vf, keys,
 
2638
            global_map=vf.get_parent_map(keys))
 
2639
        kinds = {('base',): 'knit-delta-closure',
 
2640
            ('d1',): 'knit-delta-closure-ref',
 
2641
            ('d2',): 'knit-delta-closure-ref',
 
2642
            }
 
2643
        for record in generator.get_record_stream():
 
2644
            self.assertEqual(kinds[record.key], record.storage_kind)