← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Dmitry Vasiliev
  • Date: 2009-06-29 11:02:31 UTC
  • mto: (4517.1.1 integration2)
  • mto: This revision was merged to the branch mainline in revision 4520.
  • Revision ID: dima@hlabs.spb.ru-20090629110231-tsy00fwr6aud4en3
Optimize configuration for build documentation

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
 
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
20
20
import difflib
21
21
import gzip
22
 
import sha
23
22
import sys
24
23
 
25
24
from bzrlib import (
27
26
    generate_ids,
28
27
    knit,
29
28
    multiparent,
 
29
    osutils,
30
30
    pack,
31
31
    )
32
32
from bzrlib.errors import (
42
42
    KnitSequenceMatcher,
43
43
    KnitVersionedFiles,
44
44
    PlainKnitContent,
 
45
    _VFContentMapGenerator,
45
46
    _DirectPackAccess,
46
47
    _KndxIndex,
47
48
    _KnitGraphIndex,
48
49
    _KnitKeyAccess,
49
50
    make_file_factory,
50
51
    )
51
 
from bzrlib.osutils import split_lines
52
 
from bzrlib.symbol_versioning import one_four
 
52
from bzrlib.repofmt import pack_repo
53
53
from bzrlib.tests import (
54
54
    Feature,
55
55
    KnownFailure,
56
56
    TestCase,
57
57
    TestCaseWithMemoryTransport,
58
58
    TestCaseWithTransport,
 
59
    TestNotApplicable,
59
60
    )
60
61
from bzrlib.transport import get_transport
61
62
from bzrlib.transport.memory import MemoryTransport
63
64
from bzrlib.versionedfile import (
64
65
    AbsentContentFactory,
65
66
    ConstantMapper,
 
67
    network_bytes_to_kind_and_offset,
66
68
    RecordingVersionedFilesDecorator,
67
69
    )
68
70
 
271
273
        return queue_call
272
274
 
273
275
 
 
276
class MockReadvFailingTransport(MockTransport):
 
277
    """Fail in the middle of a readv() result.
 
278
 
 
279
    This Transport will successfully yield the first two requested hunks, but
 
280
    raise NoSuchFile for the rest.
 
281
    """
 
282
 
 
283
    def readv(self, relpath, offsets):
 
284
        count = 0
 
285
        for result in MockTransport.readv(self, relpath, offsets):
 
286
            count += 1
 
287
            # we use 2 because the first offset is the pack header, the second
 
288
            # is the first actual content requset
 
289
            if count > 2:
 
290
                raise errors.NoSuchFile(relpath)
 
291
            yield result
 
292
 
 
293
 
274
294
class KnitRecordAccessTestsMixin(object):
275
295
    """Tests for getting and putting knit records."""
276
296
 
279
299
        access = self.get_access()
280
300
        memos = access.add_raw_records([('key', 10)], '1234567890')
281
301
        self.assertEqual(['1234567890'], list(access.get_raw_records(memos)))
282
 
 
 
302
 
283
303
    def test_add_several_raw_records(self):
284
304
        """add_raw_records with many records and read some back."""
285
305
        access = self.get_access()
305
325
        mapper = ConstantMapper("foo")
306
326
        access = _KnitKeyAccess(self.get_transport(), mapper)
307
327
        return access
308
 
    
 
328
 
 
329
 
 
330
class _TestException(Exception):
 
331
    """Just an exception for local tests to use."""
 
332
 
309
333
 
310
334
class TestPackKnitAccess(TestCaseWithMemoryTransport, KnitRecordAccessTestsMixin):
311
335
    """Tests for the pack based access."""
323
347
        access.set_writer(writer, index, (transport, packname))
324
348
        return access, writer
325
349
 
 
350
    def make_pack_file(self):
 
351
        """Create a pack file with 2 records."""
 
352
        access, writer = self._get_access(packname='packname', index='foo')
 
353
        memos = []
 
354
        memos.extend(access.add_raw_records([('key1', 10)], '1234567890'))
 
355
        memos.extend(access.add_raw_records([('key2', 5)], '12345'))
 
356
        writer.end()
 
357
        return memos
 
358
 
 
359
    def make_vf_for_retrying(self):
 
360
        """Create 3 packs and a reload function.
 
361
 
 
362
        Originally, 2 pack files will have the data, but one will be missing.
 
363
        And then the third will be used in place of the first two if reload()
 
364
        is called.
 
365
 
 
366
        :return: (versioned_file, reload_counter)
 
367
            versioned_file  a KnitVersionedFiles using the packs for access
 
368
        """
 
369
        tree = self.make_branch_and_memory_tree('tree')
 
370
        tree.lock_write()
 
371
        self.addCleanup(tree.unlock)
 
372
        tree.add([''], ['root-id'])
 
373
        tree.commit('one', rev_id='rev-1')
 
374
        tree.commit('two', rev_id='rev-2')
 
375
        tree.commit('three', rev_id='rev-3')
 
376
        # Pack these three revisions into another pack file, but don't remove
 
377
        # the originals
 
378
        repo = tree.branch.repository
 
379
        collection = repo._pack_collection
 
380
        collection.ensure_loaded()
 
381
        orig_packs = collection.packs
 
382
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
 
383
        new_pack = packer.pack()
 
384
        # forget about the new pack
 
385
        collection.reset()
 
386
        repo.refresh_data()
 
387
        vf = tree.branch.repository.revisions
 
388
        # Set up a reload() function that switches to using the new pack file
 
389
        new_index = new_pack.revision_index
 
390
        access_tuple = new_pack.access_tuple()
 
391
        reload_counter = [0, 0, 0]
 
392
        def reload():
 
393
            reload_counter[0] += 1
 
394
            if reload_counter[1] > 0:
 
395
                # We already reloaded, nothing more to do
 
396
                reload_counter[2] += 1
 
397
                return False
 
398
            reload_counter[1] += 1
 
399
            vf._index._graph_index._indices[:] = [new_index]
 
400
            vf._access._indices.clear()
 
401
            vf._access._indices[new_index] = access_tuple
 
402
            return True
 
403
        # Delete one of the pack files so the data will need to be reloaded. We
 
404
        # will delete the file with 'rev-2' in it
 
405
        trans, name = orig_packs[1].access_tuple()
 
406
        trans.delete(name)
 
407
        # We don't have the index trigger reloading because we want to test
 
408
        # that we reload when the .pack disappears
 
409
        vf._access._reload_func = reload
 
410
        return vf, reload_counter
 
411
 
 
412
    def make_reload_func(self, return_val=True):
 
413
        reload_called = [0]
 
414
        def reload():
 
415
            reload_called[0] += 1
 
416
            return return_val
 
417
        return reload_called, reload
 
418
 
 
419
    def make_retry_exception(self):
 
420
        # We raise a real exception so that sys.exc_info() is properly
 
421
        # populated
 
422
        try:
 
423
            raise _TestException('foobar')
 
424
        except _TestException, e:
 
425
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
 
426
                                                 exc_info=sys.exc_info())
 
427
        return retry_exc
 
428
 
326
429
    def test_read_from_several_packs(self):
327
430
        access, writer = self._get_access()
328
431
        memos = []
364
467
        writer.end()
365
468
        self.assertEqual(['1234567890'], list(access.get_raw_records(memos)))
366
469
 
 
470
    def test_missing_index_raises_retry(self):
 
471
        memos = self.make_pack_file()
 
472
        transport = self.get_transport()
 
473
        reload_called, reload_func = self.make_reload_func()
 
474
        # Note that the index key has changed from 'foo' to 'bar'
 
475
        access = _DirectPackAccess({'bar':(transport, 'packname')},
 
476
                                   reload_func=reload_func)
 
477
        e = self.assertListRaises(errors.RetryWithNewPacks,
 
478
                                  access.get_raw_records, memos)
 
479
        # Because a key was passed in which does not match our index list, we
 
480
        # assume that the listing was already reloaded
 
481
        self.assertTrue(e.reload_occurred)
 
482
        self.assertIsInstance(e.exc_info, tuple)
 
483
        self.assertIs(e.exc_info[0], KeyError)
 
484
        self.assertIsInstance(e.exc_info[1], KeyError)
 
485
 
 
486
    def test_missing_index_raises_key_error_with_no_reload(self):
 
487
        memos = self.make_pack_file()
 
488
        transport = self.get_transport()
 
489
        # Note that the index key has changed from 'foo' to 'bar'
 
490
        access = _DirectPackAccess({'bar':(transport, 'packname')})
 
491
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
 
492
 
 
493
    def test_missing_file_raises_retry(self):
 
494
        memos = self.make_pack_file()
 
495
        transport = self.get_transport()
 
496
        reload_called, reload_func = self.make_reload_func()
 
497
        # Note that the 'filename' has been changed to 'different-packname'
 
498
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
 
499
                                   reload_func=reload_func)
 
500
        e = self.assertListRaises(errors.RetryWithNewPacks,
 
501
                                  access.get_raw_records, memos)
 
502
        # The file has gone missing, so we assume we need to reload
 
503
        self.assertFalse(e.reload_occurred)
 
504
        self.assertIsInstance(e.exc_info, tuple)
 
505
        self.assertIs(e.exc_info[0], errors.NoSuchFile)
 
506
        self.assertIsInstance(e.exc_info[1], errors.NoSuchFile)
 
507
        self.assertEqual('different-packname', e.exc_info[1].path)
 
508
 
 
509
    def test_missing_file_raises_no_such_file_with_no_reload(self):
 
510
        memos = self.make_pack_file()
 
511
        transport = self.get_transport()
 
512
        # Note that the 'filename' has been changed to 'different-packname'
 
513
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
 
514
        e = self.assertListRaises(errors.NoSuchFile,
 
515
                                  access.get_raw_records, memos)
 
516
 
 
517
    def test_failing_readv_raises_retry(self):
 
518
        memos = self.make_pack_file()
 
519
        transport = self.get_transport()
 
520
        failing_transport = MockReadvFailingTransport(
 
521
                                [transport.get_bytes('packname')])
 
522
        reload_called, reload_func = self.make_reload_func()
 
523
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
 
524
                                   reload_func=reload_func)
 
525
        # Asking for a single record will not trigger the Mock failure
 
526
        self.assertEqual(['1234567890'],
 
527
            list(access.get_raw_records(memos[:1])))
 
528
        self.assertEqual(['12345'],
 
529
            list(access.get_raw_records(memos[1:2])))
 
530
        # A multiple offset readv() will fail mid-way through
 
531
        e = self.assertListRaises(errors.RetryWithNewPacks,
 
532
                                  access.get_raw_records, memos)
 
533
        # The file has gone missing, so we assume we need to reload
 
534
        self.assertFalse(e.reload_occurred)
 
535
        self.assertIsInstance(e.exc_info, tuple)
 
536
        self.assertIs(e.exc_info[0], errors.NoSuchFile)
 
537
        self.assertIsInstance(e.exc_info[1], errors.NoSuchFile)
 
538
        self.assertEqual('packname', e.exc_info[1].path)
 
539
 
 
540
    def test_failing_readv_raises_no_such_file_with_no_reload(self):
 
541
        memos = self.make_pack_file()
 
542
        transport = self.get_transport()
 
543
        failing_transport = MockReadvFailingTransport(
 
544
                                [transport.get_bytes('packname')])
 
545
        reload_called, reload_func = self.make_reload_func()
 
546
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
 
547
        # Asking for a single record will not trigger the Mock failure
 
548
        self.assertEqual(['1234567890'],
 
549
            list(access.get_raw_records(memos[:1])))
 
550
        self.assertEqual(['12345'],
 
551
            list(access.get_raw_records(memos[1:2])))
 
552
        # A multiple offset readv() will fail mid-way through
 
553
        e = self.assertListRaises(errors.NoSuchFile,
 
554
                                  access.get_raw_records, memos)
 
555
 
 
556
    def test_reload_or_raise_no_reload(self):
 
557
        access = _DirectPackAccess({}, reload_func=None)
 
558
        retry_exc = self.make_retry_exception()
 
559
        # Without a reload_func, we will just re-raise the original exception
 
560
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
 
561
 
 
562
    def test_reload_or_raise_reload_changed(self):
 
563
        reload_called, reload_func = self.make_reload_func(return_val=True)
 
564
        access = _DirectPackAccess({}, reload_func=reload_func)
 
565
        retry_exc = self.make_retry_exception()
 
566
        access.reload_or_raise(retry_exc)
 
567
        self.assertEqual([1], reload_called)
 
568
        retry_exc.reload_occurred=True
 
569
        access.reload_or_raise(retry_exc)
 
570
        self.assertEqual([2], reload_called)
 
571
 
 
572
    def test_reload_or_raise_reload_no_change(self):
 
573
        reload_called, reload_func = self.make_reload_func(return_val=False)
 
574
        access = _DirectPackAccess({}, reload_func=reload_func)
 
575
        retry_exc = self.make_retry_exception()
 
576
        # If reload_occurred is False, then we consider it an error to have
 
577
        # reload_func() return False (no changes).
 
578
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
 
579
        self.assertEqual([1], reload_called)
 
580
        retry_exc.reload_occurred=True
 
581
        # If reload_occurred is True, then we assume nothing changed because
 
582
        # it had changed earlier, but didn't change again
 
583
        access.reload_or_raise(retry_exc)
 
584
        self.assertEqual([2], reload_called)
 
585
 
 
586
    def test_annotate_retries(self):
 
587
        vf, reload_counter = self.make_vf_for_retrying()
 
588
        # It is a little bit bogus to annotate the Revision VF, but it works,
 
589
        # as we have ancestry stored there
 
590
        key = ('rev-3',)
 
591
        reload_lines = vf.annotate(key)
 
592
        self.assertEqual([1, 1, 0], reload_counter)
 
593
        plain_lines = vf.annotate(key)
 
594
        self.assertEqual([1, 1, 0], reload_counter) # No extra reloading
 
595
        if reload_lines != plain_lines:
 
596
            self.fail('Annotation was not identical with reloading.')
 
597
        # Now delete the packs-in-use, which should trigger another reload, but
 
598
        # this time we just raise an exception because we can't recover
 
599
        for trans, name in vf._access._indices.itervalues():
 
600
            trans.delete(name)
 
601
        self.assertRaises(errors.NoSuchFile, vf.annotate, key)
 
602
        self.assertEqual([2, 1, 1], reload_counter)
 
603
 
 
604
    def test__get_record_map_retries(self):
 
605
        vf, reload_counter = self.make_vf_for_retrying()
 
606
        keys = [('rev-1',), ('rev-2',), ('rev-3',)]
 
607
        records = vf._get_record_map(keys)
 
608
        self.assertEqual(keys, sorted(records.keys()))
 
609
        self.assertEqual([1, 1, 0], reload_counter)
 
610
        # Now delete the packs-in-use, which should trigger another reload, but
 
611
        # this time we just raise an exception because we can't recover
 
612
        for trans, name in vf._access._indices.itervalues():
 
613
            trans.delete(name)
 
614
        self.assertRaises(errors.NoSuchFile, vf._get_record_map, keys)
 
615
        self.assertEqual([2, 1, 1], reload_counter)
 
616
 
 
617
    def test_get_record_stream_retries(self):
 
618
        vf, reload_counter = self.make_vf_for_retrying()
 
619
        keys = [('rev-1',), ('rev-2',), ('rev-3',)]
 
620
        record_stream = vf.get_record_stream(keys, 'topological', False)
 
621
        record = record_stream.next()
 
622
        self.assertEqual(('rev-1',), record.key)
 
623
        self.assertEqual([0, 0, 0], reload_counter)
 
624
        record = record_stream.next()
 
625
        self.assertEqual(('rev-2',), record.key)
 
626
        self.assertEqual([1, 1, 0], reload_counter)
 
627
        record = record_stream.next()
 
628
        self.assertEqual(('rev-3',), record.key)
 
629
        self.assertEqual([1, 1, 0], reload_counter)
 
630
        # Now delete all pack files, and see that we raise the right error
 
631
        for trans, name in vf._access._indices.itervalues():
 
632
            trans.delete(name)
 
633
        self.assertListRaises(errors.NoSuchFile,
 
634
            vf.get_record_stream, keys, 'topological', False)
 
635
 
 
636
    def test_iter_lines_added_or_present_in_keys_retries(self):
 
637
        vf, reload_counter = self.make_vf_for_retrying()
 
638
        keys = [('rev-1',), ('rev-2',), ('rev-3',)]
 
639
        # Unfortunately, iter_lines_added_or_present_in_keys iterates the
 
640
        # result in random order (determined by the iteration order from a
 
641
        # set()), so we don't have any solid way to trigger whether data is
 
642
        # read before or after. However we tried to delete the middle node to
 
643
        # exercise the code well.
 
644
        # What we care about is that all lines are always yielded, but not
 
645
        # duplicated
 
646
        count = 0
 
647
        reload_lines = sorted(vf.iter_lines_added_or_present_in_keys(keys))
 
648
        self.assertEqual([1, 1, 0], reload_counter)
 
649
        # Now do it again, to make sure the result is equivalent
 
650
        plain_lines = sorted(vf.iter_lines_added_or_present_in_keys(keys))
 
651
        self.assertEqual([1, 1, 0], reload_counter) # No extra reloading
 
652
        self.assertEqual(plain_lines, reload_lines)
 
653
        self.assertEqual(21, len(plain_lines))
 
654
        # Now delete all pack files, and see that we raise the right error
 
655
        for trans, name in vf._access._indices.itervalues():
 
656
            trans.delete(name)
 
657
        self.assertListRaises(errors.NoSuchFile,
 
658
            vf.iter_lines_added_or_present_in_keys, keys)
 
659
        self.assertEqual([2, 1, 1], reload_counter)
 
660
 
 
661
    def test_get_record_stream_yields_disk_sorted_order(self):
 
662
        # if we get 'unordered' pick a semi-optimal order for reading. The
 
663
        # order should be grouped by pack file, and then by position in file
 
664
        repo = self.make_repository('test', format='pack-0.92')
 
665
        repo.lock_write()
 
666
        self.addCleanup(repo.unlock)
 
667
        repo.start_write_group()
 
668
        vf = repo.texts
 
669
        vf.add_lines(('f-id', 'rev-5'), [('f-id', 'rev-4')], ['lines\n'])
 
670
        vf.add_lines(('f-id', 'rev-1'), [], ['lines\n'])
 
671
        vf.add_lines(('f-id', 'rev-2'), [('f-id', 'rev-1')], ['lines\n'])
 
672
        repo.commit_write_group()
 
673
        # We inserted them as rev-5, rev-1, rev-2, we should get them back in
 
674
        # the same order
 
675
        stream = vf.get_record_stream([('f-id', 'rev-1'), ('f-id', 'rev-5'),
 
676
                                       ('f-id', 'rev-2')], 'unordered', False)
 
677
        keys = [r.key for r in stream]
 
678
        self.assertEqual([('f-id', 'rev-5'), ('f-id', 'rev-1'),
 
679
                          ('f-id', 'rev-2')], keys)
 
680
        repo.start_write_group()
 
681
        vf.add_lines(('f-id', 'rev-4'), [('f-id', 'rev-3')], ['lines\n'])
 
682
        vf.add_lines(('f-id', 'rev-3'), [('f-id', 'rev-2')], ['lines\n'])
 
683
        vf.add_lines(('f-id', 'rev-6'), [('f-id', 'rev-5')], ['lines\n'])
 
684
        repo.commit_write_group()
 
685
        # Request in random order, to make sure the output order isn't based on
 
686
        # the request
 
687
        request_keys = set(('f-id', 'rev-%d' % i) for i in range(1, 7))
 
688
        stream = vf.get_record_stream(request_keys, 'unordered', False)
 
689
        keys = [r.key for r in stream]
 
690
        # We want to get the keys back in disk order, but it doesn't matter
 
691
        # which pack we read from first. So this can come back in 2 orders
 
692
        alt1 = [('f-id', 'rev-%d' % i) for i in [4, 3, 6, 5, 1, 2]]
 
693
        alt2 = [('f-id', 'rev-%d' % i) for i in [5, 1, 2, 4, 3, 6]]
 
694
        if keys != alt1 and keys != alt2:
 
695
            self.fail('Returned key order did not match either expected order.'
 
696
                      ' expected %s or %s, not %s'
 
697
                      % (alt1, alt2, keys))
 
698
 
367
699
 
368
700
class LowLevelKnitDataTests(TestCase):
369
701
 
374
706
        gz_file.close()
375
707
        return sio.getvalue()
376
708
 
 
709
    def make_multiple_records(self):
 
710
        """Create the content for multiple records."""
 
711
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
712
        total_txt = []
 
713
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
 
714
                                        'foo\n'
 
715
                                        'bar\n'
 
716
                                        'end rev-id-1\n'
 
717
                                        % (sha1sum,))
 
718
        record_1 = (0, len(gz_txt), sha1sum)
 
719
        total_txt.append(gz_txt)
 
720
        sha1sum = osutils.sha('baz\n').hexdigest()
 
721
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
 
722
                                        'baz\n'
 
723
                                        'end rev-id-2\n'
 
724
                                        % (sha1sum,))
 
725
        record_2 = (record_1[1], len(gz_txt), sha1sum)
 
726
        total_txt.append(gz_txt)
 
727
        return total_txt, record_1, record_2
 
728
 
377
729
    def test_valid_knit_data(self):
378
 
        sha1sum = sha.new('foo\nbar\n').hexdigest()
 
730
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
379
731
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
380
732
                                        'foo\n'
381
733
                                        'bar\n'
393
745
        raw_contents = list(knit._read_records_iter_raw(records))
394
746
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
395
747
 
 
748
    def test_multiple_records_valid(self):
 
749
        total_txt, record_1, record_2 = self.make_multiple_records()
 
750
        transport = MockTransport([''.join(total_txt)])
 
751
        access = _KnitKeyAccess(transport, ConstantMapper('filename'))
 
752
        knit = KnitVersionedFiles(None, access)
 
753
        records = [(('rev-id-1',), (('rev-id-1',), record_1[0], record_1[1])),
 
754
                   (('rev-id-2',), (('rev-id-2',), record_2[0], record_2[1]))]
 
755
 
 
756
        contents = list(knit._read_records_iter(records))
 
757
        self.assertEqual([(('rev-id-1',), ['foo\n', 'bar\n'], record_1[2]),
 
758
                          (('rev-id-2',), ['baz\n'], record_2[2])],
 
759
                         contents)
 
760
 
 
761
        raw_contents = list(knit._read_records_iter_raw(records))
 
762
        self.assertEqual([(('rev-id-1',), total_txt[0], record_1[2]),
 
763
                          (('rev-id-2',), total_txt[1], record_2[2])],
 
764
                         raw_contents)
 
765
 
396
766
    def test_not_enough_lines(self):
397
 
        sha1sum = sha.new('foo\n').hexdigest()
 
767
        sha1sum = osutils.sha('foo\n').hexdigest()
398
768
        # record says 2 lines data says 1
399
769
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
400
770
                                        'foo\n'
412
782
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
413
783
 
414
784
    def test_too_many_lines(self):
415
 
        sha1sum = sha.new('foo\nbar\n').hexdigest()
 
785
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
416
786
        # record says 1 lines data says 2
417
787
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
418
788
                                        'foo\n'
431
801
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
432
802
 
433
803
    def test_mismatched_version_id(self):
434
 
        sha1sum = sha.new('foo\nbar\n').hexdigest()
 
804
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
435
805
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
436
806
                                        'foo\n'
437
807
                                        'bar\n'
450
820
            knit._read_records_iter_raw(records))
451
821
 
452
822
    def test_uncompressed_data(self):
453
 
        sha1sum = sha.new('foo\nbar\n').hexdigest()
 
823
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
454
824
        txt = ('version rev-id-1 2 %s\n'
455
825
               'foo\n'
456
826
               'bar\n'
470
840
            knit._read_records_iter_raw(records))
471
841
 
472
842
    def test_corrupted_data(self):
473
 
        sha1sum = sha.new('foo\nbar\n').hexdigest()
 
843
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
474
844
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
475
845
                                        'foo\n'
476
846
                                        'bar\n'
720
1090
            call[1][1].getvalue())
721
1091
        self.assertEqual({'create_parent_dir': True}, call[2])
722
1092
 
 
1093
    def assertTotalBuildSize(self, size, keys, positions):
 
1094
        self.assertEqual(size,
 
1095
                         knit._get_total_build_size(None, keys, positions))
 
1096
 
 
1097
    def test__get_total_build_size(self):
 
1098
        positions = {
 
1099
            ('a',): (('fulltext', False), (('a',), 0, 100), None),
 
1100
            ('b',): (('line-delta', False), (('b',), 100, 21), ('a',)),
 
1101
            ('c',): (('line-delta', False), (('c',), 121, 35), ('b',)),
 
1102
            ('d',): (('line-delta', False), (('d',), 156, 12), ('b',)),
 
1103
            }
 
1104
        self.assertTotalBuildSize(100, [('a',)], positions)
 
1105
        self.assertTotalBuildSize(121, [('b',)], positions)
 
1106
        # c needs both a & b
 
1107
        self.assertTotalBuildSize(156, [('c',)], positions)
 
1108
        # we shouldn't count 'b' twice
 
1109
        self.assertTotalBuildSize(156, [('b',), ('c',)], positions)
 
1110
        self.assertTotalBuildSize(133, [('d',)], positions)
 
1111
        self.assertTotalBuildSize(168, [('c',), ('d',)], positions)
 
1112
 
723
1113
    def test_get_position(self):
724
1114
        transport = MockTransport([
725
1115
            _KndxIndex.HEADER,
866
1256
            else:
867
1257
                raise
868
1258
 
 
1259
    def test_scan_unvalidated_index_not_implemented(self):
 
1260
        transport = MockTransport()
 
1261
        index = self.get_knit_index(transport, 'filename', 'r')
 
1262
        self.assertRaises(
 
1263
            NotImplementedError, index.scan_unvalidated_index,
 
1264
            'dummy graph_index')
 
1265
        self.assertRaises(
 
1266
            NotImplementedError, index.get_missing_compression_parents)
 
1267
 
869
1268
    def test_short_line(self):
870
1269
        transport = MockTransport([
871
1270
            _KndxIndex.HEADER,
922
1321
        return make_file_factory(annotate, mapper)(self.get_transport())
923
1322
 
924
1323
 
 
1324
class TestBadShaError(KnitTests):
 
1325
    """Tests for handling of sha errors."""
 
1326
 
 
1327
    def test_sha_exception_has_text(self):
 
1328
        # having the failed text included in the error allows for recovery.
 
1329
        source = self.make_test_knit()
 
1330
        target = self.make_test_knit(name="target")
 
1331
        if not source._max_delta_chain:
 
1332
            raise TestNotApplicable(
 
1333
                "cannot get delta-caused sha failures without deltas.")
 
1334
        # create a basis
 
1335
        basis = ('basis',)
 
1336
        broken = ('broken',)
 
1337
        source.add_lines(basis, (), ['foo\n'])
 
1338
        source.add_lines(broken, (basis,), ['foo\n', 'bar\n'])
 
1339
        # Seed target with a bad basis text
 
1340
        target.add_lines(basis, (), ['gam\n'])
 
1341
        target.insert_record_stream(
 
1342
            source.get_record_stream([broken], 'unordered', False))
 
1343
        err = self.assertRaises(errors.KnitCorrupt,
 
1344
            target.get_record_stream([broken], 'unordered', True
 
1345
            ).next().get_bytes_as, 'chunked')
 
1346
        self.assertEqual(['gam\n', 'bar\n'], err.content)
 
1347
        # Test for formatting with live data
 
1348
        self.assertStartsWith(str(err), "Knit ")
 
1349
 
 
1350
 
925
1351
class TestKnitIndex(KnitTests):
926
1352
 
927
1353
    def test_add_versions_dictionary_compresses(self):
1127
1553
            [('parent',)])])
1128
1554
        # but neither should have added data:
1129
1555
        self.assertEqual([[], [], [], []], self.caught_entries)
1130
 
        
 
1556
 
1131
1557
    def test_add_version_different_dup(self):
1132
1558
        index = self.two_graph_index(deltas=True, catch_adds=True)
1133
1559
        # change options
1134
1560
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1135
 
            [(('tip',), 'no-eol,line-delta', (None, 0, 100), [('parent',)])])
1136
 
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1137
 
            [(('tip',), 'line-delta,no-eol', (None, 0, 100), [('parent',)])])
 
1561
            [(('tip',), 'line-delta', (None, 0, 100), [('parent',)])])
1138
1562
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1139
1563
            [(('tip',), 'fulltext', (None, 0, 100), [('parent',)])])
1140
1564
        # parents
1141
1565
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1142
1566
            [(('tip',), 'fulltext,no-eol', (None, 0, 100), [])])
1143
1567
        self.assertEqual([], self.caught_entries)
1144
 
        
 
1568
 
1145
1569
    def test_add_versions_nodeltas(self):
1146
1570
        index = self.two_graph_index(catch_adds=True)
1147
1571
        index.add_records([
1189
1613
            [('parent',)])])
1190
1614
        # but neither should have added data.
1191
1615
        self.assertEqual([[], [], [], []], self.caught_entries)
1192
 
        
 
1616
 
1193
1617
    def test_add_versions_different_dup(self):
1194
1618
        index = self.two_graph_index(deltas=True, catch_adds=True)
1195
1619
        # change options
1196
1620
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1197
 
            [(('tip',), 'no-eol,line-delta', (None, 0, 100), [('parent',)])])
1198
 
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1199
 
            [(('tip',), 'line-delta,no-eol', (None, 0, 100), [('parent',)])])
 
1621
            [(('tip',), 'line-delta', (None, 0, 100), [('parent',)])])
1200
1622
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1201
1623
            [(('tip',), 'fulltext', (None, 0, 100), [('parent',)])])
1202
1624
        # parents
1205
1627
        # change options in the second record
1206
1628
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1207
1629
            [(('tip',), 'fulltext,no-eol', (None, 0, 100), [('parent',)]),
1208
 
             (('tip',), 'no-eol,line-delta', (None, 0, 100), [('parent',)])])
 
1630
             (('tip',), 'line-delta', (None, 0, 100), [('parent',)])])
1209
1631
        self.assertEqual([], self.caught_entries)
1210
1632
 
 
1633
    def make_g_index_missing_compression_parent(self):
 
1634
        graph_index = self.make_g_index('missing_comp', 2,
 
1635
            [(('tip', ), ' 100 78',
 
1636
              ([('missing-parent', ), ('ghost', )], [('missing-parent', )]))])
 
1637
        return graph_index
 
1638
 
 
1639
    def make_g_index_missing_parent(self):
 
1640
        graph_index = self.make_g_index('missing_parent', 2,
 
1641
            [(('parent', ), ' 100 78', ([], [])),
 
1642
             (('tip', ), ' 100 78',
 
1643
              ([('parent', ), ('missing-parent', )], [('parent', )])),
 
1644
              ])
 
1645
        return graph_index
 
1646
 
 
1647
    def make_g_index_no_external_refs(self):
 
1648
        graph_index = self.make_g_index('no_external_refs', 2,
 
1649
            [(('rev', ), ' 100 78',
 
1650
              ([('parent', ), ('ghost', )], []))])
 
1651
        return graph_index
 
1652
 
 
1653
    def test_add_good_unvalidated_index(self):
 
1654
        unvalidated = self.make_g_index_no_external_refs()
 
1655
        combined = CombinedGraphIndex([unvalidated])
 
1656
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1657
        index.scan_unvalidated_index(unvalidated)
 
1658
        self.assertEqual(frozenset(), index.get_missing_compression_parents())
 
1659
 
 
1660
    def test_add_missing_compression_parent_unvalidated_index(self):
 
1661
        unvalidated = self.make_g_index_missing_compression_parent()
 
1662
        combined = CombinedGraphIndex([unvalidated])
 
1663
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1664
        index.scan_unvalidated_index(unvalidated)
 
1665
        # This also checks that its only the compression parent that is
 
1666
        # examined, otherwise 'ghost' would also be reported as a missing
 
1667
        # parent.
 
1668
        self.assertEqual(
 
1669
            frozenset([('missing-parent',)]),
 
1670
            index.get_missing_compression_parents())
 
1671
 
 
1672
    def test_add_missing_noncompression_parent_unvalidated_index(self):
 
1673
        unvalidated = self.make_g_index_missing_parent()
 
1674
        combined = CombinedGraphIndex([unvalidated])
 
1675
        index = _KnitGraphIndex(combined, lambda: True, deltas=True,
 
1676
            track_external_parent_refs=True)
 
1677
        index.scan_unvalidated_index(unvalidated)
 
1678
        self.assertEqual(
 
1679
            frozenset([('missing-parent',)]), index.get_missing_parents())
 
1680
 
 
1681
    def test_track_external_parent_refs(self):
 
1682
        g_index = self.make_g_index('empty', 2, [])
 
1683
        combined = CombinedGraphIndex([g_index])
 
1684
        index = _KnitGraphIndex(combined, lambda: True, deltas=True,
 
1685
            add_callback=self.catch_add, track_external_parent_refs=True)
 
1686
        self.caught_entries = []
 
1687
        index.add_records([
 
1688
            (('new-key',), 'fulltext,no-eol', (None, 50, 60),
 
1689
             [('parent-1',), ('parent-2',)])])
 
1690
        self.assertEqual(
 
1691
            frozenset([('parent-1',), ('parent-2',)]),
 
1692
            index.get_missing_parents())
 
1693
 
 
1694
    def test_add_unvalidated_index_with_present_external_references(self):
 
1695
        index = self.two_graph_index(deltas=True)
 
1696
        # Ugly hack to get at one of the underlying GraphIndex objects that
 
1697
        # two_graph_index built.
 
1698
        unvalidated = index._graph_index._indices[1]
 
1699
        # 'parent' is an external ref of _indices[1] (unvalidated), but is
 
1700
        # present in _indices[0].
 
1701
        index.scan_unvalidated_index(unvalidated)
 
1702
        self.assertEqual(frozenset(), index.get_missing_compression_parents())
 
1703
 
 
1704
    def make_new_missing_parent_g_index(self, name):
 
1705
        missing_parent = name + '-missing-parent'
 
1706
        graph_index = self.make_g_index(name, 2,
 
1707
            [((name + 'tip', ), ' 100 78',
 
1708
              ([(missing_parent, ), ('ghost', )], [(missing_parent, )]))])
 
1709
        return graph_index
 
1710
 
 
1711
    def test_add_mulitiple_unvalidated_indices_with_missing_parents(self):
 
1712
        g_index_1 = self.make_new_missing_parent_g_index('one')
 
1713
        g_index_2 = self.make_new_missing_parent_g_index('two')
 
1714
        combined = CombinedGraphIndex([g_index_1, g_index_2])
 
1715
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1716
        index.scan_unvalidated_index(g_index_1)
 
1717
        index.scan_unvalidated_index(g_index_2)
 
1718
        self.assertEqual(
 
1719
            frozenset([('one-missing-parent',), ('two-missing-parent',)]),
 
1720
            index.get_missing_compression_parents())
 
1721
 
 
1722
    def test_add_mulitiple_unvalidated_indices_with_mutual_dependencies(self):
 
1723
        graph_index_a = self.make_g_index('one', 2,
 
1724
            [(('parent-one', ), ' 100 78', ([('non-compression-parent',)], [])),
 
1725
             (('child-of-two', ), ' 100 78',
 
1726
              ([('parent-two',)], [('parent-two',)]))])
 
1727
        graph_index_b = self.make_g_index('two', 2,
 
1728
            [(('parent-two', ), ' 100 78', ([('non-compression-parent',)], [])),
 
1729
             (('child-of-one', ), ' 100 78',
 
1730
              ([('parent-one',)], [('parent-one',)]))])
 
1731
        combined = CombinedGraphIndex([graph_index_a, graph_index_b])
 
1732
        index = _KnitGraphIndex(combined, lambda: True, deltas=True)
 
1733
        index.scan_unvalidated_index(graph_index_a)
 
1734
        index.scan_unvalidated_index(graph_index_b)
 
1735
        self.assertEqual(
 
1736
            frozenset([]), index.get_missing_compression_parents())
 
1737
 
1211
1738
 
1212
1739
class TestNoParentsGraphIndexKnit(KnitTests):
1213
1740
    """Tests for knits using _KnitGraphIndex with no parents."""
1221
1748
        size = trans.put_file(name, stream)
1222
1749
        return GraphIndex(trans, name, size)
1223
1750
 
 
1751
    def test_add_good_unvalidated_index(self):
 
1752
        unvalidated = self.make_g_index('unvalidated')
 
1753
        combined = CombinedGraphIndex([unvalidated])
 
1754
        index = _KnitGraphIndex(combined, lambda: True, parents=False)
 
1755
        index.scan_unvalidated_index(unvalidated)
 
1756
        self.assertEqual(frozenset(),
 
1757
            index.get_missing_compression_parents())
 
1758
 
1224
1759
    def test_parents_deltas_incompatible(self):
1225
1760
        index = CombinedGraphIndex([])
1226
1761
        self.assertRaises(errors.KnitError, _KnitGraphIndex, lambda:True,
1307
1842
        index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000), [])])
1308
1843
        # but neither should have added data.
1309
1844
        self.assertEqual([[], [], [], []], self.caught_entries)
1310
 
        
 
1845
 
1311
1846
    def test_add_version_different_dup(self):
1312
1847
        index = self.two_graph_index(catch_adds=True)
1313
1848
        # change options
1321
1856
        self.assertRaises(errors.KnitCorrupt, index.add_records,
1322
1857
            [(('tip',), 'fulltext,no-eol', (None, 0, 100), [('parent',)])])
1323
1858
        self.assertEqual([], self.caught_entries)
1324
 
        
 
1859
 
1325
1860
    def test_add_versions(self):
1326
1861
        index = self.two_graph_index(catch_adds=True)
1327
1862
        index.add_records([
1359
1894
        index.add_records([(('tip',), 'fulltext,no-eol', (None, 0, 1000), [])])
1360
1895
        # but neither should have added data.
1361
1896
        self.assertEqual([[], [], [], []], self.caught_entries)
1362
 
        
 
1897
 
1363
1898
    def test_add_versions_different_dup(self):
1364
1899
        index = self.two_graph_index(catch_adds=True)
1365
1900
        # change options
1379
1914
        self.assertEqual([], self.caught_entries)
1380
1915
 
1381
1916
 
 
1917
class TestKnitVersionedFiles(KnitTests):
 
1918
 
 
1919
    def assertGroupKeysForIo(self, exp_groups, keys, non_local_keys,
 
1920
                             positions, _min_buffer_size=None):
 
1921
        kvf = self.make_test_knit()
 
1922
        if _min_buffer_size is None:
 
1923
            _min_buffer_size = knit._STREAM_MIN_BUFFER_SIZE
 
1924
        self.assertEqual(exp_groups, kvf._group_keys_for_io(keys,
 
1925
                                        non_local_keys, positions,
 
1926
                                        _min_buffer_size=_min_buffer_size))
 
1927
 
 
1928
    def assertSplitByPrefix(self, expected_map, expected_prefix_order,
 
1929
                            keys):
 
1930
        split, prefix_order = KnitVersionedFiles._split_by_prefix(keys)
 
1931
        self.assertEqual(expected_map, split)
 
1932
        self.assertEqual(expected_prefix_order, prefix_order)
 
1933
 
 
1934
    def test__group_keys_for_io(self):
 
1935
        ft_detail = ('fulltext', False)
 
1936
        ld_detail = ('line-delta', False)
 
1937
        f_a = ('f', 'a')
 
1938
        f_b = ('f', 'b')
 
1939
        f_c = ('f', 'c')
 
1940
        g_a = ('g', 'a')
 
1941
        g_b = ('g', 'b')
 
1942
        g_c = ('g', 'c')
 
1943
        positions = {
 
1944
            f_a: (ft_detail, (f_a, 0, 100), None),
 
1945
            f_b: (ld_detail, (f_b, 100, 21), f_a),
 
1946
            f_c: (ld_detail, (f_c, 180, 15), f_b),
 
1947
            g_a: (ft_detail, (g_a, 121, 35), None),
 
1948
            g_b: (ld_detail, (g_b, 156, 12), g_a),
 
1949
            g_c: (ld_detail, (g_c, 195, 13), g_a),
 
1950
            }
 
1951
        self.assertGroupKeysForIo([([f_a], set())],
 
1952
                                  [f_a], [], positions)
 
1953
        self.assertGroupKeysForIo([([f_a], set([f_a]))],
 
1954
                                  [f_a], [f_a], positions)
 
1955
        self.assertGroupKeysForIo([([f_a, f_b], set([]))],
 
1956
                                  [f_a, f_b], [], positions)
 
1957
        self.assertGroupKeysForIo([([f_a, f_b], set([f_b]))],
 
1958
                                  [f_a, f_b], [f_b], positions)
 
1959
        self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
 
1960
                                  [f_a, g_a, f_b, g_b], [], positions)
 
1961
        self.assertGroupKeysForIo([([f_a, f_b, g_a, g_b], set())],
 
1962
                                  [f_a, g_a, f_b, g_b], [], positions,
 
1963
                                  _min_buffer_size=150)
 
1964
        self.assertGroupKeysForIo([([f_a, f_b], set()), ([g_a, g_b], set())],
 
1965
                                  [f_a, g_a, f_b, g_b], [], positions,
 
1966
                                  _min_buffer_size=100)
 
1967
        self.assertGroupKeysForIo([([f_c], set()), ([g_b], set())],
 
1968
                                  [f_c, g_b], [], positions,
 
1969
                                  _min_buffer_size=125)
 
1970
        self.assertGroupKeysForIo([([g_b, f_c], set())],
 
1971
                                  [g_b, f_c], [], positions,
 
1972
                                  _min_buffer_size=125)
 
1973
 
 
1974
    def test__split_by_prefix(self):
 
1975
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
1976
                                  'g': [('g', 'b'), ('g', 'a')],
 
1977
                                 }, ['f', 'g'],
 
1978
                                 [('f', 'a'), ('g', 'b'),
 
1979
                                  ('g', 'a'), ('f', 'b')])
 
1980
 
 
1981
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
1982
                                  'g': [('g', 'b'), ('g', 'a')],
 
1983
                                 }, ['f', 'g'],
 
1984
                                 [('f', 'a'), ('f', 'b'),
 
1985
                                  ('g', 'b'), ('g', 'a')])
 
1986
 
 
1987
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
1988
                                  'g': [('g', 'b'), ('g', 'a')],
 
1989
                                 }, ['f', 'g'],
 
1990
                                 [('f', 'a'), ('f', 'b'),
 
1991
                                  ('g', 'b'), ('g', 'a')])
 
1992
 
 
1993
        self.assertSplitByPrefix({'f': [('f', 'a'), ('f', 'b')],
 
1994
                                  'g': [('g', 'b'), ('g', 'a')],
 
1995
                                  '': [('a',), ('b',)]
 
1996
                                 }, ['f', 'g', ''],
 
1997
                                 [('f', 'a'), ('g', 'b'),
 
1998
                                  ('a',), ('b',),
 
1999
                                  ('g', 'a'), ('f', 'b')])
 
2000
 
 
2001
 
1382
2002
class TestStacking(KnitTests):
1383
2003
 
1384
2004
    def get_basis_and_test_knit(self):
1410
2030
        basis.calls = []
1411
2031
        test.add_lines(key_cross_border, (key_basis,), ['foo\n'])
1412
2032
        self.assertEqual('fulltext', test._index.get_method(key_cross_border))
1413
 
        self.assertEqual([("get_parent_map", set([key_basis]))], basis.calls)
 
2033
        # we don't even need to look at the basis to see that this should be
 
2034
        # stored as a fulltext
 
2035
        self.assertEqual([], basis.calls)
1414
2036
        # Subsequent adds do delta.
1415
2037
        basis.calls = []
1416
2038
        test.add_lines(key_delta, (key_cross_border,), ['foo\n'])
1437
2059
        # self.assertEqual([("annotate", key_basis)], basis.calls)
1438
2060
        self.assertEqual([('get_parent_map', set([key_basis])),
1439
2061
            ('get_parent_map', set([key_basis])),
1440
 
            ('get_parent_map', set([key_basis])),
1441
2062
            ('get_record_stream', [key_basis], 'unordered', True)],
1442
2063
            basis.calls)
1443
2064
 
1444
2065
    def test_check(self):
1445
2066
        # At the moment checking a stacked knit does implicitly check the
1446
 
        # fallback files.  
 
2067
        # fallback files.
1447
2068
        basis, test = self.get_basis_and_test_knit()
1448
2069
        test.check()
1449
2070
 
1541
2162
                True).next()
1542
2163
            self.assertEqual(record.key, result[0])
1543
2164
            self.assertEqual(record.sha1, result[1])
1544
 
            self.assertEqual(record.storage_kind, result[2])
 
2165
            # We used to check that the storage kind matched, but actually it
 
2166
            # depends on whether it was sourced from the basis, or in a single
 
2167
            # group, because asking for full texts returns proxy objects to a
 
2168
            # _ContentMapGenerator object; so checking the kind is unneeded.
1545
2169
            self.assertEqual(record.get_bytes_as('fulltext'), result[3])
1546
2170
        # It's not strictly minimal, but it seems reasonable for now for it to
1547
2171
        # ask which fallbacks have which parents.
1641
2265
        key_basis = ('bar',)
1642
2266
        key_missing = ('missing',)
1643
2267
        test.add_lines(key, (), ['foo\n'])
1644
 
        key_sha1sum = sha.new('foo\n').hexdigest()
 
2268
        key_sha1sum = osutils.sha('foo\n').hexdigest()
1645
2269
        sha1s = test.get_sha1s([key])
1646
2270
        self.assertEqual({key: key_sha1sum}, sha1s)
1647
2271
        self.assertEqual([], basis.calls)
1649
2273
        # directly (rather than via text reconstruction) so that remote servers
1650
2274
        # etc don't have to answer with full content.
1651
2275
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
1652
 
        basis_sha1sum = sha.new('foo\nbar\n').hexdigest()
 
2276
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
1653
2277
        basis.calls = []
1654
2278
        sha1s = test.get_sha1s([key, key_missing, key_basis])
1655
2279
        self.assertEqual({key: key_sha1sum,
1671
2295
        source.add_lines(key_delta, (key_basis,), ['bar\n'])
1672
2296
        stream = source.get_record_stream([key_delta], 'unordered', False)
1673
2297
        test.insert_record_stream(stream)
1674
 
        self.assertEqual([("get_parent_map", set([key_basis]))],
 
2298
        # XXX: this does somewhat too many calls in making sure of whether it
 
2299
        # has to recreate the full text.
 
2300
        self.assertEqual([("get_parent_map", set([key_basis])),
 
2301
             ('get_parent_map', set([key_basis])),
 
2302
             ('get_record_stream', [key_basis], 'unordered', True)],
1675
2303
            basis.calls)
1676
2304
        self.assertEqual({key_delta:(key_basis,)},
1677
2305
            test.get_parent_map([key_delta]))
1680
2308
 
1681
2309
    def test_iter_lines_added_or_present_in_keys(self):
1682
2310
        # Lines from the basis are returned, and lines for a given key are only
1683
 
        # returned once. 
 
2311
        # returned once.
1684
2312
        key1 = ('foo1',)
1685
2313
        key2 = ('foo2',)
1686
2314
        # all sources are asked for keys:
1738
2366
        test.add_mpdiffs([(key_delta, (key_basis,),
1739
2367
            source.get_sha1s([key_delta])[key_delta], diffs[0])])
1740
2368
        self.assertEqual([("get_parent_map", set([key_basis])),
1741
 
            ('get_record_stream', [key_basis], 'unordered', True),
1742
 
            ('get_parent_map', set([key_basis]))],
 
2369
            ('get_record_stream', [key_basis], 'unordered', True),],
1743
2370
            basis.calls)
1744
2371
        self.assertEqual({key_delta:(key_basis,)},
1745
2372
            test.get_parent_map([key_delta]))
1764
2391
                multiparent.NewText(['foo\n']),
1765
2392
                multiparent.ParentText(1, 0, 2, 1)])],
1766
2393
            diffs)
1767
 
        self.assertEqual(4, len(basis.calls))
 
2394
        self.assertEqual(3, len(basis.calls))
1768
2395
        self.assertEqual([
1769
2396
            ("get_parent_map", set([key_left, key_right])),
1770
2397
            ("get_parent_map", set([key_left, key_right])),
1771
 
            ("get_parent_map", set([key_left, key_right])),
1772
2398
            ],
1773
 
            basis.calls[:3])
1774
 
        last_call = basis.calls[3]
 
2399
            basis.calls[:-1])
 
2400
        last_call = basis.calls[-1]
1775
2401
        self.assertEqual('get_record_stream', last_call[0])
1776
2402
        self.assertEqual(set([key_left, key_right]), set(last_call[1]))
1777
2403
        self.assertEqual('unordered', last_call[2])
1778
2404
        self.assertEqual(True, last_call[3])
 
2405
 
 
2406
 
 
2407
class TestNetworkBehaviour(KnitTests):
 
2408
    """Tests for getting data out of/into knits over the network."""
 
2409
 
 
2410
    def test_include_delta_closure_generates_a_knit_delta_closure(self):
 
2411
        vf = self.make_test_knit(name='test')
 
2412
        # put in three texts, giving ft, delta, delta
 
2413
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2414
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2415
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2416
        # But heuristics could interfere, so check what happened:
 
2417
        self.assertEqual(['knit-ft-gz', 'knit-delta-gz', 'knit-delta-gz'],
 
2418
            [record.storage_kind for record in
 
2419
             vf.get_record_stream([('base',), ('d1',), ('d2',)],
 
2420
                'topological', False)])
 
2421
        # generate a stream of just the deltas include_delta_closure=True,
 
2422
        # serialise to the network, and check that we get a delta closure on the wire.
 
2423
        stream = vf.get_record_stream([('d1',), ('d2',)], 'topological', True)
 
2424
        netb = [record.get_bytes_as(record.storage_kind) for record in stream]
 
2425
        # The first bytes should be a memo from _ContentMapGenerator, and the
 
2426
        # second bytes should be empty (because its a API proxy not something
 
2427
        # for wire serialisation.
 
2428
        self.assertEqual('', netb[1])
 
2429
        bytes = netb[0]
 
2430
        kind, line_end = network_bytes_to_kind_and_offset(bytes)
 
2431
        self.assertEqual('knit-delta-closure', kind)
 
2432
 
 
2433
 
 
2434
class TestContentMapGenerator(KnitTests):
 
2435
    """Tests for ContentMapGenerator"""
 
2436
 
 
2437
    def test_get_record_stream_gives_records(self):
 
2438
        vf = self.make_test_knit(name='test')
 
2439
        # put in three texts, giving ft, delta, delta
 
2440
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2441
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2442
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2443
        keys = [('d1',), ('d2',)]
 
2444
        generator = _VFContentMapGenerator(vf, keys,
 
2445
            global_map=vf.get_parent_map(keys))
 
2446
        for record in generator.get_record_stream():
 
2447
            if record.key == ('d1',):
 
2448
                self.assertEqual('d1\n', record.get_bytes_as('fulltext'))
 
2449
            else:
 
2450
                self.assertEqual('d2\n', record.get_bytes_as('fulltext'))
 
2451
 
 
2452
    def test_get_record_stream_kinds_are_raw(self):
 
2453
        vf = self.make_test_knit(name='test')
 
2454
        # put in three texts, giving ft, delta, delta
 
2455
        vf.add_lines(('base',), (), ['base\n', 'content\n'])
 
2456
        vf.add_lines(('d1',), (('base',),), ['d1\n'])
 
2457
        vf.add_lines(('d2',), (('d1',),), ['d2\n'])
 
2458
        keys = [('base',), ('d1',), ('d2',)]
 
2459
        generator = _VFContentMapGenerator(vf, keys,
 
2460
            global_map=vf.get_parent_map(keys))
 
2461
        kinds = {('base',): 'knit-delta-closure',
 
2462
            ('d1',): 'knit-delta-closure-ref',
 
2463
            ('d2',): 'knit-delta-closure-ref',
 
2464
            }
 
2465
        for record in generator.get_record_stream():
 
2466
            self.assertEqual(kinds[record.key], record.storage_kind)