← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_btree_index.py

  • Committer: Robert Collins
  • Date: 2010-07-04 06:22:11 UTC
  • mto: This revision was merged to the branch mainline in revision 5332.
  • Revision ID: robertc@robertcollins.net-20100704062211-tk9hw6bnsn5x47fm
``bzrlib.lsprof.profile`` will no longer silently generate bad threaded
profiles when concurrent profile requests are made. Instead the profile
requests will be serialised. Reentrant requests will now deadlock.
(Robert Collins)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_btree_index.py
27
27
    lru_cache,
28
28
    osutils,
29
29
    tests,
 
30
    transport,
30
31
    )
31
32
from bzrlib.tests import (
32
33
    TestCaseWithTransport,
34
35
    multiply_tests,
35
36
    split_suite_by_condition,
36
37
    )
37
 
from bzrlib.transport import get_transport
38
38
 
39
39
 
40
40
def load_tests(standard_tests, module, loader):
280
280
 
281
281
        for node in nodes:
282
282
            builder.add_node(*node)
283
 
        transport = get_transport('trace+' + self.get_url(''))
284
 
        size = transport.put_file('index', self.time(builder.finish))
 
283
        t = transport.get_transport('trace+' + self.get_url(''))
 
284
        size = t.put_file('index', self.time(builder.finish))
285
285
        del builder
286
 
        index = btree_index.BTreeGraphIndex(transport, 'index', size)
 
286
        index = btree_index.BTreeGraphIndex(t, 'index', size)
287
287
        # Seed the metadata, we're using internal calls now.
288
288
        index.key_count()
289
289
        self.assertEqual(3, len(index._row_lengths),
409
409
        self.assertEqual(None, builder._backing_indices[2])
410
410
        self.assertEqual(16, builder._backing_indices[3].key_count())
411
411
        # Now finish, and check we got a correctly ordered tree
412
 
        transport = self.get_transport('')
413
 
        size = transport.put_file('index', builder.finish())
414
 
        index = btree_index.BTreeGraphIndex(transport, 'index', size)
 
412
        t = self.get_transport('')
 
413
        size = t.put_file('index', builder.finish())
 
414
        index = btree_index.BTreeGraphIndex(t, 'index', size)
415
415
        nodes = list(index.iter_all_entries())
416
416
        self.assertEqual(sorted(nodes), nodes)
417
417
        self.assertEqual(16, len(nodes))
607
607
        for key, value, references in nodes:
608
608
            builder.add_node(key, value, references)
609
609
        stream = builder.finish()
610
 
        trans = get_transport('trace+' + self.get_url())
 
610
        trans = transport.get_transport('trace+' + self.get_url())
611
611
        size = trans.put_file('index', stream)
612
612
        return btree_index.BTreeGraphIndex(trans, 'index', size)
613
613
 
648
648
        self.assertEqual(0, len(index._leaf_node_cache))
649
649
 
650
650
    def test_trivial_constructor(self):
651
 
        transport = get_transport('trace+' + self.get_url(''))
652
 
        index = btree_index.BTreeGraphIndex(transport, 'index', None)
 
651
        t = transport.get_transport('trace+' + self.get_url(''))
 
652
        index = btree_index.BTreeGraphIndex(t, 'index', None)
653
653
        # Checks the page size at load, but that isn't logged yet.
654
 
        self.assertEqual([], transport._activity)
 
654
        self.assertEqual([], t._activity)
655
655
 
656
656
    def test_with_size_constructor(self):
657
 
        transport = get_transport('trace+' + self.get_url(''))
658
 
        index = btree_index.BTreeGraphIndex(transport, 'index', 1)
 
657
        t = transport.get_transport('trace+' + self.get_url(''))
 
658
        index = btree_index.BTreeGraphIndex(t, 'index', 1)
659
659
        # Checks the page size at load, but that isn't logged yet.
660
 
        self.assertEqual([], transport._activity)
 
660
        self.assertEqual([], t._activity)
661
661
 
662
662
    def test_empty_key_count_no_size(self):
663
663
        builder = btree_index.BTreeBuilder(key_elements=1, reference_lists=0)
664
 
        transport = get_transport('trace+' + self.get_url(''))
665
 
        transport.put_file('index', builder.finish())
666
 
        index = btree_index.BTreeGraphIndex(transport, 'index', None)
667
 
        del transport._activity[:]
668
 
        self.assertEqual([], transport._activity)
 
664
        t = transport.get_transport('trace+' + self.get_url(''))
 
665
        t.put_file('index', builder.finish())
 
666
        index = btree_index.BTreeGraphIndex(t, 'index', None)
 
667
        del t._activity[:]
 
668
        self.assertEqual([], t._activity)
669
669
        self.assertEqual(0, index.key_count())
670
670
        # The entire index should have been requested (as we generally have the
671
671
        # size available, and doing many small readvs is inappropriate).
672
672
        # We can't tell how much was actually read here, but - check the code.
673
 
        self.assertEqual([('get', 'index')], transport._activity)
 
673
        self.assertEqual([('get', 'index')], t._activity)
674
674
 
675
675
    def test_empty_key_count(self):
676
676
        builder = btree_index.BTreeBuilder(key_elements=1, reference_lists=0)
677
 
        transport = get_transport('trace+' + self.get_url(''))
678
 
        size = transport.put_file('index', builder.finish())
 
677
        t = transport.get_transport('trace+' + self.get_url(''))
 
678
        size = t.put_file('index', builder.finish())
679
679
        self.assertEqual(72, size)
680
 
        index = btree_index.BTreeGraphIndex(transport, 'index', size)
681
 
        del transport._activity[:]
682
 
        self.assertEqual([], transport._activity)
 
680
        index = btree_index.BTreeGraphIndex(t, 'index', size)
 
681
        del t._activity[:]
 
682
        self.assertEqual([], t._activity)
683
683
        self.assertEqual(0, index.key_count())
684
684
        # The entire index should have been read, as 4K > size
685
685
        self.assertEqual([('readv', 'index', [(0, 72)], False, None)],
686
 
            transport._activity)
 
686
                         t._activity)
687
687
 
688
688
    def test_non_empty_key_count_2_2(self):
689
689
        builder = btree_index.BTreeBuilder(key_elements=2, reference_lists=2)
690
690
        nodes = self.make_nodes(35, 2, 2)
691
691
        for node in nodes:
692
692
            builder.add_node(*node)
693
 
        transport = get_transport('trace+' + self.get_url(''))
694
 
        size = transport.put_file('index', builder.finish())
695
 
        index = btree_index.BTreeGraphIndex(transport, 'index', size)
696
 
        del transport._activity[:]
697
 
        self.assertEqual([], transport._activity)
 
693
        t = transport.get_transport('trace+' + self.get_url(''))
 
694
        size = t.put_file('index', builder.finish())
 
695
        index = btree_index.BTreeGraphIndex(t, 'index', size)
 
696
        del t._activity[:]
 
697
        self.assertEqual([], t._activity)
698
698
        self.assertEqual(70, index.key_count())
699
699
        # The entire index should have been read, as it is one page long.
700
700
        self.assertEqual([('readv', 'index', [(0, size)], False, None)],
701
 
            transport._activity)
 
701
            t._activity)
702
702
        self.assertEqual(1173, size)
703
703
 
704
704
    def test_with_offset_no_size(self):
722
722
 
723
723
    def test__read_nodes_no_size_one_page_reads_once(self):
724
724
        self.make_index(nodes=[(('key',), 'value', ())])
725
 
        trans = get_transport('trace+' + self.get_url())
 
725
        trans = transport.get_transport('trace+' + self.get_url())
726
726
        index = btree_index.BTreeGraphIndex(trans, 'index', None)
727
727
        del trans._activity[:]
728
728
        nodes = dict(index._read_nodes([0]))
736
736
        index.key_count()
737
737
        num_pages = index._row_offsets[-1]
738
738
        # Reopen with a traced transport and no size
739
 
        trans = get_transport('trace+' + self.get_url())
 
739
        trans = transport.get_transport('trace+' + self.get_url())
740
740
        index = btree_index.BTreeGraphIndex(trans, 'index', None)
741
741
        del trans._activity[:]
742
742
        nodes = dict(index._read_nodes([0]))
747
747
        nodes = self.make_nodes(160, 2, 2)
748
748
        for node in nodes:
749
749
            builder.add_node(*node)
750
 
        transport = get_transport('trace+' + self.get_url(''))
751
 
        size = transport.put_file('index', builder.finish())
 
750
        t = transport.get_transport('trace+' + self.get_url(''))
 
751
        size = t.put_file('index', builder.finish())
752
752
        self.assertEqual(17692, size)
753
 
        index = btree_index.BTreeGraphIndex(transport, 'index', size)
754
 
        del transport._activity[:]
755
 
        self.assertEqual([], transport._activity)
 
753
        index = btree_index.BTreeGraphIndex(t, 'index', size)
 
754
        del t._activity[:]
 
755
        self.assertEqual([], t._activity)
756
756
        self.assertEqual(320, index.key_count())
757
757
        # The entire index should not have been read.
758
758
        self.assertEqual([('readv', 'index', [(0, 4096)], False, None)],
759
 
            transport._activity)
 
759
                         t._activity)
760
760
 
761
761
    def test_validate_one_page(self):
762
762
        builder = btree_index.BTreeBuilder(key_elements=2, reference_lists=2)
763
763
        nodes = self.make_nodes(45, 2, 2)
764
764
        for node in nodes:
765
765
            builder.add_node(*node)
766
 
        transport = get_transport('trace+' + self.get_url(''))
767
 
        size = transport.put_file('index', builder.finish())
768
 
        index = btree_index.BTreeGraphIndex(transport, 'index', size)
769
 
        del transport._activity[:]
770
 
        self.assertEqual([], transport._activity)
 
766
        t = transport.get_transport('trace+' + self.get_url(''))
 
767
        size = t.put_file('index', builder.finish())
 
768
        index = btree_index.BTreeGraphIndex(t, 'index', size)
 
769
        del t._activity[:]
 
770
        self.assertEqual([], t._activity)
771
771
        index.validate()
772
772
        # The entire index should have been read linearly.
773
773
        self.assertEqual([('readv', 'index', [(0, size)], False, None)],
774
 
            transport._activity)
 
774
                         t._activity)
775
775
        self.assertEqual(1488, size)
776
776
 
777
777
    def test_validate_two_pages(self):
779
779
        nodes = self.make_nodes(80, 2, 2)
780
780
        for node in nodes:
781
781
            builder.add_node(*node)
782
 
        transport = get_transport('trace+' + self.get_url(''))
783
 
        size = transport.put_file('index', builder.finish())
 
782
        t = transport.get_transport('trace+' + self.get_url(''))
 
783
        size = t.put_file('index', builder.finish())
784
784
        # Root page, 2 leaf pages
785
785
        self.assertEqual(9339, size)
786
 
        index = btree_index.BTreeGraphIndex(transport, 'index', size)
787
 
        del transport._activity[:]
788
 
        self.assertEqual([], transport._activity)
 
786
        index = btree_index.BTreeGraphIndex(t, 'index', size)
 
787
        del t._activity[:]
 
788
        self.assertEqual([], t._activity)
789
789
        index.validate()
790
790
        # The entire index should have been read linearly.
791
 
        self.assertEqual([('readv', 'index', [(0, 4096)], False, None),
792
 
            ('readv', 'index', [(4096, 4096), (8192, 1147)], False, None)],
793
 
            transport._activity)
 
791
        self.assertEqual(
 
792
            [('readv', 'index', [(0, 4096)], False, None),
 
793
             ('readv', 'index', [(4096, 4096), (8192, 1147)], False, None)],
 
794
            t._activity)
794
795
        # XXX: TODO: write some badly-ordered nodes, and some pointers-to-wrong
795
796
        # node and make validate find them.
796
797
 
797
798
    def test_eq_ne(self):
798
799
        # two indices are equal when constructed with the same parameters:
799
 
        transport1 = get_transport('trace+' + self.get_url(''))
800
 
        transport2 = get_transport(self.get_url(''))
801
 
        self.assertTrue(
802
 
            btree_index.BTreeGraphIndex(transport1, 'index', None) ==
803
 
            btree_index.BTreeGraphIndex(transport1, 'index', None))
804
 
        self.assertTrue(
805
 
            btree_index.BTreeGraphIndex(transport1, 'index', 20) ==
806
 
            btree_index.BTreeGraphIndex(transport1, 'index', 20))
807
 
        self.assertFalse(
808
 
            btree_index.BTreeGraphIndex(transport1, 'index', 20) ==
809
 
            btree_index.BTreeGraphIndex(transport2, 'index', 20))
810
 
        self.assertFalse(
811
 
            btree_index.BTreeGraphIndex(transport1, 'inde1', 20) ==
812
 
            btree_index.BTreeGraphIndex(transport1, 'inde2', 20))
813
 
        self.assertFalse(
814
 
            btree_index.BTreeGraphIndex(transport1, 'index', 10) ==
815
 
            btree_index.BTreeGraphIndex(transport1, 'index', 20))
816
 
        self.assertFalse(
817
 
            btree_index.BTreeGraphIndex(transport1, 'index', None) !=
818
 
            btree_index.BTreeGraphIndex(transport1, 'index', None))
819
 
        self.assertFalse(
820
 
            btree_index.BTreeGraphIndex(transport1, 'index', 20) !=
821
 
            btree_index.BTreeGraphIndex(transport1, 'index', 20))
822
 
        self.assertTrue(
823
 
            btree_index.BTreeGraphIndex(transport1, 'index', 20) !=
824
 
            btree_index.BTreeGraphIndex(transport2, 'index', 20))
825
 
        self.assertTrue(
826
 
            btree_index.BTreeGraphIndex(transport1, 'inde1', 20) !=
827
 
            btree_index.BTreeGraphIndex(transport1, 'inde2', 20))
828
 
        self.assertTrue(
829
 
            btree_index.BTreeGraphIndex(transport1, 'index', 10) !=
830
 
            btree_index.BTreeGraphIndex(transport1, 'index', 20))
 
800
        t1 = transport.get_transport('trace+' + self.get_url(''))
 
801
        t2 = transport.get_transport(self.get_url(''))
 
802
        self.assertTrue(
 
803
            btree_index.BTreeGraphIndex(t1, 'index', None) ==
 
804
            btree_index.BTreeGraphIndex(t1, 'index', None))
 
805
        self.assertTrue(
 
806
            btree_index.BTreeGraphIndex(t1, 'index', 20) ==
 
807
            btree_index.BTreeGraphIndex(t1, 'index', 20))
 
808
        self.assertFalse(
 
809
            btree_index.BTreeGraphIndex(t1, 'index', 20) ==
 
810
            btree_index.BTreeGraphIndex(t2, 'index', 20))
 
811
        self.assertFalse(
 
812
            btree_index.BTreeGraphIndex(t1, 'inde1', 20) ==
 
813
            btree_index.BTreeGraphIndex(t1, 'inde2', 20))
 
814
        self.assertFalse(
 
815
            btree_index.BTreeGraphIndex(t1, 'index', 10) ==
 
816
            btree_index.BTreeGraphIndex(t1, 'index', 20))
 
817
        self.assertFalse(
 
818
            btree_index.BTreeGraphIndex(t1, 'index', None) !=
 
819
            btree_index.BTreeGraphIndex(t1, 'index', None))
 
820
        self.assertFalse(
 
821
            btree_index.BTreeGraphIndex(t1, 'index', 20) !=
 
822
            btree_index.BTreeGraphIndex(t1, 'index', 20))
 
823
        self.assertTrue(
 
824
            btree_index.BTreeGraphIndex(t1, 'index', 20) !=
 
825
            btree_index.BTreeGraphIndex(t2, 'index', 20))
 
826
        self.assertTrue(
 
827
            btree_index.BTreeGraphIndex(t1, 'inde1', 20) !=
 
828
            btree_index.BTreeGraphIndex(t1, 'inde2', 20))
 
829
        self.assertTrue(
 
830
            btree_index.BTreeGraphIndex(t1, 'index', 10) !=
 
831
            btree_index.BTreeGraphIndex(t1, 'index', 20))
831
832
 
832
833
    def test_iter_all_only_root_no_size(self):
833
834
        self.make_index(nodes=[(('key',), 'value', ())])
834
 
        trans = get_transport('trace+' + self.get_url(''))
835
 
        index = btree_index.BTreeGraphIndex(trans, 'index', None)
836
 
        del trans._activity[:]
 
835
        t = transport.get_transport('trace+' + self.get_url(''))
 
836
        index = btree_index.BTreeGraphIndex(t, 'index', None)
 
837
        del t._activity[:]
837
838
        self.assertEqual([(('key',), 'value')],
838
839
                         [x[1:] for x in index.iter_all_entries()])
839
 
        self.assertEqual([('get', 'index')], trans._activity)
 
840
        self.assertEqual([('get', 'index')], t._activity)
840
841
 
841
842
    def test_iter_all_entries_reads(self):
842
843
        # iterating all entries reads the header, then does a linear
848
849
        nodes = self.make_nodes(10000, 2, 2)
849
850
        for node in nodes:
850
851
            builder.add_node(*node)
851
 
        transport = get_transport('trace+' + self.get_url(''))
852
 
        size = transport.put_file('index', builder.finish())
 
852
        t = transport.get_transport('trace+' + self.get_url(''))
 
853
        size = t.put_file('index', builder.finish())
853
854
        self.assertEqual(1303220, size, 'number of expected bytes in the'
854
855
                                        ' output changed')
855
856
        page_size = btree_index._PAGE_SIZE
856
857
        del builder
857
 
        index = btree_index.BTreeGraphIndex(transport, 'index', size)
858
 
        del transport._activity[:]
859
 
        self.assertEqual([], transport._activity)
 
858
        index = btree_index.BTreeGraphIndex(t, 'index', size)
 
859
        del t._activity[:]
 
860
        self.assertEqual([], t._activity)
860
861
        found_nodes = self.time(list, index.iter_all_entries())
861
862
        bare_nodes = []
862
863
        for node in found_nodes:
883
884
        readv_request[-1] = (readv_request[-1][0], 1303220 % page_size)
884
885
        expected = [('readv', 'index', [(0, page_size)], False, None),
885
886
             ('readv',  'index', readv_request, False, None)]
886
 
        if expected != transport._activity:
 
887
        if expected != t._activity:
887
888
            self.assertEqualDiff(pprint.pformat(expected),
888
889
                                 pprint.pformat(transport._activity))
889
890
 
903
904
        nodes = self.make_nodes(160, 2, 2)
904
905
        for node in nodes:
905
906
            builder.add_node(*node)
906
 
        transport = get_transport('trace+' + self.get_url(''))
907
 
        size = transport.put_file('index', builder.finish())
 
907
        t = transport.get_transport('trace+' + self.get_url(''))
 
908
        size = t.put_file('index', builder.finish())
908
909
        del builder
909
 
        index = btree_index.BTreeGraphIndex(transport, 'index', size)
910
 
        del transport._activity[:]
911
 
        self.assertEqual([], transport._activity)
 
910
        index = btree_index.BTreeGraphIndex(t, 'index', size)
 
911
        del t._activity[:]
 
912
        self.assertEqual([], t._activity)
912
913
        # search for one key
913
914
        found_nodes = list(index.iter_entries([nodes[30][0]]))
914
915
        bare_nodes = []
922
923
        # Should have read the root node, then one leaf page:
923
924
        self.assertEqual([('readv', 'index', [(0, 4096)], False, None),
924
925
             ('readv',  'index', [(8192, 4096), ], False, None)],
925
 
            transport._activity)
 
926
            t._activity)
926
927
 
927
928
    def test_iter_key_prefix_1_element_key_None(self):
928
929
        index = self.make_index()
1152
1153
        for node in nodes:
1153
1154
            builder.add_node(*node)
1154
1155
        stream = builder.finish()
1155
 
        trans = get_transport(self.get_url())
 
1156
        trans = transport.get_transport(self.get_url())
1156
1157
        size = trans.put_file('index', stream)
1157
1158
        index = btree_index.BTreeGraphIndex(trans, 'index', size)
1158
1159
        self.assertEqual(500, index.key_count())
1349
1350
        This doesn't actually create anything on disk, it just primes a
1350
1351
        BTreeGraphIndex with the recommended information.
1351
1352
        """
1352
 
        index = btree_index.BTreeGraphIndex(get_transport('memory:///'),
1353
 
                                            'test-index', size=size)
 
1353
        index = btree_index.BTreeGraphIndex(
 
1354
            transport.get_transport('memory:///'), 'test-index', size=size)
1354
1355
        if recommended_pages is not None:
1355
1356
            index._recommended_pages = recommended_pages
1356
1357
        return index