607
626
for key, value, references in nodes:
608
627
builder.add_node(key, value, references)
609
628
stream = builder.finish()
610
trans = transport.get_transport('trace+' + self.get_url())
629
trans = get_transport('trace+' + self.get_url())
611
630
size = trans.put_file('index', stream)
612
631
return btree_index.BTreeGraphIndex(trans, 'index', size)
614
def make_index_with_offset(self, ref_lists=1, key_elements=1, nodes=[],
616
builder = btree_index.BTreeBuilder(key_elements=key_elements,
617
reference_lists=ref_lists)
618
builder.add_nodes(nodes)
619
transport = self.get_transport('')
620
# NamedTemporaryFile dies on builder.finish().read(). weird.
621
temp_file = builder.finish()
622
content = temp_file.read()
625
transport.put_bytes('index', (' '*offset)+content)
626
return btree_index.BTreeGraphIndex(transport, 'index', size=size,
629
633
def test_clear_cache(self):
630
634
nodes = self.make_nodes(160, 2, 2)
631
635
index = self.make_index(ref_lists=2, key_elements=2, nodes=nodes)
648
652
self.assertEqual(0, len(index._leaf_node_cache))
650
654
def test_trivial_constructor(self):
651
t = transport.get_transport('trace+' + self.get_url(''))
652
index = btree_index.BTreeGraphIndex(t, 'index', None)
655
transport = get_transport('trace+' + self.get_url(''))
656
index = btree_index.BTreeGraphIndex(transport, 'index', None)
653
657
# Checks the page size at load, but that isn't logged yet.
654
self.assertEqual([], t._activity)
658
self.assertEqual([], transport._activity)
656
660
def test_with_size_constructor(self):
657
t = transport.get_transport('trace+' + self.get_url(''))
658
index = btree_index.BTreeGraphIndex(t, 'index', 1)
661
transport = get_transport('trace+' + self.get_url(''))
662
index = btree_index.BTreeGraphIndex(transport, 'index', 1)
659
663
# Checks the page size at load, but that isn't logged yet.
660
self.assertEqual([], t._activity)
664
self.assertEqual([], transport._activity)
662
666
def test_empty_key_count_no_size(self):
663
667
builder = btree_index.BTreeBuilder(key_elements=1, reference_lists=0)
664
t = transport.get_transport('trace+' + self.get_url(''))
665
t.put_file('index', builder.finish())
666
index = btree_index.BTreeGraphIndex(t, 'index', None)
668
self.assertEqual([], t._activity)
668
transport = get_transport('trace+' + self.get_url(''))
669
transport.put_file('index', builder.finish())
670
index = btree_index.BTreeGraphIndex(transport, 'index', None)
671
del transport._activity[:]
672
self.assertEqual([], transport._activity)
669
673
self.assertEqual(0, index.key_count())
670
674
# The entire index should have been requested (as we generally have the
671
675
# size available, and doing many small readvs is inappropriate).
672
676
# We can't tell how much was actually read here, but - check the code.
673
self.assertEqual([('get', 'index')], t._activity)
677
self.assertEqual([('get', 'index')], transport._activity)
675
679
def test_empty_key_count(self):
676
680
builder = btree_index.BTreeBuilder(key_elements=1, reference_lists=0)
677
t = transport.get_transport('trace+' + self.get_url(''))
678
size = t.put_file('index', builder.finish())
681
transport = get_transport('trace+' + self.get_url(''))
682
size = transport.put_file('index', builder.finish())
679
683
self.assertEqual(72, size)
680
index = btree_index.BTreeGraphIndex(t, 'index', size)
682
self.assertEqual([], t._activity)
684
index = btree_index.BTreeGraphIndex(transport, 'index', size)
685
del transport._activity[:]
686
self.assertEqual([], transport._activity)
683
687
self.assertEqual(0, index.key_count())
684
688
# The entire index should have been read, as 4K > size
685
689
self.assertEqual([('readv', 'index', [(0, 72)], False, None)],
688
692
def test_non_empty_key_count_2_2(self):
689
693
builder = btree_index.BTreeBuilder(key_elements=2, reference_lists=2)
690
694
nodes = self.make_nodes(35, 2, 2)
691
695
for node in nodes:
692
696
builder.add_node(*node)
693
t = transport.get_transport('trace+' + self.get_url(''))
694
size = t.put_file('index', builder.finish())
695
index = btree_index.BTreeGraphIndex(t, 'index', size)
697
self.assertEqual([], t._activity)
697
transport = get_transport('trace+' + self.get_url(''))
698
size = transport.put_file('index', builder.finish())
699
index = btree_index.BTreeGraphIndex(transport, 'index', size)
700
del transport._activity[:]
701
self.assertEqual([], transport._activity)
698
702
self.assertEqual(70, index.key_count())
699
703
# The entire index should have been read, as it is one page long.
700
704
self.assertEqual([('readv', 'index', [(0, size)], False, None)],
702
706
self.assertEqual(1173, size)
704
def test_with_offset_no_size(self):
705
index = self.make_index_with_offset(key_elements=1, ref_lists=1,
707
nodes=self.make_nodes(200, 1, 1))
708
index._size = None # throw away the size info
709
self.assertEqual(200, index.key_count())
711
def test_with_small_offset(self):
712
index = self.make_index_with_offset(key_elements=1, ref_lists=1,
714
nodes=self.make_nodes(200, 1, 1))
715
self.assertEqual(200, index.key_count())
717
def test_with_large_offset(self):
718
index = self.make_index_with_offset(key_elements=1, ref_lists=1,
720
nodes=self.make_nodes(200, 1, 1))
721
self.assertEqual(200, index.key_count())
723
708
def test__read_nodes_no_size_one_page_reads_once(self):
724
709
self.make_index(nodes=[(('key',), 'value', ())])
725
trans = transport.get_transport('trace+' + self.get_url())
710
trans = get_transport('trace+' + self.get_url())
726
711
index = btree_index.BTreeGraphIndex(trans, 'index', None)
727
712
del trans._activity[:]
728
713
nodes = dict(index._read_nodes([0]))
747
732
nodes = self.make_nodes(160, 2, 2)
748
733
for node in nodes:
749
734
builder.add_node(*node)
750
t = transport.get_transport('trace+' + self.get_url(''))
751
size = t.put_file('index', builder.finish())
735
transport = get_transport('trace+' + self.get_url(''))
736
size = transport.put_file('index', builder.finish())
752
737
self.assertEqual(17692, size)
753
index = btree_index.BTreeGraphIndex(t, 'index', size)
755
self.assertEqual([], t._activity)
738
index = btree_index.BTreeGraphIndex(transport, 'index', size)
739
del transport._activity[:]
740
self.assertEqual([], transport._activity)
756
741
self.assertEqual(320, index.key_count())
757
742
# The entire index should not have been read.
758
743
self.assertEqual([('readv', 'index', [(0, 4096)], False, None)],
761
746
def test_validate_one_page(self):
762
747
builder = btree_index.BTreeBuilder(key_elements=2, reference_lists=2)
763
748
nodes = self.make_nodes(45, 2, 2)
764
749
for node in nodes:
765
750
builder.add_node(*node)
766
t = transport.get_transport('trace+' + self.get_url(''))
767
size = t.put_file('index', builder.finish())
768
index = btree_index.BTreeGraphIndex(t, 'index', size)
770
self.assertEqual([], t._activity)
751
transport = get_transport('trace+' + self.get_url(''))
752
size = transport.put_file('index', builder.finish())
753
index = btree_index.BTreeGraphIndex(transport, 'index', size)
754
del transport._activity[:]
755
self.assertEqual([], transport._activity)
772
757
# The entire index should have been read linearly.
773
758
self.assertEqual([('readv', 'index', [(0, size)], False, None)],
775
760
self.assertEqual(1488, size)
777
762
def test_validate_two_pages(self):
779
764
nodes = self.make_nodes(80, 2, 2)
780
765
for node in nodes:
781
766
builder.add_node(*node)
782
t = transport.get_transport('trace+' + self.get_url(''))
783
size = t.put_file('index', builder.finish())
767
transport = get_transport('trace+' + self.get_url(''))
768
size = transport.put_file('index', builder.finish())
784
769
# Root page, 2 leaf pages
785
770
self.assertEqual(9339, size)
786
index = btree_index.BTreeGraphIndex(t, 'index', size)
788
self.assertEqual([], t._activity)
771
index = btree_index.BTreeGraphIndex(transport, 'index', size)
772
del transport._activity[:]
773
self.assertEqual([], transport._activity)
790
775
# The entire index should have been read linearly.
792
[('readv', 'index', [(0, 4096)], False, None),
793
('readv', 'index', [(4096, 4096), (8192, 1147)], False, None)],
776
self.assertEqual([('readv', 'index', [(0, 4096)], False, None),
777
('readv', 'index', [(4096, 4096), (8192, 1147)], False, None)],
795
779
# XXX: TODO: write some badly-ordered nodes, and some pointers-to-wrong
796
780
# node and make validate find them.
798
782
def test_eq_ne(self):
799
783
# two indices are equal when constructed with the same parameters:
800
t1 = transport.get_transport('trace+' + self.get_url(''))
801
t2 = transport.get_transport(self.get_url(''))
803
btree_index.BTreeGraphIndex(t1, 'index', None) ==
804
btree_index.BTreeGraphIndex(t1, 'index', None))
806
btree_index.BTreeGraphIndex(t1, 'index', 20) ==
807
btree_index.BTreeGraphIndex(t1, 'index', 20))
809
btree_index.BTreeGraphIndex(t1, 'index', 20) ==
810
btree_index.BTreeGraphIndex(t2, 'index', 20))
812
btree_index.BTreeGraphIndex(t1, 'inde1', 20) ==
813
btree_index.BTreeGraphIndex(t1, 'inde2', 20))
815
btree_index.BTreeGraphIndex(t1, 'index', 10) ==
816
btree_index.BTreeGraphIndex(t1, 'index', 20))
818
btree_index.BTreeGraphIndex(t1, 'index', None) !=
819
btree_index.BTreeGraphIndex(t1, 'index', None))
821
btree_index.BTreeGraphIndex(t1, 'index', 20) !=
822
btree_index.BTreeGraphIndex(t1, 'index', 20))
824
btree_index.BTreeGraphIndex(t1, 'index', 20) !=
825
btree_index.BTreeGraphIndex(t2, 'index', 20))
827
btree_index.BTreeGraphIndex(t1, 'inde1', 20) !=
828
btree_index.BTreeGraphIndex(t1, 'inde2', 20))
830
btree_index.BTreeGraphIndex(t1, 'index', 10) !=
831
btree_index.BTreeGraphIndex(t1, 'index', 20))
784
transport1 = get_transport('trace+' + self.get_url(''))
785
transport2 = get_transport(self.get_url(''))
787
btree_index.BTreeGraphIndex(transport1, 'index', None) ==
788
btree_index.BTreeGraphIndex(transport1, 'index', None))
790
btree_index.BTreeGraphIndex(transport1, 'index', 20) ==
791
btree_index.BTreeGraphIndex(transport1, 'index', 20))
793
btree_index.BTreeGraphIndex(transport1, 'index', 20) ==
794
btree_index.BTreeGraphIndex(transport2, 'index', 20))
796
btree_index.BTreeGraphIndex(transport1, 'inde1', 20) ==
797
btree_index.BTreeGraphIndex(transport1, 'inde2', 20))
799
btree_index.BTreeGraphIndex(transport1, 'index', 10) ==
800
btree_index.BTreeGraphIndex(transport1, 'index', 20))
802
btree_index.BTreeGraphIndex(transport1, 'index', None) !=
803
btree_index.BTreeGraphIndex(transport1, 'index', None))
805
btree_index.BTreeGraphIndex(transport1, 'index', 20) !=
806
btree_index.BTreeGraphIndex(transport1, 'index', 20))
808
btree_index.BTreeGraphIndex(transport1, 'index', 20) !=
809
btree_index.BTreeGraphIndex(transport2, 'index', 20))
811
btree_index.BTreeGraphIndex(transport1, 'inde1', 20) !=
812
btree_index.BTreeGraphIndex(transport1, 'inde2', 20))
814
btree_index.BTreeGraphIndex(transport1, 'index', 10) !=
815
btree_index.BTreeGraphIndex(transport1, 'index', 20))
833
817
def test_iter_all_only_root_no_size(self):
834
818
self.make_index(nodes=[(('key',), 'value', ())])
835
t = transport.get_transport('trace+' + self.get_url(''))
836
index = btree_index.BTreeGraphIndex(t, 'index', None)
819
trans = get_transport('trace+' + self.get_url(''))
820
index = btree_index.BTreeGraphIndex(trans, 'index', None)
821
del trans._activity[:]
838
822
self.assertEqual([(('key',), 'value')],
839
823
[x[1:] for x in index.iter_all_entries()])
840
self.assertEqual([('get', 'index')], t._activity)
824
self.assertEqual([('get', 'index')], trans._activity)
842
826
def test_iter_all_entries_reads(self):
843
827
# iterating all entries reads the header, then does a linear