~bzr-pqm/bzr/bzr.dev

2592.1.4 by Robert Collins
Create a GraphIndexBuilder.
1
# Copyright (C) 2007 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
17
"""Tests for indices."""
18
2592.1.5 by Robert Collins
Trivial index reading.
19
from bzrlib import errors
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
20
from bzrlib.index import *
2592.1.4 by Robert Collins
Create a GraphIndexBuilder.
21
from bzrlib.tests import TestCaseWithMemoryTransport
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
22
from bzrlib.transport import get_transport
2592.1.4 by Robert Collins
Create a GraphIndexBuilder.
23
24
25
class TestGraphIndexBuilder(TestCaseWithMemoryTransport):
26
27
    def test_build_index_empty(self):
28
        builder = GraphIndexBuilder()
29
        stream = builder.finish()
30
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
31
        self.assertEqual(
32
            "Bazaar Graph Index 1\nnode_ref_lists=0\nkey_elements=1\nlen=0\n\n",
33
            contents)
2592.1.6 by Robert Collins
Record the number of node reference lists a particular index has.
34
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
35
    def test_build_index_empty_two_element_keys(self):
36
        builder = GraphIndexBuilder(key_elements=2)
37
        stream = builder.finish()
38
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
39
        self.assertEqual(
40
            "Bazaar Graph Index 1\nnode_ref_lists=0\nkey_elements=2\nlen=0\n\n",
41
            contents)
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
42
2592.1.6 by Robert Collins
Record the number of node reference lists a particular index has.
43
    def test_build_index_one_reference_list_empty(self):
44
        builder = GraphIndexBuilder(reference_lists=1)
45
        stream = builder.finish()
46
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
47
        self.assertEqual(
48
            "Bazaar Graph Index 1\nnode_ref_lists=1\nkey_elements=1\nlen=0\n\n",
49
            contents)
2592.1.4 by Robert Collins
Create a GraphIndexBuilder.
50
2592.1.10 by Robert Collins
Make validate detect node reference parsing errors.
51
    def test_build_index_two_reference_list_empty(self):
52
        builder = GraphIndexBuilder(reference_lists=2)
53
        stream = builder.finish()
54
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
55
        self.assertEqual(
56
            "Bazaar Graph Index 1\nnode_ref_lists=2\nkey_elements=1\nlen=0\n\n",
57
            contents)
2592.1.10 by Robert Collins
Make validate detect node reference parsing errors.
58
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
59
    def test_build_index_one_node_no_refs(self):
60
        builder = GraphIndexBuilder()
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
61
        builder.add_node(('akey', ), 'data')
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
62
        stream = builder.finish()
63
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
64
        self.assertEqual(
65
            "Bazaar Graph Index 1\nnode_ref_lists=0\nkey_elements=1\nlen=1\n"
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
66
            "akey\x00\x00\x00data\n\n", contents)
67
68
    def test_build_index_one_node_no_refs_accepts_empty_reflist(self):
69
        builder = GraphIndexBuilder()
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
70
        builder.add_node(('akey', ), 'data', ())
2592.1.12 by Robert Collins
Handle basic node adds.
71
        stream = builder.finish()
72
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
73
        self.assertEqual(
74
            "Bazaar Graph Index 1\nnode_ref_lists=0\nkey_elements=1\nlen=1\n"
2592.1.43 by Robert Collins
Various index tweaks and test clarity from John's review.
75
            "akey\x00\x00\x00data\n\n", contents)
2592.1.12 by Robert Collins
Handle basic node adds.
76
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
77
    def test_build_index_one_node_2_element_keys(self):
2624.2.11 by Robert Collins
Review comments.
78
        # multipart keys are separated by \x00 - because they are fixed length,
79
        # not variable this does not cause any issues, and seems clearer to the
80
        # author.
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
81
        builder = GraphIndexBuilder(key_elements=2)
82
        builder.add_node(('akey', 'secondpart'), 'data')
83
        stream = builder.finish()
84
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
85
        self.assertEqual(
86
            "Bazaar Graph Index 1\nnode_ref_lists=0\nkey_elements=2\nlen=1\n"
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
87
            "akey\x00secondpart\x00\x00\x00data\n\n", contents)
88
2592.1.21 by Robert Collins
Empty values are ok.
89
    def test_add_node_empty_value(self):
90
        builder = GraphIndexBuilder()
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
91
        builder.add_node(('akey', ), '')
2592.1.21 by Robert Collins
Empty values are ok.
92
        stream = builder.finish()
93
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
94
        self.assertEqual(
95
            "Bazaar Graph Index 1\nnode_ref_lists=0\nkey_elements=1\nlen=1\n"
2592.1.43 by Robert Collins
Various index tweaks and test clarity from John's review.
96
            "akey\x00\x00\x00\n\n", contents)
2592.1.21 by Robert Collins
Empty values are ok.
97
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
98
    def test_build_index_nodes_sorted(self):
2592.1.17 by Robert Collins
Multi node sort order is defined.
99
        # the highest sorted node comes first.
100
        builder = GraphIndexBuilder()
101
        # use three to have a good chance of glitching dictionary hash
102
        # lookups etc. Insert in randomish order that is not correct
103
        # and not the reverse of the correct order.
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
104
        builder.add_node(('2002', ), 'data')
105
        builder.add_node(('2000', ), 'data')
106
        builder.add_node(('2001', ), 'data')
2592.1.17 by Robert Collins
Multi node sort order is defined.
107
        stream = builder.finish()
108
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
109
        self.assertEqual(
110
            "Bazaar Graph Index 1\nnode_ref_lists=0\nkey_elements=1\nlen=3\n"
2592.1.43 by Robert Collins
Various index tweaks and test clarity from John's review.
111
            "2000\x00\x00\x00data\n"
112
            "2001\x00\x00\x00data\n"
113
            "2002\x00\x00\x00data\n"
2592.1.17 by Robert Collins
Multi node sort order is defined.
114
            "\n", contents)
115
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
116
    def test_build_index_2_element_key_nodes_sorted(self):
117
        # multiple element keys are sorted first-key, second-key.
118
        builder = GraphIndexBuilder(key_elements=2)
119
        # use three values of each key element, to have a good chance of
120
        # glitching dictionary hash lookups etc. Insert in randomish order that
121
        # is not correct and not the reverse of the correct order.
122
        builder.add_node(('2002', '2002'), 'data')
123
        builder.add_node(('2002', '2000'), 'data')
124
        builder.add_node(('2002', '2001'), 'data')
125
        builder.add_node(('2000', '2002'), 'data')
126
        builder.add_node(('2000', '2000'), 'data')
127
        builder.add_node(('2000', '2001'), 'data')
128
        builder.add_node(('2001', '2002'), 'data')
129
        builder.add_node(('2001', '2000'), 'data')
130
        builder.add_node(('2001', '2001'), 'data')
131
        stream = builder.finish()
132
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
133
        self.assertEqual(
134
            "Bazaar Graph Index 1\nnode_ref_lists=0\nkey_elements=2\nlen=9\n"
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
135
            "2000\x002000\x00\x00\x00data\n"
136
            "2000\x002001\x00\x00\x00data\n"
137
            "2000\x002002\x00\x00\x00data\n"
138
            "2001\x002000\x00\x00\x00data\n"
139
            "2001\x002001\x00\x00\x00data\n"
140
            "2001\x002002\x00\x00\x00data\n"
141
            "2002\x002000\x00\x00\x00data\n"
142
            "2002\x002001\x00\x00\x00data\n"
143
            "2002\x002002\x00\x00\x00data\n"
144
            "\n", contents)
145
2592.1.19 by Robert Collins
Node references are tab separated.
146
    def test_build_index_reference_lists_are_included_one(self):
147
        builder = GraphIndexBuilder(reference_lists=1)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
148
        builder.add_node(('key', ), 'data', ([], ))
2592.1.19 by Robert Collins
Node references are tab separated.
149
        stream = builder.finish()
150
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
151
        self.assertEqual(
152
            "Bazaar Graph Index 1\nnode_ref_lists=1\nkey_elements=1\nlen=1\n"
2592.1.43 by Robert Collins
Various index tweaks and test clarity from John's review.
153
            "key\x00\x00\x00data\n"
2592.1.19 by Robert Collins
Node references are tab separated.
154
            "\n", contents)
155
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
156
    def test_build_index_reference_lists_with_2_element_keys(self):
157
        builder = GraphIndexBuilder(reference_lists=1, key_elements=2)
158
        builder.add_node(('key', 'key2'), 'data', ([], ))
159
        stream = builder.finish()
160
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
161
        self.assertEqual(
162
            "Bazaar Graph Index 1\nnode_ref_lists=1\nkey_elements=2\nlen=1\n"
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
163
            "key\x00key2\x00\x00\x00data\n"
164
            "\n", contents)
165
2592.1.19 by Robert Collins
Node references are tab separated.
166
    def test_build_index_reference_lists_are_included_two(self):
167
        builder = GraphIndexBuilder(reference_lists=2)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
168
        builder.add_node(('key', ), 'data', ([], []))
2592.1.19 by Robert Collins
Node references are tab separated.
169
        stream = builder.finish()
170
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
171
        self.assertEqual(
172
            "Bazaar Graph Index 1\nnode_ref_lists=2\nkey_elements=1\nlen=1\n"
2592.1.43 by Robert Collins
Various index tweaks and test clarity from John's review.
173
            "key\x00\x00\t\x00data\n"
2592.1.19 by Robert Collins
Node references are tab separated.
174
            "\n", contents)
175
2592.1.22 by Robert Collins
Node references are byte offsets.
176
    def test_node_references_are_byte_offsets(self):
177
        builder = GraphIndexBuilder(reference_lists=1)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
178
        builder.add_node(('reference', ), 'data', ([], ))
179
        builder.add_node(('key', ), 'data', ([('reference', )], ))
2592.1.22 by Robert Collins
Node references are byte offsets.
180
        stream = builder.finish()
181
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
182
        self.assertEqual(
183
            "Bazaar Graph Index 1\nnode_ref_lists=1\nkey_elements=1\nlen=2\n"
184
            "key\x00\x0072\x00data\n"
2592.1.40 by Robert Collins
Reverse index ordering - we do not have date prefixed revids.
185
            "reference\x00\x00\x00data\n"
2592.1.22 by Robert Collins
Node references are byte offsets.
186
            "\n", contents)
187
2592.1.23 by Robert Collins
node reference delimiting tested.
188
    def test_node_references_are_cr_delimited(self):
189
        builder = GraphIndexBuilder(reference_lists=1)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
190
        builder.add_node(('reference', ), 'data', ([], ))
191
        builder.add_node(('reference2', ), 'data', ([], ))
192
        builder.add_node(('key', ), 'data', ([('reference', ), ('reference2', )], ))
2592.1.23 by Robert Collins
node reference delimiting tested.
193
        stream = builder.finish()
194
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
195
        self.assertEqual(
196
            "Bazaar Graph Index 1\nnode_ref_lists=1\nkey_elements=1\nlen=3\n"
197
            "key\x00\x00077\r094\x00data\n"
2592.1.43 by Robert Collins
Various index tweaks and test clarity from John's review.
198
            "reference\x00\x00\x00data\n"
199
            "reference2\x00\x00\x00data\n"
2592.1.23 by Robert Collins
node reference delimiting tested.
200
            "\n", contents)
201
2592.1.24 by Robert Collins
Delimiting of multiple reference lists is by \t
202
    def test_multiple_reference_lists_are_tab_delimited(self):
203
        builder = GraphIndexBuilder(reference_lists=2)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
204
        builder.add_node(('keference', ), 'data', ([], []))
205
        builder.add_node(('rey', ), 'data', ([('keference', )], [('keference', )]))
2592.1.24 by Robert Collins
Delimiting of multiple reference lists is by \t
206
        stream = builder.finish()
207
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
208
        self.assertEqual(
209
            "Bazaar Graph Index 1\nnode_ref_lists=2\nkey_elements=1\nlen=2\n"
2592.1.43 by Robert Collins
Various index tweaks and test clarity from John's review.
210
            "keference\x00\x00\t\x00data\n"
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
211
            "rey\x00\x0059\t59\x00data\n"
2592.1.24 by Robert Collins
Delimiting of multiple reference lists is by \t
212
            "\n", contents)
213
2592.1.25 by Robert Collins
Fix and tune node offset calculation.
214
    def test_add_node_referencing_missing_key_makes_absent(self):
215
        builder = GraphIndexBuilder(reference_lists=1)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
216
        builder.add_node(('rey', ), 'data', ([('beference', ), ('aeference2', )], ))
2592.1.25 by Robert Collins
Fix and tune node offset calculation.
217
        stream = builder.finish()
218
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
219
        self.assertEqual(
220
            "Bazaar Graph Index 1\nnode_ref_lists=1\nkey_elements=1\nlen=1\n"
2592.1.43 by Robert Collins
Various index tweaks and test clarity from John's review.
221
            "aeference2\x00a\x00\x00\n"
222
            "beference\x00a\x00\x00\n"
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
223
            "rey\x00\x00074\r059\x00data\n"
2592.1.25 by Robert Collins
Fix and tune node offset calculation.
224
            "\n", contents)
225
2592.1.26 by Robert Collins
Test digit buffering is accurate.
226
    def test_node_references_three_digits(self):
227
        # test the node digit expands as needed.
228
        builder = GraphIndexBuilder(reference_lists=1)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
229
        references = [(str(val), ) for val in reversed(range(9))]
230
        builder.add_node(('2-key', ), '', (references, ))
2592.1.26 by Robert Collins
Test digit buffering is accurate.
231
        stream = builder.finish()
232
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
233
        self.assertEqual(
234
            "Bazaar Graph Index 1\nnode_ref_lists=1\nkey_elements=1\nlen=1\n"
2592.1.40 by Robert Collins
Reverse index ordering - we do not have date prefixed revids.
235
            "0\x00a\x00\x00\n"
236
            "1\x00a\x00\x00\n"
237
            "2\x00a\x00\x00\n"
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
238
            "2-key\x00\x00151\r145\r139\r133\r127\r121\r071\r065\r059\x00\n"
2592.1.40 by Robert Collins
Reverse index ordering - we do not have date prefixed revids.
239
            "3\x00a\x00\x00\n"
240
            "4\x00a\x00\x00\n"
241
            "5\x00a\x00\x00\n"
242
            "6\x00a\x00\x00\n"
243
            "7\x00a\x00\x00\n"
2592.1.26 by Robert Collins
Test digit buffering is accurate.
244
            "8\x00a\x00\x00\n"
2592.1.40 by Robert Collins
Reverse index ordering - we do not have date prefixed revids.
245
            "\n", contents)
246
247
    def test_absent_has_no_reference_overhead(self):
248
        # the offsets after an absent record should be correct when there are
249
        # >1 reference lists.
250
        builder = GraphIndexBuilder(reference_lists=2)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
251
        builder.add_node(('parent', ), '', ([('aail', ), ('zther', )], []))
2592.1.40 by Robert Collins
Reverse index ordering - we do not have date prefixed revids.
252
        stream = builder.finish()
253
        contents = stream.read()
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
254
        self.assertEqual(
255
            "Bazaar Graph Index 1\nnode_ref_lists=2\nkey_elements=1\nlen=1\n"
2592.1.40 by Robert Collins
Reverse index ordering - we do not have date prefixed revids.
256
            "aail\x00a\x00\x00\n"
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
257
            "parent\x00\x0059\r84\t\x00\n"
2592.1.40 by Robert Collins
Reverse index ordering - we do not have date prefixed revids.
258
            "zther\x00a\x00\x00\n"
2592.1.26 by Robert Collins
Test digit buffering is accurate.
259
            "\n", contents)
260
2592.1.13 by Robert Collins
Handle mismatched numbers of reference lists.
261
    def test_add_node_bad_key(self):
2592.1.12 by Robert Collins
Handle basic node adds.
262
        builder = GraphIndexBuilder()
2592.1.14 by Robert Collins
Detect bad reference key values.
263
        for bad_char in '\t\n\x0b\x0c\r\x00 ':
264
            self.assertRaises(errors.BadIndexKey, builder.add_node,
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
265
                ('a%skey' % bad_char, ), 'data')
266
        self.assertRaises(errors.BadIndexKey, builder.add_node,
267
                ('', ), 'data')
268
        self.assertRaises(errors.BadIndexKey, builder.add_node,
269
                'not-a-tuple', 'data')
270
        # not enough length
271
        self.assertRaises(errors.BadIndexKey, builder.add_node,
272
                (), 'data')
273
        # too long
274
        self.assertRaises(errors.BadIndexKey, builder.add_node,
275
                ('primary', 'secondary'), 'data')
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
276
        # secondary key elements get checked too:
277
        builder = GraphIndexBuilder(key_elements=2)
278
        for bad_char in '\t\n\x0b\x0c\r\x00 ':
279
            self.assertRaises(errors.BadIndexKey, builder.add_node,
280
                ('prefix', 'a%skey' % bad_char), 'data')
2592.1.12 by Robert Collins
Handle basic node adds.
281
2592.1.13 by Robert Collins
Handle mismatched numbers of reference lists.
282
    def test_add_node_bad_data(self):
2592.1.12 by Robert Collins
Handle basic node adds.
283
        builder = GraphIndexBuilder()
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
284
        self.assertRaises(errors.BadIndexValue, builder.add_node, ('akey', ),
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
285
            'data\naa')
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
286
        self.assertRaises(errors.BadIndexValue, builder.add_node, ('akey', ),
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
287
            'data\x00aa')
2592.1.12 by Robert Collins
Handle basic node adds.
288
2592.1.13 by Robert Collins
Handle mismatched numbers of reference lists.
289
    def test_add_node_bad_mismatched_ref_lists_length(self):
290
        builder = GraphIndexBuilder()
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
291
        self.assertRaises(errors.BadIndexValue, builder.add_node, ('akey', ),
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
292
            'data aa', ([], ))
2592.1.13 by Robert Collins
Handle mismatched numbers of reference lists.
293
        builder = GraphIndexBuilder(reference_lists=1)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
294
        self.assertRaises(errors.BadIndexValue, builder.add_node, ('akey', ),
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
295
            'data aa')
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
296
        self.assertRaises(errors.BadIndexValue, builder.add_node, ('akey', ),
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
297
            'data aa', (), )
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
298
        self.assertRaises(errors.BadIndexValue, builder.add_node, ('akey', ),
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
299
            'data aa', ([], []))
2592.1.13 by Robert Collins
Handle mismatched numbers of reference lists.
300
        builder = GraphIndexBuilder(reference_lists=2)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
301
        self.assertRaises(errors.BadIndexValue, builder.add_node, ('akey', ),
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
302
            'data aa')
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
303
        self.assertRaises(errors.BadIndexValue, builder.add_node, ('akey', ),
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
304
            'data aa', ([], ))
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
305
        self.assertRaises(errors.BadIndexValue, builder.add_node, ('akey', ),
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
306
            'data aa', ([], [], []))
2592.1.13 by Robert Collins
Handle mismatched numbers of reference lists.
307
2592.1.14 by Robert Collins
Detect bad reference key values.
308
    def test_add_node_bad_key_in_reference_lists(self):
309
        # first list, first key - trivial
310
        builder = GraphIndexBuilder(reference_lists=1)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
311
        self.assertRaises(errors.BadIndexKey, builder.add_node, ('akey', ),
312
            'data aa', ([('a key', )], ))
313
        # references keys must be tuples too
314
        self.assertRaises(errors.BadIndexKey, builder.add_node, ('akey', ),
315
            'data aa', (['not-a-tuple'], ))
316
        # not enough length
317
        self.assertRaises(errors.BadIndexKey, builder.add_node, ('akey', ),
318
            'data aa', ([()], ))
319
        # too long
320
        self.assertRaises(errors.BadIndexKey, builder.add_node, ('akey', ),
321
            'data aa', ([('primary', 'secondary')], ))
2592.1.14 by Robert Collins
Detect bad reference key values.
322
        # need to check more than the first key in the list
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
323
        self.assertRaises(errors.BadIndexKey, builder.add_node, ('akey', ),
324
            'data aa', ([('agoodkey', ), ('that is a bad key', )], ))
2592.1.14 by Robert Collins
Detect bad reference key values.
325
        # and if there is more than one list it should be getting checked
326
        # too
327
        builder = GraphIndexBuilder(reference_lists=2)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
328
        self.assertRaises(errors.BadIndexKey, builder.add_node, ('akey', ),
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
329
            'data aa', ([], ['a bad key']))
2592.1.14 by Robert Collins
Detect bad reference key values.
330
2592.1.15 by Robert Collins
Detect duplicate key insertion.
331
    def test_add_duplicate_key(self):
332
        builder = GraphIndexBuilder()
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
333
        builder.add_node(('key', ), 'data')
334
        self.assertRaises(errors.BadIndexDuplicateKey, builder.add_node, ('key', ),
2592.1.46 by Robert Collins
Make GraphIndex accept nodes as key, value, references, so that the method
335
            'data')
2592.1.15 by Robert Collins
Detect duplicate key insertion.
336
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
337
    def test_add_duplicate_key_2_elements(self):
338
        builder = GraphIndexBuilder(key_elements=2)
339
        builder.add_node(('key', 'key'), 'data')
340
        self.assertRaises(errors.BadIndexDuplicateKey, builder.add_node,
341
            ('key', 'key'), 'data')
342
2592.1.16 by Robert Collins
Can add keys after referencing them.
343
    def test_add_key_after_referencing_key(self):
344
        builder = GraphIndexBuilder(reference_lists=1)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
345
        builder.add_node(('key', ), 'data', ([('reference', )], ))
346
        builder.add_node(('reference', ), 'data', ([],))
2592.1.16 by Robert Collins
Can add keys after referencing them.
347
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
348
    def test_add_key_after_referencing_key_2_elements(self):
349
        builder = GraphIndexBuilder(reference_lists=1, key_elements=2)
350
        builder.add_node(('k', 'ey'), 'data', ([('reference', 'tokey')], ))
351
        builder.add_node(('reference', 'tokey'), 'data', ([],))
352
2592.1.5 by Robert Collins
Trivial index reading.
353
354
class TestGraphIndex(TestCaseWithMemoryTransport):
355
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
356
    def make_index(self, ref_lists=0, key_elements=1, nodes=[]):
357
        builder = GraphIndexBuilder(ref_lists, key_elements=key_elements)
2624.2.17 by Robert Collins
Review feedback.
358
        for key, value, references in nodes:
359
            builder.add_node(key, value, references)
2592.1.5 by Robert Collins
Trivial index reading.
360
        stream = builder.finish()
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
361
        trans = get_transport('trace+' + self.get_url())
2890.2.1 by Robert Collins
* ``bzrlib.index.GraphIndex`` now requires a size parameter to the
362
        size = trans.put_file('index', stream)
363
        return GraphIndex(trans, 'index', size)
2592.1.5 by Robert Collins
Trivial index reading.
364
2592.1.7 by Robert Collins
A validate that goes boom.
365
    def test_open_bad_index_no_error(self):
366
        trans = self.get_transport()
367
        trans.put_bytes('name', "not an index\n")
2890.2.1 by Robert Collins
* ``bzrlib.index.GraphIndex`` now requires a size parameter to the
368
        index = GraphIndex(trans, 'name', 13)
2592.1.7 by Robert Collins
A validate that goes boom.
369
2890.2.2 by Robert Collins
Opening an index creates a map for the parsed bytes.
370
    def test_open_sets_parsed_map_empty(self):
371
        index = self.make_index()
372
        self.assertEqual([], index._parsed_byte_map)
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
373
        self.assertEqual([], index._parsed_key_map)
374
375
    def test_first_lookup_key_via_location(self):
376
        index = self.make_index()
377
        # reset the transport log
378
        del index._transport._activity[:]
2890.2.18 by Robert Collins
Review feedback.
379
        # do a _lookup_keys_via_location call for the middle of the file, which
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
380
        # is what bisection uses.
2890.2.18 by Robert Collins
Review feedback.
381
        result = index._lookup_keys_via_location(
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
382
            [(index._size // 2, ('missing', ))])
383
        # this should have asked for a readv request, with adjust_for_latency,
384
        # and two regions: the header, and half-way into the file.
385
        self.assertEqual([
386
            ('readv', 'index', [(30, 30), (0, 200)], True, 60),
387
            ],
388
            index._transport._activity)
389
        # and the result should be that the key cannot be present, because this
390
        # is a trivial index.
391
        self.assertEqual([((index._size // 2, ('missing', )), False)],
392
            result)
393
        # And the regions of the file that have been parsed - in this case the
394
        # entire file - should be in the parsed region map.
395
        self.assertEqual([(0, 60)], index._parsed_byte_map)
396
        self.assertEqual([(None, None)], index._parsed_key_map)
397
398
    def test_parsing_parses_data_adjacent_to_parsed_regions(self):
399
        # we trim data we recieve to remove the first and trailing
400
        # partial lines, except when they start at the end/finish at the start
401
        # of a region we've alread parsed/ the end of the file. The trivial
402
        # test for this is an index with 1 key.
403
        index = self.make_index(nodes=[(('name', ), 'data', ())])
404
        # reset the transport log
405
        del index._transport._activity[:]
2890.2.18 by Robert Collins
Review feedback.
406
        result = index._lookup_keys_via_location(
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
407
            [(index._size // 2, ('missing', ))])
408
        # this should have asked for a readv request, with adjust_for_latency,
409
        # and two regions: the header, and half-way into the file.
410
        self.assertEqual([
411
            ('readv', 'index', [(36, 36), (0, 200)], True, 72),
412
            ],
413
            index._transport._activity)
414
        # and the result should be that the key cannot be present, because this
415
        # is a trivial index and we should not have to do more round trips.
416
        self.assertEqual([((index._size // 2, ('missing', )), False)],
417
            result)
418
        # The whole file should be parsed at this point.
419
        self.assertEqual([(0, 72)], index._parsed_byte_map)
420
        self.assertEqual([(None, ('name',))], index._parsed_key_map)
421
422
    def test_parsing_non_adjacent_data_trims(self):
423
        # generate a big enough index that we only read some of it on a typical
424
        # bisection lookup.
425
        nodes = []
426
        def make_key(number):
427
            return (str(number) + 'X'*100,)
428
        for counter in range(64):
429
            nodes.append((make_key(counter), 'Y'*100, ()))
430
        index = self.make_index(nodes=nodes)
2890.2.18 by Robert Collins
Review feedback.
431
        result = index._lookup_keys_via_location(
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
432
            [(index._size // 2, ('40', ))])
433
        # and the result should be that the key cannot be present, because key is
434
        # in the middle of the observed data from a 4K read - the smallest transport
435
        # will do today with this api.
436
        self.assertEqual([((index._size // 2, ('40', )), False)],
437
            result)
438
        # and we should have a parse map that includes the header and the
439
        # region that was parsed after trimming.
440
        self.assertEqual([(0, 3972), (5001, 8914)], index._parsed_byte_map)
441
        self.assertEqual([(None, make_key(26)), (make_key(31), make_key(48))],
442
            index._parsed_key_map)
443
2890.2.14 by Robert Collins
Parse more than one segment of data from a single readv response if needed.
444
    def test_parsing_data_handles_parsed_contained_regions(self):
445
        # the following patten creates a parsed region that is wholly within a
446
        # single result from the readv layer:
447
        # .... single-read (readv-minimum-size) ...
448
        # which then trims the start and end so the parsed size is < readv
449
        # miniumum.
450
        # then a dual lookup (or a reference lookup for that matter) which
451
        # abuts or overlaps the parsed region on both sides will need to 
452
        # discard the data in the middle, but parse the end as well.
453
        #
454
        # we test this by doing a single lookup to seed the data, then 
455
        # a lookup for two keys that are present, and adjacent - 
456
        # we except both to be found, and the parsed byte map to include the
457
        # locations of both keys.
458
        nodes = []
459
        def make_key(number):
460
            return (str(number) + 'X'*100,)
461
        def make_value(number):
462
            return 'Y'*100
2890.2.15 by Robert Collins
Corner case when parsing repeated sections - the bottom section of a region may not be parsed, so we need to manually advance past that.
463
        for counter in range(128):
2890.2.14 by Robert Collins
Parse more than one segment of data from a single readv response if needed.
464
            nodes.append((make_key(counter), make_value(counter), ()))
465
        index = self.make_index(nodes=nodes)
2890.2.18 by Robert Collins
Review feedback.
466
        result = index._lookup_keys_via_location(
2890.2.14 by Robert Collins
Parse more than one segment of data from a single readv response if needed.
467
            [(index._size // 2, ('40', ))])
468
        # and we should have a parse map that includes the header and the
469
        # region that was parsed after trimming.
2890.2.15 by Robert Collins
Corner case when parsing repeated sections - the bottom section of a region may not be parsed, so we need to manually advance past that.
470
        self.assertEqual([(0, 3991), (11622, 15534)], index._parsed_byte_map)
471
        self.assertEqual([(None, make_key(116)), (make_key(35), make_key(51))],
2890.2.14 by Robert Collins
Parse more than one segment of data from a single readv response if needed.
472
            index._parsed_key_map)
473
        # now ask for two keys, right before and after the parsed region
2890.2.18 by Robert Collins
Review feedback.
474
        result = index._lookup_keys_via_location(
2890.2.15 by Robert Collins
Corner case when parsing repeated sections - the bottom section of a region may not be parsed, so we need to manually advance past that.
475
            [(11450, make_key(34)), (15534, make_key(52))])
2890.2.14 by Robert Collins
Parse more than one segment of data from a single readv response if needed.
476
        self.assertEqual([
2890.2.15 by Robert Collins
Corner case when parsing repeated sections - the bottom section of a region may not be parsed, so we need to manually advance past that.
477
            ((11450, make_key(34)), (index, make_key(34), make_value(34))),
478
            ((15534, make_key(52)), (index, make_key(52), make_value(52))),
2890.2.14 by Robert Collins
Parse more than one segment of data from a single readv response if needed.
479
            ],
480
            result)
2890.2.15 by Robert Collins
Corner case when parsing repeated sections - the bottom section of a region may not be parsed, so we need to manually advance past that.
481
        self.assertEqual([(0, 3991), (9975, 17799)], index._parsed_byte_map)
2890.2.14 by Robert Collins
Parse more than one segment of data from a single readv response if needed.
482
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
483
    def test_lookup_missing_key_answers_without_io_when_map_permits(self):
484
        # generate a big enough index that we only read some of it on a typical
485
        # bisection lookup.
486
        nodes = []
487
        def make_key(number):
488
            return (str(number) + 'X'*100,)
489
        for counter in range(64):
490
            nodes.append((make_key(counter), 'Y'*100, ()))
491
        index = self.make_index(nodes=nodes)
492
        # lookup the keys in the middle of the file
2890.2.18 by Robert Collins
Review feedback.
493
        result =index._lookup_keys_via_location(
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
494
            [(index._size // 2, ('40', ))])
495
        # check the parse map, this determines the test validity
496
        self.assertEqual([(0, 3972), (5001, 8914)], index._parsed_byte_map)
497
        self.assertEqual([(None, make_key(26)), (make_key(31), make_key(48))],
498
            index._parsed_key_map)
499
        # reset the transport log
500
        del index._transport._activity[:]
501
        # now looking up a key in the portion of the file already parsed should
502
        # not create a new transport request, and should return False (cannot
503
        # be in the index) - even when the byte location we ask for is outside
504
        # the parsed region
505
        # 
2890.2.18 by Robert Collins
Review feedback.
506
        result = index._lookup_keys_via_location(
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
507
            [(4000, ('40', ))])
508
        self.assertEqual([((4000, ('40', )), False)],
509
            result)
510
        self.assertEqual([], index._transport._activity)
511
512
    def test_lookup_present_key_answers_without_io_when_map_permits(self):
513
        # generate a big enough index that we only read some of it on a typical
514
        # bisection lookup.
515
        nodes = []
516
        def make_key(number):
517
            return (str(number) + 'X'*100,)
518
        def make_value(number):
519
            return str(number) + 'Y'*100
520
        for counter in range(64):
521
            nodes.append((make_key(counter), make_value(counter), ()))
522
        index = self.make_index(nodes=nodes)
523
        # lookup the keys in the middle of the file
2890.2.18 by Robert Collins
Review feedback.
524
        result =index._lookup_keys_via_location(
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
525
            [(index._size // 2, ('40', ))])
526
        # check the parse map, this determines the test validity
527
        self.assertEqual([(0, 4008), (5046, 8996)], index._parsed_byte_map)
528
        self.assertEqual([(None, make_key(26)), (make_key(31), make_key(48))],
529
            index._parsed_key_map)
530
        # reset the transport log
531
        del index._transport._activity[:]
532
        # now looking up a key in the portion of the file already parsed should
533
        # not create a new transport request, and should return False (cannot
534
        # be in the index) - even when the byte location we ask for is outside
535
        # the parsed region
536
        # 
2890.2.18 by Robert Collins
Review feedback.
537
        result = index._lookup_keys_via_location([(4000, make_key(40))])
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
538
        self.assertEqual(
539
            [((4000, make_key(40)), (index, make_key(40), make_value(40)))],
540
            result)
541
        self.assertEqual([], index._transport._activity)
542
543
    def test_lookup_key_below_probed_area(self):
544
        # generate a big enough index that we only read some of it on a typical
545
        # bisection lookup.
546
        nodes = []
547
        def make_key(number):
548
            return (str(number) + 'X'*100,)
549
        for counter in range(64):
550
            nodes.append((make_key(counter), 'Y'*100, ()))
551
        index = self.make_index(nodes=nodes)
552
        # ask for the key in the middle, but a key that is located in the
553
        # unparsed region before the middle.
2890.2.18 by Robert Collins
Review feedback.
554
        result =index._lookup_keys_via_location(
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
555
            [(index._size // 2, ('30', ))])
556
        # check the parse map, this determines the test validity
557
        self.assertEqual([(0, 3972), (5001, 8914)], index._parsed_byte_map)
558
        self.assertEqual([(None, make_key(26)), (make_key(31), make_key(48))],
559
            index._parsed_key_map)
560
        self.assertEqual([((index._size // 2, ('30', )), -1)],
561
            result)
562
563
    def test_lookup_key_above_probed_area(self):
564
        # generate a big enough index that we only read some of it on a typical
565
        # bisection lookup.
566
        nodes = []
567
        def make_key(number):
568
            return (str(number) + 'X'*100,)
569
        for counter in range(64):
570
            nodes.append((make_key(counter), 'Y'*100, ()))
571
        index = self.make_index(nodes=nodes)
572
        # ask for the key in the middle, but a key that is located in the
573
        # unparsed region after the middle.
2890.2.18 by Robert Collins
Review feedback.
574
        result =index._lookup_keys_via_location(
2890.2.5 by Robert Collins
Create a content lookup function for bisection in GraphIndex.
575
            [(index._size // 2, ('50', ))])
576
        # check the parse map, this determines the test validity
577
        self.assertEqual([(0, 3972), (5001, 8914)], index._parsed_byte_map)
578
        self.assertEqual([(None, make_key(26)), (make_key(31), make_key(48))],
579
            index._parsed_key_map)
580
        self.assertEqual([((index._size // 2, ('50', )), +1)],
581
            result)
2890.2.2 by Robert Collins
Opening an index creates a map for the parsed bytes.
582
2890.2.6 by Robert Collins
Add support for key references to the index lookup_keys_via_location bisection interface.
583
    def test_lookup_key_resolves_references(self):
584
        # generate a big enough index that we only read some of it on a typical
585
        # bisection lookup.
586
        nodes = []
587
        def make_key(number):
588
            return (str(number) + 'X'*100,)
589
        def make_value(number):
590
            return str(number) + 'Y'*100
591
        for counter in range(64):
592
            nodes.append((make_key(counter), make_value(counter),
593
                ((make_key(counter + 20),),)  ))
594
        index = self.make_index(ref_lists=1, nodes=nodes)
595
        # lookup a key in the middle that does not exist, so that when we can
596
        # check that the referred-to-keys are not accessed automatically.
2890.2.18 by Robert Collins
Review feedback.
597
        result =index._lookup_keys_via_location(
2890.2.6 by Robert Collins
Add support for key references to the index lookup_keys_via_location bisection interface.
598
            [(index._size // 2, ('40', ))])
599
        # check the parse map - only the start and middle should have been
600
        # parsed.
601
        self.assertEqual([(0, 3890), (6444, 10274)], index._parsed_byte_map)
602
        self.assertEqual([(None, make_key(25)), (make_key(37), make_key(52))],
603
            index._parsed_key_map)
604
        # and check the transport activity likewise.
605
        self.assertEqual(
606
            [('readv', 'index', [(7906, 800), (0, 200)], True, 15813)],
607
            index._transport._activity)
608
        # reset the transport log for testing the reference lookup
609
        del index._transport._activity[:]
610
        # now looking up a key in the portion of the file already parsed should
611
        # only perform IO to resolve its key references.
2890.2.18 by Robert Collins
Review feedback.
612
        result = index._lookup_keys_via_location([(4000, make_key(40))])
2890.2.6 by Robert Collins
Add support for key references to the index lookup_keys_via_location bisection interface.
613
        self.assertEqual(
614
            [((4000, make_key(40)),
615
              (index, make_key(40), make_value(40), ((make_key(60),),)))],
616
            result)
617
        self.assertEqual([('readv', 'index', [(11976, 800)], True, 15813)],
618
            index._transport._activity)
619
2592.1.5 by Robert Collins
Trivial index reading.
620
    def test_iter_all_entries_empty(self):
621
        index = self.make_index()
622
        self.assertEqual([], list(index.iter_all_entries()))
623
2592.1.27 by Robert Collins
Test missing end lines with non-empty indices.
624
    def test_iter_all_entries_simple(self):
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
625
        index = self.make_index(nodes=[(('name', ), 'data', ())])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
626
        self.assertEqual([(index, ('name', ), 'data')],
2592.1.27 by Robert Collins
Test missing end lines with non-empty indices.
627
            list(index.iter_all_entries()))
628
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
629
    def test_iter_all_entries_simple_2_elements(self):
630
        index = self.make_index(key_elements=2,
631
            nodes=[(('name', 'surname'), 'data', ())])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
632
        self.assertEqual([(index, ('name', 'surname'), 'data')],
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
633
            list(index.iter_all_entries()))
634
2592.1.28 by Robert Collins
Basic two pass iter_all_entries.
635
    def test_iter_all_entries_references_resolved(self):
636
        index = self.make_index(1, nodes=[
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
637
            (('name', ), 'data', ([('ref', )], )),
638
            (('ref', ), 'refdata', ([], ))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
639
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref',),),)),
640
            (index, ('ref', ), 'refdata', ((), ))]),
2592.1.28 by Robert Collins
Basic two pass iter_all_entries.
641
            set(index.iter_all_entries()))
642
2890.2.10 by Robert Collins
Add test coverage to ensure \r's are not mangled by bisection parsing.
643
    def test_iter_entries_references_resolved(self):
644
        index = self.make_index(1, nodes=[
645
            (('name', ), 'data', ([('ref', ), ('ref', )], )),
646
            (('ref', ), 'refdata', ([], ))])
647
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref',),('ref',)),)),
648
            (index, ('ref', ), 'refdata', ((), ))]),
649
            set(index.iter_entries([('name',), ('ref',)])))
650
651
    def test_iter_entries_references_2_refs_resolved(self):
652
        index = self.make_index(2, nodes=[
653
            (('name', ), 'data', ([('ref', )], [('ref', )])),
654
            (('ref', ), 'refdata', ([], []))])
655
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref',),), (('ref',),))),
656
            (index, ('ref', ), 'refdata', ((), ()))]),
657
            set(index.iter_entries([('name',), ('ref',)])))
658
2592.1.30 by Robert Collins
Absent entries are not yeilded.
659
    def test_iteration_absent_skipped(self):
660
        index = self.make_index(1, nodes=[
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
661
            (('name', ), 'data', ([('ref', )], ))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
662
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref',),),))]),
2592.1.30 by Robert Collins
Absent entries are not yeilded.
663
            set(index.iter_all_entries()))
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
664
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref',),),))]),
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
665
            set(index.iter_entries([('name', )])))
666
        self.assertEqual([], list(index.iter_entries([('ref', )])))
2592.1.30 by Robert Collins
Absent entries are not yeilded.
667
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
668
    def test_iteration_absent_skipped_2_element_keys(self):
669
        index = self.make_index(1, key_elements=2, nodes=[
670
            (('name', 'fin'), 'data', ([('ref', 'erence')], ))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
671
        self.assertEqual(set([(index, ('name', 'fin'), 'data', ((('ref', 'erence'),),))]),
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
672
            set(index.iter_all_entries()))
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
673
        self.assertEqual(set([(index, ('name', 'fin'), 'data', ((('ref', 'erence'),),))]),
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
674
            set(index.iter_entries([('name', 'fin')])))
675
        self.assertEqual([], list(index.iter_entries([('ref', 'erence')])))
676
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
677
    def test_iter_all_keys(self):
2592.1.29 by Robert Collins
Basic iter_entries working.
678
        index = self.make_index(1, nodes=[
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
679
            (('name', ), 'data', ([('ref', )], )),
680
            (('ref', ), 'refdata', ([], ))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
681
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref',),),)),
682
            (index, ('ref', ), 'refdata', ((), ))]),
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
683
            set(index.iter_entries([('name', ), ('ref', )])))
2592.1.29 by Robert Collins
Basic iter_entries working.
684
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
685
    def test_iter_nothing_empty(self):
2592.1.9 by Robert Collins
Iterating no keys should work too.
686
        index = self.make_index()
687
        self.assertEqual([], list(index.iter_entries([])))
688
2592.1.5 by Robert Collins
Trivial index reading.
689
    def test_iter_missing_entry_empty(self):
690
        index = self.make_index()
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
691
        self.assertEqual([], list(index.iter_entries([('a', )])))
2592.1.7 by Robert Collins
A validate that goes boom.
692
2890.2.8 by Robert Collins
Make the size of the index optionally None for the pack-names index.
693
    def test_iter_missing_entry_empty_no_size(self):
694
        index = self.make_index()
695
        index = GraphIndex(index._transport, 'index', None)
696
        self.assertEqual([], list(index.iter_entries([('a', )])))
697
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
698
    def test_iter_key_prefix_1_element_key_None(self):
699
        index = self.make_index()
700
        self.assertRaises(errors.BadIndexKey, list,
701
            index.iter_entries_prefix([(None, )]))
702
703
    def test_iter_key_prefix_wrong_length(self):
704
        index = self.make_index()
705
        self.assertRaises(errors.BadIndexKey, list,
706
            index.iter_entries_prefix([('foo', None)]))
707
        index = self.make_index(key_elements=2)
708
        self.assertRaises(errors.BadIndexKey, list,
709
            index.iter_entries_prefix([('foo', )]))
710
        self.assertRaises(errors.BadIndexKey, list,
711
            index.iter_entries_prefix([('foo', None, None)]))
712
713
    def test_iter_key_prefix_1_key_element_no_refs(self):
714
        index = self.make_index( nodes=[
715
            (('name', ), 'data', ()),
716
            (('ref', ), 'refdata', ())])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
717
        self.assertEqual(set([(index, ('name', ), 'data'),
718
            (index, ('ref', ), 'refdata')]),
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
719
            set(index.iter_entries_prefix([('name', ), ('ref', )])))
720
721
    def test_iter_key_prefix_1_key_element_refs(self):
722
        index = self.make_index(1, nodes=[
723
            (('name', ), 'data', ([('ref', )], )),
724
            (('ref', ), 'refdata', ([], ))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
725
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref',),),)),
726
            (index, ('ref', ), 'refdata', ((), ))]),
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
727
            set(index.iter_entries_prefix([('name', ), ('ref', )])))
728
729
    def test_iter_key_prefix_2_key_element_no_refs(self):
730
        index = self.make_index(key_elements=2, nodes=[
731
            (('name', 'fin1'), 'data', ()),
732
            (('name', 'fin2'), 'beta', ()),
733
            (('ref', 'erence'), 'refdata', ())])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
734
        self.assertEqual(set([(index, ('name', 'fin1'), 'data'),
735
            (index, ('ref', 'erence'), 'refdata')]),
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
736
            set(index.iter_entries_prefix([('name', 'fin1'), ('ref', 'erence')])))
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
737
        self.assertEqual(set([(index, ('name', 'fin1'), 'data'),
738
            (index, ('name', 'fin2'), 'beta')]),
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
739
            set(index.iter_entries_prefix([('name', None)])))
740
741
    def test_iter_key_prefix_2_key_element_refs(self):
742
        index = self.make_index(1, key_elements=2, nodes=[
743
            (('name', 'fin1'), 'data', ([('ref', 'erence')], )),
744
            (('name', 'fin2'), 'beta', ([], )),
745
            (('ref', 'erence'), 'refdata', ([], ))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
746
        self.assertEqual(set([(index, ('name', 'fin1'), 'data', ((('ref', 'erence'),),)),
747
            (index, ('ref', 'erence'), 'refdata', ((), ))]),
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
748
            set(index.iter_entries_prefix([('name', 'fin1'), ('ref', 'erence')])))
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
749
        self.assertEqual(set([(index, ('name', 'fin1'), 'data', ((('ref', 'erence'),),)),
750
            (index, ('name', 'fin2'), 'beta', ((), ))]),
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
751
            set(index.iter_entries_prefix([('name', None)])))
752
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
753
    def test_key_count_empty(self):
754
        index = self.make_index()
755
        self.assertEqual(0, index.key_count())
756
757
    def test_key_count_one(self):
758
        index = self.make_index(nodes=[(('name', ), '', ())])
759
        self.assertEqual(1, index.key_count())
760
761
    def test_key_count_two(self):
762
        index = self.make_index(nodes=[
763
            (('name', ), '', ()), (('foo', ), '', ())])
764
        self.assertEqual(2, index.key_count())
765
2592.1.7 by Robert Collins
A validate that goes boom.
766
    def test_validate_bad_index_errors(self):
767
        trans = self.get_transport()
768
        trans.put_bytes('name', "not an index\n")
2890.2.1 by Robert Collins
* ``bzrlib.index.GraphIndex`` now requires a size parameter to the
769
        index = GraphIndex(trans, 'name', 13)
2592.1.7 by Robert Collins
A validate that goes boom.
770
        self.assertRaises(errors.BadIndexFormatSignature, index.validate)
2592.1.8 by Robert Collins
Empty files should validate ok.
771
2592.1.10 by Robert Collins
Make validate detect node reference parsing errors.
772
    def test_validate_bad_node_refs(self):
773
        index = self.make_index(2)
774
        trans = self.get_transport()
775
        content = trans.get_bytes('index')
776
        # change the options line to end with a rather than a parseable number
777
        new_content = content[:-2] + 'a\n\n'
778
        trans.put_bytes('index', new_content)
779
        self.assertRaises(errors.BadIndexOptions, index.validate)
780
2592.1.27 by Robert Collins
Test missing end lines with non-empty indices.
781
    def test_validate_missing_end_line_empty(self):
2592.1.11 by Robert Collins
Detect truncated indices.
782
        index = self.make_index(2)
783
        trans = self.get_transport()
784
        content = trans.get_bytes('index')
785
        # truncate the last byte
786
        trans.put_bytes('index', content[:-1])
787
        self.assertRaises(errors.BadIndexData, index.validate)
788
2592.1.27 by Robert Collins
Test missing end lines with non-empty indices.
789
    def test_validate_missing_end_line_nonempty(self):
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
790
        index = self.make_index(2, nodes=[(('key', ), '', ([], []))])
2592.1.27 by Robert Collins
Test missing end lines with non-empty indices.
791
        trans = self.get_transport()
792
        content = trans.get_bytes('index')
793
        # truncate the last byte
794
        trans.put_bytes('index', content[:-1])
795
        self.assertRaises(errors.BadIndexData, index.validate)
796
2592.1.8 by Robert Collins
Empty files should validate ok.
797
    def test_validate_empty(self):
798
        index = self.make_index()
799
        index.validate()
2592.1.12 by Robert Collins
Handle basic node adds.
800
801
    def test_validate_no_refs_content(self):
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
802
        index = self.make_index(nodes=[(('key', ), 'value', ())])
2592.1.12 by Robert Collins
Handle basic node adds.
803
        index.validate()
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
804
805
806
class TestCombinedGraphIndex(TestCaseWithMemoryTransport):
807
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
808
    def make_index(self, name, ref_lists=0, key_elements=1, nodes=[]):
809
        builder = GraphIndexBuilder(ref_lists, key_elements=key_elements)
2624.2.17 by Robert Collins
Review feedback.
810
        for key, value, references in nodes:
811
            builder.add_node(key, value, references)
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
812
        stream = builder.finish()
813
        trans = self.get_transport()
2890.2.1 by Robert Collins
* ``bzrlib.index.GraphIndex`` now requires a size parameter to the
814
        size = trans.put_file(name, stream)
815
        return GraphIndex(trans, name, size)
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
816
817
    def test_open_missing_index_no_error(self):
818
        trans = self.get_transport()
2890.2.1 by Robert Collins
* ``bzrlib.index.GraphIndex`` now requires a size parameter to the
819
        index1 = GraphIndex(trans, 'missing', 100)
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
820
        index = CombinedGraphIndex([index1])
821
2592.1.37 by Robert Collins
Add CombinedGraphIndex.insert_index.
822
    def test_add_index(self):
823
        index = CombinedGraphIndex([])
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
824
        index1 = self.make_index('name', 0, nodes=[(('key', ), '', ())])
2592.1.37 by Robert Collins
Add CombinedGraphIndex.insert_index.
825
        index.insert_index(0, index1)
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
826
        self.assertEqual([(index1, ('key', ), '')], list(index.iter_all_entries()))
2592.1.37 by Robert Collins
Add CombinedGraphIndex.insert_index.
827
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
828
    def test_iter_all_entries_empty(self):
829
        index = CombinedGraphIndex([])
830
        self.assertEqual([], list(index.iter_all_entries()))
831
832
    def test_iter_all_entries_children_empty(self):
833
        index1 = self.make_index('name')
834
        index = CombinedGraphIndex([index1])
835
        self.assertEqual([], list(index.iter_all_entries()))
836
837
    def test_iter_all_entries_simple(self):
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
838
        index1 = self.make_index('name', nodes=[(('name', ), 'data', ())])
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
839
        index = CombinedGraphIndex([index1])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
840
        self.assertEqual([(index1, ('name', ), 'data')],
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
841
            list(index.iter_all_entries()))
842
843
    def test_iter_all_entries_two_indices(self):
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
844
        index1 = self.make_index('name1', nodes=[(('name', ), 'data', ())])
845
        index2 = self.make_index('name2', nodes=[(('2', ), '', ())])
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
846
        index = CombinedGraphIndex([index1, index2])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
847
        self.assertEqual([(index1, ('name', ), 'data'),
848
            (index2, ('2', ), '')],
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
849
            list(index.iter_all_entries()))
850
2592.1.39 by Robert Collins
CombinedGraphIndex.iter_entries does not need to see all entries.
851
    def test_iter_entries_two_indices_dup_key(self):
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
852
        index1 = self.make_index('name1', nodes=[(('name', ), 'data', ())])
853
        index2 = self.make_index('name2', nodes=[(('name', ), 'data', ())])
2592.1.39 by Robert Collins
CombinedGraphIndex.iter_entries does not need to see all entries.
854
        index = CombinedGraphIndex([index1, index2])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
855
        self.assertEqual([(index1, ('name', ), 'data')],
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
856
            list(index.iter_entries([('name', )])))
2592.1.39 by Robert Collins
CombinedGraphIndex.iter_entries does not need to see all entries.
857
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
858
    def test_iter_all_entries_two_indices_dup_key(self):
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
859
        index1 = self.make_index('name1', nodes=[(('name', ), 'data', ())])
860
        index2 = self.make_index('name2', nodes=[(('name', ), 'data', ())])
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
861
        index = CombinedGraphIndex([index1, index2])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
862
        self.assertEqual([(index1, ('name', ), 'data')],
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
863
            list(index.iter_all_entries()))
864
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
865
    def test_iter_key_prefix_2_key_element_refs(self):
866
        index1 = self.make_index('1', 1, key_elements=2, nodes=[
867
            (('name', 'fin1'), 'data', ([('ref', 'erence')], ))])
868
        index2 = self.make_index('2', 1, key_elements=2, nodes=[
869
            (('name', 'fin2'), 'beta', ([], )),
870
            (('ref', 'erence'), 'refdata', ([], ))])
871
        index = CombinedGraphIndex([index1, index2])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
872
        self.assertEqual(set([(index1, ('name', 'fin1'), 'data', ((('ref', 'erence'),),)),
873
            (index2, ('ref', 'erence'), 'refdata', ((), ))]),
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
874
            set(index.iter_entries_prefix([('name', 'fin1'), ('ref', 'erence')])))
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
875
        self.assertEqual(set([(index1, ('name', 'fin1'), 'data', ((('ref', 'erence'),),)),
876
            (index2, ('name', 'fin2'), 'beta', ((), ))]),
2624.2.9 by Robert Collins
Introduce multiple component keys, which is what is needed to combine multiple knit indices into one.
877
            set(index.iter_entries_prefix([('name', None)])))
878
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
879
    def test_iter_nothing_empty(self):
880
        index = CombinedGraphIndex([])
881
        self.assertEqual([], list(index.iter_entries([])))
882
883
    def test_iter_nothing_children_empty(self):
884
        index1 = self.make_index('name')
885
        index = CombinedGraphIndex([index1])
886
        self.assertEqual([], list(index.iter_entries([])))
887
888
    def test_iter_all_keys(self):
889
        index1 = self.make_index('1', 1, nodes=[
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
890
            (('name', ), 'data', ([('ref', )], ))])
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
891
        index2 = self.make_index('2', 1, nodes=[
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
892
            (('ref', ), 'refdata', ((), ))])
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
893
        index = CombinedGraphIndex([index1, index2])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
894
        self.assertEqual(set([(index1, ('name', ), 'data', ((('ref', ), ), )),
895
            (index2, ('ref', ), 'refdata', ((), ))]),
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
896
            set(index.iter_entries([('name', ), ('ref', )])))
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
897
 
898
    def test_iter_all_keys_dup_entry(self):
899
        index1 = self.make_index('1', 1, nodes=[
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
900
            (('name', ), 'data', ([('ref', )], )),
901
            (('ref', ), 'refdata', ([], ))])
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
902
        index2 = self.make_index('2', 1, nodes=[
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
903
            (('ref', ), 'refdata', ([], ))])
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
904
        index = CombinedGraphIndex([index1, index2])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
905
        self.assertEqual(set([(index1, ('name', ), 'data', ((('ref',),),)),
906
            (index1, ('ref', ), 'refdata', ((), ))]),
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
907
            set(index.iter_entries([('name', ), ('ref', )])))
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
908
 
909
    def test_iter_missing_entry_empty(self):
910
        index = CombinedGraphIndex([])
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
911
        self.assertEqual([], list(index.iter_entries([('a', )])))
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
912
913
    def test_iter_missing_entry_one_index(self):
914
        index1 = self.make_index('1')
915
        index = CombinedGraphIndex([index1])
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
916
        self.assertEqual([], list(index.iter_entries([('a', )])))
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
917
918
    def test_iter_missing_entry_two_index(self):
919
        index1 = self.make_index('1')
920
        index2 = self.make_index('2')
921
        index = CombinedGraphIndex([index1, index2])
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
922
        self.assertEqual([], list(index.iter_entries([('a', )])))
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
923
 
924
    def test_iter_entry_present_one_index_only(self):
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
925
        index1 = self.make_index('1', nodes=[(('key', ), '', ())])
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
926
        index2 = self.make_index('2', nodes=[])
927
        index = CombinedGraphIndex([index1, index2])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
928
        self.assertEqual([(index1, ('key', ), '')],
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
929
            list(index.iter_entries([('key', )])))
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
930
        # and in the other direction
931
        index = CombinedGraphIndex([index2, index1])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
932
        self.assertEqual([(index1, ('key', ), '')],
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
933
            list(index.iter_entries([('key', )])))
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
934
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
935
    def test_key_count_empty(self):
936
        index1 = self.make_index('1', nodes=[])
937
        index2 = self.make_index('2', nodes=[])
938
        index = CombinedGraphIndex([index1, index2])
939
        self.assertEqual(0, index.key_count())
940
941
    def test_key_count_sums_index_keys(self):
942
        index1 = self.make_index('1', nodes=[
943
            (('1',), '', ()),
944
            (('2',), '', ())])
945
        index2 = self.make_index('2', nodes=[(('1',), '', ())])
946
        index = CombinedGraphIndex([index1, index2])
947
        self.assertEqual(3, index.key_count())
948
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
949
    def test_validate_bad_child_index_errors(self):
950
        trans = self.get_transport()
951
        trans.put_bytes('name', "not an index\n")
2890.2.1 by Robert Collins
* ``bzrlib.index.GraphIndex`` now requires a size parameter to the
952
        index1 = GraphIndex(trans, 'name', 13)
2592.1.31 by Robert Collins
Build a combined graph index to use multiple indices at once.
953
        index = CombinedGraphIndex([index1])
954
        self.assertRaises(errors.BadIndexFormatSignature, index.validate)
955
956
    def test_validate_empty(self):
957
        index = CombinedGraphIndex([])
958
        index.validate()
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
959
960
961
class TestInMemoryGraphIndex(TestCaseWithMemoryTransport):
962
2624.2.10 by Robert Collins
Also add iter_key_prefix support to InMemoryGraphIndex.
963
    def make_index(self, ref_lists=0, key_elements=1, nodes=[]):
964
        result = InMemoryGraphIndex(ref_lists, key_elements=key_elements)
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
965
        result.add_nodes(nodes)
966
        return result
967
2624.2.1 by Robert Collins
InMemoryGraphIndex.add_nodes was inconsistent with other metods for non-node-reference indices.
968
    def test_add_nodes_no_refs(self):
969
        index = self.make_index(0)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
970
        index.add_nodes([(('name', ), 'data')])
971
        index.add_nodes([(('name2', ), ''), (('name3', ), '')])
2624.2.1 by Robert Collins
InMemoryGraphIndex.add_nodes was inconsistent with other metods for non-node-reference indices.
972
        self.assertEqual(set([
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
973
            (index, ('name', ), 'data'),
974
            (index, ('name2', ), ''),
975
            (index, ('name3', ), ''),
2624.2.1 by Robert Collins
InMemoryGraphIndex.add_nodes was inconsistent with other metods for non-node-reference indices.
976
            ]), set(index.iter_all_entries()))
977
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
978
    def test_add_nodes(self):
979
        index = self.make_index(1)
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
980
        index.add_nodes([(('name', ), 'data', ([],))])
981
        index.add_nodes([(('name2', ), '', ([],)), (('name3', ), '', ([('r', )],))])
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
982
        self.assertEqual(set([
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
983
            (index, ('name', ), 'data', ((),)),
984
            (index, ('name2', ), '', ((),)),
985
            (index, ('name3', ), '', ((('r', ), ), )),
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
986
            ]), set(index.iter_all_entries()))
987
988
    def test_iter_all_entries_empty(self):
989
        index = self.make_index()
990
        self.assertEqual([], list(index.iter_all_entries()))
991
992
    def test_iter_all_entries_simple(self):
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
993
        index = self.make_index(nodes=[(('name', ), 'data')])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
994
        self.assertEqual([(index, ('name', ), 'data')],
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
995
            list(index.iter_all_entries()))
996
997
    def test_iter_all_entries_references(self):
998
        index = self.make_index(1, nodes=[
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
999
            (('name', ), 'data', ([('ref', )], )),
1000
            (('ref', ), 'refdata', ([], ))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1001
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref', ),),)),
1002
            (index, ('ref', ), 'refdata', ((), ))]),
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
1003
            set(index.iter_all_entries()))
1004
1005
    def test_iteration_absent_skipped(self):
1006
        index = self.make_index(1, nodes=[
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
1007
            (('name', ), 'data', ([('ref', )], ))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1008
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref',),),))]),
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
1009
            set(index.iter_all_entries()))
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1010
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref',),),))]),
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
1011
            set(index.iter_entries([('name', )])))
1012
        self.assertEqual([], list(index.iter_entries([('ref', )])))
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
1013
1014
    def test_iter_all_keys(self):
1015
        index = self.make_index(1, nodes=[
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
1016
            (('name', ), 'data', ([('ref', )], )),
1017
            (('ref', ), 'refdata', ([], ))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1018
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref',),),)),
1019
            (index, ('ref', ), 'refdata', ((), ))]),
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
1020
            set(index.iter_entries([('name', ), ('ref', )])))
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
1021
2624.2.10 by Robert Collins
Also add iter_key_prefix support to InMemoryGraphIndex.
1022
    def test_iter_key_prefix_1_key_element_no_refs(self):
1023
        index = self.make_index( nodes=[
1024
            (('name', ), 'data'),
1025
            (('ref', ), 'refdata')])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1026
        self.assertEqual(set([(index, ('name', ), 'data'),
1027
            (index, ('ref', ), 'refdata')]),
2624.2.10 by Robert Collins
Also add iter_key_prefix support to InMemoryGraphIndex.
1028
            set(index.iter_entries_prefix([('name', ), ('ref', )])))
1029
1030
    def test_iter_key_prefix_1_key_element_refs(self):
1031
        index = self.make_index(1, nodes=[
1032
            (('name', ), 'data', ([('ref', )], )),
1033
            (('ref', ), 'refdata', ([], ))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1034
        self.assertEqual(set([(index, ('name', ), 'data', ((('ref',),),)),
1035
            (index, ('ref', ), 'refdata', ((), ))]),
2624.2.10 by Robert Collins
Also add iter_key_prefix support to InMemoryGraphIndex.
1036
            set(index.iter_entries_prefix([('name', ), ('ref', )])))
1037
1038
    def test_iter_key_prefix_2_key_element_no_refs(self):
1039
        index = self.make_index(key_elements=2, nodes=[
1040
            (('name', 'fin1'), 'data'),
1041
            (('name', 'fin2'), 'beta'),
1042
            (('ref', 'erence'), 'refdata')])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1043
        self.assertEqual(set([(index, ('name', 'fin1'), 'data'),
1044
            (index, ('ref', 'erence'), 'refdata')]),
2624.2.10 by Robert Collins
Also add iter_key_prefix support to InMemoryGraphIndex.
1045
            set(index.iter_entries_prefix([('name', 'fin1'), ('ref', 'erence')])))
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1046
        self.assertEqual(set([(index, ('name', 'fin1'), 'data'),
1047
            (index, ('name', 'fin2'), 'beta')]),
2624.2.10 by Robert Collins
Also add iter_key_prefix support to InMemoryGraphIndex.
1048
            set(index.iter_entries_prefix([('name', None)])))
1049
1050
    def test_iter_key_prefix_2_key_element_refs(self):
1051
        index = self.make_index(1, key_elements=2, nodes=[
1052
            (('name', 'fin1'), 'data', ([('ref', 'erence')], )),
1053
            (('name', 'fin2'), 'beta', ([], )),
1054
            (('ref', 'erence'), 'refdata', ([], ))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1055
        self.assertEqual(set([(index, ('name', 'fin1'), 'data', ((('ref', 'erence'),),)),
1056
            (index, ('ref', 'erence'), 'refdata', ((), ))]),
2624.2.10 by Robert Collins
Also add iter_key_prefix support to InMemoryGraphIndex.
1057
            set(index.iter_entries_prefix([('name', 'fin1'), ('ref', 'erence')])))
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1058
        self.assertEqual(set([(index, ('name', 'fin1'), 'data', ((('ref', 'erence'),),)),
1059
            (index, ('name', 'fin2'), 'beta', ((), ))]),
2624.2.10 by Robert Collins
Also add iter_key_prefix support to InMemoryGraphIndex.
1060
            set(index.iter_entries_prefix([('name', None)])))
1061
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
1062
    def test_iter_nothing_empty(self):
1063
        index = self.make_index()
1064
        self.assertEqual([], list(index.iter_entries([])))
1065
1066
    def test_iter_missing_entry_empty(self):
1067
        index = self.make_index()
1068
        self.assertEqual([], list(index.iter_entries(['a'])))
1069
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
1070
    def test_key_count_empty(self):
1071
        index = self.make_index()
1072
        self.assertEqual(0, index.key_count())
1073
1074
    def test_key_count_one(self):
1075
        index = self.make_index(nodes=[(('name', ), '')])
1076
        self.assertEqual(1, index.key_count())
1077
1078
    def test_key_count_two(self):
1079
        index = self.make_index(nodes=[(('name', ), ''), (('foo', ), '')])
1080
        self.assertEqual(2, index.key_count())
1081
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
1082
    def test_validate_empty(self):
1083
        index = self.make_index()
1084
        index.validate()
1085
1086
    def test_validate_no_refs_content(self):
2624.2.5 by Robert Collins
Change bzrlib.index.Index keys to be 1-tuples, not strings.
1087
        index = self.make_index(nodes=[(('key', ), 'value')])
2592.1.38 by Robert Collins
Create an InMemoryGraphIndex for temporary indexing.
1088
        index.validate()
1089
1090
2624.2.12 by Robert Collins
Create an adapter between indices with differing key lengths.
1091
class TestGraphIndexPrefixAdapter(TestCaseWithMemoryTransport):
1092
2624.2.13 by Robert Collins
Implement add_node/add_nodes to the GraphIndexPrefixAdapter.
1093
    def make_index(self, ref_lists=1, key_elements=2, nodes=[], add_callback=False):
2624.2.12 by Robert Collins
Create an adapter between indices with differing key lengths.
1094
        result = InMemoryGraphIndex(ref_lists, key_elements=key_elements)
1095
        result.add_nodes(nodes)
2624.2.13 by Robert Collins
Implement add_node/add_nodes to the GraphIndexPrefixAdapter.
1096
        if add_callback:
2624.2.17 by Robert Collins
Review feedback.
1097
            add_nodes_callback = result.add_nodes
2624.2.13 by Robert Collins
Implement add_node/add_nodes to the GraphIndexPrefixAdapter.
1098
        else:
2624.2.17 by Robert Collins
Review feedback.
1099
            add_nodes_callback = None
2624.2.13 by Robert Collins
Implement add_node/add_nodes to the GraphIndexPrefixAdapter.
1100
        adapter = GraphIndexPrefixAdapter(result, ('prefix', ), key_elements - 1,
1101
            add_nodes_callback=add_nodes_callback)
2624.2.12 by Robert Collins
Create an adapter between indices with differing key lengths.
1102
        return result, adapter
1103
2624.2.13 by Robert Collins
Implement add_node/add_nodes to the GraphIndexPrefixAdapter.
1104
    def test_add_node(self):
1105
        index, adapter = self.make_index(add_callback=True)
1106
        adapter.add_node(('key',), 'value', ((('ref',),),))
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1107
        self.assertEqual(set([(index, ('prefix', 'key'), 'value', ((('prefix', 'ref'),),))]),
2624.2.13 by Robert Collins
Implement add_node/add_nodes to the GraphIndexPrefixAdapter.
1108
            set(index.iter_all_entries()))
1109
1110
    def test_add_nodes(self):
1111
        index, adapter = self.make_index(add_callback=True)
1112
        adapter.add_nodes((
1113
            (('key',), 'value', ((('ref',),),)),
1114
            (('key2',), 'value2', ((),)),
1115
            ))
1116
        self.assertEqual(set([
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1117
            (index, ('prefix', 'key2'), 'value2', ((),)),
1118
            (index, ('prefix', 'key'), 'value', ((('prefix', 'ref'),),))
2624.2.13 by Robert Collins
Implement add_node/add_nodes to the GraphIndexPrefixAdapter.
1119
            ]),
1120
            set(index.iter_all_entries()))
1121
2624.2.12 by Robert Collins
Create an adapter between indices with differing key lengths.
1122
    def test_construct(self):
1123
        index = InMemoryGraphIndex()
1124
        adapter = GraphIndexPrefixAdapter(index, ('prefix', ), 1)
1125
1126
    def test_construct_with_callback(self):
1127
        index = InMemoryGraphIndex()
1128
        adapter = GraphIndexPrefixAdapter(index, ('prefix', ), 1, index.add_nodes)
1129
1130
    def test_iter_all_entries_cross_prefix_map_errors(self):
1131
        index, adapter = self.make_index(nodes=[
1132
            (('prefix', 'key1'), 'data1', ((('prefixaltered', 'key2'),),))])
1133
        self.assertRaises(errors.BadIndexData, list, adapter.iter_all_entries())
1134
1135
    def test_iter_all_entries(self):
1136
        index, adapter = self.make_index(nodes=[
1137
            (('notprefix', 'key1'), 'data', ((), )),
1138
            (('prefix', 'key1'), 'data1', ((), )),
1139
            (('prefix', 'key2'), 'data2', ((('prefix', 'key1'),),))])
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1140
        self.assertEqual(set([(index, ('key1', ), 'data1', ((),)),
1141
            (index, ('key2', ), 'data2', ((('key1',),),))]),
2624.2.12 by Robert Collins
Create an adapter between indices with differing key lengths.
1142
            set(adapter.iter_all_entries()))
1143
1144
    def test_iter_entries(self):
1145
        index, adapter = self.make_index(nodes=[
1146
            (('notprefix', 'key1'), 'data', ((), )),
1147
            (('prefix', 'key1'), 'data1', ((), )),
1148
            (('prefix', 'key2'), 'data2', ((('prefix', 'key1'),),))])
1149
        # ask for many - get all
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1150
        self.assertEqual(set([(index, ('key1', ), 'data1', ((),)),
1151
            (index, ('key2', ), 'data2', ((('key1', ),),))]),
2624.2.12 by Robert Collins
Create an adapter between indices with differing key lengths.
1152
            set(adapter.iter_entries([('key1', ), ('key2', )])))
1153
        # ask for one, get one
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1154
        self.assertEqual(set([(index, ('key1', ), 'data1', ((),))]),
2624.2.12 by Robert Collins
Create an adapter between indices with differing key lengths.
1155
            set(adapter.iter_entries([('key1', )])))
1156
        # ask for missing, get none
1157
        self.assertEqual(set(),
1158
            set(adapter.iter_entries([('key3', )])))
1159
1160
    def test_iter_entries_prefix(self):
1161
        index, adapter = self.make_index(key_elements=3, nodes=[
1162
            (('notprefix', 'foo', 'key1'), 'data', ((), )),
1163
            (('prefix', 'prefix2', 'key1'), 'data1', ((), )),
1164
            (('prefix', 'prefix2', 'key2'), 'data2', ((('prefix', 'prefix2', 'key1'),),))])
1165
        # ask for a prefix, get the results for just that prefix, adjusted.
2624.2.14 by Robert Collins
Add source index to the index iteration API to allow mapping back to the origin of retrieved data.
1166
        self.assertEqual(set([(index, ('prefix2', 'key1', ), 'data1', ((),)),
1167
            (index, ('prefix2', 'key2', ), 'data2', ((('prefix2', 'key1', ),),))]),
2624.2.12 by Robert Collins
Create an adapter between indices with differing key lengths.
1168
            set(adapter.iter_entries_prefix([('prefix2', None)])))
1169
2624.2.16 by Robert Collins
Add a key_count method to GraphIndex and friends, allowing optimisation of length calculations by the index.
1170
    def test_key_count_no_matching_keys(self):
1171
        index, adapter = self.make_index(nodes=[
1172
            (('notprefix', 'key1'), 'data', ((), ))])
1173
        self.assertEqual(0, adapter.key_count())
1174
1175
    def test_key_count_some_keys(self):
1176
        index, adapter = self.make_index(nodes=[
1177
            (('notprefix', 'key1'), 'data', ((), )),
1178
            (('prefix', 'key1'), 'data1', ((), )),
1179
            (('prefix', 'key2'), 'data2', ((('prefix', 'key1'),),))])
1180
        self.assertEqual(2, adapter.key_count())
1181
2624.2.12 by Robert Collins
Create an adapter between indices with differing key lengths.
1182
    def test_validate(self):
1183
        index, adapter = self.make_index()
1184
        calls = []
1185
        def validate():
1186
            calls.append('called')
1187
        index.validate = validate
1188
        adapter.validate()
1189
        self.assertEqual(['called'], calls)