← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/btree_index.py

Merge bzr.dev 4187, and revert the change to fix refcycle issues.

I apparently didn't run the smart fetch tests. Which show that we access inv+chk pages
as a fulltext, and then insert the stream, which expects to get the block as a compressed
block. :(.
Need to rethink how to do it, possibly with weakrefs.


This also brings in CommitBuilder.record_iter_changes() and the updates to btree_index
and backing indices.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/btree_index.py
180
180
        combine mem with the first and second indexes, creating a new one of
181
181
        size 4x. On the fifth create a single new one, etc.
182
182
        """
183
 
        iterators_to_combine = [self._iter_mem_nodes()]
184
 
        pos = -1
185
 
        for pos, backing in enumerate(self._backing_indices):
186
 
            if backing is None:
187
 
                pos -= 1
188
 
                break
189
 
            iterators_to_combine.append(backing.iter_all_entries())
190
 
        backing_pos = pos + 1
191
 
        new_backing_file, size = \
192
 
            self._write_nodes(self._iter_smallest(iterators_to_combine))
 
183
        if self._combine_backing_indices:
 
184
            (new_backing_file, size,
 
185
             backing_pos) = self._spill_mem_keys_and_combine()
 
186
        else:
 
187
            new_backing_file, size = self._spill_mem_keys_without_combining()
193
188
        dir_path, base_name = osutils.split(new_backing_file.name)
194
189
        # Note: The transport here isn't strictly needed, because we will use
195
190
        #       direct access to the new_backing._file object
197
192
                                      base_name, size)
198
193
        # GC will clean up the file
199
194
        new_backing._file = new_backing_file
200
 
        if len(self._backing_indices) == backing_pos:
201
 
            self._backing_indices.append(None)
202
 
        self._backing_indices[backing_pos] = new_backing
203
 
        for pos in range(backing_pos):
204
 
            self._backing_indices[pos] = None
 
195
        if self._combine_backing_indices:
 
196
            if len(self._backing_indices) == backing_pos:
 
197
                self._backing_indices.append(None)
 
198
            self._backing_indices[backing_pos] = new_backing
 
199
            for backing_pos in range(backing_pos):
 
200
                self._backing_indices[backing_pos] = None
 
201
        else:
 
202
            self._backing_indices.append(new_backing)
205
203
        self._keys = set()
206
204
        self._nodes = {}
207
205
        self._nodes_by_key = None
208
206
 
 
207
    def _spill_mem_keys_without_combining(self):
 
208
        return self._write_nodes(self._iter_mem_nodes(), allow_optimize=False)
 
209
 
 
210
    def _spill_mem_keys_and_combine(self):
 
211
        iterators_to_combine = [self._iter_mem_nodes()]
 
212
        pos = -1
 
213
        for pos, backing in enumerate(self._backing_indices):
 
214
            if backing is None:
 
215
                pos -= 1
 
216
                break
 
217
            iterators_to_combine.append(backing.iter_all_entries())
 
218
        backing_pos = pos + 1
 
219
        new_backing_file, size = \
 
220
            self._write_nodes(self._iter_smallest(iterators_to_combine),
 
221
                              allow_optimize=False)
 
222
        return new_backing_file, size, backing_pos
 
223
 
209
224
    def add_nodes(self, nodes):
210
225
        """Add nodes to the index.
211
226
 
262
277
            except StopIteration:
263
278
                current_values[pos] = None
264
279
 
265
 
    def _add_key(self, string_key, line, rows):
 
280
    def _add_key(self, string_key, line, rows, allow_optimize=True):
266
281
        """Add a key to the current chunk.
267
282
 
268
283
        :param string_key: The key to add.
269
284
        :param line: The fully serialised key and value.
 
285
        :param allow_optimize: If set to False, prevent setting the optimize
 
286
            flag when writing out. This is used by the _spill_mem_keys_to_disk
 
287
            functionality.
270
288
        """
271
289
        if rows[-1].writer is None:
272
290
            # opening a new leaf chunk;
277
295
                    length = _PAGE_SIZE
278
296
                    if internal_row.nodes == 0:
279
297
                        length -= _RESERVED_HEADER_BYTES # padded
 
298
                    if allow_optimize:
 
299
                        optimize_for_size = self._optimize_for_size
 
300
                    else:
 
301
                        optimize_for_size = False
280
302
                    internal_row.writer = chunk_writer.ChunkWriter(length, 0,
281
 
                        optimize_for_size=self._optimize_for_size)
 
303
                        optimize_for_size=optimize_for_size)
282
304
                    internal_row.writer.write(_INTERNAL_FLAG)
283
305
                    internal_row.writer.write(_INTERNAL_OFFSET +
284
306
                        str(rows[pos + 1].nodes) + "\n")
322
344
                new_row.writer.write(_INTERNAL_OFFSET +
323
345
                    str(rows[1].nodes - 1) + "\n")
324
346
                new_row.writer.write(key_line)
325
 
            self._add_key(string_key, line, rows)
 
347
            self._add_key(string_key, line, rows, allow_optimize=allow_optimize)
326
348
 
327
 
    def _write_nodes(self, node_iterator):
 
349
    def _write_nodes(self, node_iterator, allow_optimize=True):
328
350
        """Write node_iterator out as a B+Tree.
329
351
 
330
352
        :param node_iterator: An iterator of sorted nodes. Each node should
331
353
            match the output given by iter_all_entries.
 
354
        :param allow_optimize: If set to False, prevent setting the optimize
 
355
            flag when writing out. This is used by the _spill_mem_keys_to_disk
 
356
            functionality.
332
357
        :return: A file handle for a temporary file containing a B+Tree for
333
358
            the nodes.
334
359
        """
353
378
            key_count += 1
354
379
            string_key, line = _btree_serializer._flatten_node(node,
355
380
                                    self.reference_lists)
356
 
            self._add_key(string_key, line, rows)
 
381
            self._add_key(string_key, line, rows, allow_optimize=allow_optimize)
357
382
        for row in reversed(rows):
358
383
            pad = (type(row) != _LeafBuilderRow)
359
384
            row.finish_node(pad=pad)
360
 
        result = tempfile.NamedTemporaryFile()
 
385
        result = tempfile.NamedTemporaryFile(prefix='bzr-index-')
361
386
        lines = [_BTSIGNATURE]
362
387
        lines.append(_OPTION_NODE_REFS + str(self.reference_lists) + '\n')
363
388
        lines.append(_OPTION_KEY_ELEMENTS + str(self._key_length) + '\n')