← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

  • Committer: John Arbash Meinel
  • Date: 2009-03-20 15:02:05 UTC
  • mto: (3735.2.161 brisbane-core)
  • mto: This revision was merged to the branch mainline in revision 4280.
  • Revision ID: john@arbash-meinel.com-20090320150205-kcmh70biyo76p0kn
Some testing to see if we can decrease the peak memory consumption a bit.
It looks like we can, just need some more perf, etc.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
190
190
                if pb is not None:
191
191
                    pb.update(message, idx + 1, len(keys))
192
192
                yield record
 
193
                record._manager = None
 
194
                record._bytes = None
193
195
        return pb_stream()
194
196
 
195
197
    def _get_filtered_inv_stream(self, source_vf, keys, message, pb=None):
216
218
                    p_id_roots_set.add(key)
217
219
                    self._chk_p_id_roots.append(key)
218
220
                yield record
 
221
                record._manager = None
 
222
                record._bytes = None
219
223
            # We have finished processing all of the inventory records, we
220
224
            # don't need these sets anymore
221
225
            id_roots_set.clear()
301
305
                        if pb is not None:
302
306
                            pb.update('chk node', counter[0], total_keys)
303
307
                        yield record
 
308
                        record._manager = None
 
309
                        record._bytes = None
304
310
                yield next_stream()
305
311
                # Double check that we won't be emitting any keys twice
306
312
                # If we get rid of the pre-calculation of all keys, we could
312
318
                # next_keys = next_keys.intersection(remaining_keys)
313
319
                cur_keys = []
314
320
                for prefix in sorted(keys_by_search_prefix):
315
 
                    cur_keys.extend(keys_by_search_prefix[prefix])
 
321
                    cur_keys.extend(keys_by_search_prefix.pop(prefix))
316
322
        for stream in _get_referenced_stream(self._chk_id_roots,
317
323
                                             self._gather_text_refs):
318
324
            yield stream
395
401
            self.revision_keys = source_vf.keys()
396
402
        self._copy_stream(source_vf, target_vf, self.revision_keys,
397
403
                          'revisions', self._get_progress_stream, 1)
 
404
        for index in source_vf._index._graph_index._indices:
 
405
            index._leaf_node_cache.clear()
 
406
        # target_vf._index._graph_index._spill_mem_keys_to_disk()
398
407
 
399
408
    def _copy_inventory_texts(self):
400
409
        source_vf, target_vf = self._build_vfs('inventory', True, True)
401
410
        self._copy_stream(source_vf, target_vf, self.revision_keys,
402
411
                          'inventories', self._get_filtered_inv_stream, 2)
 
412
        for index in source_vf._index._graph_index._indices:
 
413
            index._leaf_node_cache.clear()
 
414
        # target_vf._index._graph_index._spill_mem_keys_to_disk()
403
415
 
404
416
    def _copy_chk_texts(self):
405
417
        source_vf, target_vf = self._build_vfs('chk', False, False)
421
433
                    pass
422
434
        finally:
423
435
            child_pb.finished()
 
436
        for index in source_vf._index._graph_index._indices:
 
437
            index._leaf_node_cache.clear()
 
438
        # target_vf._index._graph_index._spill_mem_keys_to_disk()
424
439
 
425
440
    def _copy_text_texts(self):
426
441
        source_vf, target_vf = self._build_vfs('text', True, True)
432
447
        text_keys = source_vf.keys()
433
448
        self._copy_stream(source_vf, target_vf, text_keys,
434
449
                          'text', self._get_progress_stream, 4)
 
450
        for index in source_vf._index._graph_index._indices:
 
451
            index._leaf_node_cache.clear()
 
452
        # target_vf._index._graph_index._spill_mem_keys_to_disk()
435
453
 
436
454
    def _copy_signature_texts(self):
437
455
        source_vf, target_vf = self._build_vfs('signature', False, False)
439
457
        signature_keys.intersection(self.revision_keys)
440
458
        self._copy_stream(source_vf, target_vf, signature_keys,
441
459
                          'signatures', self._get_progress_stream, 5)
 
460
        for index in source_vf._index._graph_index._indices:
 
461
            index._leaf_node_cache.clear()
 
462
        # target_vf._index._graph_index._spill_mem_keys_to_disk()
442
463
 
443
464
    def _create_pack_from_packs(self):
444
465
        self.pb.update('repacking', 0, 7)
451
472
        self._copy_text_texts()
452
473
        self._copy_signature_texts()
453
474
        self.new_pack._check_references()
 
475
        trace.debug_memory('after fetch')
454
476
        if not self._use_pack(self.new_pack):
455
477
            self.new_pack.abort()
456
478
            return None