← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to groupcompress.py

Cap group size at 20MB internal buffer. (Probably way too big).

Show diffs side-by-side

added added

removed removed

Lines of Context:
groupcompress.py
476
476
                adapters[adapter_key] = adapter
477
477
                return adapter
478
478
        adapters = {}
479
 
        compressor = GroupCompressor(self._delta)
480
479
        # This will go up to fulltexts for gc to gc fetching, which isn't
481
480
        # ideal.
 
481
        compressor = GroupCompressor(self._delta)
482
482
        keys_to_add = []
483
483
        basis_end = 0
 
484
        groups = 1
 
485
        def flush():
 
486
            compressed = zlib.compress(''.join(compressor.lines))
 
487
            index, start, length = self._access.add_raw_records(
 
488
                [(None, len(compressed))], compressed)[0]
 
489
            nodes = []
 
490
            for key, reads, refs in keys_to_add:
 
491
                nodes.append((key, "%d %d %s" % (start, length, reads), refs))
 
492
            self._index.add_records(nodes, random_id=random_id)
484
493
        for record in stream:
485
494
            # Raise an error when a record is missing.
486
495
            if record.storage_kind == 'absent':
498
507
            keys_to_add.append((record.key, '%d %d' % (basis_end, end_point),
499
508
                (record.parents,)))
500
509
            basis_end = end_point
501
 
        compressed = zlib.compress(''.join(compressor.lines))
502
 
        index, start, length = self._access.add_raw_records(
503
 
            [(None, len(compressed))], compressed)[0]
504
 
        nodes = []
505
 
        for key, reads, refs in keys_to_add:
506
 
            nodes.append((key, "%d %d %s" % (start, length, reads), refs))
507
 
        self._index.add_records(nodes, random_id=random_id)
 
510
            if basis_end > 1024 * 1024 * 20:
 
511
                flush()
 
512
                compressor = GroupCompressor(self._delta)
 
513
                keys_to_add = []
 
514
                basis_end = 0
 
515
                groups += 1
508
516
 
509
517
    def iter_lines_added_or_present_in_keys(self, keys, pb=None):
510
518
        """Iterate over the lines in the versioned files from keys.