← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/groupcompress.py

  • Committer: John Arbash Meinel
  • Date: 2009-03-20 15:02:05 UTC
  • mto: (3735.2.161 brisbane-core)
  • mto: This revision was merged to the branch mainline in revision 4280.
  • Revision ID: john@arbash-meinel.com-20090320150205-kcmh70biyo76p0kn
Some testing to see if we can decrease the peak memory consumption a bit.
It looks like we can, just need some more perf, etc.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/groupcompress.py
339
339
        :param sha1: TODO (should we validate only when sha1 is supplied?)
340
340
        :return: The bytes for the content
341
341
        """
 
342
        if start == end == 0:
 
343
            return None, ''
342
344
        # Make sure we have enough bytes for this record
343
345
        # TODO: if we didn't want to track the end of this entry, we could
344
346
        #       _ensure_content(start+enough_bytes_for_type_and_length), and
494
496
        #       get_bytes_as call? After Manager.get_record_stream() returns
495
497
        #       the object?
496
498
        self._manager = manager
 
499
        self._bytes = None
497
500
        self.storage_kind = 'groupcompress-block'
498
501
        if not first:
499
502
            self.storage_kind = 'groupcompress-block-ref'
512
515
                return self._manager._wire_bytes()
513
516
            else:
514
517
                return ''
 
518
            self._manager = None # safe?
515
519
        if storage_kind in ('fulltext', 'chunked'):
516
 
            self._manager._prepare_for_extract()
517
 
            block = self._manager._block
518
 
            _, bytes = block.extract(self.key, self._start, self._end)
 
520
            if self._bytes is None:
 
521
                # Grab the raw bytes for this entry, and break the ref-cycle
 
522
                self._manager._prepare_for_extract()
 
523
                block = self._manager._block
 
524
                _, bytes = block.extract(self.key, self._start, self._end)
 
525
                self._bytes = bytes
 
526
                self._manager = None
519
527
            if storage_kind == 'fulltext':
520
 
                return bytes
 
528
                return self._bytes
521
529
            else:
522
 
                return [bytes]
 
530
                return [self._bytes]
523
531
        raise errors.UnavailableRepresentation(self.key, storage_kind,
524
532
            self.storage_kind)
525
533
 
1298
1306
        for key in missing:
1299
1307
            yield AbsentContentFactory(key)
1300
1308
        manager = None
 
1309
        last_block = None
 
1310
        last_memo = None
1301
1311
        # TODO: This works fairly well at batching up existing groups into a
1302
1312
        #       streamable format, and possibly allowing for taking one big
1303
1313
        #       group and splitting it when it isn't fully utilized.
1321
1331
                        yield FulltextContentFactory(key, parents, sha1, bytes)
1322
1332
                    else:
1323
1333
                        index_memo, _, parents, (method, _) = locations[key]
1324
 
                        block = self._get_block(index_memo)
 
1334
                        read_memo = index_memo[0:3]
 
1335
                        if last_memo == read_memo:
 
1336
                            block = last_block
 
1337
                        else:
 
1338
                            block = self._get_block(index_memo)
 
1339
                            last_block = block
 
1340
                            last_memo = read_memo
1325
1341
                        start, end = index_memo[3:5]
1326
1342
                        if manager is None:
1327
1343
                            manager = _LazyGroupContentManager(block)