← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/versionedfile.py

  • Committer: John Arbash Meinel
  • Date: 2010-08-10 20:03:44 UTC
  • mto: This revision was merged to the branch mainline in revision 5376.
  • Revision ID: john@arbash-meinel.com-20100810200344-6muerwvkafqu7w47
Rework things a bit so the logic can be shared.

It turns out that some of the peak memory is actually during the inventory
to string to bundle translations. So re-use the refcount logic there.
This actually does show a decrease in peak memory.
Specifically 'cd bzr.dev; bzr send ../2.2' drops from 221MB peak to 156MB.

We don't speed anything up (16.5s both ways), but peak memory is quite
a bit better.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/versionedfile.py
251
251
        # should check for them
252
252
        refcounts = {}
253
253
        setdefault = refcounts.setdefault
254
 
        maybe_ghosts = set()
 
254
        just_parents = set()
255
255
        for child_key, parent_keys in parent_map.iteritems():
256
256
            if not parent_keys:
257
 
                # Ghost? We should never get here
 
257
                # parent_keys may be None if a given VersionedFile claims to
 
258
                # not support graph operations.
258
259
                continue
259
 
            maybe_ghosts.update(p for p in parent_keys if p not in needed_keys)
 
260
            just_parents.update(parent_keys)
260
261
            needed_keys.update(parent_keys)
261
262
            for p in parent_keys:
262
263
                refcounts[p] = setdefault(p, 0) + 1
263
 
        # Remove any parents that are actually ghosts
264
 
        self.ghost_parents = maybe_ghosts.difference(
265
 
                                self.vf.get_parent_map(maybe_ghosts))
 
264
        just_parents.difference_update(parent_map)
 
265
        # Remove any parents that are actually ghosts from the needed set
 
266
        self.present_parents = set(self.vf.get_parent_map(just_parents))
 
267
        self.ghost_parents = just_parents.difference(self.present_parents)
266
268
        needed_keys.difference_update(self.ghost_parents)
267
269
        self.needed_keys = needed_keys
268
270
        self.refcounts = refcounts
283
285
                    parent_lines, left_parent_blocks)
284
286
        self.diffs[key] = diff
285
287
 
286
 
    def _process_one_record(self, record):
287
 
        this_chunks = record.get_bytes_as('chunked')
288
 
        if record.key in self.parent_map:
 
288
    def _process_one_record(self, key, this_chunks):
 
289
        parent_keys = None
 
290
        if key in self.parent_map:
289
291
            # This record should be ready to diff, since we requested
290
292
            # content in 'topological' order
291
 
            parent_keys = self.parent_map.pop(record.key)
 
293
            parent_keys = self.parent_map.pop(key)
292
294
            # If a VersionedFile claims 'no-graph' support, then it may return
293
295
            # None for any parent request, so we replace it with an empty tuple
294
296
            if parent_keys is None:
315
317
            lines = osutils.chunks_to_lines(this_chunks)
316
318
            # Since we needed the lines, we'll go ahead and cache them this way
317
319
            this_chunks = lines
318
 
            self._compute_diff(record.key, parent_lines, lines)
 
320
            self._compute_diff(key, parent_lines, lines)
319
321
            del lines
320
322
        # Is this content required for any more children?
321
 
        if record.key in self.refcounts:
322
 
            self.chunks[record.key] = this_chunks
 
323
        if key in self.refcounts:
 
324
            self.chunks[key] = this_chunks
323
325
 
324
326
    def _extract_diffs(self):
325
327
        needed_keys, refcounts = self._find_needed_keys()
327
329
                                                'topological', True):
328
330
            if record.storage_kind == 'absent':
329
331
                raise errors.RevisionNotPresent(record.key, self.vf)
330
 
            self._process_one_record(record)
 
332
            self._process_one_record(record.key,
 
333
                                     record.get_bytes_as('chunked'))
331
334
        # At this point, we should have *no* remaining content
332
335
        assert not self.parent_map
333
336
        assert set(self.refcounts) == self.ghost_parents