← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2010-08-16 06:46:17 UTC
  • mfrom: (5375.1.6 chk-canonicalizer-613698)
  • Revision ID: pqm@pqm.ubuntu.com-20100816064617-wizstoapjbffkj05
(spiv) Add hidden option 'bzr reconcile --canonicalize-chks' to repair repos
 affected by bug 522637. (Andrew Bennetts)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
32
32
    revision as _mod_revision,
33
33
    trace,
34
34
    ui,
 
35
    versionedfile,
35
36
    )
36
37
from bzrlib.btree_index import (
37
38
    BTreeGraphIndex,
38
39
    BTreeBuilder,
39
40
    )
 
41
from bzrlib.decorators import needs_write_lock
40
42
from bzrlib.groupcompress import (
41
43
    _GCGraphIndex,
42
44
    GroupCompressVersionedFiles,
425
427
        self._copy_stream(source_vf, target_vf, inventory_keys,
426
428
                          'inventories', self._get_filtered_inv_stream, 2)
427
429
 
 
430
    def _get_chk_vfs_for_copy(self):
 
431
        return self._build_vfs('chk', False, False)
 
432
 
428
433
    def _copy_chk_texts(self):
429
 
        source_vf, target_vf = self._build_vfs('chk', False, False)
 
434
        source_vf, target_vf = self._get_chk_vfs_for_copy()
430
435
        # TODO: This is technically spurious... if it is a performance issue,
431
436
        #       remove it
432
437
        total_keys = source_vf.keys()
578
583
        return new_pack.data_inserted() and self._data_changed
579
584
 
580
585
 
 
586
class GCCHKCanonicalizingPacker(GCCHKPacker):
 
587
    """A packer that ensures inventories have canonical-form CHK maps.
 
588
    
 
589
    Ideally this would be part of reconcile, but it's very slow and rarely
 
590
    needed.  (It repairs repositories affected by
 
591
    https://bugs.launchpad.net/bzr/+bug/522637).
 
592
    """
 
593
 
 
594
    def __init__(self, *args, **kwargs):
 
595
        super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
 
596
        self._data_changed = False
 
597
    
 
598
    def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
 
599
        """Create and exhaust a stream, but don't insert it.
 
600
        
 
601
        This is useful to get the side-effects of generating a stream.
 
602
        """
 
603
        self.pb.update('scanning %s' % (message,), pb_offset)
 
604
        child_pb = ui.ui_factory.nested_progress_bar()
 
605
        try:
 
606
            list(vf_to_stream(source_vf, keys, message, child_pb))
 
607
        finally:
 
608
            child_pb.finished()
 
609
 
 
610
    def _copy_inventory_texts(self):
 
611
        source_vf, target_vf = self._build_vfs('inventory', True, True)
 
612
        source_chk_vf, target_chk_vf = self._get_chk_vfs_for_copy()
 
613
        inventory_keys = source_vf.keys()
 
614
        # First, copy the existing CHKs on the assumption that most of them
 
615
        # will be correct.  This will save us from having to reinsert (and
 
616
        # recompress) these records later at the cost of perhaps preserving a
 
617
        # few unused CHKs. 
 
618
        # (Iterate but don't insert _get_filtered_inv_stream to populate the
 
619
        # variables needed by GCCHKPacker._copy_chk_texts.)
 
620
        self._exhaust_stream(source_vf, inventory_keys, 'inventories',
 
621
                self._get_filtered_inv_stream, 2)
 
622
        GCCHKPacker._copy_chk_texts(self)
 
623
        # Now copy and fix the inventories, and any regenerated CHKs.
 
624
        def chk_canonicalizing_inv_stream(source_vf, keys, message, pb=None):
 
625
            return self._get_filtered_canonicalizing_inv_stream(
 
626
                source_vf, keys, message, pb, source_chk_vf, target_chk_vf)
 
627
        self._copy_stream(source_vf, target_vf, inventory_keys,
 
628
                          'inventories', chk_canonicalizing_inv_stream, 4)
 
629
 
 
630
    def _copy_chk_texts(self):
 
631
        # No-op; in this class this happens during _copy_inventory_texts.
 
632
        pass
 
633
 
 
634
    def _get_filtered_canonicalizing_inv_stream(self, source_vf, keys, message,
 
635
            pb=None, source_chk_vf=None, target_chk_vf=None):
 
636
        """Filter the texts of inventories, regenerating CHKs to make sure they
 
637
        are canonical.
 
638
        """
 
639
        total_keys = len(keys)
 
640
        target_chk_vf = versionedfile.NoDupeAddLinesDecorator(target_chk_vf)
 
641
        def _filtered_inv_stream():
 
642
            stream = source_vf.get_record_stream(keys, 'groupcompress', True)
 
643
            search_key_name = None
 
644
            for idx, record in enumerate(stream):
 
645
                # Inventories should always be with revisions; assume success.
 
646
                bytes = record.get_bytes_as('fulltext')
 
647
                chk_inv = inventory.CHKInventory.deserialise(
 
648
                    source_chk_vf, bytes, record.key)
 
649
                if pb is not None:
 
650
                    pb.update('inv', idx, total_keys)
 
651
                chk_inv.id_to_entry._ensure_root()
 
652
                if search_key_name is None:
 
653
                    # Find the name corresponding to the search_key_func
 
654
                    search_key_reg = chk_map.search_key_registry
 
655
                    for search_key_name, func in search_key_reg.iteritems():
 
656
                        if func == chk_inv.id_to_entry._search_key_func:
 
657
                            break
 
658
                canonical_inv = inventory.CHKInventory.from_inventory(
 
659
                    target_chk_vf, chk_inv,
 
660
                    maximum_size=chk_inv.id_to_entry._root_node._maximum_size,
 
661
                    search_key_name=search_key_name)
 
662
                if chk_inv.id_to_entry.key() != canonical_inv.id_to_entry.key():
 
663
                    trace.mutter(
 
664
                        'Non-canonical CHK map for id_to_entry of inv: %s '
 
665
                        '(root is %s, should be %s)' % (chk_inv.revision_id,
 
666
                        chk_inv.id_to_entry.key()[0],
 
667
                        canonical_inv.id_to_entry.key()[0]))
 
668
                    self._data_changed = True
 
669
                p_id_map = chk_inv.parent_id_basename_to_file_id
 
670
                p_id_map._ensure_root()
 
671
                canon_p_id_map = canonical_inv.parent_id_basename_to_file_id
 
672
                if p_id_map.key() != canon_p_id_map.key():
 
673
                    trace.mutter(
 
674
                        'Non-canonical CHK map for parent_id_to_basename of '
 
675
                        'inv: %s (root is %s, should be %s)'
 
676
                        % (chk_inv.revision_id, p_id_map.key()[0],
 
677
                           canon_p_id_map.key()[0]))
 
678
                    self._data_changed = True
 
679
                yield versionedfile.ChunkedContentFactory(record.key,
 
680
                        record.parents, record.sha1,
 
681
                        canonical_inv.to_lines())
 
682
            # We have finished processing all of the inventory records, we
 
683
            # don't need these sets anymore
 
684
        return _filtered_inv_stream()
 
685
 
 
686
    def _use_pack(self, new_pack):
 
687
        """Override _use_pack to check for reconcile having changed content."""
 
688
        return new_pack.data_inserted() and self._data_changed
 
689
 
 
690
 
581
691
class GCRepositoryPackCollection(RepositoryPackCollection):
582
692
 
583
693
    pack_factory = GCPack
999
1109
        finally:
1000
1110
            pb.finished()
1001
1111
 
 
1112
    @needs_write_lock
 
1113
    def reconcile_canonicalize_chks(self):
 
1114
        """Reconcile this repository to make sure all CHKs are in canonical
 
1115
        form.
 
1116
        """
 
1117
        from bzrlib.reconcile import PackReconciler
 
1118
        reconciler = PackReconciler(self, thorough=True, canonicalize_chks=True)
 
1119
        reconciler.reconcile()
 
1120
        return reconciler
 
1121
 
1002
1122
    def _reconcile_pack(self, collection, packs, extension, revs, pb):
1003
1123
        packer = GCCHKReconcilePacker(collection, packs, extension)
1004
1124
        return packer.pack(pb)
1005
1125
 
 
1126
    def _canonicalize_chks_pack(self, collection, packs, extension, revs, pb):
 
1127
        packer = GCCHKCanonicalizingPacker(collection, packs, extension, revs)
 
1128
        return packer.pack(pb)
 
1129
 
1006
1130
    def _get_source(self, to_format):
1007
1131
        """Return a source for streaming from this repository."""
1008
1132
        if self._format._serializer == to_format._serializer: