← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

https://launchpad.net/bugs/524184
NEWS section template into a separate file

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
352
352
        """Build a VersionedFiles instance on top of this group of packs."""
353
353
        index_name = index_name + '_index'
354
354
        index_to_pack = {}
355
 
        access = knit._DirectPackAccess(index_to_pack)
 
355
        access = knit._DirectPackAccess(index_to_pack,
 
356
                                        reload_func=self._reload_func)
356
357
        if for_write:
357
358
            # Use new_pack
358
359
            if self.new_pack is None:
584
585
    pack_factory = GCPack
585
586
    resumed_pack_factory = ResumedGCPack
586
587
 
 
588
    def _check_new_inventories(self):
 
589
        """Detect missing inventories or chk root entries for the new revisions
 
590
        in this write group.
 
591
 
 
592
        :returns: list of strs, summarising any problems found.  If the list is
 
593
            empty no problems were found.
 
594
        """
 
595
        # Ensure that all revisions added in this write group have:
 
596
        #   - corresponding inventories,
 
597
        #   - chk root entries for those inventories,
 
598
        #   - and any present parent inventories have their chk root
 
599
        #     entries too.
 
600
        # And all this should be independent of any fallback repository.
 
601
        problems = []
 
602
        key_deps = self.repo.revisions._index._key_dependencies
 
603
        new_revisions_keys = key_deps.get_new_keys()
 
604
        no_fallback_inv_index = self.repo.inventories._index
 
605
        no_fallback_chk_bytes_index = self.repo.chk_bytes._index
 
606
        no_fallback_texts_index = self.repo.texts._index
 
607
        inv_parent_map = no_fallback_inv_index.get_parent_map(
 
608
            new_revisions_keys)
 
609
        # Are any inventories for corresponding to the new revisions missing?
 
610
        corresponding_invs = set(inv_parent_map)
 
611
        missing_corresponding = set(new_revisions_keys)
 
612
        missing_corresponding.difference_update(corresponding_invs)
 
613
        if missing_corresponding:
 
614
            problems.append("inventories missing for revisions %s" %
 
615
                (sorted(missing_corresponding),))
 
616
            return problems
 
617
        # Are any chk root entries missing for any inventories?  This includes
 
618
        # any present parent inventories, which may be used when calculating
 
619
        # deltas for streaming.
 
620
        all_inv_keys = set(corresponding_invs)
 
621
        for parent_inv_keys in inv_parent_map.itervalues():
 
622
            all_inv_keys.update(parent_inv_keys)
 
623
        # Filter out ghost parents.
 
624
        all_inv_keys.intersection_update(
 
625
            no_fallback_inv_index.get_parent_map(all_inv_keys))
 
626
        parent_invs_only_keys = all_inv_keys.symmetric_difference(
 
627
            corresponding_invs)
 
628
        all_missing = set()
 
629
        inv_ids = [key[-1] for key in all_inv_keys]
 
630
        parent_invs_only_ids = [key[-1] for key in parent_invs_only_keys]
 
631
        root_key_info = _build_interesting_key_sets(
 
632
            self.repo, inv_ids, parent_invs_only_ids)
 
633
        expected_chk_roots = root_key_info.all_keys()
 
634
        present_chk_roots = no_fallback_chk_bytes_index.get_parent_map(
 
635
            expected_chk_roots)
 
636
        missing_chk_roots = expected_chk_roots.difference(present_chk_roots)
 
637
        if missing_chk_roots:
 
638
            problems.append("missing referenced chk root keys: %s"
 
639
                % (sorted(missing_chk_roots),))
 
640
            # Don't bother checking any further.
 
641
            return problems
 
642
        # Find all interesting chk_bytes records, and make sure they are
 
643
        # present, as well as the text keys they reference.
 
644
        chk_bytes_no_fallbacks = self.repo.chk_bytes.without_fallbacks()
 
645
        chk_bytes_no_fallbacks._search_key_func = \
 
646
            self.repo.chk_bytes._search_key_func
 
647
        chk_diff = chk_map.iter_interesting_nodes(
 
648
            chk_bytes_no_fallbacks, root_key_info.interesting_root_keys,
 
649
            root_key_info.uninteresting_root_keys)
 
650
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
 
651
        text_keys = set()
 
652
        try:
 
653
            for record in _filter_text_keys(chk_diff, text_keys, bytes_to_info):
 
654
                pass
 
655
        except errors.NoSuchRevision, e:
 
656
            # XXX: It would be nice if we could give a more precise error here.
 
657
            problems.append("missing chk node(s) for id_to_entry maps")
 
658
        chk_diff = chk_map.iter_interesting_nodes(
 
659
            chk_bytes_no_fallbacks, root_key_info.interesting_pid_root_keys,
 
660
            root_key_info.uninteresting_pid_root_keys)
 
661
        try:
 
662
            for interesting_rec, interesting_map in chk_diff:
 
663
                pass
 
664
        except errors.NoSuchRevision, e:
 
665
            problems.append(
 
666
                "missing chk node(s) for parent_id_basename_to_file_id maps")
 
667
        present_text_keys = no_fallback_texts_index.get_parent_map(text_keys)
 
668
        missing_text_keys = text_keys.difference(present_text_keys)
 
669
        if missing_text_keys:
 
670
            problems.append("missing text keys: %r"
 
671
                % (sorted(missing_text_keys),))
 
672
        return problems
 
673
 
587
674
    def _execute_pack_operations(self, pack_operations,
588
675
                                 _packer_class=GCCHKPacker,
589
676
                                 reload_func=None):
617
704
                self._remove_pack_from_memory(pack)
618
705
        # record the newly available packs and stop advertising the old
619
706
        # packs
620
 
        self._save_pack_names(clear_obsolete_packs=True)
 
707
        result = self._save_pack_names(clear_obsolete_packs=True)
621
708
        # Move the old packs out of the way now they are no longer referenced.
622
709
        for revision_count, packs in pack_operations:
623
710
            self._obsolete_packs(packs)
 
711
        return result
624
712
 
625
713
 
626
714
class CHKInventoryRepository(KnitPackRepository):
651
739
            _GCGraphIndex(self._pack_collection.revision_index.combined_index,
652
740
                add_callback=self._pack_collection.revision_index.add_callback,
653
741
                parents=True, is_locked=self.is_locked,
654
 
                track_external_parent_refs=True),
 
742
                track_external_parent_refs=True, track_new_keys=True),
655
743
            access=self._pack_collection.revision_index.data_access,
656
744
            delta=False)
657
745
        self.signatures = GroupCompressVersionedFiles(
810
898
            yield inventory.CHKInventory.deserialise(self.chk_bytes, texts[key], key)
811
899
 
812
900
    def _iter_inventory_xmls(self, revision_ids, ordering):
813
 
        # Without a native 'xml' inventory, this method doesn't make sense, so
814
 
        # make it raise to trap naughty direct users.
815
 
        raise NotImplementedError(self._iter_inventory_xmls)
 
901
        # Without a native 'xml' inventory, this method doesn't make sense.
 
902
        # However older working trees, and older bundles want it - so we supply
 
903
        # it allowing get_inventory_xml to work. Bundles currently use the
 
904
        # serializer directly; this also isn't ideal, but there isn't an xml
 
905
        # iteration interface offered at all for repositories. We could make
 
906
        # _iter_inventory_xmls be part of the contract, even if kept private.
 
907
        inv_to_str = self._serializer.write_inventory_to_string
 
908
        for inv in self.iter_inventories(revision_ids, ordering=ordering):
 
909
            yield inv_to_str(inv), inv.revision_id
816
910
 
817
911
    def _find_present_inventory_keys(self, revision_keys):
818
912
        parent_map = self.inventories.get_parent_map(revision_keys)
844
938
                                        parent_keys)
845
939
            present_parent_inv_ids = set(
846
940
                [k[-1] for k in present_parent_inv_keys])
847
 
            uninteresting_root_keys = set()
848
 
            interesting_root_keys = set()
849
941
            inventories_to_read = set(revision_ids)
850
942
            inventories_to_read.update(present_parent_inv_ids)
851
 
            for inv in self.iter_inventories(inventories_to_read):
852
 
                entry_chk_root_key = inv.id_to_entry.key()
853
 
                if inv.revision_id in present_parent_inv_ids:
854
 
                    uninteresting_root_keys.add(entry_chk_root_key)
855
 
                else:
856
 
                    interesting_root_keys.add(entry_chk_root_key)
857
 
 
 
943
            root_key_info = _build_interesting_key_sets(
 
944
                self, inventories_to_read, present_parent_inv_ids)
 
945
            interesting_root_keys = root_key_info.interesting_root_keys
 
946
            uninteresting_root_keys = root_key_info.uninteresting_root_keys
858
947
            chk_bytes = self.chk_bytes
859
948
            for record, items in chk_map.iter_interesting_nodes(chk_bytes,
860
949
                        interesting_root_keys, uninteresting_root_keys,
994
1083
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
995
1084
        chk_bytes = self.from_repository.chk_bytes
996
1085
        def _filter_id_to_entry():
997
 
            for record, items in chk_map.iter_interesting_nodes(chk_bytes,
998
 
                        self._chk_id_roots, uninteresting_root_keys):
999
 
                for name, bytes in items:
1000
 
                    # Note: we don't care about name_utf8, because we are always
1001
 
                    # rich-root = True
1002
 
                    _, file_id, revision_id = bytes_to_info(bytes)
1003
 
                    self._text_keys.add((file_id, revision_id))
 
1086
            interesting_nodes = chk_map.iter_interesting_nodes(chk_bytes,
 
1087
                        self._chk_id_roots, uninteresting_root_keys)
 
1088
            for record in _filter_text_keys(interesting_nodes, self._text_keys,
 
1089
                    bytes_to_info):
1004
1090
                if record is not None:
1005
1091
                    yield record
1006
1092
            # Consumed
1044
1130
            missing_inventory_keys.add(key[1:])
1045
1131
        if self._chk_id_roots or self._chk_p_id_roots:
1046
1132
            raise AssertionError('Cannot call get_stream_for_missing_keys'
1047
 
                ' untill all of get_stream() has been consumed.')
 
1133
                ' until all of get_stream() has been consumed.')
1048
1134
        # Yield the inventory stream, so we can find the chk stream
1049
1135
        # Some of the missing_keys will be missing because they are ghosts.
1050
1136
        # As such, we can ignore them. The Sink is required to verify there are
1057
1143
            yield stream_info
1058
1144
 
1059
1145
 
 
1146
class _InterestingKeyInfo(object):
 
1147
    def __init__(self):
 
1148
        self.interesting_root_keys = set()
 
1149
        self.interesting_pid_root_keys = set()
 
1150
        self.uninteresting_root_keys = set()
 
1151
        self.uninteresting_pid_root_keys = set()
 
1152
 
 
1153
    def all_interesting(self):
 
1154
        return self.interesting_root_keys.union(self.interesting_pid_root_keys)
 
1155
 
 
1156
    def all_uninteresting(self):
 
1157
        return self.uninteresting_root_keys.union(
 
1158
            self.uninteresting_pid_root_keys)
 
1159
 
 
1160
    def all_keys(self):
 
1161
        return self.all_interesting().union(self.all_uninteresting())
 
1162
 
 
1163
 
 
1164
def _build_interesting_key_sets(repo, inventory_ids, parent_only_inv_ids):
 
1165
    result = _InterestingKeyInfo()
 
1166
    for inv in repo.iter_inventories(inventory_ids, 'unordered'):
 
1167
        root_key = inv.id_to_entry.key()
 
1168
        pid_root_key = inv.parent_id_basename_to_file_id.key()
 
1169
        if inv.revision_id in parent_only_inv_ids:
 
1170
            result.uninteresting_root_keys.add(root_key)
 
1171
            result.uninteresting_pid_root_keys.add(pid_root_key)
 
1172
        else:
 
1173
            result.interesting_root_keys.add(root_key)
 
1174
            result.interesting_pid_root_keys.add(pid_root_key)
 
1175
    return result
 
1176
 
 
1177
 
 
1178
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_info):
 
1179
    """Iterate the result of iter_interesting_nodes, yielding the records
 
1180
    and adding to text_keys.
 
1181
    """
 
1182
    for record, items in interesting_nodes_iterable:
 
1183
        for name, bytes in items:
 
1184
            # Note: we don't care about name_utf8, because groupcompress repos
 
1185
            # are always rich-root, so there are no synthesised root records to
 
1186
            # ignore.
 
1187
            _, file_id, revision_id = bytes_to_info(bytes)
 
1188
            text_keys.add((file_id, revision_id))
 
1189
        yield record
 
1190
 
 
1191
 
 
1192
 
 
1193
 
1060
1194
class RepositoryFormatCHK1(RepositoryFormatPack):
1061
1195
    """A hashed CHK+group compress pack repository."""
1062
1196
 
1101
1235
        return ("Development repository format - rich roots, group compression"
1102
1236
            " and chk inventories")
1103
1237
 
1104
 
    def check_conversion_target(self, target_format):
1105
 
        if not target_format.rich_root_data:
1106
 
            raise errors.BadConversionTarget(
1107
 
                'Does not support rich root data.', target_format)
1108
 
        if (self.supports_tree_reference and 
1109
 
            not getattr(target_format, 'supports_tree_reference', False)):
1110
 
            raise errors.BadConversionTarget(
1111
 
                'Does not support nested trees', target_format)
1112
 
 
1113
 
 
1114
1238
 
1115
1239
class RepositoryFormatCHK2(RepositoryFormatCHK1):
1116
1240
    """A CHK repository that uses the bencode revision serializer."""
1149
1273
 
1150
1274
    def get_format_string(self):
1151
1275
        return ('Bazaar repository format 2a (needs bzr 1.16 or later)\n')
 
1276
 
 
1277
    def get_format_description(self):
 
1278
        """See RepositoryFormat.get_format_description()."""
 
1279
        return ("Repository format 2a - rich roots, group compression"
 
1280
            " and chk inventories")