← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

  • Committer: Martin Pool
  • Date: 2009-08-28 05:00:33 UTC
  • mfrom: (4659 +trunk)
  • mto: This revision was merged to the branch mainline in revision 4660.
  • Revision ID: mbp@sourcefrog.net-20090828050033-blwvnig3gdzbr1iw
merge trunk

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
1
 
# Copyright (C) 2008, 2009, 2010 Canonical Ltd
 
1
# Copyright (C) 2008, 2009 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
29
29
    knit,
30
30
    osutils,
31
31
    pack,
 
32
    remote,
32
33
    revision as _mod_revision,
33
34
    trace,
34
35
    ui,
52
53
    ResumedPack,
53
54
    Packer,
54
55
    )
55
 
from bzrlib.static_tuple import StaticTuple
56
56
 
57
57
 
58
58
class GCPack(NewPack):
352
352
        """Build a VersionedFiles instance on top of this group of packs."""
353
353
        index_name = index_name + '_index'
354
354
        index_to_pack = {}
355
 
        access = knit._DirectPackAccess(index_to_pack,
356
 
                                        reload_func=self._reload_func)
 
355
        access = knit._DirectPackAccess(index_to_pack)
357
356
        if for_write:
358
357
            # Use new_pack
359
358
            if self.new_pack is None:
585
584
    pack_factory = GCPack
586
585
    resumed_pack_factory = ResumedGCPack
587
586
 
588
 
    def _check_new_inventories(self):
589
 
        """Detect missing inventories or chk root entries for the new revisions
590
 
        in this write group.
591
 
 
592
 
        :returns: list of strs, summarising any problems found.  If the list is
593
 
            empty no problems were found.
594
 
        """
595
 
        # Ensure that all revisions added in this write group have:
596
 
        #   - corresponding inventories,
597
 
        #   - chk root entries for those inventories,
598
 
        #   - and any present parent inventories have their chk root
599
 
        #     entries too.
600
 
        # And all this should be independent of any fallback repository.
601
 
        problems = []
602
 
        key_deps = self.repo.revisions._index._key_dependencies
603
 
        new_revisions_keys = key_deps.get_new_keys()
604
 
        no_fallback_inv_index = self.repo.inventories._index
605
 
        no_fallback_chk_bytes_index = self.repo.chk_bytes._index
606
 
        no_fallback_texts_index = self.repo.texts._index
607
 
        inv_parent_map = no_fallback_inv_index.get_parent_map(
608
 
            new_revisions_keys)
609
 
        # Are any inventories for corresponding to the new revisions missing?
610
 
        corresponding_invs = set(inv_parent_map)
611
 
        missing_corresponding = set(new_revisions_keys)
612
 
        missing_corresponding.difference_update(corresponding_invs)
613
 
        if missing_corresponding:
614
 
            problems.append("inventories missing for revisions %s" %
615
 
                (sorted(missing_corresponding),))
616
 
            return problems
617
 
        # Are any chk root entries missing for any inventories?  This includes
618
 
        # any present parent inventories, which may be used when calculating
619
 
        # deltas for streaming.
620
 
        all_inv_keys = set(corresponding_invs)
621
 
        for parent_inv_keys in inv_parent_map.itervalues():
622
 
            all_inv_keys.update(parent_inv_keys)
623
 
        # Filter out ghost parents.
624
 
        all_inv_keys.intersection_update(
625
 
            no_fallback_inv_index.get_parent_map(all_inv_keys))
626
 
        parent_invs_only_keys = all_inv_keys.symmetric_difference(
627
 
            corresponding_invs)
628
 
        all_missing = set()
629
 
        inv_ids = [key[-1] for key in all_inv_keys]
630
 
        parent_invs_only_ids = [key[-1] for key in parent_invs_only_keys]
631
 
        root_key_info = _build_interesting_key_sets(
632
 
            self.repo, inv_ids, parent_invs_only_ids)
633
 
        expected_chk_roots = root_key_info.all_keys()
634
 
        present_chk_roots = no_fallback_chk_bytes_index.get_parent_map(
635
 
            expected_chk_roots)
636
 
        missing_chk_roots = expected_chk_roots.difference(present_chk_roots)
637
 
        if missing_chk_roots:
638
 
            problems.append("missing referenced chk root keys: %s"
639
 
                % (sorted(missing_chk_roots),))
640
 
            # Don't bother checking any further.
641
 
            return problems
642
 
        # Find all interesting chk_bytes records, and make sure they are
643
 
        # present, as well as the text keys they reference.
644
 
        chk_bytes_no_fallbacks = self.repo.chk_bytes.without_fallbacks()
645
 
        chk_bytes_no_fallbacks._search_key_func = \
646
 
            self.repo.chk_bytes._search_key_func
647
 
        chk_diff = chk_map.iter_interesting_nodes(
648
 
            chk_bytes_no_fallbacks, root_key_info.interesting_root_keys,
649
 
            root_key_info.uninteresting_root_keys)
650
 
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
651
 
        text_keys = set()
652
 
        try:
653
 
            for record in _filter_text_keys(chk_diff, text_keys, bytes_to_info):
654
 
                pass
655
 
        except errors.NoSuchRevision, e:
656
 
            # XXX: It would be nice if we could give a more precise error here.
657
 
            problems.append("missing chk node(s) for id_to_entry maps")
658
 
        chk_diff = chk_map.iter_interesting_nodes(
659
 
            chk_bytes_no_fallbacks, root_key_info.interesting_pid_root_keys,
660
 
            root_key_info.uninteresting_pid_root_keys)
661
 
        try:
662
 
            for interesting_rec, interesting_map in chk_diff:
663
 
                pass
664
 
        except errors.NoSuchRevision, e:
665
 
            problems.append(
666
 
                "missing chk node(s) for parent_id_basename_to_file_id maps")
667
 
        present_text_keys = no_fallback_texts_index.get_parent_map(text_keys)
668
 
        missing_text_keys = text_keys.difference(present_text_keys)
669
 
        if missing_text_keys:
670
 
            problems.append("missing text keys: %r"
671
 
                % (sorted(missing_text_keys),))
672
 
        return problems
673
 
 
674
587
    def _execute_pack_operations(self, pack_operations,
675
588
                                 _packer_class=GCCHKPacker,
676
589
                                 reload_func=None):
704
617
                self._remove_pack_from_memory(pack)
705
618
        # record the newly available packs and stop advertising the old
706
619
        # packs
707
 
        to_be_obsoleted = []
708
 
        for _, packs in pack_operations:
709
 
            to_be_obsoleted.extend(packs)
710
 
        result = self._save_pack_names(clear_obsolete_packs=True,
711
 
                                       obsolete_packs=to_be_obsoleted)
712
 
        return result
 
620
        self._save_pack_names(clear_obsolete_packs=True)
 
621
        # Move the old packs out of the way now they are no longer referenced.
 
622
        for revision_count, packs in pack_operations:
 
623
            self._obsolete_packs(packs)
713
624
 
714
625
 
715
626
class CHKInventoryRepository(KnitPackRepository):
740
651
            _GCGraphIndex(self._pack_collection.revision_index.combined_index,
741
652
                add_callback=self._pack_collection.revision_index.add_callback,
742
653
                parents=True, is_locked=self.is_locked,
743
 
                track_external_parent_refs=True, track_new_keys=True),
 
654
                track_external_parent_refs=True),
744
655
            access=self._pack_collection.revision_index.data_access,
745
656
            delta=False)
746
657
        self.signatures = GroupCompressVersionedFiles(
816
727
                                 ' no new_path %r' % (file_id,))
817
728
            if new_path == '':
818
729
                new_inv.root_id = file_id
819
 
                parent_id_basename_key = StaticTuple('', '').intern()
 
730
                parent_id_basename_key = ('', '')
820
731
            else:
821
732
                utf8_entry_name = entry.name.encode('utf-8')
822
 
                parent_id_basename_key = StaticTuple(entry.parent_id,
823
 
                                                     utf8_entry_name).intern()
 
733
                parent_id_basename_key = (entry.parent_id, utf8_entry_name)
824
734
            new_value = entry_to_bytes(entry)
825
735
            # Populate Caches?
826
736
            # new_inv._path_to_fileid_cache[new_path] = file_id
827
 
            key = StaticTuple(file_id).intern()
828
 
            id_to_entry_dict[key] = new_value
 
737
            id_to_entry_dict[(file_id,)] = new_value
829
738
            parent_id_basename_dict[parent_id_basename_key] = file_id
830
739
 
831
740
        new_inv._populate_from_dicts(self.chk_bytes, id_to_entry_dict,
881
790
            if basis_tree is not None:
882
791
                basis_tree.unlock()
883
792
 
884
 
    def _deserialise_inventory(self, revision_id, bytes):
 
793
    def deserialise_inventory(self, revision_id, bytes):
885
794
        return inventory.CHKInventory.deserialise(self.chk_bytes, bytes,
886
795
            (revision_id,))
887
796
 
903
812
    def _iter_inventory_xmls(self, revision_ids, ordering):
904
813
        # Without a native 'xml' inventory, this method doesn't make sense.
905
814
        # However older working trees, and older bundles want it - so we supply
906
 
        # it allowing _get_inventory_xml to work. Bundles currently use the
 
815
        # it allowing get_inventory_xml to work. Bundles currently use the
907
816
        # serializer directly; this also isn't ideal, but there isn't an xml
908
817
        # iteration interface offered at all for repositories. We could make
909
818
        # _iter_inventory_xmls be part of the contract, even if kept private.
941
850
                                        parent_keys)
942
851
            present_parent_inv_ids = set(
943
852
                [k[-1] for k in present_parent_inv_keys])
 
853
            uninteresting_root_keys = set()
 
854
            interesting_root_keys = set()
944
855
            inventories_to_read = set(revision_ids)
945
856
            inventories_to_read.update(present_parent_inv_ids)
946
 
            root_key_info = _build_interesting_key_sets(
947
 
                self, inventories_to_read, present_parent_inv_ids)
948
 
            interesting_root_keys = root_key_info.interesting_root_keys
949
 
            uninteresting_root_keys = root_key_info.uninteresting_root_keys
 
857
            for inv in self.iter_inventories(inventories_to_read):
 
858
                entry_chk_root_key = inv.id_to_entry.key()
 
859
                if inv.revision_id in present_parent_inv_ids:
 
860
                    uninteresting_root_keys.add(entry_chk_root_key)
 
861
                else:
 
862
                    interesting_root_keys.add(entry_chk_root_key)
 
863
 
950
864
            chk_bytes = self.chk_bytes
951
865
            for record, items in chk_map.iter_interesting_nodes(chk_bytes,
952
866
                        interesting_root_keys, uninteresting_root_keys,
953
867
                        pb=pb):
954
868
                for name, bytes in items:
955
869
                    (name_utf8, file_id, revision_id) = bytes_to_info(bytes)
956
 
                    # TODO: consider interning file_id, revision_id here, or
957
 
                    #       pushing that intern() into bytes_to_info()
958
 
                    # TODO: rich_root should always be True here, for all
959
 
                    #       repositories that support chk_bytes
960
870
                    if not rich_root and name_utf8 == '':
961
871
                        continue
962
872
                    try:
1022
932
        super(GroupCHKStreamSource, self).__init__(from_repository, to_format)
1023
933
        self._revision_keys = None
1024
934
        self._text_keys = None
1025
 
        self._text_fetch_order = 'groupcompress'
 
935
        # self._text_fetch_order = 'unordered'
1026
936
        self._chk_id_roots = None
1027
937
        self._chk_p_id_roots = None
1028
938
 
1039
949
            p_id_roots_set = set()
1040
950
            source_vf = self.from_repository.inventories
1041
951
            stream = source_vf.get_record_stream(inventory_keys,
1042
 
                                                 'groupcompress', True)
 
952
                                                 'unordered', True)
1043
953
            for record in stream:
1044
954
                if record.storage_kind == 'absent':
1045
955
                    if allow_absent:
1090
1000
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
1091
1001
        chk_bytes = self.from_repository.chk_bytes
1092
1002
        def _filter_id_to_entry():
1093
 
            interesting_nodes = chk_map.iter_interesting_nodes(chk_bytes,
1094
 
                        self._chk_id_roots, uninteresting_root_keys)
1095
 
            for record in _filter_text_keys(interesting_nodes, self._text_keys,
1096
 
                    bytes_to_info):
 
1003
            for record, items in chk_map.iter_interesting_nodes(chk_bytes,
 
1004
                        self._chk_id_roots, uninteresting_root_keys):
 
1005
                for name, bytes in items:
 
1006
                    # Note: we don't care about name_utf8, because we are always
 
1007
                    # rich-root = True
 
1008
                    _, file_id, revision_id = bytes_to_info(bytes)
 
1009
                    self._text_keys.add((file_id, revision_id))
1097
1010
                if record is not None:
1098
1011
                    yield record
1099
1012
            # Consumed
1113
1026
        for stream_info in self._fetch_revision_texts(revision_ids):
1114
1027
            yield stream_info
1115
1028
        self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1116
 
        self.from_repository.revisions.clear_cache()
1117
 
        self.from_repository.signatures.clear_cache()
1118
1029
        yield self._get_inventory_stream(self._revision_keys)
1119
 
        self.from_repository.inventories.clear_cache()
1120
1030
        # TODO: The keys to exclude might be part of the search recipe
1121
1031
        # For now, exclude all parents that are at the edge of ancestry, for
1122
1032
        # which we have inventories
1125
1035
                        self._revision_keys)
1126
1036
        for stream_info in self._get_filtered_chk_streams(parent_keys):
1127
1037
            yield stream_info
1128
 
        self.from_repository.chk_bytes.clear_cache()
1129
1038
        yield self._get_text_stream()
1130
 
        self.from_repository.texts.clear_cache()
1131
1039
 
1132
1040
    def get_stream_for_missing_keys(self, missing_keys):
1133
1041
        # missing keys can only occur when we are byte copying and not
1142
1050
            missing_inventory_keys.add(key[1:])
1143
1051
        if self._chk_id_roots or self._chk_p_id_roots:
1144
1052
            raise AssertionError('Cannot call get_stream_for_missing_keys'
1145
 
                ' until all of get_stream() has been consumed.')
 
1053
                ' untill all of get_stream() has been consumed.')
1146
1054
        # Yield the inventory stream, so we can find the chk stream
1147
1055
        # Some of the missing_keys will be missing because they are ghosts.
1148
1056
        # As such, we can ignore them. The Sink is required to verify there are
1155
1063
            yield stream_info
1156
1064
 
1157
1065
 
1158
 
class _InterestingKeyInfo(object):
1159
 
    def __init__(self):
1160
 
        self.interesting_root_keys = set()
1161
 
        self.interesting_pid_root_keys = set()
1162
 
        self.uninteresting_root_keys = set()
1163
 
        self.uninteresting_pid_root_keys = set()
1164
 
 
1165
 
    def all_interesting(self):
1166
 
        return self.interesting_root_keys.union(self.interesting_pid_root_keys)
1167
 
 
1168
 
    def all_uninteresting(self):
1169
 
        return self.uninteresting_root_keys.union(
1170
 
            self.uninteresting_pid_root_keys)
1171
 
 
1172
 
    def all_keys(self):
1173
 
        return self.all_interesting().union(self.all_uninteresting())
1174
 
 
1175
 
 
1176
 
def _build_interesting_key_sets(repo, inventory_ids, parent_only_inv_ids):
1177
 
    result = _InterestingKeyInfo()
1178
 
    for inv in repo.iter_inventories(inventory_ids, 'unordered'):
1179
 
        root_key = inv.id_to_entry.key()
1180
 
        pid_root_key = inv.parent_id_basename_to_file_id.key()
1181
 
        if inv.revision_id in parent_only_inv_ids:
1182
 
            result.uninteresting_root_keys.add(root_key)
1183
 
            result.uninteresting_pid_root_keys.add(pid_root_key)
1184
 
        else:
1185
 
            result.interesting_root_keys.add(root_key)
1186
 
            result.interesting_pid_root_keys.add(pid_root_key)
1187
 
    return result
1188
 
 
1189
 
 
1190
 
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_info):
1191
 
    """Iterate the result of iter_interesting_nodes, yielding the records
1192
 
    and adding to text_keys.
1193
 
    """
1194
 
    for record, items in interesting_nodes_iterable:
1195
 
        for name, bytes in items:
1196
 
            # Note: we don't care about name_utf8, because groupcompress repos
1197
 
            # are always rich-root, so there are no synthesised root records to
1198
 
            # ignore.
1199
 
            _, file_id, revision_id = bytes_to_info(bytes)
1200
 
            file_id = intern(file_id)
1201
 
            revision_id = intern(revision_id)
1202
 
            text_keys.add(StaticTuple(file_id, revision_id).intern())
1203
 
        yield record
1204
 
 
1205
 
 
1206
 
 
1207
 
 
1208
1066
class RepositoryFormatCHK1(RepositoryFormatPack):
1209
1067
    """A hashed CHK+group compress pack repository."""
1210
1068
 
1287
1145
 
1288
1146
    def get_format_string(self):
1289
1147
        return ('Bazaar repository format 2a (needs bzr 1.16 or later)\n')
1290
 
 
1291
 
    def get_format_description(self):
1292
 
        """See RepositoryFormat.get_format_description()."""
1293
 
        return ("Repository format 2a - rich roots, group compression"
1294
 
            " and chk inventories")