← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

  • Committer: Vincent Ladeuil
  • Date: 2010-02-11 09:27:55 UTC
  • mfrom: (5017.3.46 test-servers)
  • mto: This revision was merged to the branch mainline in revision 5030.
  • Revision ID: v.ladeuil+lp@free.fr-20100211092755-3vvu4vbwiwjjte3s
Move tests servers from bzrlib.transport to bzrlib.tests.test_server

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
1
 
# Copyright (C) 2008, 2009 Canonical Ltd
 
1
# Copyright (C) 2008, 2009, 2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
29
29
    knit,
30
30
    osutils,
31
31
    pack,
32
 
    remote,
33
32
    revision as _mod_revision,
34
33
    trace,
35
34
    ui,
53
52
    ResumedPack,
54
53
    Packer,
55
54
    )
 
55
from bzrlib.static_tuple import StaticTuple
56
56
 
57
57
 
58
58
class GCPack(NewPack):
154
154
        self._writer.begin()
155
155
        # what state is the pack in? (open, finished, aborted)
156
156
        self._state = 'open'
 
157
        # no name until we finish writing the content
 
158
        self.name = None
157
159
 
158
160
    def _check_references(self):
159
161
        """Make sure our external references are present.
350
352
        """Build a VersionedFiles instance on top of this group of packs."""
351
353
        index_name = index_name + '_index'
352
354
        index_to_pack = {}
353
 
        access = knit._DirectPackAccess(index_to_pack)
 
355
        access = knit._DirectPackAccess(index_to_pack,
 
356
                                        reload_func=self._reload_func)
354
357
        if for_write:
355
358
            # Use new_pack
356
359
            if self.new_pack is None:
410
413
 
411
414
    def _copy_inventory_texts(self):
412
415
        source_vf, target_vf = self._build_vfs('inventory', True, True)
413
 
        self._copy_stream(source_vf, target_vf, self.revision_keys,
 
416
        # It is not sufficient to just use self.revision_keys, as stacked
 
417
        # repositories can have more inventories than they have revisions.
 
418
        # One alternative would be to do something with
 
419
        # get_parent_map(self.revision_keys), but that shouldn't be any faster
 
420
        # than this.
 
421
        inventory_keys = source_vf.keys()
 
422
        missing_inventories = set(self.revision_keys).difference(inventory_keys)
 
423
        if missing_inventories:
 
424
            missing_inventories = sorted(missing_inventories)
 
425
            raise ValueError('We are missing inventories for revisions: %s'
 
426
                % (missing_inventories,))
 
427
        self._copy_stream(source_vf, target_vf, inventory_keys,
414
428
                          'inventories', self._get_filtered_inv_stream, 2)
415
429
 
416
430
    def _copy_chk_texts(self):
466
480
        if not self._use_pack(self.new_pack):
467
481
            self.new_pack.abort()
468
482
            return None
 
483
        self.new_pack.finish_content()
 
484
        if len(self.packs) == 1:
 
485
            old_pack = self.packs[0]
 
486
            if old_pack.name == self.new_pack._hash.hexdigest():
 
487
                # The single old pack was already optimally packed.
 
488
                trace.mutter('single pack %s was already optimally packed',
 
489
                    old_pack.name)
 
490
                self.new_pack.abort()
 
491
                return None
469
492
        self.pb.update('finishing repack', 6, 7)
470
493
        self.new_pack.finish()
471
494
        self._pack_collection.allocate(self.new_pack)
562
585
    pack_factory = GCPack
563
586
    resumed_pack_factory = ResumedGCPack
564
587
 
 
588
    def _check_new_inventories(self):
 
589
        """Detect missing inventories or chk root entries for the new revisions
 
590
        in this write group.
 
591
 
 
592
        :returns: list of strs, summarising any problems found.  If the list is
 
593
            empty no problems were found.
 
594
        """
 
595
        # Ensure that all revisions added in this write group have:
 
596
        #   - corresponding inventories,
 
597
        #   - chk root entries for those inventories,
 
598
        #   - and any present parent inventories have their chk root
 
599
        #     entries too.
 
600
        # And all this should be independent of any fallback repository.
 
601
        problems = []
 
602
        key_deps = self.repo.revisions._index._key_dependencies
 
603
        new_revisions_keys = key_deps.get_new_keys()
 
604
        no_fallback_inv_index = self.repo.inventories._index
 
605
        no_fallback_chk_bytes_index = self.repo.chk_bytes._index
 
606
        no_fallback_texts_index = self.repo.texts._index
 
607
        inv_parent_map = no_fallback_inv_index.get_parent_map(
 
608
            new_revisions_keys)
 
609
        # Are any inventories for corresponding to the new revisions missing?
 
610
        corresponding_invs = set(inv_parent_map)
 
611
        missing_corresponding = set(new_revisions_keys)
 
612
        missing_corresponding.difference_update(corresponding_invs)
 
613
        if missing_corresponding:
 
614
            problems.append("inventories missing for revisions %s" %
 
615
                (sorted(missing_corresponding),))
 
616
            return problems
 
617
        # Are any chk root entries missing for any inventories?  This includes
 
618
        # any present parent inventories, which may be used when calculating
 
619
        # deltas for streaming.
 
620
        all_inv_keys = set(corresponding_invs)
 
621
        for parent_inv_keys in inv_parent_map.itervalues():
 
622
            all_inv_keys.update(parent_inv_keys)
 
623
        # Filter out ghost parents.
 
624
        all_inv_keys.intersection_update(
 
625
            no_fallback_inv_index.get_parent_map(all_inv_keys))
 
626
        parent_invs_only_keys = all_inv_keys.symmetric_difference(
 
627
            corresponding_invs)
 
628
        all_missing = set()
 
629
        inv_ids = [key[-1] for key in all_inv_keys]
 
630
        parent_invs_only_ids = [key[-1] for key in parent_invs_only_keys]
 
631
        root_key_info = _build_interesting_key_sets(
 
632
            self.repo, inv_ids, parent_invs_only_ids)
 
633
        expected_chk_roots = root_key_info.all_keys()
 
634
        present_chk_roots = no_fallback_chk_bytes_index.get_parent_map(
 
635
            expected_chk_roots)
 
636
        missing_chk_roots = expected_chk_roots.difference(present_chk_roots)
 
637
        if missing_chk_roots:
 
638
            problems.append("missing referenced chk root keys: %s"
 
639
                % (sorted(missing_chk_roots),))
 
640
            # Don't bother checking any further.
 
641
            return problems
 
642
        # Find all interesting chk_bytes records, and make sure they are
 
643
        # present, as well as the text keys they reference.
 
644
        chk_bytes_no_fallbacks = self.repo.chk_bytes.without_fallbacks()
 
645
        chk_bytes_no_fallbacks._search_key_func = \
 
646
            self.repo.chk_bytes._search_key_func
 
647
        chk_diff = chk_map.iter_interesting_nodes(
 
648
            chk_bytes_no_fallbacks, root_key_info.interesting_root_keys,
 
649
            root_key_info.uninteresting_root_keys)
 
650
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
 
651
        text_keys = set()
 
652
        try:
 
653
            for record in _filter_text_keys(chk_diff, text_keys, bytes_to_info):
 
654
                pass
 
655
        except errors.NoSuchRevision, e:
 
656
            # XXX: It would be nice if we could give a more precise error here.
 
657
            problems.append("missing chk node(s) for id_to_entry maps")
 
658
        chk_diff = chk_map.iter_interesting_nodes(
 
659
            chk_bytes_no_fallbacks, root_key_info.interesting_pid_root_keys,
 
660
            root_key_info.uninteresting_pid_root_keys)
 
661
        try:
 
662
            for interesting_rec, interesting_map in chk_diff:
 
663
                pass
 
664
        except errors.NoSuchRevision, e:
 
665
            problems.append(
 
666
                "missing chk node(s) for parent_id_basename_to_file_id maps")
 
667
        present_text_keys = no_fallback_texts_index.get_parent_map(text_keys)
 
668
        missing_text_keys = text_keys.difference(present_text_keys)
 
669
        if missing_text_keys:
 
670
            problems.append("missing text keys: %r"
 
671
                % (sorted(missing_text_keys),))
 
672
        return problems
 
673
 
565
674
    def _execute_pack_operations(self, pack_operations,
566
675
                                 _packer_class=GCCHKPacker,
567
676
                                 reload_func=None):
580
689
            packer = GCCHKPacker(self, packs, '.autopack',
581
690
                                 reload_func=reload_func)
582
691
            try:
583
 
                packer.pack()
 
692
                result = packer.pack()
584
693
            except errors.RetryWithNewPacks:
585
694
                # An exception is propagating out of this context, make sure
586
695
                # this packer has cleaned up. Packer() doesn't set its new_pack
589
698
                if packer.new_pack is not None:
590
699
                    packer.new_pack.abort()
591
700
                raise
 
701
            if result is None:
 
702
                return
592
703
            for pack in packs:
593
704
                self._remove_pack_from_memory(pack)
594
705
        # record the newly available packs and stop advertising the old
595
706
        # packs
596
 
        self._save_pack_names(clear_obsolete_packs=True)
597
 
        # Move the old packs out of the way now they are no longer referenced.
598
 
        for revision_count, packs in pack_operations:
599
 
            self._obsolete_packs(packs)
 
707
        to_be_obsoleted = []
 
708
        for _, packs in pack_operations:
 
709
            to_be_obsoleted.extend(packs)
 
710
        result = self._save_pack_names(clear_obsolete_packs=True,
 
711
                                       obsolete_packs=to_be_obsoleted)
 
712
        return result
600
713
 
601
714
 
602
715
class CHKInventoryRepository(KnitPackRepository):
627
740
            _GCGraphIndex(self._pack_collection.revision_index.combined_index,
628
741
                add_callback=self._pack_collection.revision_index.add_callback,
629
742
                parents=True, is_locked=self.is_locked,
630
 
                track_external_parent_refs=True),
 
743
                track_external_parent_refs=True, track_new_keys=True),
631
744
            access=self._pack_collection.revision_index.data_access,
632
745
            delta=False)
633
746
        self.signatures = GroupCompressVersionedFiles(
703
816
                                 ' no new_path %r' % (file_id,))
704
817
            if new_path == '':
705
818
                new_inv.root_id = file_id
706
 
                parent_id_basename_key = ('', '')
 
819
                parent_id_basename_key = StaticTuple('', '').intern()
707
820
            else:
708
821
                utf8_entry_name = entry.name.encode('utf-8')
709
 
                parent_id_basename_key = (entry.parent_id, utf8_entry_name)
 
822
                parent_id_basename_key = StaticTuple(entry.parent_id,
 
823
                                                     utf8_entry_name).intern()
710
824
            new_value = entry_to_bytes(entry)
711
825
            # Populate Caches?
712
826
            # new_inv._path_to_fileid_cache[new_path] = file_id
713
 
            id_to_entry_dict[(file_id,)] = new_value
 
827
            key = StaticTuple(file_id).intern()
 
828
            id_to_entry_dict[key] = new_value
714
829
            parent_id_basename_dict[parent_id_basename_key] = file_id
715
830
 
716
831
        new_inv._populate_from_dicts(self.chk_bytes, id_to_entry_dict,
766
881
            if basis_tree is not None:
767
882
                basis_tree.unlock()
768
883
 
769
 
    def deserialise_inventory(self, revision_id, bytes):
 
884
    def _deserialise_inventory(self, revision_id, bytes):
770
885
        return inventory.CHKInventory.deserialise(self.chk_bytes, bytes,
771
886
            (revision_id,))
772
887
 
773
 
    def _iter_inventories(self, revision_ids):
 
888
    def _iter_inventories(self, revision_ids, ordering):
774
889
        """Iterate over many inventory objects."""
 
890
        if ordering is None:
 
891
            ordering = 'unordered'
775
892
        keys = [(revision_id,) for revision_id in revision_ids]
776
 
        stream = self.inventories.get_record_stream(keys, 'unordered', True)
 
893
        stream = self.inventories.get_record_stream(keys, ordering, True)
777
894
        texts = {}
778
895
        for record in stream:
779
896
            if record.storage_kind != 'absent':
783
900
        for key in keys:
784
901
            yield inventory.CHKInventory.deserialise(self.chk_bytes, texts[key], key)
785
902
 
786
 
    def _iter_inventory_xmls(self, revision_ids):
787
 
        # Without a native 'xml' inventory, this method doesn't make sense, so
788
 
        # make it raise to trap naughty direct users.
789
 
        raise NotImplementedError(self._iter_inventory_xmls)
 
903
    def _iter_inventory_xmls(self, revision_ids, ordering):
 
904
        # Without a native 'xml' inventory, this method doesn't make sense.
 
905
        # However older working trees, and older bundles want it - so we supply
 
906
        # it allowing _get_inventory_xml to work. Bundles currently use the
 
907
        # serializer directly; this also isn't ideal, but there isn't an xml
 
908
        # iteration interface offered at all for repositories. We could make
 
909
        # _iter_inventory_xmls be part of the contract, even if kept private.
 
910
        inv_to_str = self._serializer.write_inventory_to_string
 
911
        for inv in self.iter_inventories(revision_ids, ordering=ordering):
 
912
            yield inv_to_str(inv), inv.revision_id
790
913
 
791
914
    def _find_present_inventory_keys(self, revision_keys):
792
915
        parent_map = self.inventories.get_parent_map(revision_keys)
818
941
                                        parent_keys)
819
942
            present_parent_inv_ids = set(
820
943
                [k[-1] for k in present_parent_inv_keys])
821
 
            uninteresting_root_keys = set()
822
 
            interesting_root_keys = set()
823
944
            inventories_to_read = set(revision_ids)
824
945
            inventories_to_read.update(present_parent_inv_ids)
825
 
            for inv in self.iter_inventories(inventories_to_read):
826
 
                entry_chk_root_key = inv.id_to_entry.key()
827
 
                if inv.revision_id in present_parent_inv_ids:
828
 
                    uninteresting_root_keys.add(entry_chk_root_key)
829
 
                else:
830
 
                    interesting_root_keys.add(entry_chk_root_key)
831
 
 
 
946
            root_key_info = _build_interesting_key_sets(
 
947
                self, inventories_to_read, present_parent_inv_ids)
 
948
            interesting_root_keys = root_key_info.interesting_root_keys
 
949
            uninteresting_root_keys = root_key_info.uninteresting_root_keys
832
950
            chk_bytes = self.chk_bytes
833
951
            for record, items in chk_map.iter_interesting_nodes(chk_bytes,
834
952
                        interesting_root_keys, uninteresting_root_keys,
835
953
                        pb=pb):
836
954
                for name, bytes in items:
837
955
                    (name_utf8, file_id, revision_id) = bytes_to_info(bytes)
 
956
                    # TODO: consider interning file_id, revision_id here, or
 
957
                    #       pushing that intern() into bytes_to_info()
 
958
                    # TODO: rich_root should always be True here, for all
 
959
                    #       repositories that support chk_bytes
838
960
                    if not rich_root and name_utf8 == '':
839
961
                        continue
840
962
                    try:
883
1005
 
884
1006
    def _get_source(self, to_format):
885
1007
        """Return a source for streaming from this repository."""
886
 
        if isinstance(to_format, remote.RemoteRepositoryFormat):
887
 
            # Can't just check attributes on to_format with the current code,
888
 
            # work around this:
889
 
            to_format._ensure_real()
890
 
            to_format = to_format._custom_format
891
 
        if to_format.__class__ is self._format.__class__:
 
1008
        if self._format._serializer == to_format._serializer:
892
1009
            # We must be exactly the same format, otherwise stuff like the chk
893
 
            # page layout might be different
 
1010
            # page layout might be different.
 
1011
            # Actually, this test is just slightly looser than exact so that
 
1012
            # CHK2 <-> 2a transfers will work.
894
1013
            return GroupCHKStreamSource(self, to_format)
895
1014
        return super(CHKInventoryRepository, self)._get_source(to_format)
896
1015
 
971
1090
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
972
1091
        chk_bytes = self.from_repository.chk_bytes
973
1092
        def _filter_id_to_entry():
974
 
            for record, items in chk_map.iter_interesting_nodes(chk_bytes,
975
 
                        self._chk_id_roots, uninteresting_root_keys):
976
 
                for name, bytes in items:
977
 
                    # Note: we don't care about name_utf8, because we are always
978
 
                    # rich-root = True
979
 
                    _, file_id, revision_id = bytes_to_info(bytes)
980
 
                    self._text_keys.add((file_id, revision_id))
 
1093
            interesting_nodes = chk_map.iter_interesting_nodes(chk_bytes,
 
1094
                        self._chk_id_roots, uninteresting_root_keys)
 
1095
            for record in _filter_text_keys(interesting_nodes, self._text_keys,
 
1096
                    bytes_to_info):
981
1097
                if record is not None:
982
1098
                    yield record
983
1099
            # Consumed
997
1113
        for stream_info in self._fetch_revision_texts(revision_ids):
998
1114
            yield stream_info
999
1115
        self._revision_keys = [(rev_id,) for rev_id in revision_ids]
 
1116
        self.from_repository.revisions.clear_cache()
 
1117
        self.from_repository.signatures.clear_cache()
1000
1118
        yield self._get_inventory_stream(self._revision_keys)
 
1119
        self.from_repository.inventories.clear_cache()
1001
1120
        # TODO: The keys to exclude might be part of the search recipe
1002
1121
        # For now, exclude all parents that are at the edge of ancestry, for
1003
1122
        # which we have inventories
1006
1125
                        self._revision_keys)
1007
1126
        for stream_info in self._get_filtered_chk_streams(parent_keys):
1008
1127
            yield stream_info
 
1128
        self.from_repository.chk_bytes.clear_cache()
1009
1129
        yield self._get_text_stream()
 
1130
        self.from_repository.texts.clear_cache()
1010
1131
 
1011
1132
    def get_stream_for_missing_keys(self, missing_keys):
1012
1133
        # missing keys can only occur when we are byte copying and not
1021
1142
            missing_inventory_keys.add(key[1:])
1022
1143
        if self._chk_id_roots or self._chk_p_id_roots:
1023
1144
            raise AssertionError('Cannot call get_stream_for_missing_keys'
1024
 
                ' untill all of get_stream() has been consumed.')
 
1145
                ' until all of get_stream() has been consumed.')
1025
1146
        # Yield the inventory stream, so we can find the chk stream
1026
1147
        # Some of the missing_keys will be missing because they are ghosts.
1027
1148
        # As such, we can ignore them. The Sink is required to verify there are
1034
1155
            yield stream_info
1035
1156
 
1036
1157
 
 
1158
class _InterestingKeyInfo(object):
 
1159
    def __init__(self):
 
1160
        self.interesting_root_keys = set()
 
1161
        self.interesting_pid_root_keys = set()
 
1162
        self.uninteresting_root_keys = set()
 
1163
        self.uninteresting_pid_root_keys = set()
 
1164
 
 
1165
    def all_interesting(self):
 
1166
        return self.interesting_root_keys.union(self.interesting_pid_root_keys)
 
1167
 
 
1168
    def all_uninteresting(self):
 
1169
        return self.uninteresting_root_keys.union(
 
1170
            self.uninteresting_pid_root_keys)
 
1171
 
 
1172
    def all_keys(self):
 
1173
        return self.all_interesting().union(self.all_uninteresting())
 
1174
 
 
1175
 
 
1176
def _build_interesting_key_sets(repo, inventory_ids, parent_only_inv_ids):
 
1177
    result = _InterestingKeyInfo()
 
1178
    for inv in repo.iter_inventories(inventory_ids, 'unordered'):
 
1179
        root_key = inv.id_to_entry.key()
 
1180
        pid_root_key = inv.parent_id_basename_to_file_id.key()
 
1181
        if inv.revision_id in parent_only_inv_ids:
 
1182
            result.uninteresting_root_keys.add(root_key)
 
1183
            result.uninteresting_pid_root_keys.add(pid_root_key)
 
1184
        else:
 
1185
            result.interesting_root_keys.add(root_key)
 
1186
            result.interesting_pid_root_keys.add(pid_root_key)
 
1187
    return result
 
1188
 
 
1189
 
 
1190
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_info):
 
1191
    """Iterate the result of iter_interesting_nodes, yielding the records
 
1192
    and adding to text_keys.
 
1193
    """
 
1194
    for record, items in interesting_nodes_iterable:
 
1195
        for name, bytes in items:
 
1196
            # Note: we don't care about name_utf8, because groupcompress repos
 
1197
            # are always rich-root, so there are no synthesised root records to
 
1198
            # ignore.
 
1199
            _, file_id, revision_id = bytes_to_info(bytes)
 
1200
            file_id = intern(file_id)
 
1201
            revision_id = intern(revision_id)
 
1202
            text_keys.add(StaticTuple(file_id, revision_id).intern())
 
1203
        yield record
 
1204
 
 
1205
 
 
1206
 
 
1207
 
1037
1208
class RepositoryFormatCHK1(RepositoryFormatPack):
1038
1209
    """A hashed CHK+group compress pack repository."""
1039
1210
 
1078
1249
        return ("Development repository format - rich roots, group compression"
1079
1250
            " and chk inventories")
1080
1251
 
1081
 
    def check_conversion_target(self, target_format):
1082
 
        if not target_format.rich_root_data:
1083
 
            raise errors.BadConversionTarget(
1084
 
                'Does not support rich root data.', target_format)
1085
 
        if (self.supports_tree_reference and 
1086
 
            not getattr(target_format, 'supports_tree_reference', False)):
1087
 
            raise errors.BadConversionTarget(
1088
 
                'Does not support nested trees', target_format)
1089
 
 
1090
 
 
1091
1252
 
1092
1253
class RepositoryFormatCHK2(RepositoryFormatCHK1):
1093
1254
    """A CHK repository that uses the bencode revision serializer."""
1110
1271
 
1111
1272
class RepositoryFormat2a(RepositoryFormatCHK2):
1112
1273
    """A CHK repository that uses the bencode revision serializer.
1113
 
    
 
1274
 
1114
1275
    This is the same as RepositoryFormatCHK2 but with a public name.
1115
1276
    """
1116
1277
 
1126
1287
 
1127
1288
    def get_format_string(self):
1128
1289
        return ('Bazaar repository format 2a (needs bzr 1.16 or later)\n')
 
1290
 
 
1291
    def get_format_description(self):
 
1292
        """See RepositoryFormat.get_format_description()."""
 
1293
        return ("Repository format 2a - rich roots, group compression"
 
1294
            " and chk inventories")