← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2010-09-01 08:02:42 UTC
  • mfrom: (5390.3.3 faster-revert-593560)
  • Revision ID: pqm@pqm.ubuntu.com-20100901080242-esg62ody4frwmy66
(spiv) Avoid repeatedly calling self.target.all_file_ids() in
 InterTree.iter_changes. (Andrew Bennetts)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
1
 
# Copyright (C) 2008, 2009 Canonical Ltd
 
1
# Copyright (C) 2008, 2009, 2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
29
29
    knit,
30
30
    osutils,
31
31
    pack,
32
 
    remote,
33
32
    revision as _mod_revision,
34
33
    trace,
35
34
    ui,
 
35
    versionedfile,
36
36
    )
37
37
from bzrlib.btree_index import (
38
38
    BTreeGraphIndex,
39
39
    BTreeBuilder,
40
40
    )
 
41
from bzrlib.decorators import needs_write_lock
41
42
from bzrlib.groupcompress import (
42
43
    _GCGraphIndex,
43
44
    GroupCompressVersionedFiles,
53
54
    ResumedPack,
54
55
    Packer,
55
56
    )
 
57
from bzrlib.static_tuple import StaticTuple
56
58
 
57
59
 
58
60
class GCPack(NewPack):
154
156
        self._writer.begin()
155
157
        # what state is the pack in? (open, finished, aborted)
156
158
        self._state = 'open'
 
159
        # no name until we finish writing the content
 
160
        self.name = None
157
161
 
158
162
    def _check_references(self):
159
163
        """Make sure our external references are present.
260
264
        remaining_keys = set(keys)
261
265
        counter = [0]
262
266
        if self._gather_text_refs:
263
 
            bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
264
267
            self._text_refs = set()
265
268
        def _get_referenced_stream(root_keys, parse_leaf_nodes=False):
266
269
            cur_keys = root_keys
287
290
                    # Store is None, because we know we have a LeafNode, and we
288
291
                    # just want its entries
289
292
                    for file_id, bytes in node.iteritems(None):
290
 
                        name_utf8, file_id, revision_id = bytes_to_info(bytes)
291
 
                        self._text_refs.add((file_id, revision_id))
 
293
                        self._text_refs.add(chk_map._bytes_to_text_key(bytes))
292
294
                def next_stream():
293
295
                    stream = source_vf.get_record_stream(cur_keys,
294
296
                                                         'as-requested', True)
350
352
        """Build a VersionedFiles instance on top of this group of packs."""
351
353
        index_name = index_name + '_index'
352
354
        index_to_pack = {}
353
 
        access = knit._DirectPackAccess(index_to_pack)
 
355
        access = knit._DirectPackAccess(index_to_pack,
 
356
                                        reload_func=self._reload_func)
354
357
        if for_write:
355
358
            # Use new_pack
356
359
            if self.new_pack is None:
410
413
 
411
414
    def _copy_inventory_texts(self):
412
415
        source_vf, target_vf = self._build_vfs('inventory', True, True)
413
 
        self._copy_stream(source_vf, target_vf, self.revision_keys,
 
416
        # It is not sufficient to just use self.revision_keys, as stacked
 
417
        # repositories can have more inventories than they have revisions.
 
418
        # One alternative would be to do something with
 
419
        # get_parent_map(self.revision_keys), but that shouldn't be any faster
 
420
        # than this.
 
421
        inventory_keys = source_vf.keys()
 
422
        missing_inventories = set(self.revision_keys).difference(inventory_keys)
 
423
        if missing_inventories:
 
424
            missing_inventories = sorted(missing_inventories)
 
425
            raise ValueError('We are missing inventories for revisions: %s'
 
426
                % (missing_inventories,))
 
427
        self._copy_stream(source_vf, target_vf, inventory_keys,
414
428
                          'inventories', self._get_filtered_inv_stream, 2)
415
429
 
 
430
    def _get_chk_vfs_for_copy(self):
 
431
        return self._build_vfs('chk', False, False)
 
432
 
416
433
    def _copy_chk_texts(self):
417
 
        source_vf, target_vf = self._build_vfs('chk', False, False)
 
434
        source_vf, target_vf = self._get_chk_vfs_for_copy()
418
435
        # TODO: This is technically spurious... if it is a performance issue,
419
436
        #       remove it
420
437
        total_keys = source_vf.keys()
466
483
        if not self._use_pack(self.new_pack):
467
484
            self.new_pack.abort()
468
485
            return None
 
486
        self.new_pack.finish_content()
 
487
        if len(self.packs) == 1:
 
488
            old_pack = self.packs[0]
 
489
            if old_pack.name == self.new_pack._hash.hexdigest():
 
490
                # The single old pack was already optimally packed.
 
491
                trace.mutter('single pack %s was already optimally packed',
 
492
                    old_pack.name)
 
493
                self.new_pack.abort()
 
494
                return None
469
495
        self.pb.update('finishing repack', 6, 7)
470
496
        self.new_pack.finish()
471
497
        self._pack_collection.allocate(self.new_pack)
557
583
        return new_pack.data_inserted() and self._data_changed
558
584
 
559
585
 
 
586
class GCCHKCanonicalizingPacker(GCCHKPacker):
 
587
    """A packer that ensures inventories have canonical-form CHK maps.
 
588
    
 
589
    Ideally this would be part of reconcile, but it's very slow and rarely
 
590
    needed.  (It repairs repositories affected by
 
591
    https://bugs.launchpad.net/bzr/+bug/522637).
 
592
    """
 
593
 
 
594
    def __init__(self, *args, **kwargs):
 
595
        super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
 
596
        self._data_changed = False
 
597
    
 
598
    def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
 
599
        """Create and exhaust a stream, but don't insert it.
 
600
        
 
601
        This is useful to get the side-effects of generating a stream.
 
602
        """
 
603
        self.pb.update('scanning %s' % (message,), pb_offset)
 
604
        child_pb = ui.ui_factory.nested_progress_bar()
 
605
        try:
 
606
            list(vf_to_stream(source_vf, keys, message, child_pb))
 
607
        finally:
 
608
            child_pb.finished()
 
609
 
 
610
    def _copy_inventory_texts(self):
 
611
        source_vf, target_vf = self._build_vfs('inventory', True, True)
 
612
        source_chk_vf, target_chk_vf = self._get_chk_vfs_for_copy()
 
613
        inventory_keys = source_vf.keys()
 
614
        # First, copy the existing CHKs on the assumption that most of them
 
615
        # will be correct.  This will save us from having to reinsert (and
 
616
        # recompress) these records later at the cost of perhaps preserving a
 
617
        # few unused CHKs. 
 
618
        # (Iterate but don't insert _get_filtered_inv_stream to populate the
 
619
        # variables needed by GCCHKPacker._copy_chk_texts.)
 
620
        self._exhaust_stream(source_vf, inventory_keys, 'inventories',
 
621
                self._get_filtered_inv_stream, 2)
 
622
        GCCHKPacker._copy_chk_texts(self)
 
623
        # Now copy and fix the inventories, and any regenerated CHKs.
 
624
        def chk_canonicalizing_inv_stream(source_vf, keys, message, pb=None):
 
625
            return self._get_filtered_canonicalizing_inv_stream(
 
626
                source_vf, keys, message, pb, source_chk_vf, target_chk_vf)
 
627
        self._copy_stream(source_vf, target_vf, inventory_keys,
 
628
                          'inventories', chk_canonicalizing_inv_stream, 4)
 
629
 
 
630
    def _copy_chk_texts(self):
 
631
        # No-op; in this class this happens during _copy_inventory_texts.
 
632
        pass
 
633
 
 
634
    def _get_filtered_canonicalizing_inv_stream(self, source_vf, keys, message,
 
635
            pb=None, source_chk_vf=None, target_chk_vf=None):
 
636
        """Filter the texts of inventories, regenerating CHKs to make sure they
 
637
        are canonical.
 
638
        """
 
639
        total_keys = len(keys)
 
640
        target_chk_vf = versionedfile.NoDupeAddLinesDecorator(target_chk_vf)
 
641
        def _filtered_inv_stream():
 
642
            stream = source_vf.get_record_stream(keys, 'groupcompress', True)
 
643
            search_key_name = None
 
644
            for idx, record in enumerate(stream):
 
645
                # Inventories should always be with revisions; assume success.
 
646
                bytes = record.get_bytes_as('fulltext')
 
647
                chk_inv = inventory.CHKInventory.deserialise(
 
648
                    source_chk_vf, bytes, record.key)
 
649
                if pb is not None:
 
650
                    pb.update('inv', idx, total_keys)
 
651
                chk_inv.id_to_entry._ensure_root()
 
652
                if search_key_name is None:
 
653
                    # Find the name corresponding to the search_key_func
 
654
                    search_key_reg = chk_map.search_key_registry
 
655
                    for search_key_name, func in search_key_reg.iteritems():
 
656
                        if func == chk_inv.id_to_entry._search_key_func:
 
657
                            break
 
658
                canonical_inv = inventory.CHKInventory.from_inventory(
 
659
                    target_chk_vf, chk_inv,
 
660
                    maximum_size=chk_inv.id_to_entry._root_node._maximum_size,
 
661
                    search_key_name=search_key_name)
 
662
                if chk_inv.id_to_entry.key() != canonical_inv.id_to_entry.key():
 
663
                    trace.mutter(
 
664
                        'Non-canonical CHK map for id_to_entry of inv: %s '
 
665
                        '(root is %s, should be %s)' % (chk_inv.revision_id,
 
666
                        chk_inv.id_to_entry.key()[0],
 
667
                        canonical_inv.id_to_entry.key()[0]))
 
668
                    self._data_changed = True
 
669
                p_id_map = chk_inv.parent_id_basename_to_file_id
 
670
                p_id_map._ensure_root()
 
671
                canon_p_id_map = canonical_inv.parent_id_basename_to_file_id
 
672
                if p_id_map.key() != canon_p_id_map.key():
 
673
                    trace.mutter(
 
674
                        'Non-canonical CHK map for parent_id_to_basename of '
 
675
                        'inv: %s (root is %s, should be %s)'
 
676
                        % (chk_inv.revision_id, p_id_map.key()[0],
 
677
                           canon_p_id_map.key()[0]))
 
678
                    self._data_changed = True
 
679
                yield versionedfile.ChunkedContentFactory(record.key,
 
680
                        record.parents, record.sha1,
 
681
                        canonical_inv.to_lines())
 
682
            # We have finished processing all of the inventory records, we
 
683
            # don't need these sets anymore
 
684
        return _filtered_inv_stream()
 
685
 
 
686
    def _use_pack(self, new_pack):
 
687
        """Override _use_pack to check for reconcile having changed content."""
 
688
        return new_pack.data_inserted() and self._data_changed
 
689
 
 
690
 
560
691
class GCRepositoryPackCollection(RepositoryPackCollection):
561
692
 
562
693
    pack_factory = GCPack
563
694
    resumed_pack_factory = ResumedGCPack
564
695
 
 
696
    def _check_new_inventories(self):
 
697
        """Detect missing inventories or chk root entries for the new revisions
 
698
        in this write group.
 
699
 
 
700
        :returns: list of strs, summarising any problems found.  If the list is
 
701
            empty no problems were found.
 
702
        """
 
703
        # Ensure that all revisions added in this write group have:
 
704
        #   - corresponding inventories,
 
705
        #   - chk root entries for those inventories,
 
706
        #   - and any present parent inventories have their chk root
 
707
        #     entries too.
 
708
        # And all this should be independent of any fallback repository.
 
709
        problems = []
 
710
        key_deps = self.repo.revisions._index._key_dependencies
 
711
        new_revisions_keys = key_deps.get_new_keys()
 
712
        no_fallback_inv_index = self.repo.inventories._index
 
713
        no_fallback_chk_bytes_index = self.repo.chk_bytes._index
 
714
        no_fallback_texts_index = self.repo.texts._index
 
715
        inv_parent_map = no_fallback_inv_index.get_parent_map(
 
716
            new_revisions_keys)
 
717
        # Are any inventories for corresponding to the new revisions missing?
 
718
        corresponding_invs = set(inv_parent_map)
 
719
        missing_corresponding = set(new_revisions_keys)
 
720
        missing_corresponding.difference_update(corresponding_invs)
 
721
        if missing_corresponding:
 
722
            problems.append("inventories missing for revisions %s" %
 
723
                (sorted(missing_corresponding),))
 
724
            return problems
 
725
        # Are any chk root entries missing for any inventories?  This includes
 
726
        # any present parent inventories, which may be used when calculating
 
727
        # deltas for streaming.
 
728
        all_inv_keys = set(corresponding_invs)
 
729
        for parent_inv_keys in inv_parent_map.itervalues():
 
730
            all_inv_keys.update(parent_inv_keys)
 
731
        # Filter out ghost parents.
 
732
        all_inv_keys.intersection_update(
 
733
            no_fallback_inv_index.get_parent_map(all_inv_keys))
 
734
        parent_invs_only_keys = all_inv_keys.symmetric_difference(
 
735
            corresponding_invs)
 
736
        all_missing = set()
 
737
        inv_ids = [key[-1] for key in all_inv_keys]
 
738
        parent_invs_only_ids = [key[-1] for key in parent_invs_only_keys]
 
739
        root_key_info = _build_interesting_key_sets(
 
740
            self.repo, inv_ids, parent_invs_only_ids)
 
741
        expected_chk_roots = root_key_info.all_keys()
 
742
        present_chk_roots = no_fallback_chk_bytes_index.get_parent_map(
 
743
            expected_chk_roots)
 
744
        missing_chk_roots = expected_chk_roots.difference(present_chk_roots)
 
745
        if missing_chk_roots:
 
746
            problems.append("missing referenced chk root keys: %s"
 
747
                % (sorted(missing_chk_roots),))
 
748
            # Don't bother checking any further.
 
749
            return problems
 
750
        # Find all interesting chk_bytes records, and make sure they are
 
751
        # present, as well as the text keys they reference.
 
752
        chk_bytes_no_fallbacks = self.repo.chk_bytes.without_fallbacks()
 
753
        chk_bytes_no_fallbacks._search_key_func = \
 
754
            self.repo.chk_bytes._search_key_func
 
755
        chk_diff = chk_map.iter_interesting_nodes(
 
756
            chk_bytes_no_fallbacks, root_key_info.interesting_root_keys,
 
757
            root_key_info.uninteresting_root_keys)
 
758
        text_keys = set()
 
759
        try:
 
760
            for record in _filter_text_keys(chk_diff, text_keys,
 
761
                                            chk_map._bytes_to_text_key):
 
762
                pass
 
763
        except errors.NoSuchRevision, e:
 
764
            # XXX: It would be nice if we could give a more precise error here.
 
765
            problems.append("missing chk node(s) for id_to_entry maps")
 
766
        chk_diff = chk_map.iter_interesting_nodes(
 
767
            chk_bytes_no_fallbacks, root_key_info.interesting_pid_root_keys,
 
768
            root_key_info.uninteresting_pid_root_keys)
 
769
        try:
 
770
            for interesting_rec, interesting_map in chk_diff:
 
771
                pass
 
772
        except errors.NoSuchRevision, e:
 
773
            problems.append(
 
774
                "missing chk node(s) for parent_id_basename_to_file_id maps")
 
775
        present_text_keys = no_fallback_texts_index.get_parent_map(text_keys)
 
776
        missing_text_keys = text_keys.difference(present_text_keys)
 
777
        if missing_text_keys:
 
778
            problems.append("missing text keys: %r"
 
779
                % (sorted(missing_text_keys),))
 
780
        return problems
 
781
 
565
782
    def _execute_pack_operations(self, pack_operations,
566
783
                                 _packer_class=GCCHKPacker,
567
784
                                 reload_func=None):
580
797
            packer = GCCHKPacker(self, packs, '.autopack',
581
798
                                 reload_func=reload_func)
582
799
            try:
583
 
                packer.pack()
 
800
                result = packer.pack()
584
801
            except errors.RetryWithNewPacks:
585
802
                # An exception is propagating out of this context, make sure
586
803
                # this packer has cleaned up. Packer() doesn't set its new_pack
589
806
                if packer.new_pack is not None:
590
807
                    packer.new_pack.abort()
591
808
                raise
 
809
            if result is None:
 
810
                return
592
811
            for pack in packs:
593
812
                self._remove_pack_from_memory(pack)
594
813
        # record the newly available packs and stop advertising the old
595
814
        # packs
596
 
        self._save_pack_names(clear_obsolete_packs=True)
597
 
        # Move the old packs out of the way now they are no longer referenced.
598
 
        for revision_count, packs in pack_operations:
599
 
            self._obsolete_packs(packs)
 
815
        to_be_obsoleted = []
 
816
        for _, packs in pack_operations:
 
817
            to_be_obsoleted.extend(packs)
 
818
        result = self._save_pack_names(clear_obsolete_packs=True,
 
819
                                       obsolete_packs=to_be_obsoleted)
 
820
        return result
600
821
 
601
822
 
602
823
class CHKInventoryRepository(KnitPackRepository):
627
848
            _GCGraphIndex(self._pack_collection.revision_index.combined_index,
628
849
                add_callback=self._pack_collection.revision_index.add_callback,
629
850
                parents=True, is_locked=self.is_locked,
630
 
                track_external_parent_refs=True),
 
851
                track_external_parent_refs=True, track_new_keys=True),
631
852
            access=self._pack_collection.revision_index.data_access,
632
853
            delta=False)
633
854
        self.signatures = GroupCompressVersionedFiles(
703
924
                                 ' no new_path %r' % (file_id,))
704
925
            if new_path == '':
705
926
                new_inv.root_id = file_id
706
 
                parent_id_basename_key = ('', '')
 
927
                parent_id_basename_key = StaticTuple('', '').intern()
707
928
            else:
708
929
                utf8_entry_name = entry.name.encode('utf-8')
709
 
                parent_id_basename_key = (entry.parent_id, utf8_entry_name)
 
930
                parent_id_basename_key = StaticTuple(entry.parent_id,
 
931
                                                     utf8_entry_name).intern()
710
932
            new_value = entry_to_bytes(entry)
711
933
            # Populate Caches?
712
934
            # new_inv._path_to_fileid_cache[new_path] = file_id
713
 
            id_to_entry_dict[(file_id,)] = new_value
 
935
            key = StaticTuple(file_id).intern()
 
936
            id_to_entry_dict[key] = new_value
714
937
            parent_id_basename_dict[parent_id_basename_key] = file_id
715
938
 
716
939
        new_inv._populate_from_dicts(self.chk_bytes, id_to_entry_dict,
749
972
        if basis_inv is None:
750
973
            if basis_revision_id == _mod_revision.NULL_REVISION:
751
974
                new_inv = self._create_inv_from_null(delta, new_revision_id)
 
975
                if new_inv.root_id is None:
 
976
                    raise errors.RootMissing()
752
977
                inv_lines = new_inv.to_lines()
753
978
                return self._inventory_add_lines(new_revision_id, parents,
754
979
                    inv_lines, check_content=False), new_inv
766
991
            if basis_tree is not None:
767
992
                basis_tree.unlock()
768
993
 
769
 
    def deserialise_inventory(self, revision_id, bytes):
 
994
    def _deserialise_inventory(self, revision_id, bytes):
770
995
        return inventory.CHKInventory.deserialise(self.chk_bytes, bytes,
771
996
            (revision_id,))
772
997
 
773
 
    def _iter_inventories(self, revision_ids):
 
998
    def _iter_inventories(self, revision_ids, ordering):
774
999
        """Iterate over many inventory objects."""
 
1000
        if ordering is None:
 
1001
            ordering = 'unordered'
775
1002
        keys = [(revision_id,) for revision_id in revision_ids]
776
 
        stream = self.inventories.get_record_stream(keys, 'unordered', True)
 
1003
        stream = self.inventories.get_record_stream(keys, ordering, True)
777
1004
        texts = {}
778
1005
        for record in stream:
779
1006
            if record.storage_kind != 'absent':
783
1010
        for key in keys:
784
1011
            yield inventory.CHKInventory.deserialise(self.chk_bytes, texts[key], key)
785
1012
 
786
 
    def _iter_inventory_xmls(self, revision_ids):
787
 
        # Without a native 'xml' inventory, this method doesn't make sense, so
788
 
        # make it raise to trap naughty direct users.
789
 
        raise NotImplementedError(self._iter_inventory_xmls)
 
1013
    def _iter_inventory_xmls(self, revision_ids, ordering):
 
1014
        # Without a native 'xml' inventory, this method doesn't make sense.
 
1015
        # However older working trees, and older bundles want it - so we supply
 
1016
        # it allowing _get_inventory_xml to work. Bundles currently use the
 
1017
        # serializer directly; this also isn't ideal, but there isn't an xml
 
1018
        # iteration interface offered at all for repositories. We could make
 
1019
        # _iter_inventory_xmls be part of the contract, even if kept private.
 
1020
        inv_to_str = self._serializer.write_inventory_to_string
 
1021
        for inv in self.iter_inventories(revision_ids, ordering=ordering):
 
1022
            yield inv_to_str(inv), inv.revision_id
790
1023
 
791
1024
    def _find_present_inventory_keys(self, revision_keys):
792
1025
        parent_map = self.inventories.get_parent_map(revision_keys)
818
1051
                                        parent_keys)
819
1052
            present_parent_inv_ids = set(
820
1053
                [k[-1] for k in present_parent_inv_keys])
821
 
            uninteresting_root_keys = set()
822
 
            interesting_root_keys = set()
823
1054
            inventories_to_read = set(revision_ids)
824
1055
            inventories_to_read.update(present_parent_inv_ids)
825
 
            for inv in self.iter_inventories(inventories_to_read):
826
 
                entry_chk_root_key = inv.id_to_entry.key()
827
 
                if inv.revision_id in present_parent_inv_ids:
828
 
                    uninteresting_root_keys.add(entry_chk_root_key)
829
 
                else:
830
 
                    interesting_root_keys.add(entry_chk_root_key)
831
 
 
 
1056
            root_key_info = _build_interesting_key_sets(
 
1057
                self, inventories_to_read, present_parent_inv_ids)
 
1058
            interesting_root_keys = root_key_info.interesting_root_keys
 
1059
            uninteresting_root_keys = root_key_info.uninteresting_root_keys
832
1060
            chk_bytes = self.chk_bytes
833
1061
            for record, items in chk_map.iter_interesting_nodes(chk_bytes,
834
1062
                        interesting_root_keys, uninteresting_root_keys,
835
1063
                        pb=pb):
836
1064
                for name, bytes in items:
837
1065
                    (name_utf8, file_id, revision_id) = bytes_to_info(bytes)
 
1066
                    # TODO: consider interning file_id, revision_id here, or
 
1067
                    #       pushing that intern() into bytes_to_info()
 
1068
                    # TODO: rich_root should always be True here, for all
 
1069
                    #       repositories that support chk_bytes
838
1070
                    if not rich_root and name_utf8 == '':
839
1071
                        continue
840
1072
                    try:
877
1109
        finally:
878
1110
            pb.finished()
879
1111
 
 
1112
    @needs_write_lock
 
1113
    def reconcile_canonicalize_chks(self):
 
1114
        """Reconcile this repository to make sure all CHKs are in canonical
 
1115
        form.
 
1116
        """
 
1117
        from bzrlib.reconcile import PackReconciler
 
1118
        reconciler = PackReconciler(self, thorough=True, canonicalize_chks=True)
 
1119
        reconciler.reconcile()
 
1120
        return reconciler
 
1121
 
880
1122
    def _reconcile_pack(self, collection, packs, extension, revs, pb):
881
1123
        packer = GCCHKReconcilePacker(collection, packs, extension)
882
1124
        return packer.pack(pb)
883
1125
 
 
1126
    def _canonicalize_chks_pack(self, collection, packs, extension, revs, pb):
 
1127
        packer = GCCHKCanonicalizingPacker(collection, packs, extension, revs)
 
1128
        return packer.pack(pb)
 
1129
 
884
1130
    def _get_source(self, to_format):
885
1131
        """Return a source for streaming from this repository."""
886
 
        if isinstance(to_format, remote.RemoteRepositoryFormat):
887
 
            # Can't just check attributes on to_format with the current code,
888
 
            # work around this:
889
 
            to_format._ensure_real()
890
 
            to_format = to_format._custom_format
891
 
        if to_format.__class__ is self._format.__class__:
 
1132
        if self._format._serializer == to_format._serializer:
892
1133
            # We must be exactly the same format, otherwise stuff like the chk
893
 
            # page layout might be different
 
1134
            # page layout might be different.
 
1135
            # Actually, this test is just slightly looser than exact so that
 
1136
            # CHK2 <-> 2a transfers will work.
894
1137
            return GroupCHKStreamSource(self, to_format)
895
1138
        return super(CHKInventoryRepository, self)._get_source(to_format)
896
1139
 
968
1211
                uninteresting_root_keys.add(inv.id_to_entry.key())
969
1212
                uninteresting_pid_root_keys.add(
970
1213
                    inv.parent_id_basename_to_file_id.key())
971
 
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
972
1214
        chk_bytes = self.from_repository.chk_bytes
973
1215
        def _filter_id_to_entry():
974
 
            for record, items in chk_map.iter_interesting_nodes(chk_bytes,
975
 
                        self._chk_id_roots, uninteresting_root_keys):
976
 
                for name, bytes in items:
977
 
                    # Note: we don't care about name_utf8, because we are always
978
 
                    # rich-root = True
979
 
                    _, file_id, revision_id = bytes_to_info(bytes)
980
 
                    self._text_keys.add((file_id, revision_id))
 
1216
            interesting_nodes = chk_map.iter_interesting_nodes(chk_bytes,
 
1217
                        self._chk_id_roots, uninteresting_root_keys)
 
1218
            for record in _filter_text_keys(interesting_nodes, self._text_keys,
 
1219
                    chk_map._bytes_to_text_key):
981
1220
                if record is not None:
982
1221
                    yield record
983
1222
            # Consumed
993
1232
        yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
994
1233
 
995
1234
    def get_stream(self, search):
 
1235
        def wrap_and_count(pb, rc, stream):
 
1236
            """Yield records from stream while showing progress."""
 
1237
            count = 0
 
1238
            for record in stream:
 
1239
                if count == rc.STEP:
 
1240
                    rc.increment(count)
 
1241
                    pb.update('Estimate', rc.current, rc.max)
 
1242
                    count = 0
 
1243
                count += 1
 
1244
                yield record
 
1245
 
996
1246
        revision_ids = search.get_keys()
 
1247
        pb = ui.ui_factory.nested_progress_bar()
 
1248
        rc = self._record_counter
 
1249
        self._record_counter.setup(len(revision_ids))
997
1250
        for stream_info in self._fetch_revision_texts(revision_ids):
998
 
            yield stream_info
 
1251
            yield (stream_info[0],
 
1252
                wrap_and_count(pb, rc, stream_info[1]))
999
1253
        self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1000
 
        yield self._get_inventory_stream(self._revision_keys)
 
1254
        self.from_repository.revisions.clear_cache()
 
1255
        self.from_repository.signatures.clear_cache()
 
1256
        s = self._get_inventory_stream(self._revision_keys)
 
1257
        yield (s[0], wrap_and_count(pb, rc, s[1]))
 
1258
        self.from_repository.inventories.clear_cache()
1001
1259
        # TODO: The keys to exclude might be part of the search recipe
1002
1260
        # For now, exclude all parents that are at the edge of ancestry, for
1003
1261
        # which we have inventories
1005
1263
        parent_keys = from_repo._find_parent_keys_of_revisions(
1006
1264
                        self._revision_keys)
1007
1265
        for stream_info in self._get_filtered_chk_streams(parent_keys):
1008
 
            yield stream_info
1009
 
        yield self._get_text_stream()
 
1266
            yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))
 
1267
        self.from_repository.chk_bytes.clear_cache()
 
1268
        s = self._get_text_stream()
 
1269
        yield (s[0], wrap_and_count(pb, rc, s[1]))
 
1270
        self.from_repository.texts.clear_cache()
 
1271
        pb.update('Done', rc.max, rc.max)
 
1272
        pb.finished()
1010
1273
 
1011
1274
    def get_stream_for_missing_keys(self, missing_keys):
1012
1275
        # missing keys can only occur when we are byte copying and not
1021
1284
            missing_inventory_keys.add(key[1:])
1022
1285
        if self._chk_id_roots or self._chk_p_id_roots:
1023
1286
            raise AssertionError('Cannot call get_stream_for_missing_keys'
1024
 
                ' untill all of get_stream() has been consumed.')
 
1287
                ' until all of get_stream() has been consumed.')
1025
1288
        # Yield the inventory stream, so we can find the chk stream
1026
1289
        # Some of the missing_keys will be missing because they are ghosts.
1027
1290
        # As such, we can ignore them. The Sink is required to verify there are
1034
1297
            yield stream_info
1035
1298
 
1036
1299
 
 
1300
class _InterestingKeyInfo(object):
 
1301
    def __init__(self):
 
1302
        self.interesting_root_keys = set()
 
1303
        self.interesting_pid_root_keys = set()
 
1304
        self.uninteresting_root_keys = set()
 
1305
        self.uninteresting_pid_root_keys = set()
 
1306
 
 
1307
    def all_interesting(self):
 
1308
        return self.interesting_root_keys.union(self.interesting_pid_root_keys)
 
1309
 
 
1310
    def all_uninteresting(self):
 
1311
        return self.uninteresting_root_keys.union(
 
1312
            self.uninteresting_pid_root_keys)
 
1313
 
 
1314
    def all_keys(self):
 
1315
        return self.all_interesting().union(self.all_uninteresting())
 
1316
 
 
1317
 
 
1318
def _build_interesting_key_sets(repo, inventory_ids, parent_only_inv_ids):
 
1319
    result = _InterestingKeyInfo()
 
1320
    for inv in repo.iter_inventories(inventory_ids, 'unordered'):
 
1321
        root_key = inv.id_to_entry.key()
 
1322
        pid_root_key = inv.parent_id_basename_to_file_id.key()
 
1323
        if inv.revision_id in parent_only_inv_ids:
 
1324
            result.uninteresting_root_keys.add(root_key)
 
1325
            result.uninteresting_pid_root_keys.add(pid_root_key)
 
1326
        else:
 
1327
            result.interesting_root_keys.add(root_key)
 
1328
            result.interesting_pid_root_keys.add(pid_root_key)
 
1329
    return result
 
1330
 
 
1331
 
 
1332
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_text_key):
 
1333
    """Iterate the result of iter_interesting_nodes, yielding the records
 
1334
    and adding to text_keys.
 
1335
    """
 
1336
    text_keys_update = text_keys.update
 
1337
    for record, items in interesting_nodes_iterable:
 
1338
        text_keys_update([bytes_to_text_key(b) for n,b in items])
 
1339
        yield record
 
1340
 
 
1341
 
 
1342
 
 
1343
 
1037
1344
class RepositoryFormatCHK1(RepositoryFormatPack):
1038
1345
    """A hashed CHK+group compress pack repository."""
1039
1346
 
1078
1385
        return ("Development repository format - rich roots, group compression"
1079
1386
            " and chk inventories")
1080
1387
 
1081
 
    def check_conversion_target(self, target_format):
1082
 
        if not target_format.rich_root_data:
1083
 
            raise errors.BadConversionTarget(
1084
 
                'Does not support rich root data.', target_format)
1085
 
        if (self.supports_tree_reference and 
1086
 
            not getattr(target_format, 'supports_tree_reference', False)):
1087
 
            raise errors.BadConversionTarget(
1088
 
                'Does not support nested trees', target_format)
1089
 
 
1090
 
 
1091
1388
 
1092
1389
class RepositoryFormatCHK2(RepositoryFormatCHK1):
1093
1390
    """A CHK repository that uses the bencode revision serializer."""
1110
1407
 
1111
1408
class RepositoryFormat2a(RepositoryFormatCHK2):
1112
1409
    """A CHK repository that uses the bencode revision serializer.
1113
 
    
 
1410
 
1114
1411
    This is the same as RepositoryFormatCHK2 but with a public name.
1115
1412
    """
1116
1413
 
1126
1423
 
1127
1424
    def get_format_string(self):
1128
1425
        return ('Bazaar repository format 2a (needs bzr 1.16 or later)\n')
 
1426
 
 
1427
    def get_format_description(self):
 
1428
        """See RepositoryFormat.get_format_description()."""
 
1429
        return ("Repository format 2a - rich roots, group compression"
 
1430
            " and chk inventories")