← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

  • Committer: John Arbash Meinel
  • Date: 2010-02-17 17:11:16 UTC
  • mfrom: (4797.2.17 2.1)
  • mto: (4797.2.18 2.1)
  • mto: This revision was merged to the branch mainline in revision 5055.
  • Revision ID: john@arbash-meinel.com-20100217171116-h7t9223ystbnx5h8
merge bzr.2.1 in preparation for NEWS entry.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
29
29
    knit,
30
30
    osutils,
31
31
    pack,
 
32
    remote,
32
33
    revision as _mod_revision,
33
34
    trace,
34
35
    ui,
35
 
    versionedfile,
36
36
    )
37
37
from bzrlib.btree_index import (
38
38
    BTreeGraphIndex,
39
39
    BTreeBuilder,
40
40
    )
41
 
from bzrlib.decorators import needs_write_lock
42
41
from bzrlib.groupcompress import (
43
42
    _GCGraphIndex,
44
43
    GroupCompressVersionedFiles,
264
263
        remaining_keys = set(keys)
265
264
        counter = [0]
266
265
        if self._gather_text_refs:
 
266
            bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
267
267
            self._text_refs = set()
268
268
        def _get_referenced_stream(root_keys, parse_leaf_nodes=False):
269
269
            cur_keys = root_keys
290
290
                    # Store is None, because we know we have a LeafNode, and we
291
291
                    # just want its entries
292
292
                    for file_id, bytes in node.iteritems(None):
293
 
                        self._text_refs.add(chk_map._bytes_to_text_key(bytes))
 
293
                        name_utf8, file_id, revision_id = bytes_to_info(bytes)
 
294
                        self._text_refs.add((file_id, revision_id))
294
295
                def next_stream():
295
296
                    stream = source_vf.get_record_stream(cur_keys,
296
297
                                                         'as-requested', True)
427
428
        self._copy_stream(source_vf, target_vf, inventory_keys,
428
429
                          'inventories', self._get_filtered_inv_stream, 2)
429
430
 
430
 
    def _get_chk_vfs_for_copy(self):
431
 
        return self._build_vfs('chk', False, False)
432
 
 
433
431
    def _copy_chk_texts(self):
434
 
        source_vf, target_vf = self._get_chk_vfs_for_copy()
 
432
        source_vf, target_vf = self._build_vfs('chk', False, False)
435
433
        # TODO: This is technically spurious... if it is a performance issue,
436
434
        #       remove it
437
435
        total_keys = source_vf.keys()
583
581
        return new_pack.data_inserted() and self._data_changed
584
582
 
585
583
 
586
 
class GCCHKCanonicalizingPacker(GCCHKPacker):
587
 
    """A packer that ensures inventories have canonical-form CHK maps.
588
 
    
589
 
    Ideally this would be part of reconcile, but it's very slow and rarely
590
 
    needed.  (It repairs repositories affected by
591
 
    https://bugs.launchpad.net/bzr/+bug/522637).
592
 
    """
593
 
 
594
 
    def __init__(self, *args, **kwargs):
595
 
        super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
596
 
        self._data_changed = False
597
 
    
598
 
    def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
599
 
        """Create and exhaust a stream, but don't insert it.
600
 
        
601
 
        This is useful to get the side-effects of generating a stream.
602
 
        """
603
 
        self.pb.update('scanning %s' % (message,), pb_offset)
604
 
        child_pb = ui.ui_factory.nested_progress_bar()
605
 
        try:
606
 
            list(vf_to_stream(source_vf, keys, message, child_pb))
607
 
        finally:
608
 
            child_pb.finished()
609
 
 
610
 
    def _copy_inventory_texts(self):
611
 
        source_vf, target_vf = self._build_vfs('inventory', True, True)
612
 
        source_chk_vf, target_chk_vf = self._get_chk_vfs_for_copy()
613
 
        inventory_keys = source_vf.keys()
614
 
        # First, copy the existing CHKs on the assumption that most of them
615
 
        # will be correct.  This will save us from having to reinsert (and
616
 
        # recompress) these records later at the cost of perhaps preserving a
617
 
        # few unused CHKs. 
618
 
        # (Iterate but don't insert _get_filtered_inv_stream to populate the
619
 
        # variables needed by GCCHKPacker._copy_chk_texts.)
620
 
        self._exhaust_stream(source_vf, inventory_keys, 'inventories',
621
 
                self._get_filtered_inv_stream, 2)
622
 
        GCCHKPacker._copy_chk_texts(self)
623
 
        # Now copy and fix the inventories, and any regenerated CHKs.
624
 
        def chk_canonicalizing_inv_stream(source_vf, keys, message, pb=None):
625
 
            return self._get_filtered_canonicalizing_inv_stream(
626
 
                source_vf, keys, message, pb, source_chk_vf, target_chk_vf)
627
 
        self._copy_stream(source_vf, target_vf, inventory_keys,
628
 
                          'inventories', chk_canonicalizing_inv_stream, 4)
629
 
 
630
 
    def _copy_chk_texts(self):
631
 
        # No-op; in this class this happens during _copy_inventory_texts.
632
 
        pass
633
 
 
634
 
    def _get_filtered_canonicalizing_inv_stream(self, source_vf, keys, message,
635
 
            pb=None, source_chk_vf=None, target_chk_vf=None):
636
 
        """Filter the texts of inventories, regenerating CHKs to make sure they
637
 
        are canonical.
638
 
        """
639
 
        total_keys = len(keys)
640
 
        target_chk_vf = versionedfile.NoDupeAddLinesDecorator(target_chk_vf)
641
 
        def _filtered_inv_stream():
642
 
            stream = source_vf.get_record_stream(keys, 'groupcompress', True)
643
 
            search_key_name = None
644
 
            for idx, record in enumerate(stream):
645
 
                # Inventories should always be with revisions; assume success.
646
 
                bytes = record.get_bytes_as('fulltext')
647
 
                chk_inv = inventory.CHKInventory.deserialise(
648
 
                    source_chk_vf, bytes, record.key)
649
 
                if pb is not None:
650
 
                    pb.update('inv', idx, total_keys)
651
 
                chk_inv.id_to_entry._ensure_root()
652
 
                if search_key_name is None:
653
 
                    # Find the name corresponding to the search_key_func
654
 
                    search_key_reg = chk_map.search_key_registry
655
 
                    for search_key_name, func in search_key_reg.iteritems():
656
 
                        if func == chk_inv.id_to_entry._search_key_func:
657
 
                            break
658
 
                canonical_inv = inventory.CHKInventory.from_inventory(
659
 
                    target_chk_vf, chk_inv,
660
 
                    maximum_size=chk_inv.id_to_entry._root_node._maximum_size,
661
 
                    search_key_name=search_key_name)
662
 
                if chk_inv.id_to_entry.key() != canonical_inv.id_to_entry.key():
663
 
                    trace.mutter(
664
 
                        'Non-canonical CHK map for id_to_entry of inv: %s '
665
 
                        '(root is %s, should be %s)' % (chk_inv.revision_id,
666
 
                        chk_inv.id_to_entry.key()[0],
667
 
                        canonical_inv.id_to_entry.key()[0]))
668
 
                    self._data_changed = True
669
 
                p_id_map = chk_inv.parent_id_basename_to_file_id
670
 
                p_id_map._ensure_root()
671
 
                canon_p_id_map = canonical_inv.parent_id_basename_to_file_id
672
 
                if p_id_map.key() != canon_p_id_map.key():
673
 
                    trace.mutter(
674
 
                        'Non-canonical CHK map for parent_id_to_basename of '
675
 
                        'inv: %s (root is %s, should be %s)'
676
 
                        % (chk_inv.revision_id, p_id_map.key()[0],
677
 
                           canon_p_id_map.key()[0]))
678
 
                    self._data_changed = True
679
 
                yield versionedfile.ChunkedContentFactory(record.key,
680
 
                        record.parents, record.sha1,
681
 
                        canonical_inv.to_lines())
682
 
            # We have finished processing all of the inventory records, we
683
 
            # don't need these sets anymore
684
 
        return _filtered_inv_stream()
685
 
 
686
 
    def _use_pack(self, new_pack):
687
 
        """Override _use_pack to check for reconcile having changed content."""
688
 
        return new_pack.data_inserted() and self._data_changed
689
 
 
690
 
 
691
584
class GCRepositoryPackCollection(RepositoryPackCollection):
692
585
 
693
586
    pack_factory = GCPack
755
648
        chk_diff = chk_map.iter_interesting_nodes(
756
649
            chk_bytes_no_fallbacks, root_key_info.interesting_root_keys,
757
650
            root_key_info.uninteresting_root_keys)
 
651
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
758
652
        text_keys = set()
759
653
        try:
760
 
            for record in _filter_text_keys(chk_diff, text_keys,
761
 
                                            chk_map._bytes_to_text_key):
 
654
            for record in _filter_text_keys(chk_diff, text_keys, bytes_to_info):
762
655
                pass
763
656
        except errors.NoSuchRevision, e:
764
657
            # XXX: It would be nice if we could give a more precise error here.
972
865
        if basis_inv is None:
973
866
            if basis_revision_id == _mod_revision.NULL_REVISION:
974
867
                new_inv = self._create_inv_from_null(delta, new_revision_id)
975
 
                if new_inv.root_id is None:
976
 
                    raise errors.RootMissing()
977
868
                inv_lines = new_inv.to_lines()
978
869
                return self._inventory_add_lines(new_revision_id, parents,
979
870
                    inv_lines, check_content=False), new_inv
991
882
            if basis_tree is not None:
992
883
                basis_tree.unlock()
993
884
 
994
 
    def _deserialise_inventory(self, revision_id, bytes):
 
885
    def deserialise_inventory(self, revision_id, bytes):
995
886
        return inventory.CHKInventory.deserialise(self.chk_bytes, bytes,
996
887
            (revision_id,))
997
888
 
1013
904
    def _iter_inventory_xmls(self, revision_ids, ordering):
1014
905
        # Without a native 'xml' inventory, this method doesn't make sense.
1015
906
        # However older working trees, and older bundles want it - so we supply
1016
 
        # it allowing _get_inventory_xml to work. Bundles currently use the
 
907
        # it allowing get_inventory_xml to work. Bundles currently use the
1017
908
        # serializer directly; this also isn't ideal, but there isn't an xml
1018
909
        # iteration interface offered at all for repositories. We could make
1019
910
        # _iter_inventory_xmls be part of the contract, even if kept private.
1109
1000
        finally:
1110
1001
            pb.finished()
1111
1002
 
1112
 
    @needs_write_lock
1113
 
    def reconcile_canonicalize_chks(self):
1114
 
        """Reconcile this repository to make sure all CHKs are in canonical
1115
 
        form.
1116
 
        """
1117
 
        from bzrlib.reconcile import PackReconciler
1118
 
        reconciler = PackReconciler(self, thorough=True, canonicalize_chks=True)
1119
 
        reconciler.reconcile()
1120
 
        return reconciler
1121
 
 
1122
1003
    def _reconcile_pack(self, collection, packs, extension, revs, pb):
1123
1004
        packer = GCCHKReconcilePacker(collection, packs, extension)
1124
1005
        return packer.pack(pb)
1125
1006
 
1126
 
    def _canonicalize_chks_pack(self, collection, packs, extension, revs, pb):
1127
 
        packer = GCCHKCanonicalizingPacker(collection, packs, extension, revs)
1128
 
        return packer.pack(pb)
1129
 
 
1130
1007
    def _get_source(self, to_format):
1131
1008
        """Return a source for streaming from this repository."""
1132
1009
        if self._format._serializer == to_format._serializer:
1211
1088
                uninteresting_root_keys.add(inv.id_to_entry.key())
1212
1089
                uninteresting_pid_root_keys.add(
1213
1090
                    inv.parent_id_basename_to_file_id.key())
 
1091
        bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
1214
1092
        chk_bytes = self.from_repository.chk_bytes
1215
1093
        def _filter_id_to_entry():
1216
1094
            interesting_nodes = chk_map.iter_interesting_nodes(chk_bytes,
1217
1095
                        self._chk_id_roots, uninteresting_root_keys)
1218
1096
            for record in _filter_text_keys(interesting_nodes, self._text_keys,
1219
 
                    chk_map._bytes_to_text_key):
 
1097
                    bytes_to_info):
1220
1098
                if record is not None:
1221
1099
                    yield record
1222
1100
            # Consumed
1232
1110
        yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
1233
1111
 
1234
1112
    def get_stream(self, search):
1235
 
        def wrap_and_count(pb, rc, stream):
1236
 
            """Yield records from stream while showing progress."""
1237
 
            count = 0
1238
 
            for record in stream:
1239
 
                if count == rc.STEP:
1240
 
                    rc.increment(count)
1241
 
                    pb.update('Estimate', rc.current, rc.max)
1242
 
                    count = 0
1243
 
                count += 1
1244
 
                yield record
1245
 
 
1246
1113
        revision_ids = search.get_keys()
1247
 
        pb = ui.ui_factory.nested_progress_bar()
1248
 
        rc = self._record_counter
1249
 
        self._record_counter.setup(len(revision_ids))
1250
1114
        for stream_info in self._fetch_revision_texts(revision_ids):
1251
 
            yield (stream_info[0],
1252
 
                wrap_and_count(pb, rc, stream_info[1]))
 
1115
            yield stream_info
1253
1116
        self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1254
1117
        self.from_repository.revisions.clear_cache()
1255
1118
        self.from_repository.signatures.clear_cache()
1256
 
        s = self._get_inventory_stream(self._revision_keys)
1257
 
        yield (s[0], wrap_and_count(pb, rc, s[1]))
 
1119
        yield self._get_inventory_stream(self._revision_keys)
1258
1120
        self.from_repository.inventories.clear_cache()
1259
1121
        # TODO: The keys to exclude might be part of the search recipe
1260
1122
        # For now, exclude all parents that are at the edge of ancestry, for
1263
1125
        parent_keys = from_repo._find_parent_keys_of_revisions(
1264
1126
                        self._revision_keys)
1265
1127
        for stream_info in self._get_filtered_chk_streams(parent_keys):
1266
 
            yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))
 
1128
            yield stream_info
1267
1129
        self.from_repository.chk_bytes.clear_cache()
1268
 
        s = self._get_text_stream()
1269
 
        yield (s[0], wrap_and_count(pb, rc, s[1]))
 
1130
        yield self._get_text_stream()
1270
1131
        self.from_repository.texts.clear_cache()
1271
 
        pb.update('Done', rc.max, rc.max)
1272
 
        pb.finished()
1273
1132
 
1274
1133
    def get_stream_for_missing_keys(self, missing_keys):
1275
1134
        # missing keys can only occur when we are byte copying and not
1329
1188
    return result
1330
1189
 
1331
1190
 
1332
 
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_text_key):
 
1191
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_info):
1333
1192
    """Iterate the result of iter_interesting_nodes, yielding the records
1334
1193
    and adding to text_keys.
1335
1194
    """
1336
 
    text_keys_update = text_keys.update
1337
1195
    for record, items in interesting_nodes_iterable:
1338
 
        text_keys_update([bytes_to_text_key(b) for n,b in items])
 
1196
        for name, bytes in items:
 
1197
            # Note: we don't care about name_utf8, because groupcompress repos
 
1198
            # are always rich-root, so there are no synthesised root records to
 
1199
            # ignore.
 
1200
            _, file_id, revision_id = bytes_to_info(bytes)
 
1201
            file_id = intern(file_id)
 
1202
            revision_id = intern(revision_id)
 
1203
            text_keys.add(StaticTuple(file_id, revision_id).intern())
1339
1204
        yield record
1340
1205
 
1341
1206