← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2010-09-01 08:02:42 UTC
  • mfrom: (5390.3.3 faster-revert-593560)
  • Revision ID: pqm@pqm.ubuntu.com-20100901080242-esg62ody4frwmy66
(spiv) Avoid repeatedly calling self.target.all_file_ids() in
 InterTree.iter_changes. (Andrew Bennetts)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
1
 
# Copyright (C) 2008-2011 Canonical Ltd
 
1
# Copyright (C) 2008, 2009, 2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
16
16
 
17
17
"""Repository formats using CHK inventories and groupcompress compression."""
18
18
 
19
 
from __future__ import absolute_import
20
 
 
21
19
import time
22
20
 
23
21
from bzrlib import (
24
 
    controldir,
 
22
    bzrdir,
25
23
    chk_map,
26
24
    chk_serializer,
27
25
    debug,
28
26
    errors,
29
27
    index as _mod_index,
30
28
    inventory,
 
29
    knit,
31
30
    osutils,
32
31
    pack,
33
32
    revision as _mod_revision,
45
44
    GroupCompressVersionedFiles,
46
45
    )
47
46
from bzrlib.repofmt.pack_repo import (
48
 
    _DirectPackAccess,
49
47
    Pack,
50
48
    NewPack,
51
 
    PackRepository,
 
49
    KnitPackRepository,
 
50
    KnitPackStreamSource,
52
51
    PackRootCommitBuilder,
53
52
    RepositoryPackCollection,
54
53
    RepositoryFormatPack,
55
54
    ResumedPack,
56
55
    Packer,
57
56
    )
58
 
from bzrlib.vf_repository import (
59
 
    StreamSource,
60
 
    )
61
57
from bzrlib.static_tuple import StaticTuple
62
58
 
63
59
 
356
352
        """Build a VersionedFiles instance on top of this group of packs."""
357
353
        index_name = index_name + '_index'
358
354
        index_to_pack = {}
359
 
        access = _DirectPackAccess(index_to_pack,
360
 
                                   reload_func=self._reload_func)
 
355
        access = knit._DirectPackAccess(index_to_pack,
 
356
                                        reload_func=self._reload_func)
361
357
        if for_write:
362
358
            # Use new_pack
363
359
            if self.new_pack is None:
425
421
        inventory_keys = source_vf.keys()
426
422
        missing_inventories = set(self.revision_keys).difference(inventory_keys)
427
423
        if missing_inventories:
428
 
            # Go back to the original repo, to see if these are really missing
429
 
            # https://bugs.launchpad.net/bzr/+bug/437003
430
 
            # If we are packing a subset of the repo, it is fine to just have
431
 
            # the data in another Pack file, which is not included in this pack
432
 
            # operation.
433
 
            inv_index = self._pack_collection.repo.inventories._index
434
 
            pmap = inv_index.get_parent_map(missing_inventories)
435
 
            really_missing = missing_inventories.difference(pmap)
436
 
            if really_missing:
437
 
                missing_inventories = sorted(really_missing)
438
 
                raise ValueError('We are missing inventories for revisions: %s'
439
 
                    % (missing_inventories,))
 
424
            missing_inventories = sorted(missing_inventories)
 
425
            raise ValueError('We are missing inventories for revisions: %s'
 
426
                % (missing_inventories,))
440
427
        self._copy_stream(source_vf, target_vf, inventory_keys,
441
428
                          'inventories', self._get_filtered_inv_stream, 2)
442
429
 
607
594
    def __init__(self, *args, **kwargs):
608
595
        super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
609
596
        self._data_changed = False
610
 
 
 
597
    
611
598
    def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
612
599
        """Create and exhaust a stream, but don't insert it.
613
 
 
 
600
        
614
601
        This is useful to get the side-effects of generating a stream.
615
602
        """
616
603
        self.pb.update('scanning %s' % (message,), pb_offset)
705
692
 
706
693
    pack_factory = GCPack
707
694
    resumed_pack_factory = ResumedGCPack
708
 
    normal_packer_class = GCCHKPacker
709
 
    optimising_packer_class = GCCHKPacker
710
695
 
711
696
    def _check_new_inventories(self):
712
697
        """Detect missing inventories or chk root entries for the new revisions
758
743
            expected_chk_roots)
759
744
        missing_chk_roots = expected_chk_roots.difference(present_chk_roots)
760
745
        if missing_chk_roots:
761
 
            problems.append(
762
 
                "missing referenced chk root keys: %s."
763
 
                "Run 'bzr reconcile --canonicalize-chks' on the affected "
764
 
                "repository."
 
746
            problems.append("missing referenced chk root keys: %s"
765
747
                % (sorted(missing_chk_roots),))
766
748
            # Don't bother checking any further.
767
749
            return problems
797
779
                % (sorted(missing_text_keys),))
798
780
        return problems
799
781
 
800
 
 
801
 
class CHKInventoryRepository(PackRepository):
802
 
    """subclass of PackRepository that uses CHK based inventories."""
 
782
    def _execute_pack_operations(self, pack_operations,
 
783
                                 _packer_class=GCCHKPacker,
 
784
                                 reload_func=None):
 
785
        """Execute a series of pack operations.
 
786
 
 
787
        :param pack_operations: A list of [revision_count, packs_to_combine].
 
788
        :param _packer_class: The class of packer to use (default: Packer).
 
789
        :return: None.
 
790
        """
 
791
        # XXX: Copied across from RepositoryPackCollection simply because we
 
792
        #      want to override the _packer_class ... :(
 
793
        for revision_count, packs in pack_operations:
 
794
            # we may have no-ops from the setup logic
 
795
            if len(packs) == 0:
 
796
                continue
 
797
            packer = GCCHKPacker(self, packs, '.autopack',
 
798
                                 reload_func=reload_func)
 
799
            try:
 
800
                result = packer.pack()
 
801
            except errors.RetryWithNewPacks:
 
802
                # An exception is propagating out of this context, make sure
 
803
                # this packer has cleaned up. Packer() doesn't set its new_pack
 
804
                # state into the RepositoryPackCollection object, so we only
 
805
                # have access to it directly here.
 
806
                if packer.new_pack is not None:
 
807
                    packer.new_pack.abort()
 
808
                raise
 
809
            if result is None:
 
810
                return
 
811
            for pack in packs:
 
812
                self._remove_pack_from_memory(pack)
 
813
        # record the newly available packs and stop advertising the old
 
814
        # packs
 
815
        to_be_obsoleted = []
 
816
        for _, packs in pack_operations:
 
817
            to_be_obsoleted.extend(packs)
 
818
        result = self._save_pack_names(clear_obsolete_packs=True,
 
819
                                       obsolete_packs=to_be_obsoleted)
 
820
        return result
 
821
 
 
822
 
 
823
class CHKInventoryRepository(KnitPackRepository):
 
824
    """subclass of KnitPackRepository that uses CHK based inventories."""
803
825
 
804
826
    def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
805
827
        _serializer):
806
828
        """Overridden to change pack collection class."""
807
 
        super(CHKInventoryRepository, self).__init__(_format, a_bzrdir,
808
 
            control_files, _commit_builder_class, _serializer)
 
829
        KnitPackRepository.__init__(self, _format, a_bzrdir, control_files,
 
830
            _commit_builder_class, _serializer)
 
831
        # and now replace everything it did :)
809
832
        index_transport = self._transport.clone('indices')
810
833
        self._pack_collection = GCRepositoryPackCollection(self,
811
834
            self._transport, index_transport,
957
980
            else:
958
981
                basis_tree = self.revision_tree(basis_revision_id)
959
982
                basis_tree.lock_read()
960
 
                basis_inv = basis_tree.root_inventory
 
983
                basis_inv = basis_tree.inventory
961
984
        try:
962
985
            result = basis_inv.create_by_apply_delta(delta, new_revision_id,
963
986
                propagate_caches=propagate_caches)
983
1006
            if record.storage_kind != 'absent':
984
1007
                texts[record.key] = record.get_bytes_as('fulltext')
985
1008
            else:
986
 
                texts[record.key] = None
 
1009
                raise errors.NoSuchRevision(self, record.key)
987
1010
        for key in keys:
988
 
            bytes = texts[key]
989
 
            if bytes is None:
990
 
                yield (None, key[-1])
991
 
            else:
992
 
                yield (inventory.CHKInventory.deserialise(
993
 
                    self.chk_bytes, bytes, key), key[-1])
 
1011
            yield inventory.CHKInventory.deserialise(self.chk_bytes, texts[key], key)
994
1012
 
995
 
    def _get_inventory_xml(self, revision_id):
996
 
        """Get serialized inventory as a string."""
 
1013
    def _iter_inventory_xmls(self, revision_ids, ordering):
997
1014
        # Without a native 'xml' inventory, this method doesn't make sense.
998
1015
        # However older working trees, and older bundles want it - so we supply
999
1016
        # it allowing _get_inventory_xml to work. Bundles currently use the
1000
1017
        # serializer directly; this also isn't ideal, but there isn't an xml
1001
 
        # iteration interface offered at all for repositories.
1002
 
        return self._serializer.write_inventory_to_string(
1003
 
            self.get_inventory(revision_id))
 
1018
        # iteration interface offered at all for repositories. We could make
 
1019
        # _iter_inventory_xmls be part of the contract, even if kept private.
 
1020
        inv_to_str = self._serializer.write_inventory_to_string
 
1021
        for inv in self.iter_inventories(revision_ids, ordering=ordering):
 
1022
            yield inv_to_str(inv), inv.revision_id
1004
1023
 
1005
1024
    def _find_present_inventory_keys(self, revision_keys):
1006
1025
        parent_map = self.inventories.get_parent_map(revision_keys)
1118
1137
            return GroupCHKStreamSource(self, to_format)
1119
1138
        return super(CHKInventoryRepository, self)._get_source(to_format)
1120
1139
 
1121
 
    def _find_inconsistent_revision_parents(self, revisions_iterator=None):
1122
 
        """Find revisions with different parent lists in the revision object
1123
 
        and in the index graph.
1124
 
 
1125
 
        :param revisions_iterator: None, or an iterator of (revid,
1126
 
            Revision-or-None). This iterator controls the revisions checked.
1127
 
        :returns: an iterator yielding tuples of (revison-id, parents-in-index,
1128
 
            parents-in-revision).
1129
 
        """
1130
 
        if not self.is_locked():
1131
 
            raise AssertionError()
1132
 
        vf = self.revisions
1133
 
        if revisions_iterator is None:
1134
 
            revisions_iterator = self._iter_revisions(None)
1135
 
        for revid, revision in revisions_iterator:
1136
 
            if revision is None:
1137
 
                pass
1138
 
            parent_map = vf.get_parent_map([(revid,)])
1139
 
            parents_according_to_index = tuple(parent[-1] for parent in
1140
 
                parent_map[(revid,)])
1141
 
            parents_according_to_revision = tuple(revision.parent_ids)
1142
 
            if parents_according_to_index != parents_according_to_revision:
1143
 
                yield (revid, parents_according_to_index,
1144
 
                    parents_according_to_revision)
1145
 
 
1146
 
    def _check_for_inconsistent_revision_parents(self):
1147
 
        inconsistencies = list(self._find_inconsistent_revision_parents())
1148
 
        if inconsistencies:
1149
 
            raise errors.BzrCheckError(
1150
 
                "Revision index has inconsistent parents.")
1151
 
 
1152
 
 
1153
 
class GroupCHKStreamSource(StreamSource):
 
1140
 
 
1141
class GroupCHKStreamSource(KnitPackStreamSource):
1154
1142
    """Used when both the source and target repo are GroupCHK repos."""
1155
1143
 
1156
1144
    def __init__(self, from_repository, to_format):
1243
1231
            self._chk_p_id_roots = None
1244
1232
        yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
1245
1233
 
1246
 
    def _get_text_stream(self):
1247
 
        # Note: We know we don't have to handle adding root keys, because both
1248
 
        # the source and target are the identical network name.
1249
 
        text_stream = self.from_repository.texts.get_record_stream(
1250
 
                        self._text_keys, self._text_fetch_order, False)
1251
 
        return ('texts', text_stream)
1252
 
 
1253
1234
    def get_stream(self, search):
1254
1235
        def wrap_and_count(pb, rc, stream):
1255
1236
            """Yield records from stream while showing progress."""
1270
1251
            yield (stream_info[0],
1271
1252
                wrap_and_count(pb, rc, stream_info[1]))
1272
1253
        self._revision_keys = [(rev_id,) for rev_id in revision_ids]
 
1254
        self.from_repository.revisions.clear_cache()
 
1255
        self.from_repository.signatures.clear_cache()
 
1256
        s = self._get_inventory_stream(self._revision_keys)
 
1257
        yield (s[0], wrap_and_count(pb, rc, s[1]))
 
1258
        self.from_repository.inventories.clear_cache()
1273
1259
        # TODO: The keys to exclude might be part of the search recipe
1274
1260
        # For now, exclude all parents that are at the edge of ancestry, for
1275
1261
        # which we have inventories
1276
1262
        from_repo = self.from_repository
1277
1263
        parent_keys = from_repo._find_parent_keys_of_revisions(
1278
1264
                        self._revision_keys)
1279
 
        self.from_repository.revisions.clear_cache()
1280
 
        self.from_repository.signatures.clear_cache()
1281
 
        # Clear the repo's get_parent_map cache too.
1282
 
        self.from_repository._unstacked_provider.disable_cache()
1283
 
        self.from_repository._unstacked_provider.enable_cache()
1284
 
        s = self._get_inventory_stream(self._revision_keys)
1285
 
        yield (s[0], wrap_and_count(pb, rc, s[1]))
1286
 
        self.from_repository.inventories.clear_cache()
1287
1265
        for stream_info in self._get_filtered_chk_streams(parent_keys):
1288
1266
            yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))
1289
1267
        self.from_repository.chk_bytes.clear_cache()
1361
1339
        yield record
1362
1340
 
1363
1341
 
1364
 
class RepositoryFormat2a(RepositoryFormatPack):
1365
 
    """A CHK repository that uses the bencode revision serializer."""
 
1342
 
 
1343
 
 
1344
class RepositoryFormatCHK1(RepositoryFormatPack):
 
1345
    """A hashed CHK+group compress pack repository."""
1366
1346
 
1367
1347
    repository_class = CHKInventoryRepository
1368
1348
    supports_external_lookups = True
1369
1349
    supports_chks = True
 
1350
    # For right now, setting this to True gives us InterModel1And2 rather
 
1351
    # than InterDifferingSerializer
1370
1352
    _commit_builder_class = PackRootCommitBuilder
1371
1353
    rich_root_data = True
1372
 
    _serializer = chk_serializer.chk_bencode_serializer
 
1354
    _serializer = chk_serializer.chk_serializer_255_bigpage
1373
1355
    _commit_inv_deltas = True
1374
1356
    # What index classes to use
1375
1357
    index_builder_class = BTreeBuilder
1386
1368
    pack_compresses = True
1387
1369
 
1388
1370
    def _get_matching_bzrdir(self):
1389
 
        return controldir.format_registry.make_bzrdir('2a')
1390
 
 
1391
 
    def _ignore_setting_bzrdir(self, format):
1392
 
        pass
1393
 
 
1394
 
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1395
 
 
1396
 
    @classmethod
1397
 
    def get_format_string(cls):
 
1371
        return bzrdir.format_registry.make_bzrdir('development6-rich-root')
 
1372
 
 
1373
    def _ignore_setting_bzrdir(self, format):
 
1374
        pass
 
1375
 
 
1376
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
 
1377
 
 
1378
    def get_format_string(self):
 
1379
        """See RepositoryFormat.get_format_string()."""
 
1380
        return ('Bazaar development format - group compression and chk inventory'
 
1381
                ' (needs bzr.dev from 1.14)\n')
 
1382
 
 
1383
    def get_format_description(self):
 
1384
        """See RepositoryFormat.get_format_description()."""
 
1385
        return ("Development repository format - rich roots, group compression"
 
1386
            " and chk inventories")
 
1387
 
 
1388
 
 
1389
class RepositoryFormatCHK2(RepositoryFormatCHK1):
 
1390
    """A CHK repository that uses the bencode revision serializer."""
 
1391
 
 
1392
    _serializer = chk_serializer.chk_bencode_serializer
 
1393
 
 
1394
    def _get_matching_bzrdir(self):
 
1395
        return bzrdir.format_registry.make_bzrdir('development7-rich-root')
 
1396
 
 
1397
    def _ignore_setting_bzrdir(self, format):
 
1398
        pass
 
1399
 
 
1400
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
 
1401
 
 
1402
    def get_format_string(self):
 
1403
        """See RepositoryFormat.get_format_string()."""
 
1404
        return ('Bazaar development format - chk repository with bencode '
 
1405
                'revision serialization (needs bzr.dev from 1.16)\n')
 
1406
 
 
1407
 
 
1408
class RepositoryFormat2a(RepositoryFormatCHK2):
 
1409
    """A CHK repository that uses the bencode revision serializer.
 
1410
 
 
1411
    This is the same as RepositoryFormatCHK2 but with a public name.
 
1412
    """
 
1413
 
 
1414
    _serializer = chk_serializer.chk_bencode_serializer
 
1415
 
 
1416
    def _get_matching_bzrdir(self):
 
1417
        return bzrdir.format_registry.make_bzrdir('2a')
 
1418
 
 
1419
    def _ignore_setting_bzrdir(self, format):
 
1420
        pass
 
1421
 
 
1422
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
 
1423
 
 
1424
    def get_format_string(self):
1398
1425
        return ('Bazaar repository format 2a (needs bzr 1.16 or later)\n')
1399
1426
 
1400
1427
    def get_format_description(self):
1401
1428
        """See RepositoryFormat.get_format_description()."""
1402
1429
        return ("Repository format 2a - rich roots, group compression"
1403
1430
            " and chk inventories")
1404
 
 
1405
 
 
1406
 
class RepositoryFormat2aSubtree(RepositoryFormat2a):
1407
 
    """A 2a repository format that supports nested trees.
1408
 
 
1409
 
    """
1410
 
 
1411
 
    def _get_matching_bzrdir(self):
1412
 
        return controldir.format_registry.make_bzrdir('development-subtree')
1413
 
 
1414
 
    def _ignore_setting_bzrdir(self, format):
1415
 
        pass
1416
 
 
1417
 
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1418
 
 
1419
 
    @classmethod
1420
 
    def get_format_string(cls):
1421
 
        return ('Bazaar development format 8\n')
1422
 
 
1423
 
    def get_format_description(self):
1424
 
        """See RepositoryFormat.get_format_description()."""
1425
 
        return ("Development repository format 8 - nested trees, "
1426
 
                "group compression and chk inventories")
1427
 
 
1428
 
    experimental = True
1429
 
    supports_tree_reference = True