← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

  • Committer: Vincent Ladeuil
  • Date: 2016-02-01 16:52:49 UTC
  • mto: (6614.2.1 trunk)
  • mto: This revision was merged to the branch mainline in revision 6615.
  • Revision ID: v.ladeuil+lp@free.fr-20160201165249-pjd165c0e37whpb4
Fix assertNotEquals being deprecated by using assertNotEqual.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
1
 
# Copyright (C) 2008, 2009, 2010 Canonical Ltd
 
1
# Copyright (C) 2008-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
16
16
 
17
17
"""Repository formats using CHK inventories and groupcompress compression."""
18
18
 
 
19
from __future__ import absolute_import
 
20
 
19
21
import time
20
22
 
21
23
from bzrlib import (
22
 
    bzrdir,
 
24
    controldir,
23
25
    chk_map,
24
26
    chk_serializer,
25
27
    debug,
26
28
    errors,
27
29
    index as _mod_index,
28
30
    inventory,
29
 
    knit,
30
31
    osutils,
31
32
    pack,
32
33
    revision as _mod_revision,
44
45
    GroupCompressVersionedFiles,
45
46
    )
46
47
from bzrlib.repofmt.pack_repo import (
 
48
    _DirectPackAccess,
47
49
    Pack,
48
50
    NewPack,
49
 
    KnitPackRepository,
50
 
    KnitPackStreamSource,
 
51
    PackRepository,
51
52
    PackRootCommitBuilder,
52
53
    RepositoryPackCollection,
53
54
    RepositoryFormatPack,
54
55
    ResumedPack,
55
56
    Packer,
56
57
    )
 
58
from bzrlib.vf_repository import (
 
59
    StreamSource,
 
60
    )
57
61
from bzrlib.static_tuple import StaticTuple
58
62
 
59
63
 
352
356
        """Build a VersionedFiles instance on top of this group of packs."""
353
357
        index_name = index_name + '_index'
354
358
        index_to_pack = {}
355
 
        access = knit._DirectPackAccess(index_to_pack,
356
 
                                        reload_func=self._reload_func)
 
359
        access = _DirectPackAccess(index_to_pack,
 
360
                                   reload_func=self._reload_func)
357
361
        if for_write:
358
362
            # Use new_pack
359
363
            if self.new_pack is None:
421
425
        inventory_keys = source_vf.keys()
422
426
        missing_inventories = set(self.revision_keys).difference(inventory_keys)
423
427
        if missing_inventories:
424
 
            missing_inventories = sorted(missing_inventories)
425
 
            raise ValueError('We are missing inventories for revisions: %s'
426
 
                % (missing_inventories,))
 
428
            # Go back to the original repo, to see if these are really missing
 
429
            # https://bugs.launchpad.net/bzr/+bug/437003
 
430
            # If we are packing a subset of the repo, it is fine to just have
 
431
            # the data in another Pack file, which is not included in this pack
 
432
            # operation.
 
433
            inv_index = self._pack_collection.repo.inventories._index
 
434
            pmap = inv_index.get_parent_map(missing_inventories)
 
435
            really_missing = missing_inventories.difference(pmap)
 
436
            if really_missing:
 
437
                missing_inventories = sorted(really_missing)
 
438
                raise ValueError('We are missing inventories for revisions: %s'
 
439
                    % (missing_inventories,))
427
440
        self._copy_stream(source_vf, target_vf, inventory_keys,
428
441
                          'inventories', self._get_filtered_inv_stream, 2)
429
442
 
594
607
    def __init__(self, *args, **kwargs):
595
608
        super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
596
609
        self._data_changed = False
597
 
    
 
610
 
598
611
    def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
599
612
        """Create and exhaust a stream, but don't insert it.
600
 
        
 
613
 
601
614
        This is useful to get the side-effects of generating a stream.
602
615
        """
603
616
        self.pb.update('scanning %s' % (message,), pb_offset)
692
705
 
693
706
    pack_factory = GCPack
694
707
    resumed_pack_factory = ResumedGCPack
 
708
    normal_packer_class = GCCHKPacker
 
709
    optimising_packer_class = GCCHKPacker
695
710
 
696
711
    def _check_new_inventories(self):
697
712
        """Detect missing inventories or chk root entries for the new revisions
743
758
            expected_chk_roots)
744
759
        missing_chk_roots = expected_chk_roots.difference(present_chk_roots)
745
760
        if missing_chk_roots:
746
 
            problems.append("missing referenced chk root keys: %s"
 
761
            problems.append(
 
762
                "missing referenced chk root keys: %s."
 
763
                "Run 'bzr reconcile --canonicalize-chks' on the affected "
 
764
                "repository."
747
765
                % (sorted(missing_chk_roots),))
748
766
            # Don't bother checking any further.
749
767
            return problems
779
797
                % (sorted(missing_text_keys),))
780
798
        return problems
781
799
 
782
 
    def _execute_pack_operations(self, pack_operations,
783
 
                                 _packer_class=GCCHKPacker,
784
 
                                 reload_func=None):
785
 
        """Execute a series of pack operations.
786
 
 
787
 
        :param pack_operations: A list of [revision_count, packs_to_combine].
788
 
        :param _packer_class: The class of packer to use (default: Packer).
789
 
        :return: None.
790
 
        """
791
 
        # XXX: Copied across from RepositoryPackCollection simply because we
792
 
        #      want to override the _packer_class ... :(
793
 
        for revision_count, packs in pack_operations:
794
 
            # we may have no-ops from the setup logic
795
 
            if len(packs) == 0:
796
 
                continue
797
 
            packer = GCCHKPacker(self, packs, '.autopack',
798
 
                                 reload_func=reload_func)
799
 
            try:
800
 
                result = packer.pack()
801
 
            except errors.RetryWithNewPacks:
802
 
                # An exception is propagating out of this context, make sure
803
 
                # this packer has cleaned up. Packer() doesn't set its new_pack
804
 
                # state into the RepositoryPackCollection object, so we only
805
 
                # have access to it directly here.
806
 
                if packer.new_pack is not None:
807
 
                    packer.new_pack.abort()
808
 
                raise
809
 
            if result is None:
810
 
                return
811
 
            for pack in packs:
812
 
                self._remove_pack_from_memory(pack)
813
 
        # record the newly available packs and stop advertising the old
814
 
        # packs
815
 
        to_be_obsoleted = []
816
 
        for _, packs in pack_operations:
817
 
            to_be_obsoleted.extend(packs)
818
 
        result = self._save_pack_names(clear_obsolete_packs=True,
819
 
                                       obsolete_packs=to_be_obsoleted)
820
 
        return result
821
 
 
822
 
 
823
 
class CHKInventoryRepository(KnitPackRepository):
824
 
    """subclass of KnitPackRepository that uses CHK based inventories."""
 
800
 
 
801
class CHKInventoryRepository(PackRepository):
 
802
    """subclass of PackRepository that uses CHK based inventories."""
825
803
 
826
804
    def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
827
805
        _serializer):
828
806
        """Overridden to change pack collection class."""
829
 
        KnitPackRepository.__init__(self, _format, a_bzrdir, control_files,
830
 
            _commit_builder_class, _serializer)
831
 
        # and now replace everything it did :)
 
807
        super(CHKInventoryRepository, self).__init__(_format, a_bzrdir,
 
808
            control_files, _commit_builder_class, _serializer)
832
809
        index_transport = self._transport.clone('indices')
833
810
        self._pack_collection = GCRepositoryPackCollection(self,
834
811
            self._transport, index_transport,
980
957
            else:
981
958
                basis_tree = self.revision_tree(basis_revision_id)
982
959
                basis_tree.lock_read()
983
 
                basis_inv = basis_tree.inventory
 
960
                basis_inv = basis_tree.root_inventory
984
961
        try:
985
962
            result = basis_inv.create_by_apply_delta(delta, new_revision_id,
986
963
                propagate_caches=propagate_caches)
1006
983
            if record.storage_kind != 'absent':
1007
984
                texts[record.key] = record.get_bytes_as('fulltext')
1008
985
            else:
1009
 
                raise errors.NoSuchRevision(self, record.key)
 
986
                texts[record.key] = None
1010
987
        for key in keys:
1011
 
            yield inventory.CHKInventory.deserialise(self.chk_bytes, texts[key], key)
 
988
            bytes = texts[key]
 
989
            if bytes is None:
 
990
                yield (None, key[-1])
 
991
            else:
 
992
                yield (inventory.CHKInventory.deserialise(
 
993
                    self.chk_bytes, bytes, key), key[-1])
1012
994
 
1013
 
    def _iter_inventory_xmls(self, revision_ids, ordering):
 
995
    def _get_inventory_xml(self, revision_id):
 
996
        """Get serialized inventory as a string."""
1014
997
        # Without a native 'xml' inventory, this method doesn't make sense.
1015
998
        # However older working trees, and older bundles want it - so we supply
1016
999
        # it allowing _get_inventory_xml to work. Bundles currently use the
1017
1000
        # serializer directly; this also isn't ideal, but there isn't an xml
1018
 
        # iteration interface offered at all for repositories. We could make
1019
 
        # _iter_inventory_xmls be part of the contract, even if kept private.
1020
 
        inv_to_str = self._serializer.write_inventory_to_string
1021
 
        for inv in self.iter_inventories(revision_ids, ordering=ordering):
1022
 
            yield inv_to_str(inv), inv.revision_id
 
1001
        # iteration interface offered at all for repositories.
 
1002
        return self._serializer.write_inventory_to_string(
 
1003
            self.get_inventory(revision_id))
1023
1004
 
1024
1005
    def _find_present_inventory_keys(self, revision_keys):
1025
1006
        parent_map = self.inventories.get_parent_map(revision_keys)
1137
1118
            return GroupCHKStreamSource(self, to_format)
1138
1119
        return super(CHKInventoryRepository, self)._get_source(to_format)
1139
1120
 
1140
 
 
1141
 
class GroupCHKStreamSource(KnitPackStreamSource):
 
1121
    def _find_inconsistent_revision_parents(self, revisions_iterator=None):
 
1122
        """Find revisions with different parent lists in the revision object
 
1123
        and in the index graph.
 
1124
 
 
1125
        :param revisions_iterator: None, or an iterator of (revid,
 
1126
            Revision-or-None). This iterator controls the revisions checked.
 
1127
        :returns: an iterator yielding tuples of (revison-id, parents-in-index,
 
1128
            parents-in-revision).
 
1129
        """
 
1130
        if not self.is_locked():
 
1131
            raise AssertionError()
 
1132
        vf = self.revisions
 
1133
        if revisions_iterator is None:
 
1134
            revisions_iterator = self._iter_revisions(None)
 
1135
        for revid, revision in revisions_iterator:
 
1136
            if revision is None:
 
1137
                pass
 
1138
            parent_map = vf.get_parent_map([(revid,)])
 
1139
            parents_according_to_index = tuple(parent[-1] for parent in
 
1140
                parent_map[(revid,)])
 
1141
            parents_according_to_revision = tuple(revision.parent_ids)
 
1142
            if parents_according_to_index != parents_according_to_revision:
 
1143
                yield (revid, parents_according_to_index,
 
1144
                    parents_according_to_revision)
 
1145
 
 
1146
    def _check_for_inconsistent_revision_parents(self):
 
1147
        inconsistencies = list(self._find_inconsistent_revision_parents())
 
1148
        if inconsistencies:
 
1149
            raise errors.BzrCheckError(
 
1150
                "Revision index has inconsistent parents.")
 
1151
 
 
1152
 
 
1153
class GroupCHKStreamSource(StreamSource):
1142
1154
    """Used when both the source and target repo are GroupCHK repos."""
1143
1155
 
1144
1156
    def __init__(self, from_repository, to_format):
1231
1243
            self._chk_p_id_roots = None
1232
1244
        yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
1233
1245
 
 
1246
    def _get_text_stream(self):
 
1247
        # Note: We know we don't have to handle adding root keys, because both
 
1248
        # the source and target are the identical network name.
 
1249
        text_stream = self.from_repository.texts.get_record_stream(
 
1250
                        self._text_keys, self._text_fetch_order, False)
 
1251
        return ('texts', text_stream)
 
1252
 
1234
1253
    def get_stream(self, search):
1235
1254
        def wrap_and_count(pb, rc, stream):
1236
1255
            """Yield records from stream while showing progress."""
1251
1270
            yield (stream_info[0],
1252
1271
                wrap_and_count(pb, rc, stream_info[1]))
1253
1272
        self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1254
 
        self.from_repository.revisions.clear_cache()
1255
 
        self.from_repository.signatures.clear_cache()
1256
 
        s = self._get_inventory_stream(self._revision_keys)
1257
 
        yield (s[0], wrap_and_count(pb, rc, s[1]))
1258
 
        self.from_repository.inventories.clear_cache()
1259
1273
        # TODO: The keys to exclude might be part of the search recipe
1260
1274
        # For now, exclude all parents that are at the edge of ancestry, for
1261
1275
        # which we have inventories
1262
1276
        from_repo = self.from_repository
1263
1277
        parent_keys = from_repo._find_parent_keys_of_revisions(
1264
1278
                        self._revision_keys)
 
1279
        self.from_repository.revisions.clear_cache()
 
1280
        self.from_repository.signatures.clear_cache()
 
1281
        # Clear the repo's get_parent_map cache too.
 
1282
        self.from_repository._unstacked_provider.disable_cache()
 
1283
        self.from_repository._unstacked_provider.enable_cache()
 
1284
        s = self._get_inventory_stream(self._revision_keys)
 
1285
        yield (s[0], wrap_and_count(pb, rc, s[1]))
 
1286
        self.from_repository.inventories.clear_cache()
1265
1287
        for stream_info in self._get_filtered_chk_streams(parent_keys):
1266
1288
            yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))
1267
1289
        self.from_repository.chk_bytes.clear_cache()
1339
1361
        yield record
1340
1362
 
1341
1363
 
1342
 
 
1343
 
 
1344
 
class RepositoryFormatCHK1(RepositoryFormatPack):
1345
 
    """A hashed CHK+group compress pack repository."""
 
1364
class RepositoryFormat2a(RepositoryFormatPack):
 
1365
    """A CHK repository that uses the bencode revision serializer."""
1346
1366
 
1347
1367
    repository_class = CHKInventoryRepository
1348
1368
    supports_external_lookups = True
1349
1369
    supports_chks = True
1350
 
    # For right now, setting this to True gives us InterModel1And2 rather
1351
 
    # than InterDifferingSerializer
1352
1370
    _commit_builder_class = PackRootCommitBuilder
1353
1371
    rich_root_data = True
1354
 
    _serializer = chk_serializer.chk_serializer_255_bigpage
 
1372
    _serializer = chk_serializer.chk_bencode_serializer
1355
1373
    _commit_inv_deltas = True
1356
1374
    # What index classes to use
1357
1375
    index_builder_class = BTreeBuilder
1368
1386
    pack_compresses = True
1369
1387
 
1370
1388
    def _get_matching_bzrdir(self):
1371
 
        return bzrdir.format_registry.make_bzrdir('development6-rich-root')
1372
 
 
1373
 
    def _ignore_setting_bzrdir(self, format):
1374
 
        pass
1375
 
 
1376
 
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1377
 
 
1378
 
    def get_format_string(self):
1379
 
        """See RepositoryFormat.get_format_string()."""
1380
 
        return ('Bazaar development format - group compression and chk inventory'
1381
 
                ' (needs bzr.dev from 1.14)\n')
1382
 
 
1383
 
    def get_format_description(self):
1384
 
        """See RepositoryFormat.get_format_description()."""
1385
 
        return ("Development repository format - rich roots, group compression"
1386
 
            " and chk inventories")
1387
 
 
1388
 
 
1389
 
class RepositoryFormatCHK2(RepositoryFormatCHK1):
1390
 
    """A CHK repository that uses the bencode revision serializer."""
1391
 
 
1392
 
    _serializer = chk_serializer.chk_bencode_serializer
1393
 
 
1394
 
    def _get_matching_bzrdir(self):
1395
 
        return bzrdir.format_registry.make_bzrdir('development7-rich-root')
1396
 
 
1397
 
    def _ignore_setting_bzrdir(self, format):
1398
 
        pass
1399
 
 
1400
 
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1401
 
 
1402
 
    def get_format_string(self):
1403
 
        """See RepositoryFormat.get_format_string()."""
1404
 
        return ('Bazaar development format - chk repository with bencode '
1405
 
                'revision serialization (needs bzr.dev from 1.16)\n')
1406
 
 
1407
 
 
1408
 
class RepositoryFormat2a(RepositoryFormatCHK2):
1409
 
    """A CHK repository that uses the bencode revision serializer.
1410
 
 
1411
 
    This is the same as RepositoryFormatCHK2 but with a public name.
1412
 
    """
1413
 
 
1414
 
    _serializer = chk_serializer.chk_bencode_serializer
1415
 
 
1416
 
    def _get_matching_bzrdir(self):
1417
 
        return bzrdir.format_registry.make_bzrdir('2a')
1418
 
 
1419
 
    def _ignore_setting_bzrdir(self, format):
1420
 
        pass
1421
 
 
1422
 
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1423
 
 
1424
 
    def get_format_string(self):
 
1389
        return controldir.format_registry.make_bzrdir('2a')
 
1390
 
 
1391
    def _ignore_setting_bzrdir(self, format):
 
1392
        pass
 
1393
 
 
1394
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
 
1395
 
 
1396
    @classmethod
 
1397
    def get_format_string(cls):
1425
1398
        return ('Bazaar repository format 2a (needs bzr 1.16 or later)\n')
1426
1399
 
1427
1400
    def get_format_description(self):
1436
1409
    """
1437
1410
 
1438
1411
    def _get_matching_bzrdir(self):
1439
 
        return bzrdir.format_registry.make_bzrdir('development-subtree')
 
1412
        return controldir.format_registry.make_bzrdir('development-subtree')
1440
1413
 
1441
1414
    def _ignore_setting_bzrdir(self, format):
1442
1415
        pass
1443
1416
 
1444
1417
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1445
1418
 
1446
 
    def get_format_string(self):
 
1419
    @classmethod
 
1420
    def get_format_string(cls):
1447
1421
        return ('Bazaar development format 8\n')
1448
1422
 
1449
1423
    def get_format_description(self):