← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/groupcompress_repo.py

  • Committer: Jelmer Vernooij
  • Date: 2011-12-16 16:40:10 UTC
  • mto: This revision was merged to the branch mainline in revision 6391.
  • Revision ID: jelmer@samba.org-20111216164010-z3hy00xrnclnkf7a
Update tests.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/groupcompress_repo.py
1
 
# Copyright (C) 2008, 2009, 2010 Canonical Ltd
 
1
# Copyright (C) 2008-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
26
26
    errors,
27
27
    index as _mod_index,
28
28
    inventory,
29
 
    knit,
30
29
    osutils,
31
30
    pack,
32
31
    revision as _mod_revision,
44
43
    GroupCompressVersionedFiles,
45
44
    )
46
45
from bzrlib.repofmt.pack_repo import (
 
46
    _DirectPackAccess,
47
47
    Pack,
48
48
    NewPack,
49
 
    KnitPackRepository,
50
 
    KnitPackStreamSource,
 
49
    PackRepository,
51
50
    PackRootCommitBuilder,
52
51
    RepositoryPackCollection,
53
52
    RepositoryFormatPack,
54
53
    ResumedPack,
55
54
    Packer,
56
55
    )
 
56
from bzrlib.vf_repository import (
 
57
    StreamSource,
 
58
    )
57
59
from bzrlib.static_tuple import StaticTuple
58
60
 
59
61
 
352
354
        """Build a VersionedFiles instance on top of this group of packs."""
353
355
        index_name = index_name + '_index'
354
356
        index_to_pack = {}
355
 
        access = knit._DirectPackAccess(index_to_pack,
356
 
                                        reload_func=self._reload_func)
 
357
        access = _DirectPackAccess(index_to_pack,
 
358
                                   reload_func=self._reload_func)
357
359
        if for_write:
358
360
            # Use new_pack
359
361
            if self.new_pack is None:
421
423
        inventory_keys = source_vf.keys()
422
424
        missing_inventories = set(self.revision_keys).difference(inventory_keys)
423
425
        if missing_inventories:
424
 
            missing_inventories = sorted(missing_inventories)
425
 
            raise ValueError('We are missing inventories for revisions: %s'
426
 
                % (missing_inventories,))
 
426
            # Go back to the original repo, to see if these are really missing
 
427
            # https://bugs.launchpad.net/bzr/+bug/437003
 
428
            # If we are packing a subset of the repo, it is fine to just have
 
429
            # the data in another Pack file, which is not included in this pack
 
430
            # operation.
 
431
            inv_index = self._pack_collection.repo.inventories._index
 
432
            pmap = inv_index.get_parent_map(missing_inventories)
 
433
            really_missing = missing_inventories.difference(pmap)
 
434
            if really_missing:
 
435
                missing_inventories = sorted(really_missing)
 
436
                raise ValueError('We are missing inventories for revisions: %s'
 
437
                    % (missing_inventories,))
427
438
        self._copy_stream(source_vf, target_vf, inventory_keys,
428
439
                          'inventories', self._get_filtered_inv_stream, 2)
429
440
 
594
605
    def __init__(self, *args, **kwargs):
595
606
        super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
596
607
        self._data_changed = False
597
 
    
 
608
 
598
609
    def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
599
610
        """Create and exhaust a stream, but don't insert it.
600
 
        
 
611
 
601
612
        This is useful to get the side-effects of generating a stream.
602
613
        """
603
614
        self.pb.update('scanning %s' % (message,), pb_offset)
692
703
 
693
704
    pack_factory = GCPack
694
705
    resumed_pack_factory = ResumedGCPack
 
706
    normal_packer_class = GCCHKPacker
 
707
    optimising_packer_class = GCCHKPacker
695
708
 
696
709
    def _check_new_inventories(self):
697
710
        """Detect missing inventories or chk root entries for the new revisions
779
792
                % (sorted(missing_text_keys),))
780
793
        return problems
781
794
 
782
 
    def _execute_pack_operations(self, pack_operations,
783
 
                                 _packer_class=GCCHKPacker,
784
 
                                 reload_func=None):
785
 
        """Execute a series of pack operations.
786
 
 
787
 
        :param pack_operations: A list of [revision_count, packs_to_combine].
788
 
        :param _packer_class: The class of packer to use (default: Packer).
789
 
        :return: None.
790
 
        """
791
 
        # XXX: Copied across from RepositoryPackCollection simply because we
792
 
        #      want to override the _packer_class ... :(
793
 
        for revision_count, packs in pack_operations:
794
 
            # we may have no-ops from the setup logic
795
 
            if len(packs) == 0:
796
 
                continue
797
 
            packer = GCCHKPacker(self, packs, '.autopack',
798
 
                                 reload_func=reload_func)
799
 
            try:
800
 
                result = packer.pack()
801
 
            except errors.RetryWithNewPacks:
802
 
                # An exception is propagating out of this context, make sure
803
 
                # this packer has cleaned up. Packer() doesn't set its new_pack
804
 
                # state into the RepositoryPackCollection object, so we only
805
 
                # have access to it directly here.
806
 
                if packer.new_pack is not None:
807
 
                    packer.new_pack.abort()
808
 
                raise
809
 
            if result is None:
810
 
                return
811
 
            for pack in packs:
812
 
                self._remove_pack_from_memory(pack)
813
 
        # record the newly available packs and stop advertising the old
814
 
        # packs
815
 
        to_be_obsoleted = []
816
 
        for _, packs in pack_operations:
817
 
            to_be_obsoleted.extend(packs)
818
 
        result = self._save_pack_names(clear_obsolete_packs=True,
819
 
                                       obsolete_packs=to_be_obsoleted)
820
 
        return result
821
 
 
822
 
 
823
 
class CHKInventoryRepository(KnitPackRepository):
824
 
    """subclass of KnitPackRepository that uses CHK based inventories."""
 
795
 
 
796
class CHKInventoryRepository(PackRepository):
 
797
    """subclass of PackRepository that uses CHK based inventories."""
825
798
 
826
799
    def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
827
800
        _serializer):
828
801
        """Overridden to change pack collection class."""
829
 
        KnitPackRepository.__init__(self, _format, a_bzrdir, control_files,
830
 
            _commit_builder_class, _serializer)
831
 
        # and now replace everything it did :)
 
802
        super(CHKInventoryRepository, self).__init__(_format, a_bzrdir,
 
803
            control_files, _commit_builder_class, _serializer)
832
804
        index_transport = self._transport.clone('indices')
833
805
        self._pack_collection = GCRepositoryPackCollection(self,
834
806
            self._transport, index_transport,
1006
978
            if record.storage_kind != 'absent':
1007
979
                texts[record.key] = record.get_bytes_as('fulltext')
1008
980
            else:
1009
 
                raise errors.NoSuchRevision(self, record.key)
 
981
                texts[record.key] = None
1010
982
        for key in keys:
1011
 
            yield inventory.CHKInventory.deserialise(self.chk_bytes, texts[key], key)
 
983
            bytes = texts[key]
 
984
            if bytes is None:
 
985
                yield (None, key[-1])
 
986
            else:
 
987
                yield (inventory.CHKInventory.deserialise(
 
988
                    self.chk_bytes, bytes, key), key[-1])
1012
989
 
1013
 
    def _iter_inventory_xmls(self, revision_ids, ordering):
 
990
    def _get_inventory_xml(self, revision_id):
 
991
        """Get serialized inventory as a string."""
1014
992
        # Without a native 'xml' inventory, this method doesn't make sense.
1015
993
        # However older working trees, and older bundles want it - so we supply
1016
994
        # it allowing _get_inventory_xml to work. Bundles currently use the
1017
995
        # serializer directly; this also isn't ideal, but there isn't an xml
1018
 
        # iteration interface offered at all for repositories. We could make
1019
 
        # _iter_inventory_xmls be part of the contract, even if kept private.
1020
 
        inv_to_str = self._serializer.write_inventory_to_string
1021
 
        for inv in self.iter_inventories(revision_ids, ordering=ordering):
1022
 
            yield inv_to_str(inv), inv.revision_id
 
996
        # iteration interface offered at all for repositories.
 
997
        return self._serializer.write_inventory_to_string(
 
998
            self.get_inventory(revision_id))
1023
999
 
1024
1000
    def _find_present_inventory_keys(self, revision_keys):
1025
1001
        parent_map = self.inventories.get_parent_map(revision_keys)
1137
1113
            return GroupCHKStreamSource(self, to_format)
1138
1114
        return super(CHKInventoryRepository, self)._get_source(to_format)
1139
1115
 
1140
 
 
1141
 
class GroupCHKStreamSource(KnitPackStreamSource):
 
1116
    def _find_inconsistent_revision_parents(self, revisions_iterator=None):
 
1117
        """Find revisions with different parent lists in the revision object
 
1118
        and in the index graph.
 
1119
 
 
1120
        :param revisions_iterator: None, or an iterator of (revid,
 
1121
            Revision-or-None). This iterator controls the revisions checked.
 
1122
        :returns: an iterator yielding tuples of (revison-id, parents-in-index,
 
1123
            parents-in-revision).
 
1124
        """
 
1125
        if not self.is_locked():
 
1126
            raise AssertionError()
 
1127
        vf = self.revisions
 
1128
        if revisions_iterator is None:
 
1129
            revisions_iterator = self._iter_revisions(None)
 
1130
        for revid, revision in revisions_iterator:
 
1131
            if revision is None:
 
1132
                pass
 
1133
            parent_map = vf.get_parent_map([(revid,)])
 
1134
            parents_according_to_index = tuple(parent[-1] for parent in
 
1135
                parent_map[(revid,)])
 
1136
            parents_according_to_revision = tuple(revision.parent_ids)
 
1137
            if parents_according_to_index != parents_according_to_revision:
 
1138
                yield (revid, parents_according_to_index,
 
1139
                    parents_according_to_revision)
 
1140
 
 
1141
    def _check_for_inconsistent_revision_parents(self):
 
1142
        inconsistencies = list(self._find_inconsistent_revision_parents())
 
1143
        if inconsistencies:
 
1144
            raise errors.BzrCheckError(
 
1145
                "Revision index has inconsistent parents.")
 
1146
 
 
1147
 
 
1148
class GroupCHKStreamSource(StreamSource):
1142
1149
    """Used when both the source and target repo are GroupCHK repos."""
1143
1150
 
1144
1151
    def __init__(self, from_repository, to_format):
1231
1238
            self._chk_p_id_roots = None
1232
1239
        yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
1233
1240
 
 
1241
    def _get_text_stream(self):
 
1242
        # Note: We know we don't have to handle adding root keys, because both
 
1243
        # the source and target are the identical network name.
 
1244
        text_stream = self.from_repository.texts.get_record_stream(
 
1245
                        self._text_keys, self._text_fetch_order, False)
 
1246
        return ('texts', text_stream)
 
1247
 
1234
1248
    def get_stream(self, search):
1235
1249
        def wrap_and_count(pb, rc, stream):
1236
1250
            """Yield records from stream while showing progress."""
1251
1265
            yield (stream_info[0],
1252
1266
                wrap_and_count(pb, rc, stream_info[1]))
1253
1267
        self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1254
 
        self.from_repository.revisions.clear_cache()
1255
 
        self.from_repository.signatures.clear_cache()
1256
 
        s = self._get_inventory_stream(self._revision_keys)
1257
 
        yield (s[0], wrap_and_count(pb, rc, s[1]))
1258
 
        self.from_repository.inventories.clear_cache()
1259
1268
        # TODO: The keys to exclude might be part of the search recipe
1260
1269
        # For now, exclude all parents that are at the edge of ancestry, for
1261
1270
        # which we have inventories
1262
1271
        from_repo = self.from_repository
1263
1272
        parent_keys = from_repo._find_parent_keys_of_revisions(
1264
1273
                        self._revision_keys)
 
1274
        self.from_repository.revisions.clear_cache()
 
1275
        self.from_repository.signatures.clear_cache()
 
1276
        # Clear the repo's get_parent_map cache too.
 
1277
        self.from_repository._unstacked_provider.disable_cache()
 
1278
        self.from_repository._unstacked_provider.enable_cache()
 
1279
        s = self._get_inventory_stream(self._revision_keys)
 
1280
        yield (s[0], wrap_and_count(pb, rc, s[1]))
 
1281
        self.from_repository.inventories.clear_cache()
1265
1282
        for stream_info in self._get_filtered_chk_streams(parent_keys):
1266
1283
            yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))
1267
1284
        self.from_repository.chk_bytes.clear_cache()
1339
1356
        yield record
1340
1357
 
1341
1358
 
1342
 
 
1343
 
 
1344
 
class RepositoryFormatCHK1(RepositoryFormatPack):
1345
 
    """A hashed CHK+group compress pack repository."""
 
1359
class RepositoryFormat2a(RepositoryFormatPack):
 
1360
    """A CHK repository that uses the bencode revision serializer."""
1346
1361
 
1347
1362
    repository_class = CHKInventoryRepository
1348
1363
    supports_external_lookups = True
1349
1364
    supports_chks = True
1350
 
    # For right now, setting this to True gives us InterModel1And2 rather
1351
 
    # than InterDifferingSerializer
1352
1365
    _commit_builder_class = PackRootCommitBuilder
1353
1366
    rich_root_data = True
1354
 
    _serializer = chk_serializer.chk_serializer_255_bigpage
 
1367
    _serializer = chk_serializer.chk_bencode_serializer
1355
1368
    _commit_inv_deltas = True
1356
1369
    # What index classes to use
1357
1370
    index_builder_class = BTreeBuilder
1368
1381
    pack_compresses = True
1369
1382
 
1370
1383
    def _get_matching_bzrdir(self):
1371
 
        return bzrdir.format_registry.make_bzrdir('development6-rich-root')
1372
 
 
1373
 
    def _ignore_setting_bzrdir(self, format):
1374
 
        pass
1375
 
 
1376
 
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1377
 
 
1378
 
    def get_format_string(self):
1379
 
        """See RepositoryFormat.get_format_string()."""
1380
 
        return ('Bazaar development format - group compression and chk inventory'
1381
 
                ' (needs bzr.dev from 1.14)\n')
1382
 
 
1383
 
    def get_format_description(self):
1384
 
        """See RepositoryFormat.get_format_description()."""
1385
 
        return ("Development repository format - rich roots, group compression"
1386
 
            " and chk inventories")
1387
 
 
1388
 
 
1389
 
class RepositoryFormatCHK2(RepositoryFormatCHK1):
1390
 
    """A CHK repository that uses the bencode revision serializer."""
1391
 
 
1392
 
    _serializer = chk_serializer.chk_bencode_serializer
1393
 
 
1394
 
    def _get_matching_bzrdir(self):
1395
 
        return bzrdir.format_registry.make_bzrdir('development7-rich-root')
1396
 
 
1397
 
    def _ignore_setting_bzrdir(self, format):
1398
 
        pass
1399
 
 
1400
 
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1401
 
 
1402
 
    def get_format_string(self):
1403
 
        """See RepositoryFormat.get_format_string()."""
1404
 
        return ('Bazaar development format - chk repository with bencode '
1405
 
                'revision serialization (needs bzr.dev from 1.16)\n')
1406
 
 
1407
 
 
1408
 
class RepositoryFormat2a(RepositoryFormatCHK2):
1409
 
    """A CHK repository that uses the bencode revision serializer.
1410
 
 
1411
 
    This is the same as RepositoryFormatCHK2 but with a public name.
1412
 
    """
1413
 
 
1414
 
    _serializer = chk_serializer.chk_bencode_serializer
1415
 
 
1416
 
    def _get_matching_bzrdir(self):
1417
1384
        return bzrdir.format_registry.make_bzrdir('2a')
1418
1385
 
1419
1386
    def _ignore_setting_bzrdir(self, format):
1421
1388
 
1422
1389
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1423
1390
 
1424
 
    def get_format_string(self):
 
1391
    @classmethod
 
1392
    def get_format_string(cls):
1425
1393
        return ('Bazaar repository format 2a (needs bzr 1.16 or later)\n')
1426
1394
 
1427
1395
    def get_format_description(self):
1443
1411
 
1444
1412
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1445
1413
 
1446
 
    def get_format_string(self):
 
1414
    @classmethod
 
1415
    def get_format_string(cls):
1447
1416
        return ('Bazaar development format 8\n')
1448
1417
 
1449
1418
    def get_format_description(self):