352
354
"""Build a VersionedFiles instance on top of this group of packs."""
353
355
index_name = index_name + '_index'
354
356
index_to_pack = {}
355
access = knit._DirectPackAccess(index_to_pack,
356
reload_func=self._reload_func)
357
access = _DirectPackAccess(index_to_pack,
358
reload_func=self._reload_func)
359
361
if self.new_pack is None:
421
423
inventory_keys = source_vf.keys()
422
424
missing_inventories = set(self.revision_keys).difference(inventory_keys)
423
425
if missing_inventories:
424
missing_inventories = sorted(missing_inventories)
425
raise ValueError('We are missing inventories for revisions: %s'
426
% (missing_inventories,))
426
# Go back to the original repo, to see if these are really missing
427
# https://bugs.launchpad.net/bzr/+bug/437003
428
# If we are packing a subset of the repo, it is fine to just have
429
# the data in another Pack file, which is not included in this pack
431
inv_index = self._pack_collection.repo.inventories._index
432
pmap = inv_index.get_parent_map(missing_inventories)
433
really_missing = missing_inventories.difference(pmap)
435
missing_inventories = sorted(really_missing)
436
raise ValueError('We are missing inventories for revisions: %s'
437
% (missing_inventories,))
427
438
self._copy_stream(source_vf, target_vf, inventory_keys,
428
439
'inventories', self._get_filtered_inv_stream, 2)
594
605
def __init__(self, *args, **kwargs):
595
606
super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
596
607
self._data_changed = False
598
609
def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
599
610
"""Create and exhaust a stream, but don't insert it.
601
612
This is useful to get the side-effects of generating a stream.
603
614
self.pb.update('scanning %s' % (message,), pb_offset)
779
792
% (sorted(missing_text_keys),))
782
def _execute_pack_operations(self, pack_operations,
783
_packer_class=GCCHKPacker,
785
"""Execute a series of pack operations.
787
:param pack_operations: A list of [revision_count, packs_to_combine].
788
:param _packer_class: The class of packer to use (default: Packer).
791
# XXX: Copied across from RepositoryPackCollection simply because we
792
# want to override the _packer_class ... :(
793
for revision_count, packs in pack_operations:
794
# we may have no-ops from the setup logic
797
packer = GCCHKPacker(self, packs, '.autopack',
798
reload_func=reload_func)
800
result = packer.pack()
801
except errors.RetryWithNewPacks:
802
# An exception is propagating out of this context, make sure
803
# this packer has cleaned up. Packer() doesn't set its new_pack
804
# state into the RepositoryPackCollection object, so we only
805
# have access to it directly here.
806
if packer.new_pack is not None:
807
packer.new_pack.abort()
812
self._remove_pack_from_memory(pack)
813
# record the newly available packs and stop advertising the old
816
for _, packs in pack_operations:
817
to_be_obsoleted.extend(packs)
818
result = self._save_pack_names(clear_obsolete_packs=True,
819
obsolete_packs=to_be_obsoleted)
823
class CHKInventoryRepository(KnitPackRepository):
824
"""subclass of KnitPackRepository that uses CHK based inventories."""
796
class CHKInventoryRepository(PackRepository):
797
"""subclass of PackRepository that uses CHK based inventories."""
826
799
def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
828
801
"""Overridden to change pack collection class."""
829
KnitPackRepository.__init__(self, _format, a_bzrdir, control_files,
830
_commit_builder_class, _serializer)
831
# and now replace everything it did :)
802
super(CHKInventoryRepository, self).__init__(_format, a_bzrdir,
803
control_files, _commit_builder_class, _serializer)
832
804
index_transport = self._transport.clone('indices')
833
805
self._pack_collection = GCRepositoryPackCollection(self,
834
806
self._transport, index_transport,
1137
1109
return GroupCHKStreamSource(self, to_format)
1138
1110
return super(CHKInventoryRepository, self)._get_source(to_format)
1141
class GroupCHKStreamSource(KnitPackStreamSource):
1112
def _find_inconsistent_revision_parents(self, revisions_iterator=None):
1113
"""Find revisions with different parent lists in the revision object
1114
and in the index graph.
1116
:param revisions_iterator: None, or an iterator of (revid,
1117
Revision-or-None). This iterator controls the revisions checked.
1118
:returns: an iterator yielding tuples of (revison-id, parents-in-index,
1119
parents-in-revision).
1121
if not self.is_locked():
1122
raise AssertionError()
1124
if revisions_iterator is None:
1125
revisions_iterator = self._iter_revisions(None)
1126
for revid, revision in revisions_iterator:
1127
if revision is None:
1129
parent_map = vf.get_parent_map([(revid,)])
1130
parents_according_to_index = tuple(parent[-1] for parent in
1131
parent_map[(revid,)])
1132
parents_according_to_revision = tuple(revision.parent_ids)
1133
if parents_according_to_index != parents_according_to_revision:
1134
yield (revid, parents_according_to_index,
1135
parents_according_to_revision)
1137
def _check_for_inconsistent_revision_parents(self):
1138
inconsistencies = list(self._find_inconsistent_revision_parents())
1140
raise errors.BzrCheckError(
1141
"Revision index has inconsistent parents.")
1144
class GroupCHKStreamSource(StreamSource):
1142
1145
"""Used when both the source and target repo are GroupCHK repos."""
1144
1147
def __init__(self, from_repository, to_format):
1231
1234
self._chk_p_id_roots = None
1232
1235
yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
1237
def _get_text_stream(self):
1238
# Note: We know we don't have to handle adding root keys, because both
1239
# the source and target are the identical network name.
1240
text_stream = self.from_repository.texts.get_record_stream(
1241
self._text_keys, self._text_fetch_order, False)
1242
return ('texts', text_stream)
1234
1244
def get_stream(self, search):
1235
1245
def wrap_and_count(pb, rc, stream):
1236
1246
"""Yield records from stream while showing progress."""
1251
1261
yield (stream_info[0],
1252
1262
wrap_and_count(pb, rc, stream_info[1]))
1253
1263
self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1254
self.from_repository.revisions.clear_cache()
1255
self.from_repository.signatures.clear_cache()
1256
s = self._get_inventory_stream(self._revision_keys)
1257
yield (s[0], wrap_and_count(pb, rc, s[1]))
1258
self.from_repository.inventories.clear_cache()
1259
1264
# TODO: The keys to exclude might be part of the search recipe
1260
1265
# For now, exclude all parents that are at the edge of ancestry, for
1261
1266
# which we have inventories
1262
1267
from_repo = self.from_repository
1263
1268
parent_keys = from_repo._find_parent_keys_of_revisions(
1264
1269
self._revision_keys)
1270
self.from_repository.revisions.clear_cache()
1271
self.from_repository.signatures.clear_cache()
1272
# Clear the repo's get_parent_map cache too.
1273
self.from_repository._unstacked_provider.disable_cache()
1274
self.from_repository._unstacked_provider.enable_cache()
1275
s = self._get_inventory_stream(self._revision_keys)
1276
yield (s[0], wrap_and_count(pb, rc, s[1]))
1277
self.from_repository.inventories.clear_cache()
1265
1278
for stream_info in self._get_filtered_chk_streams(parent_keys):
1266
1279
yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))
1267
1280
self.from_repository.chk_bytes.clear_cache()
1344
class RepositoryFormatCHK1(RepositoryFormatPack):
1345
"""A hashed CHK+group compress pack repository."""
1355
class RepositoryFormat2a(RepositoryFormatPack):
1356
"""A CHK repository that uses the bencode revision serializer."""
1347
1358
repository_class = CHKInventoryRepository
1348
1359
supports_external_lookups = True
1349
1360
supports_chks = True
1350
# For right now, setting this to True gives us InterModel1And2 rather
1351
# than InterDifferingSerializer
1352
1361
_commit_builder_class = PackRootCommitBuilder
1353
1362
rich_root_data = True
1354
_serializer = chk_serializer.chk_serializer_255_bigpage
1363
_serializer = chk_serializer.chk_bencode_serializer
1355
1364
_commit_inv_deltas = True
1356
1365
# What index classes to use
1357
1366
index_builder_class = BTreeBuilder
1368
1377
pack_compresses = True
1370
1379
def _get_matching_bzrdir(self):
1371
return bzrdir.format_registry.make_bzrdir('development6-rich-root')
1373
def _ignore_setting_bzrdir(self, format):
1376
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1378
def get_format_string(self):
1379
"""See RepositoryFormat.get_format_string()."""
1380
return ('Bazaar development format - group compression and chk inventory'
1381
' (needs bzr.dev from 1.14)\n')
1383
def get_format_description(self):
1384
"""See RepositoryFormat.get_format_description()."""
1385
return ("Development repository format - rich roots, group compression"
1386
" and chk inventories")
1389
class RepositoryFormatCHK2(RepositoryFormatCHK1):
1390
"""A CHK repository that uses the bencode revision serializer."""
1392
_serializer = chk_serializer.chk_bencode_serializer
1394
def _get_matching_bzrdir(self):
1395
return bzrdir.format_registry.make_bzrdir('development7-rich-root')
1397
def _ignore_setting_bzrdir(self, format):
1400
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1402
def get_format_string(self):
1403
"""See RepositoryFormat.get_format_string()."""
1404
return ('Bazaar development format - chk repository with bencode '
1405
'revision serialization (needs bzr.dev from 1.16)\n')
1408
class RepositoryFormat2a(RepositoryFormatCHK2):
1409
"""A CHK repository that uses the bencode revision serializer.
1411
This is the same as RepositoryFormatCHK2 but with a public name.
1414
_serializer = chk_serializer.chk_bencode_serializer
1416
def _get_matching_bzrdir(self):
1417
1380
return bzrdir.format_registry.make_bzrdir('2a')
1419
1382
def _ignore_setting_bzrdir(self, format):
1428
1391
"""See RepositoryFormat.get_format_description()."""
1429
1392
return ("Repository format 2a - rich roots, group compression"
1430
1393
" and chk inventories")
1396
class RepositoryFormat2aSubtree(RepositoryFormat2a):
1397
"""A 2a repository format that supports nested trees.
1401
def _get_matching_bzrdir(self):
1402
return bzrdir.format_registry.make_bzrdir('development-subtree')
1404
def _ignore_setting_bzrdir(self, format):
1407
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1409
def get_format_string(self):
1410
return ('Bazaar development format 8\n')
1412
def get_format_description(self):
1413
"""See RepositoryFormat.get_format_description()."""
1414
return ("Development repository format 8 - nested trees, "
1415
"group compression and chk inventories")
1418
supports_tree_reference = True
added
removed
bzrlib/repofmt/groupcompress_repo.py
