423
422
inventory_keys = source_vf.keys()
424
423
missing_inventories = set(self.revision_keys).difference(inventory_keys)
425
424
if missing_inventories:
426
# Go back to the original repo, to see if these are really missing
427
# https://bugs.launchpad.net/bzr/+bug/437003
428
# If we are packing a subset of the repo, it is fine to just have
429
# the data in another Pack file, which is not included in this pack
431
inv_index = self._pack_collection.repo.inventories._index
432
pmap = inv_index.get_parent_map(missing_inventories)
433
really_missing = missing_inventories.difference(pmap)
435
missing_inventories = sorted(really_missing)
436
raise ValueError('We are missing inventories for revisions: %s'
437
% (missing_inventories,))
425
missing_inventories = sorted(missing_inventories)
426
raise ValueError('We are missing inventories for revisions: %s'
427
% (missing_inventories,))
438
428
self._copy_stream(source_vf, target_vf, inventory_keys,
439
429
'inventories', self._get_filtered_inv_stream, 2)
441
def _get_chk_vfs_for_copy(self):
442
return self._build_vfs('chk', False, False)
444
431
def _copy_chk_texts(self):
445
source_vf, target_vf = self._get_chk_vfs_for_copy()
432
source_vf, target_vf = self._build_vfs('chk', False, False)
446
433
# TODO: This is technically spurious... if it is a performance issue,
448
435
total_keys = source_vf.keys()
594
581
return new_pack.data_inserted() and self._data_changed
597
class GCCHKCanonicalizingPacker(GCCHKPacker):
598
"""A packer that ensures inventories have canonical-form CHK maps.
600
Ideally this would be part of reconcile, but it's very slow and rarely
601
needed. (It repairs repositories affected by
602
https://bugs.launchpad.net/bzr/+bug/522637).
605
def __init__(self, *args, **kwargs):
606
super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
607
self._data_changed = False
609
def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
610
"""Create and exhaust a stream, but don't insert it.
612
This is useful to get the side-effects of generating a stream.
614
self.pb.update('scanning %s' % (message,), pb_offset)
615
child_pb = ui.ui_factory.nested_progress_bar()
617
list(vf_to_stream(source_vf, keys, message, child_pb))
621
def _copy_inventory_texts(self):
622
source_vf, target_vf = self._build_vfs('inventory', True, True)
623
source_chk_vf, target_chk_vf = self._get_chk_vfs_for_copy()
624
inventory_keys = source_vf.keys()
625
# First, copy the existing CHKs on the assumption that most of them
626
# will be correct. This will save us from having to reinsert (and
627
# recompress) these records later at the cost of perhaps preserving a
629
# (Iterate but don't insert _get_filtered_inv_stream to populate the
630
# variables needed by GCCHKPacker._copy_chk_texts.)
631
self._exhaust_stream(source_vf, inventory_keys, 'inventories',
632
self._get_filtered_inv_stream, 2)
633
GCCHKPacker._copy_chk_texts(self)
634
# Now copy and fix the inventories, and any regenerated CHKs.
635
def chk_canonicalizing_inv_stream(source_vf, keys, message, pb=None):
636
return self._get_filtered_canonicalizing_inv_stream(
637
source_vf, keys, message, pb, source_chk_vf, target_chk_vf)
638
self._copy_stream(source_vf, target_vf, inventory_keys,
639
'inventories', chk_canonicalizing_inv_stream, 4)
641
def _copy_chk_texts(self):
642
# No-op; in this class this happens during _copy_inventory_texts.
645
def _get_filtered_canonicalizing_inv_stream(self, source_vf, keys, message,
646
pb=None, source_chk_vf=None, target_chk_vf=None):
647
"""Filter the texts of inventories, regenerating CHKs to make sure they
650
total_keys = len(keys)
651
target_chk_vf = versionedfile.NoDupeAddLinesDecorator(target_chk_vf)
652
def _filtered_inv_stream():
653
stream = source_vf.get_record_stream(keys, 'groupcompress', True)
654
search_key_name = None
655
for idx, record in enumerate(stream):
656
# Inventories should always be with revisions; assume success.
657
bytes = record.get_bytes_as('fulltext')
658
chk_inv = inventory.CHKInventory.deserialise(
659
source_chk_vf, bytes, record.key)
661
pb.update('inv', idx, total_keys)
662
chk_inv.id_to_entry._ensure_root()
663
if search_key_name is None:
664
# Find the name corresponding to the search_key_func
665
search_key_reg = chk_map.search_key_registry
666
for search_key_name, func in search_key_reg.iteritems():
667
if func == chk_inv.id_to_entry._search_key_func:
669
canonical_inv = inventory.CHKInventory.from_inventory(
670
target_chk_vf, chk_inv,
671
maximum_size=chk_inv.id_to_entry._root_node._maximum_size,
672
search_key_name=search_key_name)
673
if chk_inv.id_to_entry.key() != canonical_inv.id_to_entry.key():
675
'Non-canonical CHK map for id_to_entry of inv: %s '
676
'(root is %s, should be %s)' % (chk_inv.revision_id,
677
chk_inv.id_to_entry.key()[0],
678
canonical_inv.id_to_entry.key()[0]))
679
self._data_changed = True
680
p_id_map = chk_inv.parent_id_basename_to_file_id
681
p_id_map._ensure_root()
682
canon_p_id_map = canonical_inv.parent_id_basename_to_file_id
683
if p_id_map.key() != canon_p_id_map.key():
685
'Non-canonical CHK map for parent_id_to_basename of '
686
'inv: %s (root is %s, should be %s)'
687
% (chk_inv.revision_id, p_id_map.key()[0],
688
canon_p_id_map.key()[0]))
689
self._data_changed = True
690
yield versionedfile.ChunkedContentFactory(record.key,
691
record.parents, record.sha1,
692
canonical_inv.to_lines())
693
# We have finished processing all of the inventory records, we
694
# don't need these sets anymore
695
return _filtered_inv_stream()
697
def _use_pack(self, new_pack):
698
"""Override _use_pack to check for reconcile having changed content."""
699
return new_pack.data_inserted() and self._data_changed
702
584
class GCRepositoryPackCollection(RepositoryPackCollection):
704
586
pack_factory = GCPack
705
587
resumed_pack_factory = ResumedGCPack
706
normal_packer_class = GCCHKPacker
707
optimising_packer_class = GCCHKPacker
709
589
def _check_new_inventories(self):
710
590
"""Detect missing inventories or chk root entries for the new revisions
792
672
% (sorted(missing_text_keys),))
796
class CHKInventoryRepository(PackRepository):
797
"""subclass of PackRepository that uses CHK based inventories."""
675
def _execute_pack_operations(self, pack_operations,
676
_packer_class=GCCHKPacker,
678
"""Execute a series of pack operations.
680
:param pack_operations: A list of [revision_count, packs_to_combine].
681
:param _packer_class: The class of packer to use (default: Packer).
684
# XXX: Copied across from RepositoryPackCollection simply because we
685
# want to override the _packer_class ... :(
686
for revision_count, packs in pack_operations:
687
# we may have no-ops from the setup logic
690
packer = GCCHKPacker(self, packs, '.autopack',
691
reload_func=reload_func)
693
result = packer.pack()
694
except errors.RetryWithNewPacks:
695
# An exception is propagating out of this context, make sure
696
# this packer has cleaned up. Packer() doesn't set its new_pack
697
# state into the RepositoryPackCollection object, so we only
698
# have access to it directly here.
699
if packer.new_pack is not None:
700
packer.new_pack.abort()
705
self._remove_pack_from_memory(pack)
706
# record the newly available packs and stop advertising the old
708
result = self._save_pack_names(clear_obsolete_packs=True)
709
# Move the old packs out of the way now they are no longer referenced.
710
for revision_count, packs in pack_operations:
711
self._obsolete_packs(packs)
715
class CHKInventoryRepository(KnitPackRepository):
716
"""subclass of KnitPackRepository that uses CHK based inventories."""
799
718
def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
801
720
"""Overridden to change pack collection class."""
802
super(CHKInventoryRepository, self).__init__(_format, a_bzrdir,
803
control_files, _commit_builder_class, _serializer)
721
KnitPackRepository.__init__(self, _format, a_bzrdir, control_files,
722
_commit_builder_class, _serializer)
723
# and now replace everything it did :)
804
724
index_transport = self._transport.clone('indices')
805
725
self._pack_collection = GCRepositoryPackCollection(self,
806
726
self._transport, index_transport,
1109
1013
return GroupCHKStreamSource(self, to_format)
1110
1014
return super(CHKInventoryRepository, self)._get_source(to_format)
1112
def _find_inconsistent_revision_parents(self, revisions_iterator=None):
1113
"""Find revisions with different parent lists in the revision object
1114
and in the index graph.
1116
:param revisions_iterator: None, or an iterator of (revid,
1117
Revision-or-None). This iterator controls the revisions checked.
1118
:returns: an iterator yielding tuples of (revison-id, parents-in-index,
1119
parents-in-revision).
1121
if not self.is_locked():
1122
raise AssertionError()
1124
if revisions_iterator is None:
1125
revisions_iterator = self._iter_revisions(None)
1126
for revid, revision in revisions_iterator:
1127
if revision is None:
1129
parent_map = vf.get_parent_map([(revid,)])
1130
parents_according_to_index = tuple(parent[-1] for parent in
1131
parent_map[(revid,)])
1132
parents_according_to_revision = tuple(revision.parent_ids)
1133
if parents_according_to_index != parents_according_to_revision:
1134
yield (revid, parents_according_to_index,
1135
parents_according_to_revision)
1137
def _check_for_inconsistent_revision_parents(self):
1138
inconsistencies = list(self._find_inconsistent_revision_parents())
1140
raise errors.BzrCheckError(
1141
"Revision index has inconsistent parents.")
1144
class GroupCHKStreamSource(StreamSource):
1017
class GroupCHKStreamSource(KnitPackStreamSource):
1145
1018
"""Used when both the source and target repo are GroupCHK repos."""
1147
1020
def __init__(self, from_repository, to_format):
1234
1108
self._chk_p_id_roots = None
1235
1109
yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
1237
def _get_text_stream(self):
1238
# Note: We know we don't have to handle adding root keys, because both
1239
# the source and target are the identical network name.
1240
text_stream = self.from_repository.texts.get_record_stream(
1241
self._text_keys, self._text_fetch_order, False)
1242
return ('texts', text_stream)
1244
1111
def get_stream(self, search):
1245
def wrap_and_count(pb, rc, stream):
1246
"""Yield records from stream while showing progress."""
1248
for record in stream:
1249
if count == rc.STEP:
1251
pb.update('Estimate', rc.current, rc.max)
1256
1112
revision_ids = search.get_keys()
1257
pb = ui.ui_factory.nested_progress_bar()
1258
rc = self._record_counter
1259
self._record_counter.setup(len(revision_ids))
1260
1113
for stream_info in self._fetch_revision_texts(revision_ids):
1261
yield (stream_info[0],
1262
wrap_and_count(pb, rc, stream_info[1]))
1263
1115
self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1116
self.from_repository.revisions.clear_cache()
1117
self.from_repository.signatures.clear_cache()
1118
yield self._get_inventory_stream(self._revision_keys)
1119
self.from_repository.inventories.clear_cache()
1264
1120
# TODO: The keys to exclude might be part of the search recipe
1265
1121
# For now, exclude all parents that are at the edge of ancestry, for
1266
1122
# which we have inventories
1267
1123
from_repo = self.from_repository
1268
1124
parent_keys = from_repo._find_parent_keys_of_revisions(
1269
1125
self._revision_keys)
1270
self.from_repository.revisions.clear_cache()
1271
self.from_repository.signatures.clear_cache()
1272
# Clear the repo's get_parent_map cache too.
1273
self.from_repository._unstacked_provider.disable_cache()
1274
self.from_repository._unstacked_provider.enable_cache()
1275
s = self._get_inventory_stream(self._revision_keys)
1276
yield (s[0], wrap_and_count(pb, rc, s[1]))
1277
self.from_repository.inventories.clear_cache()
1278
1126
for stream_info in self._get_filtered_chk_streams(parent_keys):
1279
yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))
1280
1128
self.from_repository.chk_bytes.clear_cache()
1281
s = self._get_text_stream()
1282
yield (s[0], wrap_and_count(pb, rc, s[1]))
1129
yield self._get_text_stream()
1283
1130
self.from_repository.texts.clear_cache()
1284
pb.update('Done', rc.max, rc.max)
1287
1132
def get_stream_for_missing_keys(self, missing_keys):
1288
1133
# missing keys can only occur when we are byte copying and not
1345
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_text_key):
1190
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_info):
1346
1191
"""Iterate the result of iter_interesting_nodes, yielding the records
1347
1192
and adding to text_keys.
1349
text_keys_update = text_keys.update
1350
1194
for record, items in interesting_nodes_iterable:
1351
text_keys_update([bytes_to_text_key(b) for n,b in items])
1195
for name, bytes in items:
1196
# Note: we don't care about name_utf8, because groupcompress repos
1197
# are always rich-root, so there are no synthesised root records to
1199
_, file_id, revision_id = bytes_to_info(bytes)
1200
file_id = intern(file_id)
1201
revision_id = intern(revision_id)
1202
text_keys.add(StaticTuple(file_id, revision_id).intern())
1355
class RepositoryFormat2a(RepositoryFormatPack):
1356
"""A CHK repository that uses the bencode revision serializer."""
1208
class RepositoryFormatCHK1(RepositoryFormatPack):
1209
"""A hashed CHK+group compress pack repository."""
1358
1211
repository_class = CHKInventoryRepository
1359
1212
supports_external_lookups = True
1360
1213
supports_chks = True
1214
# For right now, setting this to True gives us InterModel1And2 rather
1215
# than InterDifferingSerializer
1361
1216
_commit_builder_class = PackRootCommitBuilder
1362
1217
rich_root_data = True
1363
_serializer = chk_serializer.chk_bencode_serializer
1218
_serializer = chk_serializer.chk_serializer_255_bigpage
1364
1219
_commit_inv_deltas = True
1365
1220
# What index classes to use
1366
1221
index_builder_class = BTreeBuilder
1377
1232
pack_compresses = True
1379
1234
def _get_matching_bzrdir(self):
1235
return bzrdir.format_registry.make_bzrdir('development6-rich-root')
1237
def _ignore_setting_bzrdir(self, format):
1240
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1242
def get_format_string(self):
1243
"""See RepositoryFormat.get_format_string()."""
1244
return ('Bazaar development format - group compression and chk inventory'
1245
' (needs bzr.dev from 1.14)\n')
1247
def get_format_description(self):
1248
"""See RepositoryFormat.get_format_description()."""
1249
return ("Development repository format - rich roots, group compression"
1250
" and chk inventories")
1253
class RepositoryFormatCHK2(RepositoryFormatCHK1):
1254
"""A CHK repository that uses the bencode revision serializer."""
1256
_serializer = chk_serializer.chk_bencode_serializer
1258
def _get_matching_bzrdir(self):
1259
return bzrdir.format_registry.make_bzrdir('development7-rich-root')
1261
def _ignore_setting_bzrdir(self, format):
1264
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1266
def get_format_string(self):
1267
"""See RepositoryFormat.get_format_string()."""
1268
return ('Bazaar development format - chk repository with bencode '
1269
'revision serialization (needs bzr.dev from 1.16)\n')
1272
class RepositoryFormat2a(RepositoryFormatCHK2):
1273
"""A CHK repository that uses the bencode revision serializer.
1275
This is the same as RepositoryFormatCHK2 but with a public name.
1278
_serializer = chk_serializer.chk_bencode_serializer
1280
def _get_matching_bzrdir(self):
1380
1281
return bzrdir.format_registry.make_bzrdir('2a')
1382
1283
def _ignore_setting_bzrdir(self, format):
added
removed
bzrlib/repofmt/groupcompress_repo.py
