583
580
return new_pack.data_inserted() and self._data_changed
586
class GCCHKCanonicalizingPacker(GCCHKPacker):
587
"""A packer that ensures inventories have canonical-form CHK maps.
589
Ideally this would be part of reconcile, but it's very slow and rarely
590
needed. (It repairs repositories affected by
591
https://bugs.launchpad.net/bzr/+bug/522637).
594
def __init__(self, *args, **kwargs):
595
super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
596
self._data_changed = False
598
def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
599
"""Create and exhaust a stream, but don't insert it.
601
This is useful to get the side-effects of generating a stream.
603
self.pb.update('scanning %s' % (message,), pb_offset)
604
child_pb = ui.ui_factory.nested_progress_bar()
606
list(vf_to_stream(source_vf, keys, message, child_pb))
610
def _copy_inventory_texts(self):
611
source_vf, target_vf = self._build_vfs('inventory', True, True)
612
source_chk_vf, target_chk_vf = self._get_chk_vfs_for_copy()
613
inventory_keys = source_vf.keys()
614
# First, copy the existing CHKs on the assumption that most of them
615
# will be correct. This will save us from having to reinsert (and
616
# recompress) these records later at the cost of perhaps preserving a
618
# (Iterate but don't insert _get_filtered_inv_stream to populate the
619
# variables needed by GCCHKPacker._copy_chk_texts.)
620
self._exhaust_stream(source_vf, inventory_keys, 'inventories',
621
self._get_filtered_inv_stream, 2)
622
GCCHKPacker._copy_chk_texts(self)
623
# Now copy and fix the inventories, and any regenerated CHKs.
624
def chk_canonicalizing_inv_stream(source_vf, keys, message, pb=None):
625
return self._get_filtered_canonicalizing_inv_stream(
626
source_vf, keys, message, pb, source_chk_vf, target_chk_vf)
627
self._copy_stream(source_vf, target_vf, inventory_keys,
628
'inventories', chk_canonicalizing_inv_stream, 4)
630
def _copy_chk_texts(self):
631
# No-op; in this class this happens during _copy_inventory_texts.
634
def _get_filtered_canonicalizing_inv_stream(self, source_vf, keys, message,
635
pb=None, source_chk_vf=None, target_chk_vf=None):
636
"""Filter the texts of inventories, regenerating CHKs to make sure they
639
total_keys = len(keys)
640
target_chk_vf = versionedfile.NoDupeAddLinesDecorator(target_chk_vf)
641
def _filtered_inv_stream():
642
stream = source_vf.get_record_stream(keys, 'groupcompress', True)
643
search_key_name = None
644
for idx, record in enumerate(stream):
645
# Inventories should always be with revisions; assume success.
646
bytes = record.get_bytes_as('fulltext')
647
chk_inv = inventory.CHKInventory.deserialise(
648
source_chk_vf, bytes, record.key)
650
pb.update('inv', idx, total_keys)
651
chk_inv.id_to_entry._ensure_root()
652
if search_key_name is None:
653
# Find the name corresponding to the search_key_func
654
search_key_reg = chk_map.search_key_registry
655
for search_key_name, func in search_key_reg.iteritems():
656
if func == chk_inv.id_to_entry._search_key_func:
658
canonical_inv = inventory.CHKInventory.from_inventory(
659
target_chk_vf, chk_inv,
660
maximum_size=chk_inv.id_to_entry._root_node._maximum_size,
661
search_key_name=search_key_name)
662
if chk_inv.id_to_entry.key() != canonical_inv.id_to_entry.key():
664
'Non-canonical CHK map for id_to_entry of inv: %s '
665
'(root is %s, should be %s)' % (chk_inv.revision_id,
666
chk_inv.id_to_entry.key()[0],
667
canonical_inv.id_to_entry.key()[0]))
668
self._data_changed = True
669
p_id_map = chk_inv.parent_id_basename_to_file_id
670
p_id_map._ensure_root()
671
canon_p_id_map = canonical_inv.parent_id_basename_to_file_id
672
if p_id_map.key() != canon_p_id_map.key():
674
'Non-canonical CHK map for parent_id_to_basename of '
675
'inv: %s (root is %s, should be %s)'
676
% (chk_inv.revision_id, p_id_map.key()[0],
677
canon_p_id_map.key()[0]))
678
self._data_changed = True
679
yield versionedfile.ChunkedContentFactory(record.key,
680
record.parents, record.sha1,
681
canonical_inv.to_lines())
682
# We have finished processing all of the inventory records, we
683
# don't need these sets anymore
684
return _filtered_inv_stream()
686
def _use_pack(self, new_pack):
687
"""Override _use_pack to check for reconcile having changed content."""
688
return new_pack.data_inserted() and self._data_changed
691
583
class GCRepositoryPackCollection(RepositoryPackCollection):
693
585
pack_factory = GCPack
1113
def reconcile_canonicalize_chks(self):
1114
"""Reconcile this repository to make sure all CHKs are in canonical
1117
from bzrlib.reconcile import PackReconciler
1118
reconciler = PackReconciler(self, thorough=True, canonicalize_chks=True)
1119
reconciler.reconcile()
1122
1002
def _reconcile_pack(self, collection, packs, extension, revs, pb):
1123
1003
packer = GCCHKReconcilePacker(collection, packs, extension)
1124
1004
return packer.pack(pb)
1126
def _canonicalize_chks_pack(self, collection, packs, extension, revs, pb):
1127
packer = GCCHKCanonicalizingPacker(collection, packs, extension, revs)
1128
return packer.pack(pb)
1130
1006
def _get_source(self, to_format):
1131
1007
"""Return a source for streaming from this repository."""
1132
1008
if self._format._serializer == to_format._serializer:
1232
1109
yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
1234
1111
def get_stream(self, search):
1235
def wrap_and_count(pb, rc, stream):
1236
"""Yield records from stream while showing progress."""
1238
for record in stream:
1239
if count == rc.STEP:
1241
pb.update('Estimate', rc.current, rc.max)
1246
1112
revision_ids = search.get_keys()
1247
pb = ui.ui_factory.nested_progress_bar()
1248
rc = self._record_counter
1249
self._record_counter.setup(len(revision_ids))
1250
1113
for stream_info in self._fetch_revision_texts(revision_ids):
1251
yield (stream_info[0],
1252
wrap_and_count(pb, rc, stream_info[1]))
1253
1115
self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1254
1116
self.from_repository.revisions.clear_cache()
1255
1117
self.from_repository.signatures.clear_cache()
1256
s = self._get_inventory_stream(self._revision_keys)
1257
yield (s[0], wrap_and_count(pb, rc, s[1]))
1118
yield self._get_inventory_stream(self._revision_keys)
1258
1119
self.from_repository.inventories.clear_cache()
1259
1120
# TODO: The keys to exclude might be part of the search recipe
1260
1121
# For now, exclude all parents that are at the edge of ancestry, for
1332
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_text_key):
1190
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_info):
1333
1191
"""Iterate the result of iter_interesting_nodes, yielding the records
1334
1192
and adding to text_keys.
1336
text_keys_update = text_keys.update
1337
1194
for record, items in interesting_nodes_iterable:
1338
text_keys_update([bytes_to_text_key(b) for n,b in items])
1195
for name, bytes in items:
1196
# Note: we don't care about name_utf8, because groupcompress repos
1197
# are always rich-root, so there are no synthesised root records to
1199
_, file_id, revision_id = bytes_to_info(bytes)
1200
file_id = intern(file_id)
1201
revision_id = intern(revision_id)
1202
text_keys.add(StaticTuple(file_id, revision_id).intern())
added
removed
bzrlib/repofmt/groupcompress_repo.py
