425
421
inventory_keys = source_vf.keys()
426
422
missing_inventories = set(self.revision_keys).difference(inventory_keys)
427
423
if missing_inventories:
428
# Go back to the original repo, to see if these are really missing
429
# https://bugs.launchpad.net/bzr/+bug/437003
430
# If we are packing a subset of the repo, it is fine to just have
431
# the data in another Pack file, which is not included in this pack
433
inv_index = self._pack_collection.repo.inventories._index
434
pmap = inv_index.get_parent_map(missing_inventories)
435
really_missing = missing_inventories.difference(pmap)
437
missing_inventories = sorted(really_missing)
438
raise ValueError('We are missing inventories for revisions: %s'
439
% (missing_inventories,))
424
missing_inventories = sorted(missing_inventories)
425
raise ValueError('We are missing inventories for revisions: %s'
426
% (missing_inventories,))
440
427
self._copy_stream(source_vf, target_vf, inventory_keys,
441
428
'inventories', self._get_filtered_inv_stream, 2)
443
def _get_chk_vfs_for_copy(self):
444
return self._build_vfs('chk', False, False)
446
430
def _copy_chk_texts(self):
447
source_vf, target_vf = self._get_chk_vfs_for_copy()
431
source_vf, target_vf = self._build_vfs('chk', False, False)
448
432
# TODO: This is technically spurious... if it is a performance issue,
450
434
total_keys = source_vf.keys()
596
580
return new_pack.data_inserted() and self._data_changed
599
class GCCHKCanonicalizingPacker(GCCHKPacker):
600
"""A packer that ensures inventories have canonical-form CHK maps.
602
Ideally this would be part of reconcile, but it's very slow and rarely
603
needed. (It repairs repositories affected by
604
https://bugs.launchpad.net/bzr/+bug/522637).
607
def __init__(self, *args, **kwargs):
608
super(GCCHKCanonicalizingPacker, self).__init__(*args, **kwargs)
609
self._data_changed = False
611
def _exhaust_stream(self, source_vf, keys, message, vf_to_stream, pb_offset):
612
"""Create and exhaust a stream, but don't insert it.
614
This is useful to get the side-effects of generating a stream.
616
self.pb.update('scanning %s' % (message,), pb_offset)
617
child_pb = ui.ui_factory.nested_progress_bar()
619
list(vf_to_stream(source_vf, keys, message, child_pb))
623
def _copy_inventory_texts(self):
624
source_vf, target_vf = self._build_vfs('inventory', True, True)
625
source_chk_vf, target_chk_vf = self._get_chk_vfs_for_copy()
626
inventory_keys = source_vf.keys()
627
# First, copy the existing CHKs on the assumption that most of them
628
# will be correct. This will save us from having to reinsert (and
629
# recompress) these records later at the cost of perhaps preserving a
631
# (Iterate but don't insert _get_filtered_inv_stream to populate the
632
# variables needed by GCCHKPacker._copy_chk_texts.)
633
self._exhaust_stream(source_vf, inventory_keys, 'inventories',
634
self._get_filtered_inv_stream, 2)
635
GCCHKPacker._copy_chk_texts(self)
636
# Now copy and fix the inventories, and any regenerated CHKs.
637
def chk_canonicalizing_inv_stream(source_vf, keys, message, pb=None):
638
return self._get_filtered_canonicalizing_inv_stream(
639
source_vf, keys, message, pb, source_chk_vf, target_chk_vf)
640
self._copy_stream(source_vf, target_vf, inventory_keys,
641
'inventories', chk_canonicalizing_inv_stream, 4)
643
def _copy_chk_texts(self):
644
# No-op; in this class this happens during _copy_inventory_texts.
647
def _get_filtered_canonicalizing_inv_stream(self, source_vf, keys, message,
648
pb=None, source_chk_vf=None, target_chk_vf=None):
649
"""Filter the texts of inventories, regenerating CHKs to make sure they
652
total_keys = len(keys)
653
target_chk_vf = versionedfile.NoDupeAddLinesDecorator(target_chk_vf)
654
def _filtered_inv_stream():
655
stream = source_vf.get_record_stream(keys, 'groupcompress', True)
656
search_key_name = None
657
for idx, record in enumerate(stream):
658
# Inventories should always be with revisions; assume success.
659
bytes = record.get_bytes_as('fulltext')
660
chk_inv = inventory.CHKInventory.deserialise(
661
source_chk_vf, bytes, record.key)
663
pb.update('inv', idx, total_keys)
664
chk_inv.id_to_entry._ensure_root()
665
if search_key_name is None:
666
# Find the name corresponding to the search_key_func
667
search_key_reg = chk_map.search_key_registry
668
for search_key_name, func in search_key_reg.iteritems():
669
if func == chk_inv.id_to_entry._search_key_func:
671
canonical_inv = inventory.CHKInventory.from_inventory(
672
target_chk_vf, chk_inv,
673
maximum_size=chk_inv.id_to_entry._root_node._maximum_size,
674
search_key_name=search_key_name)
675
if chk_inv.id_to_entry.key() != canonical_inv.id_to_entry.key():
677
'Non-canonical CHK map for id_to_entry of inv: %s '
678
'(root is %s, should be %s)' % (chk_inv.revision_id,
679
chk_inv.id_to_entry.key()[0],
680
canonical_inv.id_to_entry.key()[0]))
681
self._data_changed = True
682
p_id_map = chk_inv.parent_id_basename_to_file_id
683
p_id_map._ensure_root()
684
canon_p_id_map = canonical_inv.parent_id_basename_to_file_id
685
if p_id_map.key() != canon_p_id_map.key():
687
'Non-canonical CHK map for parent_id_to_basename of '
688
'inv: %s (root is %s, should be %s)'
689
% (chk_inv.revision_id, p_id_map.key()[0],
690
canon_p_id_map.key()[0]))
691
self._data_changed = True
692
yield versionedfile.ChunkedContentFactory(record.key,
693
record.parents, record.sha1,
694
canonical_inv.to_lines())
695
# We have finished processing all of the inventory records, we
696
# don't need these sets anymore
697
return _filtered_inv_stream()
699
def _use_pack(self, new_pack):
700
"""Override _use_pack to check for reconcile having changed content."""
701
return new_pack.data_inserted() and self._data_changed
704
583
class GCRepositoryPackCollection(RepositoryPackCollection):
706
585
pack_factory = GCPack
707
586
resumed_pack_factory = ResumedGCPack
708
normal_packer_class = GCCHKPacker
709
optimising_packer_class = GCCHKPacker
711
588
def _check_new_inventories(self):
712
589
"""Detect missing inventories or chk root entries for the new revisions
794
671
% (sorted(missing_text_keys),))
798
class CHKInventoryRepository(PackRepository):
799
"""subclass of PackRepository that uses CHK based inventories."""
674
def _execute_pack_operations(self, pack_operations,
675
_packer_class=GCCHKPacker,
677
"""Execute a series of pack operations.
679
:param pack_operations: A list of [revision_count, packs_to_combine].
680
:param _packer_class: The class of packer to use (default: Packer).
683
# XXX: Copied across from RepositoryPackCollection simply because we
684
# want to override the _packer_class ... :(
685
for revision_count, packs in pack_operations:
686
# we may have no-ops from the setup logic
689
packer = GCCHKPacker(self, packs, '.autopack',
690
reload_func=reload_func)
692
result = packer.pack()
693
except errors.RetryWithNewPacks:
694
# An exception is propagating out of this context, make sure
695
# this packer has cleaned up. Packer() doesn't set its new_pack
696
# state into the RepositoryPackCollection object, so we only
697
# have access to it directly here.
698
if packer.new_pack is not None:
699
packer.new_pack.abort()
704
self._remove_pack_from_memory(pack)
705
# record the newly available packs and stop advertising the old
707
result = self._save_pack_names(clear_obsolete_packs=True)
708
# Move the old packs out of the way now they are no longer referenced.
709
for revision_count, packs in pack_operations:
710
self._obsolete_packs(packs)
714
class CHKInventoryRepository(KnitPackRepository):
715
"""subclass of KnitPackRepository that uses CHK based inventories."""
801
717
def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
803
719
"""Overridden to change pack collection class."""
804
super(CHKInventoryRepository, self).__init__(_format, a_bzrdir,
805
control_files, _commit_builder_class, _serializer)
720
KnitPackRepository.__init__(self, _format, a_bzrdir, control_files,
721
_commit_builder_class, _serializer)
722
# and now replace everything it did :)
806
723
index_transport = self._transport.clone('indices')
807
724
self._pack_collection = GCRepositoryPackCollection(self,
808
725
self._transport, index_transport,
980
895
if record.storage_kind != 'absent':
981
896
texts[record.key] = record.get_bytes_as('fulltext')
983
texts[record.key] = None
898
raise errors.NoSuchRevision(self, record.key)
987
yield (None, key[-1])
989
yield (inventory.CHKInventory.deserialise(
990
self.chk_bytes, bytes, key), key[-1])
900
yield inventory.CHKInventory.deserialise(self.chk_bytes, texts[key], key)
992
def _get_inventory_xml(self, revision_id):
993
"""Get serialized inventory as a string."""
902
def _iter_inventory_xmls(self, revision_ids, ordering):
994
903
# Without a native 'xml' inventory, this method doesn't make sense.
995
904
# However older working trees, and older bundles want it - so we supply
996
# it allowing _get_inventory_xml to work. Bundles currently use the
905
# it allowing get_inventory_xml to work. Bundles currently use the
997
906
# serializer directly; this also isn't ideal, but there isn't an xml
998
# iteration interface offered at all for repositories.
999
return self._serializer.write_inventory_to_string(
1000
self.get_inventory(revision_id))
907
# iteration interface offered at all for repositories. We could make
908
# _iter_inventory_xmls be part of the contract, even if kept private.
909
inv_to_str = self._serializer.write_inventory_to_string
910
for inv in self.iter_inventories(revision_ids, ordering=ordering):
911
yield inv_to_str(inv), inv.revision_id
1002
913
def _find_present_inventory_keys(self, revision_keys):
1003
914
parent_map = self.inventories.get_parent_map(revision_keys)
1115
1012
return GroupCHKStreamSource(self, to_format)
1116
1013
return super(CHKInventoryRepository, self)._get_source(to_format)
1118
def _find_inconsistent_revision_parents(self, revisions_iterator=None):
1119
"""Find revisions with different parent lists in the revision object
1120
and in the index graph.
1122
:param revisions_iterator: None, or an iterator of (revid,
1123
Revision-or-None). This iterator controls the revisions checked.
1124
:returns: an iterator yielding tuples of (revison-id, parents-in-index,
1125
parents-in-revision).
1127
if not self.is_locked():
1128
raise AssertionError()
1130
if revisions_iterator is None:
1131
revisions_iterator = self._iter_revisions(None)
1132
for revid, revision in revisions_iterator:
1133
if revision is None:
1135
parent_map = vf.get_parent_map([(revid,)])
1136
parents_according_to_index = tuple(parent[-1] for parent in
1137
parent_map[(revid,)])
1138
parents_according_to_revision = tuple(revision.parent_ids)
1139
if parents_according_to_index != parents_according_to_revision:
1140
yield (revid, parents_according_to_index,
1141
parents_according_to_revision)
1143
def _check_for_inconsistent_revision_parents(self):
1144
inconsistencies = list(self._find_inconsistent_revision_parents())
1146
raise errors.BzrCheckError(
1147
"Revision index has inconsistent parents.")
1150
class GroupCHKStreamSource(StreamSource):
1016
class GroupCHKStreamSource(KnitPackStreamSource):
1151
1017
"""Used when both the source and target repo are GroupCHK repos."""
1153
1019
def __init__(self, from_repository, to_format):
1240
1107
self._chk_p_id_roots = None
1241
1108
yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
1243
def _get_text_stream(self):
1244
# Note: We know we don't have to handle adding root keys, because both
1245
# the source and target are the identical network name.
1246
text_stream = self.from_repository.texts.get_record_stream(
1247
self._text_keys, self._text_fetch_order, False)
1248
return ('texts', text_stream)
1250
1110
def get_stream(self, search):
1251
def wrap_and_count(pb, rc, stream):
1252
"""Yield records from stream while showing progress."""
1254
for record in stream:
1255
if count == rc.STEP:
1257
pb.update('Estimate', rc.current, rc.max)
1262
1111
revision_ids = search.get_keys()
1263
pb = ui.ui_factory.nested_progress_bar()
1264
rc = self._record_counter
1265
self._record_counter.setup(len(revision_ids))
1266
1112
for stream_info in self._fetch_revision_texts(revision_ids):
1267
yield (stream_info[0],
1268
wrap_and_count(pb, rc, stream_info[1]))
1269
1114
self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1115
self.from_repository.revisions.clear_cache()
1116
self.from_repository.signatures.clear_cache()
1117
yield self._get_inventory_stream(self._revision_keys)
1118
self.from_repository.inventories.clear_cache()
1270
1119
# TODO: The keys to exclude might be part of the search recipe
1271
1120
# For now, exclude all parents that are at the edge of ancestry, for
1272
1121
# which we have inventories
1273
1122
from_repo = self.from_repository
1274
1123
parent_keys = from_repo._find_parent_keys_of_revisions(
1275
1124
self._revision_keys)
1276
self.from_repository.revisions.clear_cache()
1277
self.from_repository.signatures.clear_cache()
1278
# Clear the repo's get_parent_map cache too.
1279
self.from_repository._unstacked_provider.disable_cache()
1280
self.from_repository._unstacked_provider.enable_cache()
1281
s = self._get_inventory_stream(self._revision_keys)
1282
yield (s[0], wrap_and_count(pb, rc, s[1]))
1283
self.from_repository.inventories.clear_cache()
1284
1125
for stream_info in self._get_filtered_chk_streams(parent_keys):
1285
yield (stream_info[0], wrap_and_count(pb, rc, stream_info[1]))
1286
1127
self.from_repository.chk_bytes.clear_cache()
1287
s = self._get_text_stream()
1288
yield (s[0], wrap_and_count(pb, rc, s[1]))
1128
yield self._get_text_stream()
1289
1129
self.from_repository.texts.clear_cache()
1290
pb.update('Done', rc.max, rc.max)
1293
1131
def get_stream_for_missing_keys(self, missing_keys):
1294
1132
# missing keys can only occur when we are byte copying and not
1351
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_text_key):
1189
def _filter_text_keys(interesting_nodes_iterable, text_keys, bytes_to_info):
1352
1190
"""Iterate the result of iter_interesting_nodes, yielding the records
1353
1191
and adding to text_keys.
1355
text_keys_update = text_keys.update
1356
1193
for record, items in interesting_nodes_iterable:
1357
text_keys_update([bytes_to_text_key(b) for n,b in items])
1194
for name, bytes in items:
1195
# Note: we don't care about name_utf8, because groupcompress repos
1196
# are always rich-root, so there are no synthesised root records to
1198
_, file_id, revision_id = bytes_to_info(bytes)
1199
file_id = intern(file_id)
1200
revision_id = intern(revision_id)
1201
text_keys.add(StaticTuple(file_id, revision_id).intern())
1361
class RepositoryFormat2a(RepositoryFormatPack):
1362
"""A CHK repository that uses the bencode revision serializer."""
1207
class RepositoryFormatCHK1(RepositoryFormatPack):
1208
"""A hashed CHK+group compress pack repository."""
1364
1210
repository_class = CHKInventoryRepository
1365
1211
supports_external_lookups = True
1366
1212
supports_chks = True
1213
# For right now, setting this to True gives us InterModel1And2 rather
1214
# than InterDifferingSerializer
1367
1215
_commit_builder_class = PackRootCommitBuilder
1368
1216
rich_root_data = True
1369
_serializer = chk_serializer.chk_bencode_serializer
1217
_serializer = chk_serializer.chk_serializer_255_bigpage
1370
1218
_commit_inv_deltas = True
1371
1219
# What index classes to use
1372
1220
index_builder_class = BTreeBuilder
1383
1231
pack_compresses = True
1385
1233
def _get_matching_bzrdir(self):
1386
return controldir.format_registry.make_bzrdir('2a')
1388
def _ignore_setting_bzrdir(self, format):
1391
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1394
def get_format_string(cls):
1234
return bzrdir.format_registry.make_bzrdir('development6-rich-root')
1236
def _ignore_setting_bzrdir(self, format):
1239
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1241
def get_format_string(self):
1242
"""See RepositoryFormat.get_format_string()."""
1243
return ('Bazaar development format - group compression and chk inventory'
1244
' (needs bzr.dev from 1.14)\n')
1246
def get_format_description(self):
1247
"""See RepositoryFormat.get_format_description()."""
1248
return ("Development repository format - rich roots, group compression"
1249
" and chk inventories")
1252
class RepositoryFormatCHK2(RepositoryFormatCHK1):
1253
"""A CHK repository that uses the bencode revision serializer."""
1255
_serializer = chk_serializer.chk_bencode_serializer
1257
def _get_matching_bzrdir(self):
1258
return bzrdir.format_registry.make_bzrdir('development7-rich-root')
1260
def _ignore_setting_bzrdir(self, format):
1263
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1265
def get_format_string(self):
1266
"""See RepositoryFormat.get_format_string()."""
1267
return ('Bazaar development format - chk repository with bencode '
1268
'revision serialization (needs bzr.dev from 1.16)\n')
1271
class RepositoryFormat2a(RepositoryFormatCHK2):
1272
"""A CHK repository that uses the bencode revision serializer.
1274
This is the same as RepositoryFormatCHK2 but with a public name.
1277
_serializer = chk_serializer.chk_bencode_serializer
1279
def _get_matching_bzrdir(self):
1280
return bzrdir.format_registry.make_bzrdir('2a')
1282
def _ignore_setting_bzrdir(self, format):
1285
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1287
def get_format_string(self):
1395
1288
return ('Bazaar repository format 2a (needs bzr 1.16 or later)\n')
1397
1290
def get_format_description(self):
1398
1291
"""See RepositoryFormat.get_format_description()."""
1399
1292
return ("Repository format 2a - rich roots, group compression"
1400
1293
" and chk inventories")
1403
class RepositoryFormat2aSubtree(RepositoryFormat2a):
1404
"""A 2a repository format that supports nested trees.
1408
def _get_matching_bzrdir(self):
1409
return controldir.format_registry.make_bzrdir('development-subtree')
1411
def _ignore_setting_bzrdir(self, format):
1414
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1417
def get_format_string(cls):
1418
return ('Bazaar development format 8\n')
1420
def get_format_description(self):
1421
"""See RepositoryFormat.get_format_description()."""
1422
return ("Development repository format 8 - nested trees, "
1423
"group compression and chk inventories")
1426
supports_tree_reference = True
added
removed
bzrlib/repofmt/groupcompress_repo.py
