87
87
# have a regular 2-list index giving parents and compression
89
89
index_builder_class(reference_lists=1),
90
# Texts: per file graph, for all fileids - so one reference list
91
# and two elements in the key tuple.
90
# Texts: compression and per file graph, for all fileids - so two
91
# reference lists and two elements in the key tuple.
92
92
index_builder_class(reference_lists=1, key_elements=2),
93
93
# Signatures: Just blobs to store, no compression, no parents
165
163
have deltas based on a fallback repository.
166
164
(See <https://bugs.launchpad.net/bzr/+bug/288751>)
168
# Groupcompress packs don't have any external references, arguably CHK
169
# pages have external references, but we cannot 'cheaply' determine
170
# them without actually walking all of the chk pages.
173
class ResumedGCPack(ResumedPack):
175
def _check_references(self):
176
"""Make sure our external compression parents are present."""
177
# See GCPack._check_references for why this is empty
179
def _get_external_refs(self, index):
180
# GC repositories don't have compression parents external to a given
166
# Groupcompress packs don't have any external references
185
169
class GCCHKPacker(Packer):
413
391
def _copy_inventory_texts(self):
414
392
source_vf, target_vf = self._build_vfs('inventory', True, True)
415
# It is not sufficient to just use self.revision_keys, as stacked
416
# repositories can have more inventories than they have revisions.
417
# One alternative would be to do something with
418
# get_parent_map(self.revision_keys), but that shouldn't be any faster
420
inventory_keys = source_vf.keys()
421
missing_inventories = set(self.revision_keys).difference(inventory_keys)
422
if missing_inventories:
423
missing_inventories = sorted(missing_inventories)
424
raise ValueError('We are missing inventories for revisions: %s'
425
% (missing_inventories,))
426
self._copy_stream(source_vf, target_vf, inventory_keys,
393
self._copy_stream(source_vf, target_vf, self.revision_keys,
427
394
'inventories', self._get_filtered_inv_stream, 2)
429
396
def _copy_chk_texts(self):
456
423
# is grabbing too many keys...
457
424
text_keys = source_vf.keys()
458
425
self._copy_stream(source_vf, target_vf, text_keys,
459
'texts', self._get_progress_stream, 4)
426
'text', self._get_progress_stream, 4)
461
428
def _copy_signature_texts(self):
462
429
source_vf, target_vf = self._build_vfs('signature', False, False)
644
604
self.inventories = GroupCompressVersionedFiles(
645
605
_GCGraphIndex(self._pack_collection.inventory_index.combined_index,
646
606
add_callback=self._pack_collection.inventory_index.add_callback,
647
parents=True, is_locked=self.is_locked,
648
inconsistency_fatal=False),
607
parents=True, is_locked=self.is_locked),
649
608
access=self._pack_collection.inventory_index.data_access)
650
609
self.revisions = GroupCompressVersionedFiles(
651
610
_GCGraphIndex(self._pack_collection.revision_index.combined_index,
652
611
add_callback=self._pack_collection.revision_index.add_callback,
653
parents=True, is_locked=self.is_locked,
654
track_external_parent_refs=True),
612
parents=True, is_locked=self.is_locked),
655
613
access=self._pack_collection.revision_index.data_access,
657
615
self.signatures = GroupCompressVersionedFiles(
658
616
_GCGraphIndex(self._pack_collection.signature_index.combined_index,
659
617
add_callback=self._pack_collection.signature_index.add_callback,
660
parents=False, is_locked=self.is_locked,
661
inconsistency_fatal=False),
618
parents=False, is_locked=self.is_locked),
662
619
access=self._pack_collection.signature_index.data_access,
664
621
self.texts = GroupCompressVersionedFiles(
665
622
_GCGraphIndex(self._pack_collection.text_index.combined_index,
666
623
add_callback=self._pack_collection.text_index.add_callback,
667
parents=True, is_locked=self.is_locked,
668
inconsistency_fatal=False),
624
parents=True, is_locked=self.is_locked),
669
625
access=self._pack_collection.text_index.data_access)
670
626
# No parents, individual CHK pages don't have specific ancestry
671
627
self.chk_bytes = GroupCompressVersionedFiles(
672
628
_GCGraphIndex(self._pack_collection.chk_index.combined_index,
673
629
add_callback=self._pack_collection.chk_index.add_callback,
674
parents=False, is_locked=self.is_locked,
675
inconsistency_fatal=False),
630
parents=False, is_locked=self.is_locked),
676
631
access=self._pack_collection.chk_index.data_access)
677
search_key_name = self._format._serializer.search_key_name
678
search_key_func = chk_map.search_key_registry.get(search_key_name)
679
self.chk_bytes._search_key_func = search_key_func
680
632
# True when the repository object is 'write locked' (as opposed to the
681
633
# physical lock only taken out around changes to the pack-names list.)
682
634
# Another way to represent this would be a decorator around the control
705
657
return self._inventory_add_lines(revision_id, parents,
706
658
inv_lines, check_content=False)
708
def _create_inv_from_null(self, delta, revision_id):
709
"""This will mutate new_inv directly.
711
This is a simplified form of create_by_apply_delta which knows that all
712
the old values must be None, so everything is a create.
714
serializer = self._format._serializer
715
new_inv = inventory.CHKInventory(serializer.search_key_name)
716
new_inv.revision_id = revision_id
717
entry_to_bytes = new_inv._entry_to_bytes
718
id_to_entry_dict = {}
719
parent_id_basename_dict = {}
720
for old_path, new_path, file_id, entry in delta:
721
if old_path is not None:
722
raise ValueError('Invalid delta, somebody tried to delete %r'
723
' from the NULL_REVISION'
724
% ((old_path, file_id),))
726
raise ValueError('Invalid delta, delta from NULL_REVISION has'
727
' no new_path %r' % (file_id,))
729
new_inv.root_id = file_id
730
parent_id_basename_key = ('', '')
732
utf8_entry_name = entry.name.encode('utf-8')
733
parent_id_basename_key = (entry.parent_id, utf8_entry_name)
734
new_value = entry_to_bytes(entry)
736
# new_inv._path_to_fileid_cache[new_path] = file_id
737
id_to_entry_dict[(file_id,)] = new_value
738
parent_id_basename_dict[parent_id_basename_key] = file_id
740
new_inv._populate_from_dicts(self.chk_bytes, id_to_entry_dict,
741
parent_id_basename_dict, maximum_size=serializer.maximum_size)
744
660
def add_inventory_by_delta(self, basis_revision_id, delta, new_revision_id,
745
661
parents, basis_inv=None, propagate_caches=False):
746
662
"""Add a new inventory expressed as a delta against another revision.
766
682
repository format specific) of the serialized inventory, and the
767
683
resulting inventory.
685
if basis_revision_id == _mod_revision.NULL_REVISION:
686
return KnitPackRepository.add_inventory_by_delta(self,
687
basis_revision_id, delta, new_revision_id, parents)
769
688
if not self.is_in_write_group():
770
689
raise AssertionError("%r not in write group" % (self,))
771
690
_mod_revision.check_not_reserved_id(new_revision_id)
773
if basis_inv is None:
774
if basis_revision_id == _mod_revision.NULL_REVISION:
775
new_inv = self._create_inv_from_null(delta, new_revision_id)
776
inv_lines = new_inv.to_lines()
777
return self._inventory_add_lines(new_revision_id, parents,
778
inv_lines, check_content=False), new_inv
780
basis_tree = self.revision_tree(basis_revision_id)
781
basis_tree.lock_read()
691
basis_tree = self.revision_tree(basis_revision_id)
692
basis_tree.lock_read()
694
if basis_inv is None:
782
695
basis_inv = basis_tree.inventory
784
696
result = basis_inv.create_by_apply_delta(delta, new_revision_id,
785
697
propagate_caches=propagate_caches)
786
698
inv_lines = result.to_lines()
787
699
return self._inventory_add_lines(new_revision_id, parents,
788
700
inv_lines, check_content=False), result
790
if basis_tree is not None:
793
def deserialise_inventory(self, revision_id, bytes):
794
return inventory.CHKInventory.deserialise(self.chk_bytes, bytes,
797
def _iter_inventories(self, revision_ids, ordering):
704
def _iter_inventories(self, revision_ids):
798
705
"""Iterate over many inventory objects."""
800
ordering = 'unordered'
801
706
keys = [(revision_id,) for revision_id in revision_ids]
802
stream = self.inventories.get_record_stream(keys, ordering, True)
707
stream = self.inventories.get_record_stream(keys, 'unordered', True)
804
709
for record in stream:
805
710
if record.storage_kind != 'absent':
810
715
yield inventory.CHKInventory.deserialise(self.chk_bytes, texts[key], key)
812
def _iter_inventory_xmls(self, revision_ids, ordering):
813
# Without a native 'xml' inventory, this method doesn't make sense.
814
# However older working trees, and older bundles want it - so we supply
815
# it allowing get_inventory_xml to work. Bundles currently use the
816
# serializer directly; this also isn't ideal, but there isn't an xml
817
# iteration interface offered at all for repositories. We could make
818
# _iter_inventory_xmls be part of the contract, even if kept private.
819
inv_to_str = self._serializer.write_inventory_to_string
820
for inv in self.iter_inventories(revision_ids, ordering=ordering):
821
yield inv_to_str(inv), inv.revision_id
823
def _find_present_inventory_keys(self, revision_keys):
824
parent_map = self.inventories.get_parent_map(revision_keys)
825
present_inventory_keys = set(k for k in parent_map)
826
return present_inventory_keys
717
def _iter_inventory_xmls(self, revision_ids):
718
# Without a native 'xml' inventory, this method doesn't make sense, so
719
# make it raise to trap naughty direct users.
720
raise NotImplementedError(self._iter_inventory_xmls)
722
def _find_revision_outside_set(self, revision_ids):
723
revision_set = frozenset(revision_ids)
724
for revid in revision_ids:
725
parent_ids = self.get_parent_map([revid]).get(revid, ())
726
for parent in parent_ids:
727
if parent in revision_set:
728
# Parent is not outside the set
730
if parent not in self.get_parent_map([parent]):
734
return _mod_revision.NULL_REVISION
736
def _find_file_keys_to_fetch(self, revision_ids, pb):
737
rich_root = self.supports_rich_root()
738
revision_outside_set = self._find_revision_outside_set(revision_ids)
739
if revision_outside_set == _mod_revision.NULL_REVISION:
740
uninteresting_root_keys = set()
742
uninteresting_inv = self.get_inventory(revision_outside_set)
743
uninteresting_root_keys = set([uninteresting_inv.id_to_entry.key()])
744
interesting_root_keys = set()
745
for idx, inv in enumerate(self.iter_inventories(revision_ids)):
746
interesting_root_keys.add(inv.id_to_entry.key())
747
revision_ids = frozenset(revision_ids)
748
file_id_revisions = {}
749
bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
750
for record, items in chk_map.iter_interesting_nodes(self.chk_bytes,
751
interesting_root_keys, uninteresting_root_keys,
753
# This is cheating a bit to use the last grabbed 'inv', but it
755
for name, bytes in items:
756
(name_utf8, file_id, revision_id) = bytes_to_info(bytes)
757
if not rich_root and name_utf8 == '':
759
if revision_id in revision_ids:
760
# Would we rather build this up into file_id => revision
763
file_id_revisions[file_id].add(revision_id)
765
file_id_revisions[file_id] = set([revision_id])
766
for file_id, revisions in file_id_revisions.iteritems():
767
yield ('file', file_id, revisions)
828
769
def fileids_altered_by_revision_ids(self, revision_ids, _inv_weave=None):
829
770
"""Find the file ids and versions affected by revisions.
835
776
revision_ids. Each altered file-ids has the exact revision_ids that
836
777
altered it listed explicitly.
838
rich_root = self.supports_rich_root()
839
bytes_to_info = inventory.CHKInventory._bytes_to_utf8name_key
840
file_id_revisions = {}
779
rich_roots = self.supports_rich_root()
841
781
pb = ui.ui_factory.nested_progress_bar()
843
revision_keys = [(r,) for r in revision_ids]
844
parent_keys = self._find_parent_keys_of_revisions(revision_keys)
845
# TODO: instead of using _find_present_inventory_keys, change the
846
# code paths to allow missing inventories to be tolerated.
847
# However, we only want to tolerate missing parent
848
# inventories, not missing inventories for revision_ids
849
present_parent_inv_keys = self._find_present_inventory_keys(
851
present_parent_inv_ids = set(
852
[k[-1] for k in present_parent_inv_keys])
853
uninteresting_root_keys = set()
854
interesting_root_keys = set()
855
inventories_to_read = set(revision_ids)
856
inventories_to_read.update(present_parent_inv_ids)
857
for inv in self.iter_inventories(inventories_to_read):
858
entry_chk_root_key = inv.id_to_entry.key()
859
if inv.revision_id in present_parent_inv_ids:
860
uninteresting_root_keys.add(entry_chk_root_key)
862
interesting_root_keys.add(entry_chk_root_key)
864
chk_bytes = self.chk_bytes
865
for record, items in chk_map.iter_interesting_nodes(chk_bytes,
866
interesting_root_keys, uninteresting_root_keys,
868
for name, bytes in items:
869
(name_utf8, file_id, revision_id) = bytes_to_info(bytes)
870
if not rich_root and name_utf8 == '':
873
file_id_revisions[file_id].add(revision_id)
875
file_id_revisions[file_id] = set([revision_id])
783
total = len(revision_ids)
784
for pos, inv in enumerate(self.iter_inventories(revision_ids)):
785
pb.update("Finding text references", pos, total)
786
for entry in inv.iter_just_entries():
787
if entry.revision != inv.revision_id:
789
if not rich_roots and entry.file_id == inv.root_id:
791
alterations = result.setdefault(entry.file_id, set([]))
792
alterations.add(entry.revision)
878
return file_id_revisions
880
797
def find_text_key_references(self):
881
798
"""Find the text key references within the repository.
916
833
def _get_source(self, to_format):
917
834
"""Return a source for streaming from this repository."""
918
if self._format._serializer == to_format._serializer:
835
if isinstance(to_format, remote.RemoteRepositoryFormat):
836
# Can't just check attributes on to_format with the current code,
838
to_format._ensure_real()
839
to_format = to_format._custom_format
840
if to_format.__class__ is self._format.__class__:
919
841
# We must be exactly the same format, otherwise stuff like the chk
920
# page layout might be different.
921
# Actually, this test is just slightly looser than exact so that
922
# CHK2 <-> 2a transfers will work.
842
# page layout might be different
923
843
return GroupCHKStreamSource(self, to_format)
924
844
return super(CHKInventoryRepository, self)._get_source(to_format)
927
class GroupCHKStreamSource(KnitPackStreamSource):
846
def suspend_write_group(self):
847
raise errors.UnsuspendableWriteGroup(self)
849
def _resume_write_group(self, tokens):
850
raise errors.UnsuspendableWriteGroup(self)
853
class GroupCHKStreamSource(repository.StreamSource):
928
854
"""Used when both the source and target repo are GroupCHK repos."""
930
856
def __init__(self, from_repository, to_format):
932
858
super(GroupCHKStreamSource, self).__init__(from_repository, to_format)
933
859
self._revision_keys = None
934
860
self._text_keys = None
935
self._text_fetch_order = 'groupcompress'
936
861
self._chk_id_roots = None
937
862
self._chk_p_id_roots = None
939
def _get_inventory_stream(self, inventory_keys, allow_absent=False):
864
def _get_filtered_inv_stream(self):
940
865
"""Get a stream of inventory texts.
942
867
When this function returns, self._chk_id_roots and self._chk_p_id_roots
977
897
p_id_roots_set.clear()
978
898
return ('inventories', _filtered_inv_stream())
980
def _get_filtered_chk_streams(self, excluded_revision_keys):
900
def _get_filtered_chk_streams(self, excluded_keys):
981
901
self._text_keys = set()
982
excluded_revision_keys.discard(_mod_revision.NULL_REVISION)
983
if not excluded_revision_keys:
902
excluded_keys.discard(_mod_revision.NULL_REVISION)
903
if not excluded_keys:
984
904
uninteresting_root_keys = set()
985
905
uninteresting_pid_root_keys = set()
987
# filter out any excluded revisions whose inventories are not
989
# TODO: Update Repository.iter_inventories() to add
990
# ignore_missing=True
991
present_keys = self.from_repository._find_present_inventory_keys(
992
excluded_revision_keys)
993
present_ids = [k[-1] for k in present_keys]
994
907
uninteresting_root_keys = set()
995
908
uninteresting_pid_root_keys = set()
996
for inv in self.from_repository.iter_inventories(present_ids):
909
for inv in self.from_repository.iter_inventories(excluded_keys):
997
910
uninteresting_root_keys.add(inv.id_to_entry.key())
998
911
uninteresting_pid_root_keys.add(
999
912
inv.parent_id_basename_to_file_id.key())
1009
922
self._text_keys.add((file_id, revision_id))
1010
923
if record is not None:
1013
self._chk_id_roots = None
1014
925
yield 'chk_bytes', _filter_id_to_entry()
1015
926
def _get_parent_id_basename_to_file_id_pages():
1016
927
for record, items in chk_map.iter_interesting_nodes(chk_bytes,
1017
928
self._chk_p_id_roots, uninteresting_pid_root_keys):
1018
929
if record is not None:
1021
self._chk_p_id_roots = None
1022
931
yield 'chk_bytes', _get_parent_id_basename_to_file_id_pages()
933
def _get_text_stream(self):
934
# Note: We know we don't have to handle adding root keys, because both
935
# the source and target are GCCHK, and those always support rich-roots
936
# We may want to request as 'unordered', in case the source has done a
938
return ('texts', self.from_repository.texts.get_record_stream(
939
self._text_keys, 'groupcompress', False))
1024
941
def get_stream(self, search):
1025
942
revision_ids = search.get_keys()
1026
943
for stream_info in self._fetch_revision_texts(revision_ids):
1027
944
yield stream_info
1028
945
self._revision_keys = [(rev_id,) for rev_id in revision_ids]
1029
yield self._get_inventory_stream(self._revision_keys)
1030
# TODO: The keys to exclude might be part of the search recipe
1031
# For now, exclude all parents that are at the edge of ancestry, for
1032
# which we have inventories
1033
from_repo = self.from_repository
1034
parent_keys = from_repo._find_parent_keys_of_revisions(
1035
self._revision_keys)
1036
for stream_info in self._get_filtered_chk_streams(parent_keys):
946
yield self._get_filtered_inv_stream()
947
# The keys to exclude are part of the search recipe
948
_, _, exclude_keys, _ = search.get_recipe()
949
for stream_info in self._get_filtered_chk_streams(exclude_keys):
1037
950
yield stream_info
1038
951
yield self._get_text_stream()
1040
def get_stream_for_missing_keys(self, missing_keys):
1041
# missing keys can only occur when we are byte copying and not
1042
# translating (because translation means we don't send
1043
# unreconstructable deltas ever).
1044
missing_inventory_keys = set()
1045
for key in missing_keys:
1046
if key[0] != 'inventories':
1047
raise AssertionError('The only missing keys we should'
1048
' be filling in are inventory keys, not %s'
1050
missing_inventory_keys.add(key[1:])
1051
if self._chk_id_roots or self._chk_p_id_roots:
1052
raise AssertionError('Cannot call get_stream_for_missing_keys'
1053
' untill all of get_stream() has been consumed.')
1054
# Yield the inventory stream, so we can find the chk stream
1055
# Some of the missing_keys will be missing because they are ghosts.
1056
# As such, we can ignore them. The Sink is required to verify there are
1057
# no unavailable texts when the ghost inventories are not filled in.
1058
yield self._get_inventory_stream(missing_inventory_keys,
1060
# We use the empty set for excluded_revision_keys, to make it clear
1061
# that we want to transmit all referenced chk pages.
1062
for stream_info in self._get_filtered_chk_streams(set()):
1066
954
class RepositoryFormatCHK1(RepositoryFormatPack):
1067
955
"""A hashed CHK+group compress pack repository."""
1069
957
repository_class = CHKInventoryRepository
1070
supports_external_lookups = True
1071
958
supports_chks = True
1072
959
# For right now, setting this to True gives us InterModel1And2 rather
1073
960
# than InterDifferingSerializer
1107
993
return ("Development repository format - rich roots, group compression"
1108
994
" and chk inventories")
1111
class RepositoryFormatCHK2(RepositoryFormatCHK1):
1112
"""A CHK repository that uses the bencode revision serializer."""
1114
_serializer = chk_serializer.chk_bencode_serializer
1116
def _get_matching_bzrdir(self):
1117
return bzrdir.format_registry.make_bzrdir('development7-rich-root')
1119
def _ignore_setting_bzrdir(self, format):
1122
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1124
def get_format_string(self):
1125
"""See RepositoryFormat.get_format_string()."""
1126
return ('Bazaar development format - chk repository with bencode '
1127
'revision serialization (needs bzr.dev from 1.16)\n')
1130
class RepositoryFormat2a(RepositoryFormatCHK2):
1131
"""A CHK repository that uses the bencode revision serializer.
1133
This is the same as RepositoryFormatCHK2 but with a public name.
1136
_serializer = chk_serializer.chk_bencode_serializer
1138
def _get_matching_bzrdir(self):
1139
return bzrdir.format_registry.make_bzrdir('2a')
1141
def _ignore_setting_bzrdir(self, format):
1144
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
1146
def get_format_string(self):
1147
return ('Bazaar repository format 2a (needs bzr 1.16 or later)\n')
996
def check_conversion_target(self, target_format):
997
if not target_format.rich_root_data:
998
raise errors.BadConversionTarget(
999
'Does not support rich root data.', target_format)
1000
if not getattr(target_format, 'supports_tree_reference', False):
1001
raise errors.BadConversionTarget(
1002
'Does not support nested trees', target_format)
added
removed
bzrlib/repofmt/groupcompress_repo.py
