« back to all changes in this revision
Viewing changes to bzrlib/repofmt/pack_repo.py
- browse files at revision 5757.8.6
- compare with another revision
- compare with revision 1551.12.2
- download diff
- download tarball
- view history from revision 5757.8.6
- Committer: Jelmer Vernooij
- Date: 2011-04-05 23:57:17 UTC
- mfrom: (5757.7.5 knitpackrepo-6)
- mto: This revision was merged to the branch mainline in revision 5801.
- Revision ID: jelmer@samba.org-20110405235717-874fdflj5lon4sle
Merge knitpackrepo-6.
added
removed
bzrlib/repofmt/pack_repo.py
658
658
# _copy_inventory_texts
659
659
self._text_filter = None
660
660
661
def _pack_map_and_index_list(self, index_attribute):
662
"""Convert a list of packs to an index pack map and index list.
663
664
:param index_attribute: The attribute that the desired index is found
665
on.
666
:return: A tuple (map, list) where map contains the dict from
667
index:pack_tuple, and list contains the indices in the preferred
668
access order.
669
"""
670
indices = []
671
pack_map = {}
672
for pack_obj in self.packs:
673
index = getattr(pack_obj, index_attribute)
674
indices.append(index)
675
pack_map[index] = pack_obj
676
return pack_map, indices
677
678
def _index_contents(self, indices, key_filter=None):
679
"""Get an iterable of the index contents from a pack_map.
680
681
:param indices: The list of indices to query
682
:param key_filter: An optional filter to limit the keys returned.
683
"""
684
all_index = CombinedGraphIndex(indices)
685
if key_filter is None:
686
return all_index.iter_all_entries()
687
else:
688
return all_index.iter_entries(key_filter)
689
690
661
def pack(self, pb=None):
691
662
"""Create a new pack by reading data from other packs.
692
663
740
711
new_pack.signature_index.set_optimize(combine_backing_indices=False)
741
712
return new_pack
742
713
743
def _update_pack_order(self, entries, index_to_pack_map):
744
"""Determine how we want our packs to be ordered.
745
746
This changes the sort order of the self.packs list so that packs unused
747
by 'entries' will be at the end of the list, so that future requests
748
can avoid probing them. Used packs will be at the front of the
749
self.packs list, in the order of their first use in 'entries'.
750
751
:param entries: A list of (index, ...) tuples
752
:param index_to_pack_map: A mapping from index objects to pack objects.
753
"""
754
packs = []
755
seen_indexes = set()
756
for entry in entries:
757
index = entry[0]
758
if index not in seen_indexes:
759
packs.append(index_to_pack_map[index])
760
seen_indexes.add(index)
761
if len(packs) == len(self.packs):
762
if 'pack' in debug.debug_flags:
763
mutter('Not changing pack list, all packs used.')
764
return
765
seen_packs = set(packs)
766
for pack in self.packs:
767
if pack not in seen_packs:
768
packs.append(pack)
769
seen_packs.add(pack)
770
if 'pack' in debug.debug_flags:
771
old_names = [p.access_tuple()[1] for p in self.packs]
772
new_names = [p.access_tuple()[1] for p in packs]
773
mutter('Reordering packs\nfrom: %s\n to: %s',
774
old_names, new_names)
775
self.packs = packs
776
777
714
def _copy_revision_texts(self):
778
715
"""Copy revision data to the new pack."""
779
716
raise NotImplementedError(self._copy_revision_texts)
793
730
def _create_pack_from_packs(self):
794
731
raise NotImplementedError(self._create_pack_from_packs)
795
732
796
def _least_readv_node_readv(self, nodes):
797
"""Generate request groups for nodes using the least readv's.
798
799
:param nodes: An iterable of graph index nodes.
800
:return: Total node count and an iterator of the data needed to perform
801
readvs to obtain the data for nodes. Each item yielded by the
802
iterator is a tuple with:
803
index, readv_vector, node_vector. readv_vector is a list ready to
804
hand to the transport readv method, and node_vector is a list of
805
(key, eol_flag, references) for the node retrieved by the
806
matching readv_vector.
807
"""
808
# group by pack so we do one readv per pack
809
nodes = sorted(nodes)
810
total = len(nodes)
811
request_groups = {}
812
for index, key, value, references in nodes:
813
if index not in request_groups:
814
request_groups[index] = []
815
request_groups[index].append((key, value, references))
816
result = []
817
for index, items in request_groups.iteritems():
818
pack_readv_requests = []
819
for key, value, references in items:
820
# ---- KnitGraphIndex.get_position
821
bits = value[1:].split(' ')
822
offset, length = int(bits[0]), int(bits[1])
823
pack_readv_requests.append(
824
((offset, length), (key, value[0], references)))
825
# linear scan up the pack to maximum range combining.
826
pack_readv_requests.sort()
827
# split out the readv and the node data.
828
pack_readv = [readv for readv, node in pack_readv_requests]
829
node_vector = [node for readv, node in pack_readv_requests]
830
result.append((index, pack_readv, node_vector))
831
return total, result
832
833
733
def _log_copied_texts(self):
834
734
if 'pack' in debug.debug_flags:
835
735
mutter('%s: create_pack: file texts copied: %s%s %d items t+%6.3fs',
838
738
self.new_pack.text_index.key_count(),
839
739
time.time() - self.new_pack.start_time)
840
740
841
def _revision_node_readv(self, revision_nodes):
842
"""Return the total revisions and the readv's to issue.
843
844
:param revision_nodes: The revision index contents for the packs being
845
incorporated into the new pack.
846
:return: As per _least_readv_node_readv.
847
"""
848
return self._least_readv_node_readv(revision_nodes)
849
850
741
def _use_pack(self, new_pack):
851
742
"""Return True if new_pack should be used.
852
743
856
747
return new_pack.data_inserted()
857
748
858
749
859
class OptimisingPacker(Packer):
860
"""A packer which spends more time to create better disk layouts."""
861
862
def _revision_node_readv(self, revision_nodes):
863
"""Return the total revisions and the readv's to issue.
864
865
This sort places revisions in topological order with the ancestors
866
after the children.
867
868
:param revision_nodes: The revision index contents for the packs being
869
incorporated into the new pack.
870
:return: As per _least_readv_node_readv.
871
"""
872
# build an ancestors dict
873
ancestors = {}
874
by_key = {}
875
for index, key, value, references in revision_nodes:
876
ancestors[key] = references[0]
877
by_key[key] = (index, value, references)
878
order = tsort.topo_sort(ancestors)
879
total = len(order)
880
# Single IO is pathological, but it will work as a starting point.
881
requests = []
882
for key in reversed(order):
883
index, value, references = by_key[key]
884
# ---- KnitGraphIndex.get_position
885
bits = value[1:].split(' ')
886
offset, length = int(bits[0]), int(bits[1])
887
requests.append(
888
(index, [(offset, length)], [(key, value[0], references)]))
889
# TODO: combine requests in the same index that are in ascending order.
890
return total, requests
891
892
def open_pack(self):
893
"""Open a pack for the pack we are creating."""
894
new_pack = super(OptimisingPacker, self).open_pack()
895
# Turn on the optimization flags for all the index builders.
896
new_pack.revision_index.set_optimize(for_size=True)
897
new_pack.inventory_index.set_optimize(for_size=True)
898
new_pack.text_index.set_optimize(for_size=True)
899
new_pack.signature_index.set_optimize(for_size=True)
900
return new_pack
901
902
903
750
class RepositoryPackCollection(object):
904
751
"""Management of packs within a repository.
905
752
906
753
:ivar _names: map of {pack_name: (index_size,)}
907
754
"""
908
755
909
pack_factory = NewPack
910
resumed_pack_factory = ResumedPack
756
pack_factory = None
757
resumed_pack_factory = None
758
normal_packer_class = None
759
optimising_packer_class = None
911
760
912
761
def __init__(self, repo, transport, index_transport, upload_transport,
913
762
pack_transport, index_builder_class, index_class,
1054
903
'containing %d revisions. Packing %d files into %d affecting %d'
1055
904
' revisions', self, total_packs, total_revisions, num_old_packs,
1056
905
num_new_packs, num_revs_affected)
1057
result = self._execute_pack_operations(pack_operations,
906
result = self._execute_pack_operations(pack_operations, packer_class=self.normal_packer_class,
1058
907
reload_func=self._restart_autopack)
1059
908
mutter('Auto-packing repository %s completed', self)
1060
909
return result
1061
910
1062
def _execute_pack_operations(self, pack_operations, _packer_class=Packer,
1063
reload_func=None):
911
def _execute_pack_operations(self, pack_operations, packer_class, reload_func=None):
1064
912
"""Execute a series of pack operations.
1065
913
1066
914
:param pack_operations: A list of [revision_count, packs_to_combine].
1067
:param _packer_class: The class of packer to use (default: Packer).
915
:param _packer_class: The class of packer to use
1068
916
:return: The new pack names.
1069
917
"""
1070
918
for revision_count, packs in pack_operations:
1071
919
# we may have no-ops from the setup logic
1072
920
if len(packs) == 0:
1073
921
continue
1074
packer = _packer_class(self, packs, '.autopack',
922
packer = packer_class(self, packs, '.autopack',
1075
923
reload_func=reload_func)
1076
924
try:
1077
925
packer.pack()
1144
992
# or this pack was included in the hint.
1145
993
pack_operations[-1][0] += pack.get_revision_count()
1146
994
pack_operations[-1][1].append(pack)
1147
self._execute_pack_operations(pack_operations, OptimisingPacker,
995
self._execute_pack_operations(pack_operations,
996
packer_class=self.optimising_packer_class,
1148
997
reload_func=self._restart_pack_operations)
1149
998
1150
999
def plan_autopack_combinations(self, existing_packs, pack_distribution):
Loggerhead is a web-based interface for Breezy
Version: 2.0.1