« back to all changes in this revision
Viewing changes to bzrlib/repofmt/pack_repo.py
- browse files at revision 3350.6.4
- compare with another revision
- compare with revision 140
- download diff
- download tarball
- view history from revision 3350.6.4
- Committer: Robert Collins
- Date: 2008-06-11 04:20:16 UTC
- mto: (3221.12.10 Development1) (3517.4.1 annotate)
- mto: This revision was merged to the branch mainline in revision 3510.
- Revision ID: robertc@robertcollins.net-20080611042016-661dds0v1rjz542g
First cut at pluralised VersionedFiles. Some rather massive API incompatabilities, primarily because of the difficulty of coherence among competing stores.
- files removed:
- bzrlib/store/revision/__init__.py
- bzrlib/tests/interversionedfile_implementations
- bzrlib/tests/revisionstore_implementations
- files modified:
- bzrlib/annotate.py
added
removed
bzrlib/repofmt/pack_repo.py
34
34
CombinedGraphIndex,
35
35
GraphIndexPrefixAdapter,
36
36
)
37
from bzrlib.knit import KnitGraphIndex, _PackAccess, _KnitData
38
from bzrlib.osutils import rand_chars
37
from bzrlib.knit import (
38
KnitPlainFactory,
39
KnitVersionedFiles,
40
_KnitGraphIndex,
41
_DirectPackAccess,
42
)
43
from bzrlib.osutils import rand_chars, split_lines
39
44
from bzrlib.pack import ContainerWriter
40
45
from bzrlib.store import revision
41
46
from bzrlib import tsort
43
48
from bzrlib import (
44
49
bzrdir,
45
50
errors,
46
knit,
47
51
lockable_files,
48
52
lockdir,
49
53
osutils,
64
68
RootCommitBuilder,
65
69
)
66
70
import bzrlib.revision as _mod_revision
67
from bzrlib.store.revision.knit import KnitRevisionStore
68
71
from bzrlib.store.versioned import VersionedFileStore
69
72
from bzrlib.trace import (
70
73
mutter,
90
93
self._file_graph = graph.Graph(
91
94
repository._pack_collection.text_index.combined_index)
92
95
93
def _add_text_to_weave(self, file_id, new_lines, parents, nostore_sha):
94
return self.repository._pack_collection._add_text_to_weave(file_id,
95
self._new_revision_id, new_lines, parents, nostore_sha,
96
self.random_revid)
97
98
96
def _heads(self, file_id, revision_ids):
99
97
keys = [(file_id, revision_id) for revision_id in revision_ids]
100
98
return set([key[1] for key in self._file_graph.heads(keys)])
116
114
self._file_graph = graph.Graph(
117
115
repository._pack_collection.text_index.combined_index)
118
116
119
def _add_text_to_weave(self, file_id, new_lines, parents, nostore_sha):
120
return self.repository._pack_collection._add_text_to_weave(file_id,
121
self._new_revision_id, new_lines, parents, nostore_sha,
122
self.random_revid)
123
124
117
def _heads(self, file_id, revision_ids):
125
118
keys = [(file_id, revision_id) for revision_id in revision_ids]
126
119
return set([key[1] for key in self._file_graph.heads(keys)])
457
450
"""Create an AggregateIndex."""
458
451
self.index_to_pack = {}
459
452
self.combined_index = CombinedGraphIndex([])
460
self.knit_access = _PackAccess(self.index_to_pack)
453
self.data_access = _DirectPackAccess(self.index_to_pack)
454
self.add_callback = None
461
455
462
456
def replace_indices(self, index_to_pack, indices):
463
457
"""Replace the current mappings with fresh ones.
509
503
# allow writing: queue writes to a new index
510
504
self.add_index(index, pack)
511
505
# Updates the index to packs mapping as a side effect,
512
self.knit_access.set_writer(pack._writer, index, pack.access_tuple())
506
self.data_access.set_writer(pack._writer, index, pack.access_tuple())
513
507
self.add_callback = index.add_nodes
514
508
515
509
def clear(self):
516
510
"""Reset all the aggregate data to nothing."""
517
self.knit_access.set_writer(None, None, (None, None))
511
self.data_access.set_writer(None, None, (None, None))
518
512
self.index_to_pack.clear()
519
513
del self.combined_index._indices[:]
520
514
self.add_callback = None
530
524
if (self.add_callback is not None and
531
525
getattr(index, 'add_nodes', None) == self.add_callback):
532
526
self.add_callback = None
533
self.knit_access.set_writer(None, None, (None, None))
527
self.data_access.set_writer(None, None, (None, None))
534
528
535
529
536
530
class Packer(object):
589
583
return None
590
584
else:
591
585
self.revision_ids = frozenset(self.revision_ids)
586
self.revision_keys = frozenset((revid,) for revid in
587
self.revision_ids)
592
588
if pb is None:
593
589
self.pb = ui.ui_factory.nested_progress_bar()
594
590
else:
763
759
764
760
def _do_copy_nodes(self, nodes, index_map, writer, write_index, pb):
765
761
# for record verification
766
knit_data = _KnitData(None)
762
knit = KnitVersionedFiles(None, None)
767
763
# plan a readv on each source pack:
768
764
# group by pack
769
765
nodes = sorted(nodes)
795
791
izip(reader.iter_records(), pack_readv_requests):
796
792
raw_data = read_func(None)
797
793
# check the header only
798
df, _ = knit_data._parse_record_header(key[-1], raw_data)
794
df, _ = knit._parse_record_header(key, raw_data)
799
795
df.close()
800
796
pos, size = writer.add_bytes_record(raw_data, names)
801
797
write_index.add_node(key, eol_flag + "%d %d" % (pos, size))
824
820
def _do_copy_nodes_graph(self, index_map, writer, write_index,
825
821
output_lines, pb, readv_group_iter, total_items):
826
822
# for record verification
827
knit_data = _KnitData(None)
823
knit = KnitVersionedFiles(None, None)
828
824
# for line extraction when requested (inventories only)
829
825
if output_lines:
830
factory = knit.KnitPlainFactory()
826
factory = KnitPlainFactory()
831
827
record_index = 0
832
828
pb.update("Copied record", record_index, total_items)
833
829
for index, readv_vector, node_vector in readv_group_iter:
837
833
for (names, read_func), (key, eol_flag, references) in \
838
834
izip(reader.iter_records(), node_vector):
839
835
raw_data = read_func(None)
840
version_id = key[-1]
841
836
if output_lines:
842
837
# read the entire thing
843
content, _ = knit_data._parse_record(version_id, raw_data)
838
content, _ = knit._parse_record(key[-1], raw_data)
844
839
if len(references[-1]) == 0:
845
840
line_iterator = factory.get_fulltext_content(content)
846
841
else:
847
842
line_iterator = factory.get_linedelta_content(content)
848
843
for line in line_iterator:
849
yield line, version_id
844
yield line, key
850
845
else:
851
846
# check the header only
852
df, _ = knit_data._parse_record_header(version_id, raw_data)
847
df, _ = knit._parse_record_header(key, raw_data)
853
848
df.close()
854
849
pos, size = writer.add_bytes_record(raw_data, names)
855
850
write_index.add_node(key, eol_flag + "%d %d" % (pos, size), references)
911
906
"""Use up the inv_lines generator and setup a text key filter."""
912
907
repo = self._pack_collection.repo
913
908
fileid_revisions = repo._find_file_ids_from_xml_inventory_lines(
914
inv_lines, self.revision_ids)
909
inv_lines, self.revision_keys)
915
910
text_filter = []
916
911
for fileid, file_revids in fileid_revisions.iteritems():
917
912
text_filter.extend([(fileid, file_revid) for file_revid in file_revids])
1064
1059
self.new_pack.text_index,
1065
1060
('blank', ), 1,
1066
1061
add_nodes_callback=self.new_pack.text_index.add_nodes)
1067
knit_index = KnitGraphIndex(file_id_index,
1068
add_callback=file_id_index.add_nodes,
1069
deltas=True, parents=True)
1070
output_knit = knit.KnitVersionedFile('reconcile-texts',
1071
self._pack_collection.transport,
1072
index=knit_index,
1073
access_method=_PackAccess(
1074
{self.new_pack.text_index:self.new_pack.access_tuple()},
1075
(self.new_pack._writer, self.new_pack.text_index)),
1076
factory=knit.KnitPlainFactory())
1062
data_access = _DirectPackAccess(
1063
{self.new_pack.text_index:self.new_pack.access_tuple()})
1064
data_access.set_writer(self.new_pack._writer, self.new_pack.text_index,
1065
self.new_pack.access_tuple())
1066
output_texts = KnitVersionedFiles(
1067
_KnitGraphIndex(self.new_pack.text_index,
1068
add_callback=self.new_pack.text_index.add_nodes,
1069
deltas=True, parents=True, is_locked=repo.is_locked),
1070
data_access=data_access, max_delta_chain=200)
1077
1071
for key, parent_keys in bad_texts:
1078
1072
# We refer to the new pack to delta data being output.
1079
1073
# A possible improvement would be to catch errors on short reads
1086
1080
raise errors.BzrError('Mismatched key parent %r:%r' %
1087
1081
(key, parent_keys))
1088
1082
parents.append(parent_key[1])
1089
source_weave = repo.weave_store.get_weave(key[0], transaction)
1090
text_lines = source_weave.get_lines(key[1])
1091
# adapt the 'knit' to the current file_id.
1092
file_id_index = GraphIndexPrefixAdapter(
1093
self.new_pack.text_index,
1094
(key[0], ), 1,
1095
add_nodes_callback=self.new_pack.text_index.add_nodes)
1096
knit_index._graph_index = file_id_index
1097
knit_index._add_callback = file_id_index.add_nodes
1098
output_knit.add_lines_with_ghosts(
1099
key[1], parents, text_lines, random_id=True, check_content=False)
1083
text_lines = split_lines(repo.texts.get_record_stream(
1084
[key], 'unordered', True).next().get_bytes_as('fulltext'))
1085
output_texts.add_lines(key, parent_keys, text_lines,
1086
random_id=True, check_content=False)
1100
1087
# 5) check that nothing inserted has a reference outside the keyspace.
1101
1088
missing_text_keys = self.new_pack._external_compression_parents_of_texts()
1102
1089
if missing_text_keys:
1166
1153
self.text_index.add_index(pack.text_index, pack)
1167
1154
self.signature_index.add_index(pack.signature_index, pack)
1168
1155
1169
def _add_text_to_weave(self, file_id, revision_id, new_lines, parents,
1170
nostore_sha, random_revid):
1171
file_id_index = GraphIndexPrefixAdapter(
1172
self.text_index.combined_index,
1173
(file_id, ), 1,
1174
add_nodes_callback=self.text_index.add_callback)
1175
self.repo._text_knit._index._graph_index = file_id_index
1176
self.repo._text_knit._index._add_callback = file_id_index.add_nodes
1177
return self.repo._text_knit.add_lines_with_ghosts(
1178
revision_id, parents, new_lines, nostore_sha=nostore_sha,
1179
random_id=random_revid, check_content=False)[0:2]
1180
1181
1156
def all_packs(self):
1182
1157
"""Return a list of all the Pack objects this repository has.
1183
1158
1641
1616
self.signature_index.add_writable_index(self._new_pack.signature_index,
1642
1617
self._new_pack)
1643
1618
1644
# reused revision and signature knits may need updating
1645
#
1646
# "Hysterical raisins. client code in bzrlib grabs those knits outside
1647
# of write groups and then mutates it inside the write group."
1648
if self.repo._revision_knit is not None:
1649
self.repo._revision_knit._index._add_callback = \
1650
self.revision_index.add_callback
1651
if self.repo._signature_knit is not None:
1652
self.repo._signature_knit._index._add_callback = \
1653
self.signature_index.add_callback
1654
# create a reused knit object for text addition in commit.
1655
self.repo._text_knit = self.repo.weave_store.get_weave_or_empty(
1656
'all-texts', None)
1619
self.repo.inventories._index._add_callback = self.inventory_index.add_callback
1620
self.repo.revisions._index._add_callback = self.revision_index.add_callback
1621
self.repo.signatures._index._add_callback = self.signature_index.add_callback
1622
self.repo.texts._index._add_callback = self.text_index.add_callback
1657
1623
1658
1624
def _abort_write_group(self):
1659
1625
# FIXME: just drop the transient index.
1681
1647
self.repo._text_knit = None
1682
1648
1683
1649
1684
class KnitPackRevisionStore(KnitRevisionStore):
1685
"""An object to adapt access from RevisionStore's to use KnitPacks.
1686
1687
This class works by replacing the original RevisionStore.
1688
We need to do this because the KnitPackRevisionStore is less
1689
isolated in its layering - it uses services from the repo.
1690
"""
1691
1692
def __init__(self, repo, transport, revisionstore):
1693
"""Create a KnitPackRevisionStore on repo with revisionstore.
1694
1695
This will store its state in the Repository, use the
1696
indices to provide a KnitGraphIndex,
1697
and at the end of transactions write new indices.
1698
"""
1699
KnitRevisionStore.__init__(self, revisionstore.versioned_file_store)
1700
self.repo = repo
1701
self._serializer = revisionstore._serializer
1702
self.transport = transport
1703
1704
def get_revision_file(self, transaction):
1705
"""Get the revision versioned file object."""
1706
if getattr(self.repo, '_revision_knit', None) is not None:
1707
return self.repo._revision_knit
1708
self.repo._pack_collection.ensure_loaded()
1709
add_callback = self.repo._pack_collection.revision_index.add_callback
1710
# setup knit specific objects
1711
knit_index = KnitGraphIndex(
1712
self.repo._pack_collection.revision_index.combined_index,
1713
add_callback=add_callback)
1714
self.repo._revision_knit = knit.KnitVersionedFile(
1715
'revisions', self.transport.clone('..'),
1716
self.repo.control_files._file_mode,
1717
create=False,
1718
index=knit_index, delta=False, factory=knit.KnitPlainFactory(),
1719
access_method=self.repo._pack_collection.revision_index.knit_access)
1720
return self.repo._revision_knit
1721
1722
def get_signature_file(self, transaction):
1723
"""Get the signature versioned file object."""
1724
if getattr(self.repo, '_signature_knit', None) is not None:
1725
return self.repo._signature_knit
1726
self.repo._pack_collection.ensure_loaded()
1727
add_callback = self.repo._pack_collection.signature_index.add_callback
1728
# setup knit specific objects
1729
knit_index = KnitGraphIndex(
1730
self.repo._pack_collection.signature_index.combined_index,
1731
add_callback=add_callback, parents=False)
1732
self.repo._signature_knit = knit.KnitVersionedFile(
1733
'signatures', self.transport.clone('..'),
1734
self.repo.control_files._file_mode,
1735
create=False,
1736
index=knit_index, delta=False, factory=knit.KnitPlainFactory(),
1737
access_method=self.repo._pack_collection.signature_index.knit_access)
1738
return self.repo._signature_knit
1739
1740
1741
class KnitPackTextStore(VersionedFileStore):
1742
"""Presents a TextStore abstraction on top of packs.
1743
1744
This class works by replacing the original VersionedFileStore.
1745
We need to do this because the KnitPackRevisionStore is less
1746
isolated in its layering - it uses services from the repo and shares them
1747
with all the data written in a single write group.
1748
"""
1749
1750
def __init__(self, repo, transport, weavestore):
1751
"""Create a KnitPackTextStore on repo with weavestore.
1752
1753
This will store its state in the Repository, use the
1754
indices FileNames to provide a KnitGraphIndex,
1755
and at the end of transactions write new indices.
1756
"""
1757
# don't call base class constructor - it's not suitable.
1758
# no transient data stored in the transaction
1759
# cache.
1760
self._precious = False
1761
self.repo = repo
1762
self.transport = transport
1763
self.weavestore = weavestore
1764
# XXX for check() which isn't updated yet
1765
self._transport = weavestore._transport
1766
1767
def get_weave_or_empty(self, file_id, transaction):
1768
"""Get a 'Knit' backed by the .tix indices.
1769
1770
The transaction parameter is ignored.
1771
"""
1772
self.repo._pack_collection.ensure_loaded()
1773
add_callback = self.repo._pack_collection.text_index.add_callback
1774
# setup knit specific objects
1775
file_id_index = GraphIndexPrefixAdapter(
1776
self.repo._pack_collection.text_index.combined_index,
1777
(file_id, ), 1, add_nodes_callback=add_callback)
1778
knit_index = KnitGraphIndex(file_id_index,
1779
add_callback=file_id_index.add_nodes,
1780
deltas=True, parents=True)
1781
return knit.KnitVersionedFile('text:' + file_id,
1782
self.transport.clone('..'),
1783
None,
1784
index=knit_index,
1785
access_method=self.repo._pack_collection.text_index.knit_access,
1786
factory=knit.KnitPlainFactory())
1787
1788
get_weave = get_weave_or_empty
1789
1790
def __iter__(self):
1791
"""Generate a list of the fileids inserted, for use by check."""
1792
self.repo._pack_collection.ensure_loaded()
1793
ids = set()
1794
for index, key, value, refs in \
1795
self.repo._pack_collection.text_index.combined_index.iter_all_entries():
1796
ids.add(key[0])
1797
return iter(ids)
1798
1799
1800
class InventoryKnitThunk(object):
1801
"""An object to manage thunking get_inventory_weave to pack based knits."""
1802
1803
def __init__(self, repo, transport):
1804
"""Create an InventoryKnitThunk for repo at transport.
1805
1806
This will store its state in the Repository, use the
1807
indices FileNames to provide a KnitGraphIndex,
1808
and at the end of transactions write a new index..
1809
"""
1810
self.repo = repo
1811
self.transport = transport
1812
1813
def get_weave(self):
1814
"""Get a 'Knit' that contains inventory data."""
1815
self.repo._pack_collection.ensure_loaded()
1816
add_callback = self.repo._pack_collection.inventory_index.add_callback
1817
# setup knit specific objects
1818
knit_index = KnitGraphIndex(
1819
self.repo._pack_collection.inventory_index.combined_index,
1820
add_callback=add_callback, deltas=True, parents=True)
1821
return knit.KnitVersionedFile(
1822
'inventory', self.transport.clone('..'),
1823
self.repo.control_files._file_mode,
1824
create=False,
1825
index=knit_index, delta=True, factory=knit.KnitPlainFactory(),
1826
access_method=self.repo._pack_collection.inventory_index.knit_access)
1827
1828
1829
1650
class KnitPackRepository(KnitRepository):
1830
1651
"""Experimental graph-knit using repository."""
1831
1652
1832
def __init__(self, _format, a_bzrdir, control_files, _revision_store,
1833
control_store, text_store, _commit_builder_class, _serializer):
1834
KnitRepository.__init__(self, _format, a_bzrdir, control_files,
1835
_revision_store, control_store, text_store, _commit_builder_class,
1836
_serializer)
1653
def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
1654
_serializer):
1837
1655
index_transport = control_files._transport.clone('indices')
1838
1656
self._pack_collection = RepositoryPackCollection(self, control_files._transport,
1839
1657
index_transport,
1840
1658
control_files._transport.clone('upload'),
1841
1659
control_files._transport.clone('packs'))
1842
self._revision_store = KnitPackRevisionStore(self, index_transport, self._revision_store)
1843
self.weave_store = KnitPackTextStore(self, index_transport, self.weave_store)
1844
self._inv_thunk = InventoryKnitThunk(self, index_transport)
1660
KnitRepository.__init__(self, _format, a_bzrdir, control_files,
1661
_commit_builder_class, _serializer)
1662
self.inventories = KnitVersionedFiles(
1663
_KnitGraphIndex(self._pack_collection.inventory_index.combined_index,
1664
add_callback=self._pack_collection.inventory_index.add_callback,
1665
deltas=True, parents=True, is_locked=self.is_locked),
1666
data_access=self._pack_collection.inventory_index.data_access,
1667
max_delta_chain=200)
1668
self.revisions = KnitVersionedFiles(
1669
_KnitGraphIndex(self._pack_collection.revision_index.combined_index,
1670
add_callback=self._pack_collection.revision_index.add_callback,
1671
deltas=False, parents=True, is_locked=self.is_locked),
1672
data_access=self._pack_collection.revision_index.data_access,
1673
max_delta_chain=0)
1674
self.signatures = KnitVersionedFiles(
1675
_KnitGraphIndex(self._pack_collection.signature_index.combined_index,
1676
add_callback=self._pack_collection.signature_index.add_callback,
1677
deltas=False, parents=False, is_locked=self.is_locked),
1678
data_access=self._pack_collection.signature_index.data_access,
1679
max_delta_chain=0)
1680
self.texts = KnitVersionedFiles(
1681
_KnitGraphIndex(self._pack_collection.text_index.combined_index,
1682
add_callback=self._pack_collection.text_index.add_callback,
1683
deltas=True, parents=True, is_locked=self.is_locked),
1684
data_access=self._pack_collection.text_index.data_access,
1685
max_delta_chain=200)
1845
1686
# True when the repository object is 'write locked' (as opposed to the
1846
1687
# physical lock only taken out around changes to the pack-names list.)
1847
1688
# Another way to represent this would be a decorator around the control
1957
1798
def _commit_write_group(self):
1958
1799
return self._pack_collection._commit_write_group()
1959
1800
1960
def get_inventory_weave(self):
1961
return self._inv_thunk.get_weave()
1962
1963
1801
def get_transaction(self):
1964
1802
if self._write_lock_count:
1965
1803
return self._transaction
2059
1897
# External references are not supported in pack repositories yet.
2060
1898
supports_external_lookups = False
2061
1899
2062
def _get_control_store(self, repo_transport, control_files):
2063
"""Return the control store for this repository."""
2064
return VersionedFileStore(
2065
repo_transport,
2066
prefixed=False,
2067
file_mode=control_files._file_mode,
2068
versionedfile_class=knit.make_file_knit,
2069
versionedfile_kwargs={'factory': knit.KnitPlainFactory()},
2070
)
2071
2072
def _get_revision_store(self, repo_transport, control_files):
2073
"""See RepositoryFormat._get_revision_store()."""
2074
versioned_file_store = VersionedFileStore(
2075
repo_transport,
2076
file_mode=control_files._file_mode,
2077
prefixed=False,
2078
precious=True,
2079
versionedfile_class=knit.make_file_knit,
2080
versionedfile_kwargs={'delta': False,
2081
'factory': knit.KnitPlainFactory(),
2082
},
2083
escaped=False,
2084
)
2085
return KnitRevisionStore(versioned_file_store)
2086
2087
def _get_text_store(self, transport, control_files):
2088
"""See RepositoryFormat._get_text_store()."""
2089
return self._get_versioned_file_store('knits',
2090
transport,
2091
control_files,
2092
versionedfile_class=knit.make_file_knit,
2093
versionedfile_kwargs={
2094
'create_parent_dir': True,
2095
'delay_create': True,
2096
'dir_mode': control_files._dir_mode,
2097
},
2098
escaped=True)
2099
2100
1900
def initialize(self, a_bzrdir, shared=False):
2101
1901
"""Create a pack based repository.
2102
1902
2129
1929
repo_transport = a_bzrdir.get_repository_transport(None)
2130
1930
control_files = lockable_files.LockableFiles(repo_transport,
2131
1931
'lock', lockdir.LockDir)
2132
text_store = self._get_text_store(repo_transport, control_files)
2133
control_store = self._get_control_store(repo_transport, control_files)
2134
_revision_store = self._get_revision_store(repo_transport, control_files)
2135
1932
return self.repository_class(_format=self,
2136
1933
a_bzrdir=a_bzrdir,
2137
1934
control_files=control_files,
2138
_revision_store=_revision_store,
2139
control_store=control_store,
2140
text_store=text_store,
2141
1935
_commit_builder_class=self._commit_builder_class,
2142
1936
_serializer=self._serializer)
2143
1937
Loggerhead is a web-based interface for Breezy
Version: 2.0.1