« back to all changes in this revision
Viewing changes to bzrlib/repofmt/pack_repo.py
- browse files at revision 3510
- compare with another revision
- compare with revision 334
- download diff
- download tarball
- view history from revision 3510
- Committer: Canonical.com Patch Queue Manager
- Date: 2008-06-25 10:36:36 UTC
- mfrom: (3350.6.12 versionedfiles)
- Revision ID: pqm@pqm.ubuntu.com-20080625103636-6kxh4e1gmyn82f50
(mbp for robertc) VersionedFiles refactoring
- files removed:
- bzrlib/store/revision/__init__.py
- bzrlib/tests/interversionedfile_implementations
- bzrlib/tests/revisionstore_implementations
- files modified:
- NEWS
added
removed
bzrlib/repofmt/pack_repo.py
34
34
CombinedGraphIndex,
35
35
GraphIndexPrefixAdapter,
36
36
)
37
from bzrlib.knit import KnitGraphIndex, _PackAccess, _KnitData
38
from bzrlib.osutils import rand_chars
37
from bzrlib.knit import (
38
KnitPlainFactory,
39
KnitVersionedFiles,
40
_KnitGraphIndex,
41
_DirectPackAccess,
42
)
43
from bzrlib.osutils import rand_chars, split_lines
39
44
from bzrlib.pack import ContainerWriter
40
45
from bzrlib.store import revision
41
46
from bzrlib import tsort
43
48
from bzrlib import (
44
49
bzrdir,
45
50
errors,
46
knit,
47
51
lockable_files,
48
52
lockdir,
49
53
osutils,
64
68
RootCommitBuilder,
65
69
)
66
70
import bzrlib.revision as _mod_revision
67
from bzrlib.store.revision.knit import KnitRevisionStore
68
71
from bzrlib.store.versioned import VersionedFileStore
69
72
from bzrlib.trace import (
70
73
mutter,
90
93
self._file_graph = graph.Graph(
91
94
repository._pack_collection.text_index.combined_index)
92
95
93
def _add_text_to_weave(self, file_id, new_lines, parents, nostore_sha):
94
return self.repository._pack_collection._add_text_to_weave(file_id,
95
self._new_revision_id, new_lines, parents, nostore_sha,
96
self.random_revid)
97
98
96
def _heads(self, file_id, revision_ids):
99
97
keys = [(file_id, revision_id) for revision_id in revision_ids]
100
98
return set([key[1] for key in self._file_graph.heads(keys)])
116
114
self._file_graph = graph.Graph(
117
115
repository._pack_collection.text_index.combined_index)
118
116
119
def _add_text_to_weave(self, file_id, new_lines, parents, nostore_sha):
120
return self.repository._pack_collection._add_text_to_weave(file_id,
121
self._new_revision_id, new_lines, parents, nostore_sha,
122
self.random_revid)
123
124
117
def _heads(self, file_id, revision_ids):
125
118
keys = [(file_id, revision_id) for revision_id in revision_ids]
126
119
return set([key[1] for key in self._file_graph.heads(keys)])
457
450
"""Create an AggregateIndex."""
458
451
self.index_to_pack = {}
459
452
self.combined_index = CombinedGraphIndex([])
460
self.knit_access = _PackAccess(self.index_to_pack)
453
self.data_access = _DirectPackAccess(self.index_to_pack)
454
self.add_callback = None
461
455
462
456
def replace_indices(self, index_to_pack, indices):
463
457
"""Replace the current mappings with fresh ones.
509
503
# allow writing: queue writes to a new index
510
504
self.add_index(index, pack)
511
505
# Updates the index to packs mapping as a side effect,
512
self.knit_access.set_writer(pack._writer, index, pack.access_tuple())
506
self.data_access.set_writer(pack._writer, index, pack.access_tuple())
513
507
self.add_callback = index.add_nodes
514
508
515
509
def clear(self):
516
510
"""Reset all the aggregate data to nothing."""
517
self.knit_access.set_writer(None, None, (None, None))
511
self.data_access.set_writer(None, None, (None, None))
518
512
self.index_to_pack.clear()
519
513
del self.combined_index._indices[:]
520
514
self.add_callback = None
530
524
if (self.add_callback is not None and
531
525
getattr(index, 'add_nodes', None) == self.add_callback):
532
526
self.add_callback = None
533
self.knit_access.set_writer(None, None, (None, None))
527
self.data_access.set_writer(None, None, (None, None))
534
528
535
529
536
530
class Packer(object):
589
583
return None
590
584
else:
591
585
self.revision_ids = frozenset(self.revision_ids)
586
self.revision_keys = frozenset((revid,) for revid in
587
self.revision_ids)
592
588
if pb is None:
593
589
self.pb = ui.ui_factory.nested_progress_bar()
594
590
else:
763
759
764
760
def _do_copy_nodes(self, nodes, index_map, writer, write_index, pb):
765
761
# for record verification
766
knit_data = _KnitData(None)
762
knit = KnitVersionedFiles(None, None)
767
763
# plan a readv on each source pack:
768
764
# group by pack
769
765
nodes = sorted(nodes)
795
791
izip(reader.iter_records(), pack_readv_requests):
796
792
raw_data = read_func(None)
797
793
# check the header only
798
df, _ = knit_data._parse_record_header(key[-1], raw_data)
794
df, _ = knit._parse_record_header(key, raw_data)
799
795
df.close()
800
796
pos, size = writer.add_bytes_record(raw_data, names)
801
797
write_index.add_node(key, eol_flag + "%d %d" % (pos, size))
824
820
def _do_copy_nodes_graph(self, index_map, writer, write_index,
825
821
output_lines, pb, readv_group_iter, total_items):
826
822
# for record verification
827
knit_data = _KnitData(None)
823
knit = KnitVersionedFiles(None, None)
828
824
# for line extraction when requested (inventories only)
829
825
if output_lines:
830
factory = knit.KnitPlainFactory()
826
factory = KnitPlainFactory()
831
827
record_index = 0
832
828
pb.update("Copied record", record_index, total_items)
833
829
for index, readv_vector, node_vector in readv_group_iter:
837
833
for (names, read_func), (key, eol_flag, references) in \
838
834
izip(reader.iter_records(), node_vector):
839
835
raw_data = read_func(None)
840
version_id = key[-1]
841
836
if output_lines:
842
837
# read the entire thing
843
content, _ = knit_data._parse_record(version_id, raw_data)
838
content, _ = knit._parse_record(key[-1], raw_data)
844
839
if len(references[-1]) == 0:
845
840
line_iterator = factory.get_fulltext_content(content)
846
841
else:
847
842
line_iterator = factory.get_linedelta_content(content)
848
843
for line in line_iterator:
849
yield line, version_id
844
yield line, key
850
845
else:
851
846
# check the header only
852
df, _ = knit_data._parse_record_header(version_id, raw_data)
847
df, _ = knit._parse_record_header(key, raw_data)
853
848
df.close()
854
849
pos, size = writer.add_bytes_record(raw_data, names)
855
850
write_index.add_node(key, eol_flag + "%d %d" % (pos, size), references)
911
906
"""Use up the inv_lines generator and setup a text key filter."""
912
907
repo = self._pack_collection.repo
913
908
fileid_revisions = repo._find_file_ids_from_xml_inventory_lines(
914
inv_lines, self.revision_ids)
909
inv_lines, self.revision_keys)
915
910
text_filter = []
916
911
for fileid, file_revids in fileid_revisions.iteritems():
917
912
text_filter.extend([(fileid, file_revid) for file_revid in file_revids])
1064
1059
self.new_pack.text_index,
1065
1060
('blank', ), 1,
1066
1061
add_nodes_callback=self.new_pack.text_index.add_nodes)
1067
knit_index = KnitGraphIndex(file_id_index,
1068
add_callback=file_id_index.add_nodes,
1069
deltas=True, parents=True)
1070
output_knit = knit.KnitVersionedFile('reconcile-texts',
1071
self._pack_collection.transport,
1072
index=knit_index,
1073
access_method=_PackAccess(
1074
{self.new_pack.text_index:self.new_pack.access_tuple()},
1075
(self.new_pack._writer, self.new_pack.text_index)),
1076
factory=knit.KnitPlainFactory())
1062
data_access = _DirectPackAccess(
1063
{self.new_pack.text_index:self.new_pack.access_tuple()})
1064
data_access.set_writer(self.new_pack._writer, self.new_pack.text_index,
1065
self.new_pack.access_tuple())
1066
output_texts = KnitVersionedFiles(
1067
_KnitGraphIndex(self.new_pack.text_index,
1068
add_callback=self.new_pack.text_index.add_nodes,
1069
deltas=True, parents=True, is_locked=repo.is_locked),
1070
data_access=data_access, max_delta_chain=200)
1077
1071
for key, parent_keys in bad_texts:
1078
1072
# We refer to the new pack to delta data being output.
1079
1073
# A possible improvement would be to catch errors on short reads
1086
1080
raise errors.BzrError('Mismatched key parent %r:%r' %
1087
1081
(key, parent_keys))
1088
1082
parents.append(parent_key[1])
1089
source_weave = repo.weave_store.get_weave(key[0], transaction)
1090
text_lines = source_weave.get_lines(key[1])
1091
# adapt the 'knit' to the current file_id.
1092
file_id_index = GraphIndexPrefixAdapter(
1093
self.new_pack.text_index,
1094
(key[0], ), 1,
1095
add_nodes_callback=self.new_pack.text_index.add_nodes)
1096
knit_index._graph_index = file_id_index
1097
knit_index._add_callback = file_id_index.add_nodes
1098
output_knit.add_lines_with_ghosts(
1099
key[1], parents, text_lines, random_id=True, check_content=False)
1083
text_lines = split_lines(repo.texts.get_record_stream(
1084
[key], 'unordered', True).next().get_bytes_as('fulltext'))
1085
output_texts.add_lines(key, parent_keys, text_lines,
1086
random_id=True, check_content=False)
1100
1087
# 5) check that nothing inserted has a reference outside the keyspace.
1101
1088
missing_text_keys = self.new_pack._external_compression_parents_of_texts()
1102
1089
if missing_text_keys:
1166
1153
self.text_index.add_index(pack.text_index, pack)
1167
1154
self.signature_index.add_index(pack.signature_index, pack)
1168
1155
1169
def _add_text_to_weave(self, file_id, revision_id, new_lines, parents,
1170
nostore_sha, random_revid):
1171
file_id_index = GraphIndexPrefixAdapter(
1172
self.text_index.combined_index,
1173
(file_id, ), 1,
1174
add_nodes_callback=self.text_index.add_callback)
1175
self.repo._text_knit._index._graph_index = file_id_index
1176
self.repo._text_knit._index._add_callback = file_id_index.add_nodes
1177
return self.repo._text_knit.add_lines_with_ghosts(
1178
revision_id, parents, new_lines, nostore_sha=nostore_sha,
1179
random_id=random_revid, check_content=False)[0:2]
1180
1181
1156
def all_packs(self):
1182
1157
"""Return a list of all the Pack objects this repository has.
1183
1158
1649
1624
self.signature_index.add_writable_index(self._new_pack.signature_index,
1650
1625
self._new_pack)
1651
1626
1652
# reused revision and signature knits may need updating
1653
#
1654
# "Hysterical raisins. client code in bzrlib grabs those knits outside
1655
# of write groups and then mutates it inside the write group."
1656
if self.repo._revision_knit is not None:
1657
self.repo._revision_knit._index._add_callback = \
1658
self.revision_index.add_callback
1659
if self.repo._signature_knit is not None:
1660
self.repo._signature_knit._index._add_callback = \
1661
self.signature_index.add_callback
1662
# create a reused knit object for text addition in commit.
1663
self.repo._text_knit = self.repo.weave_store.get_weave_or_empty(
1664
'all-texts', None)
1627
self.repo.inventories._index._add_callback = self.inventory_index.add_callback
1628
self.repo.revisions._index._add_callback = self.revision_index.add_callback
1629
self.repo.signatures._index._add_callback = self.signature_index.add_callback
1630
self.repo.texts._index._add_callback = self.text_index.add_callback
1665
1631
1666
1632
def _abort_write_group(self):
1667
1633
# FIXME: just drop the transient index.
1689
1655
self.repo._text_knit = None
1690
1656
1691
1657
1692
class KnitPackRevisionStore(KnitRevisionStore):
1693
"""An object to adapt access from RevisionStore's to use KnitPacks.
1694
1695
This class works by replacing the original RevisionStore.
1696
We need to do this because the KnitPackRevisionStore is less
1697
isolated in its layering - it uses services from the repo.
1698
"""
1699
1700
def __init__(self, repo, transport, revisionstore):
1701
"""Create a KnitPackRevisionStore on repo with revisionstore.
1702
1703
This will store its state in the Repository, use the
1704
indices to provide a KnitGraphIndex,
1705
and at the end of transactions write new indices.
1706
"""
1707
KnitRevisionStore.__init__(self, revisionstore.versioned_file_store)
1708
self.repo = repo
1709
self._serializer = revisionstore._serializer
1710
self.transport = transport
1711
1712
def get_revision_file(self, transaction):
1713
"""Get the revision versioned file object."""
1714
if getattr(self.repo, '_revision_knit', None) is not None:
1715
return self.repo._revision_knit
1716
self.repo._pack_collection.ensure_loaded()
1717
add_callback = self.repo._pack_collection.revision_index.add_callback
1718
# setup knit specific objects
1719
knit_index = KnitGraphIndex(
1720
self.repo._pack_collection.revision_index.combined_index,
1721
add_callback=add_callback)
1722
self.repo._revision_knit = knit.KnitVersionedFile(
1723
'revisions', self.transport.clone('..'),
1724
self.repo.bzrdir._get_file_mode(),
1725
create=False,
1726
index=knit_index, delta=False, factory=knit.KnitPlainFactory(),
1727
access_method=self.repo._pack_collection.revision_index.knit_access)
1728
return self.repo._revision_knit
1729
1730
def get_signature_file(self, transaction):
1731
"""Get the signature versioned file object."""
1732
if getattr(self.repo, '_signature_knit', None) is not None:
1733
return self.repo._signature_knit
1734
self.repo._pack_collection.ensure_loaded()
1735
add_callback = self.repo._pack_collection.signature_index.add_callback
1736
# setup knit specific objects
1737
knit_index = KnitGraphIndex(
1738
self.repo._pack_collection.signature_index.combined_index,
1739
add_callback=add_callback, parents=False)
1740
self.repo._signature_knit = knit.KnitVersionedFile(
1741
'signatures', self.transport.clone('..'),
1742
self.repo.bzrdir._get_file_mode(),
1743
create=False,
1744
index=knit_index, delta=False, factory=knit.KnitPlainFactory(),
1745
access_method=self.repo._pack_collection.signature_index.knit_access)
1746
return self.repo._signature_knit
1747
1748
1749
class KnitPackTextStore(VersionedFileStore):
1750
"""Presents a TextStore abstraction on top of packs.
1751
1752
This class works by replacing the original VersionedFileStore.
1753
We need to do this because the KnitPackRevisionStore is less
1754
isolated in its layering - it uses services from the repo and shares them
1755
with all the data written in a single write group.
1756
"""
1757
1758
def __init__(self, repo, transport, weavestore):
1759
"""Create a KnitPackTextStore on repo with weavestore.
1760
1761
This will store its state in the Repository, use the
1762
indices FileNames to provide a KnitGraphIndex,
1763
and at the end of transactions write new indices.
1764
"""
1765
# don't call base class constructor - it's not suitable.
1766
# no transient data stored in the transaction
1767
# cache.
1768
self._precious = False
1769
self.repo = repo
1770
self.transport = transport
1771
self.weavestore = weavestore
1772
# XXX for check() which isn't updated yet
1773
self._transport = weavestore._transport
1774
1775
def get_weave_or_empty(self, file_id, transaction):
1776
"""Get a 'Knit' backed by the .tix indices.
1777
1778
The transaction parameter is ignored.
1779
"""
1780
self.repo._pack_collection.ensure_loaded()
1781
add_callback = self.repo._pack_collection.text_index.add_callback
1782
# setup knit specific objects
1783
file_id_index = GraphIndexPrefixAdapter(
1784
self.repo._pack_collection.text_index.combined_index,
1785
(file_id, ), 1, add_nodes_callback=add_callback)
1786
knit_index = KnitGraphIndex(file_id_index,
1787
add_callback=file_id_index.add_nodes,
1788
deltas=True, parents=True)
1789
return knit.KnitVersionedFile('text:' + file_id,
1790
self.transport.clone('..'),
1791
None,
1792
index=knit_index,
1793
access_method=self.repo._pack_collection.text_index.knit_access,
1794
factory=knit.KnitPlainFactory())
1795
1796
get_weave = get_weave_or_empty
1797
1798
def __iter__(self):
1799
"""Generate a list of the fileids inserted, for use by check."""
1800
self.repo._pack_collection.ensure_loaded()
1801
ids = set()
1802
for index, key, value, refs in \
1803
self.repo._pack_collection.text_index.combined_index.iter_all_entries():
1804
ids.add(key[0])
1805
return iter(ids)
1806
1807
1808
class InventoryKnitThunk(object):
1809
"""An object to manage thunking get_inventory_weave to pack based knits."""
1810
1811
def __init__(self, repo, transport):
1812
"""Create an InventoryKnitThunk for repo at transport.
1813
1814
This will store its state in the Repository, use the
1815
indices FileNames to provide a KnitGraphIndex,
1816
and at the end of transactions write a new index..
1817
"""
1818
self.repo = repo
1819
self.transport = transport
1820
1821
def get_weave(self):
1822
"""Get a 'Knit' that contains inventory data."""
1823
self.repo._pack_collection.ensure_loaded()
1824
add_callback = self.repo._pack_collection.inventory_index.add_callback
1825
# setup knit specific objects
1826
knit_index = KnitGraphIndex(
1827
self.repo._pack_collection.inventory_index.combined_index,
1828
add_callback=add_callback, deltas=True, parents=True)
1829
return knit.KnitVersionedFile(
1830
'inventory', self.transport.clone('..'),
1831
self.repo.bzrdir._get_file_mode(),
1832
create=False,
1833
index=knit_index, delta=True, factory=knit.KnitPlainFactory(),
1834
access_method=self.repo._pack_collection.inventory_index.knit_access)
1835
1836
1837
1658
class KnitPackRepository(KnitRepository):
1838
"""Repository with knit objects stored inside pack containers."""
1839
1840
def __init__(self, _format, a_bzrdir, control_files, _revision_store,
1841
control_store, text_store, _commit_builder_class, _serializer):
1659
"""Repository with knit objects stored inside pack containers.
1660
1661
The layering for a KnitPackRepository is:
1662
1663
Graph | HPSS | Repository public layer |
1664
===================================================
1665
Tuple based apis below, string based, and key based apis above
1666
---------------------------------------------------
1667
KnitVersionedFiles
1668
Provides .texts, .revisions etc
1669
This adapts the N-tuple keys to physical knit records which only have a
1670
single string identifier (for historical reasons), which in older formats
1671
was always the revision_id, and in the mapped code for packs is always
1672
the last element of key tuples.
1673
---------------------------------------------------
1674
GraphIndex
1675
A separate GraphIndex is used for each of the
1676
texts/inventories/revisions/signatures contained within each individual
1677
pack file. The GraphIndex layer works in N-tuples and is unaware of any
1678
semantic value.
1679
===================================================
1680
1681
"""
1682
1683
def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
1684
_serializer):
1842
1685
KnitRepository.__init__(self, _format, a_bzrdir, control_files,
1843
_revision_store, control_store, text_store, _commit_builder_class,
1844
_serializer)
1686
_commit_builder_class, _serializer)
1845
1687
index_transport = self._transport.clone('indices')
1846
self._pack_collection = RepositoryPackCollection(self,
1847
self._transport,
1688
self._pack_collection = RepositoryPackCollection(self, self._transport,
1848
1689
index_transport,
1849
1690
self._transport.clone('upload'),
1850
1691
self._transport.clone('packs'))
1851
self._revision_store = KnitPackRevisionStore(self, index_transport, self._revision_store)
1852
self.weave_store = KnitPackTextStore(self, index_transport, self.weave_store)
1853
self._inv_thunk = InventoryKnitThunk(self, index_transport)
1692
self.inventories = KnitVersionedFiles(
1693
_KnitGraphIndex(self._pack_collection.inventory_index.combined_index,
1694
add_callback=self._pack_collection.inventory_index.add_callback,
1695
deltas=True, parents=True, is_locked=self.is_locked),
1696
data_access=self._pack_collection.inventory_index.data_access,
1697
max_delta_chain=200)
1698
self.revisions = KnitVersionedFiles(
1699
_KnitGraphIndex(self._pack_collection.revision_index.combined_index,
1700
add_callback=self._pack_collection.revision_index.add_callback,
1701
deltas=False, parents=True, is_locked=self.is_locked),
1702
data_access=self._pack_collection.revision_index.data_access,
1703
max_delta_chain=0)
1704
self.signatures = KnitVersionedFiles(
1705
_KnitGraphIndex(self._pack_collection.signature_index.combined_index,
1706
add_callback=self._pack_collection.signature_index.add_callback,
1707
deltas=False, parents=False, is_locked=self.is_locked),
1708
data_access=self._pack_collection.signature_index.data_access,
1709
max_delta_chain=0)
1710
self.texts = KnitVersionedFiles(
1711
_KnitGraphIndex(self._pack_collection.text_index.combined_index,
1712
add_callback=self._pack_collection.text_index.add_callback,
1713
deltas=True, parents=True, is_locked=self.is_locked),
1714
data_access=self._pack_collection.text_index.data_access,
1715
max_delta_chain=200)
1854
1716
# True when the repository object is 'write locked' (as opposed to the
1855
1717
# physical lock only taken out around changes to the pack-names list.)
1856
1718
# Another way to represent this would be a decorator around the control
1968
1830
def _commit_write_group(self):
1969
1831
return self._pack_collection._commit_write_group()
1970
1832
1971
def get_inventory_weave(self):
1972
return self._inv_thunk.get_weave()
1973
1974
1833
def get_transaction(self):
1975
1834
if self._write_lock_count:
1976
1835
return self._transaction
2070
1929
# External references are not supported in pack repositories yet.
2071
1930
supports_external_lookups = False
2072
1931
2073
def _get_control_store(self, repo_transport, control_files):
2074
"""Return the control store for this repository."""
2075
return VersionedFileStore(
2076
repo_transport,
2077
prefixed=False,
2078
file_mode=control_files._file_mode,
2079
versionedfile_class=knit.make_file_knit,
2080
versionedfile_kwargs={'factory': knit.KnitPlainFactory()},
2081
)
2082
2083
def _get_revision_store(self, repo_transport, control_files):
2084
"""See RepositoryFormat._get_revision_store()."""
2085
versioned_file_store = VersionedFileStore(
2086
repo_transport,
2087
file_mode=control_files._file_mode,
2088
prefixed=False,
2089
precious=True,
2090
versionedfile_class=knit.make_file_knit,
2091
versionedfile_kwargs={'delta': False,
2092
'factory': knit.KnitPlainFactory(),
2093
},
2094
escaped=True,
2095
)
2096
return KnitRevisionStore(versioned_file_store)
2097
2098
def _get_text_store(self, transport, control_files):
2099
"""See RepositoryFormat._get_text_store()."""
2100
return self._get_versioned_file_store('knits',
2101
transport,
2102
control_files,
2103
versionedfile_class=knit.make_file_knit,
2104
versionedfile_kwargs={
2105
'create_parent_dir': True,
2106
'delay_create': True,
2107
'dir_mode': control_files._dir_mode,
2108
},
2109
escaped=True)
2110
2111
1932
def initialize(self, a_bzrdir, shared=False):
2112
1933
"""Create a pack based repository.
2113
1934
2139
1960
else:
2140
1961
repo_transport = a_bzrdir.get_repository_transport(None)
2141
1962
control_files = lockable_files.LockableFiles(repo_transport,
2142
'lock', lockdir.LockDir)
2143
text_store = self._get_text_store(repo_transport, control_files)
2144
control_store = self._get_control_store(repo_transport, control_files)
2145
_revision_store = self._get_revision_store(repo_transport, control_files)
1963
'lock', lockdir.LockDir)
2146
1964
return self.repository_class(_format=self,
2147
1965
a_bzrdir=a_bzrdir,
2148
1966
control_files=control_files,
2149
_revision_store=_revision_store,
2150
control_store=control_store,
2151
text_store=text_store,
2152
1967
_commit_builder_class=self._commit_builder_class,
2153
1968
_serializer=self._serializer)
2154
1969
Loggerhead is a web-based interface for Breezy
Version: 2.0.1