13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
from __future__ import absolute_import
17
19
from bzrlib.lazy_import import lazy_import
18
20
lazy_import(globals(), """
22
from bzrlib.store import revision
23
from bzrlib.store.revision.knit import KnitRevisionStore
30
revision as _mod_revision,
39
39
from bzrlib.decorators import needs_read_lock, needs_write_lock
40
40
from bzrlib.repository import (
43
MetaDirRepositoryFormat,
47
import bzrlib.revision as _mod_revision
48
from bzrlib.store.versioned import VersionedFileStore
49
from bzrlib.trace import mutter, mutter_callsite
50
from bzrlib.util import bencode
43
RepositoryFormatMetaDir,
45
from bzrlib.vf_repository import (
46
InterSameDataRepository,
47
MetaDirVersionedFileRepository,
48
MetaDirVersionedFileRepositoryFormat,
49
VersionedFileCommitBuilder,
50
VersionedFileRootCommitBuilder,
52
from bzrlib import symbol_versioning
53
55
class _KnitParentsProvider(object):
58
60
def __repr__(self):
59
61
return 'KnitParentsProvider(%r)' % self._knit
61
def get_parents(self, revision_ids):
63
for revision_id in revision_ids:
63
def get_parent_map(self, keys):
64
"""See graph.StackedParentsProvider.get_parent_map"""
66
for revision_id in keys:
67
if revision_id is None:
68
raise ValueError('get_parent_map(None) is not valid')
64
69
if revision_id == _mod_revision.NULL_REVISION:
70
parent_map[revision_id] = ()
68
parents = self._knit.get_parents_with_ghosts(revision_id)
74
self._knit.get_parents_with_ghosts(revision_id))
69
75
except errors.RevisionNotPresent:
72
78
if len(parents) == 0:
73
parents = [_mod_revision.NULL_REVISION]
74
parents_list.append(parents)
78
class KnitRepository(MetaDirRepository):
79
parents = (_mod_revision.NULL_REVISION,)
80
parent_map[revision_id] = parents
84
class _KnitsParentsProvider(object):
86
def __init__(self, knit, prefix=()):
87
"""Create a parent provider for string keys mapped to tuple keys."""
92
return 'KnitsParentsProvider(%r)' % self._knit
94
def get_parent_map(self, keys):
95
"""See graph.StackedParentsProvider.get_parent_map"""
96
parent_map = self._knit.get_parent_map(
97
[self._prefix + (key,) for key in keys])
99
for key, parents in parent_map.items():
101
if len(parents) == 0:
102
parents = (_mod_revision.NULL_REVISION,)
104
parents = tuple(parent[-1] for parent in parents)
105
result[revid] = parents
106
for revision_id in keys:
107
if revision_id == _mod_revision.NULL_REVISION:
108
result[revision_id] = ()
112
class KnitRepository(MetaDirVersionedFileRepository):
79
113
"""Knit format repository."""
81
115
# These attributes are inherited from the Repository base class. Setting
85
119
_commit_builder_class = None
86
120
_serializer = None
88
def __init__(self, _format, a_bzrdir, control_files, _revision_store,
89
control_store, text_store, _commit_builder_class, _serializer):
90
MetaDirRepository.__init__(self, _format, a_bzrdir, control_files,
91
_revision_store, control_store, text_store)
122
def __init__(self, _format, a_bzrdir, control_files, _commit_builder_class,
124
super(KnitRepository, self).__init__(_format, a_bzrdir, control_files)
92
125
self._commit_builder_class = _commit_builder_class
93
126
self._serializer = _serializer
94
127
self._reconcile_fixes_text_parents = True
96
def _warn_if_deprecated(self):
97
# This class isn't deprecated
100
def _inventory_add_lines(self, inv_vf, revid, parents, lines, check_content):
101
return inv_vf.add_lines_with_ghosts(revid, parents, lines,
102
check_content=check_content)[0]
105
130
def _all_revision_ids(self):
106
131
"""See Repository.all_revision_ids()."""
107
# Knits get the revision graph from the index of the revision knit, so
108
# it's always possible even if they're on an unlistable transport.
109
return self._revision_store.all_revision_ids(self.get_transaction())
111
def fileid_involved_between_revs(self, from_revid, to_revid):
112
"""Find file_id(s) which are involved in the changes between revisions.
114
This determines the set of revisions which are involved, and then
115
finds all file ids affected by those revisions.
117
vf = self._get_revision_vf()
118
from_set = set(vf.get_ancestry(from_revid))
119
to_set = set(vf.get_ancestry(to_revid))
120
changed = to_set.difference(from_set)
121
return self._fileid_involved_by_set(changed)
123
def fileid_involved(self, last_revid=None):
124
"""Find all file_ids modified in the ancestry of last_revid.
126
:param last_revid: If None, last_revision() will be used.
129
changed = set(self.all_revision_ids())
131
changed = set(self.get_ancestry(last_revid))
134
return self._fileid_involved_by_set(changed)
137
def get_ancestry(self, revision_id, topo_sorted=True):
138
"""Return a list of revision-ids integrated by a revision.
140
This is topologically sorted, unless 'topo_sorted' is specified as
143
if _mod_revision.is_null(revision_id):
145
vf = self._get_revision_vf()
147
return [None] + vf.get_ancestry(revision_id, topo_sorted)
148
except errors.RevisionNotPresent:
149
raise errors.NoSuchRevision(self, revision_id)
152
def get_data_stream(self, revision_ids):
153
"""See Repository.get_data_stream."""
154
item_keys = self.item_keys_introduced_by(revision_ids)
155
for knit_kind, file_id, versions in item_keys:
157
if knit_kind == 'file':
158
name = ('file', file_id)
159
knit = self.weave_store.get_weave_or_empty(
160
file_id, self.get_transaction())
161
elif knit_kind == 'inventory':
162
knit = self.get_inventory_weave()
163
elif knit_kind == 'revisions':
164
knit = self._revision_store.get_revision_file(
165
self.get_transaction())
166
elif knit_kind == 'signatures':
167
knit = self._revision_store.get_signature_file(
168
self.get_transaction())
170
raise AssertionError('Unknown knit kind %r' % (knit_kind,))
171
yield name, _get_stream_as_bytes(knit, versions)
132
return [key[0] for key in self.revisions.keys()]
134
def _activate_new_inventory(self):
135
"""Put a replacement inventory.new into use as inventories."""
136
# Copy the content across
138
t.copy('inventory.new.kndx', 'inventory.kndx')
140
t.copy('inventory.new.knit', 'inventory.knit')
141
except errors.NoSuchFile:
142
# empty inventories knit
143
t.delete('inventory.knit')
144
# delete the temp inventory
145
t.delete('inventory.new.kndx')
147
t.delete('inventory.new.knit')
148
except errors.NoSuchFile:
149
# empty inventories knit
151
# Force index reload (sanity check)
152
self.inventories._index._reset_cache()
153
self.inventories.keys()
155
def _backup_inventory(self):
157
t.copy('inventory.kndx', 'inventory.backup.kndx')
158
t.copy('inventory.knit', 'inventory.backup.knit')
160
def _move_file_id(self, from_id, to_id):
161
t = self._transport.clone('knits')
162
from_rel_url = self.texts._index._mapper.map((from_id, None))
163
to_rel_url = self.texts._index._mapper.map((to_id, None))
164
# We expect both files to always exist in this case.
165
for suffix in ('.knit', '.kndx'):
166
t.rename(from_rel_url + suffix, to_rel_url + suffix)
168
def _remove_file_id(self, file_id):
169
t = self._transport.clone('knits')
170
rel_url = self.texts._index._mapper.map((file_id, None))
171
for suffix in ('.kndx', '.knit'):
173
t.delete(rel_url + suffix)
174
except errors.NoSuchFile:
177
def _temp_inventories(self):
178
result = self._format._get_inventories(self._transport, self,
180
# Reconciling when the output has no revisions would result in no
181
# writes - but we want to ensure there is an inventory for
182
# compatibility with older clients that don't lazy-load.
183
result.get_parent_map([('A',)])
174
187
def get_revision(self, revision_id):
176
189
revision_id = osutils.safe_revision_id(revision_id)
177
190
return self.get_revision_reconcile(revision_id)
180
def get_revision_graph(self, revision_id=None):
181
"""Return a dictionary containing the revision graph.
183
:param revision_id: The revision_id to get a graph from. If None, then
184
the entire revision graph is returned. This is a deprecated mode of
185
operation and will be removed in the future.
186
:return: a dictionary of revision_id->revision_parents_list.
188
if 'evil' in debug.debug_flags:
190
"get_revision_graph scales with size of history.")
191
# special case NULL_REVISION
192
if revision_id == _mod_revision.NULL_REVISION:
194
a_weave = self._get_revision_vf()
195
if revision_id is None:
196
return a_weave.get_graph()
197
if revision_id not in a_weave:
198
raise errors.NoSuchRevision(self, revision_id)
200
# add what can be reached from revision_id
201
return a_weave.get_graph([revision_id])
204
def get_revision_graph_with_ghosts(self, revision_ids=None):
205
"""Return a graph of the revisions with ghosts marked as applicable.
207
:param revision_ids: an iterable of revisions to graph or None for all.
208
:return: a Graph object with the graph reachable from revision_ids.
210
if 'evil' in debug.debug_flags:
212
"get_revision_graph_with_ghosts scales with size of history.")
213
result = deprecated_graph.Graph()
214
vf = self._get_revision_vf()
215
versions = set(vf.versions())
217
pending = set(self.all_revision_ids())
220
pending = set(revision_ids)
221
# special case NULL_REVISION
222
if _mod_revision.NULL_REVISION in pending:
223
pending.remove(_mod_revision.NULL_REVISION)
224
required = set(pending)
227
revision_id = pending.pop()
228
if not revision_id in versions:
229
if revision_id in required:
230
raise errors.NoSuchRevision(self, revision_id)
232
result.add_ghost(revision_id)
233
# mark it as done so we don't try for it again.
234
done.add(revision_id)
236
parent_ids = vf.get_parents_with_ghosts(revision_id)
237
for parent_id in parent_ids:
238
# is this queued or done ?
239
if (parent_id not in pending and
240
parent_id not in done):
242
pending.add(parent_id)
243
result.add_node(revision_id, parent_ids)
244
done.add(revision_id)
247
def _get_revision_vf(self):
248
""":return: a versioned file containing the revisions."""
249
vf = self._revision_store.get_revision_file(self.get_transaction())
252
def _get_history_vf(self):
253
"""Get a versionedfile whose history graph reflects all revisions.
255
For knit repositories, this is the revision knit.
257
return self._get_revision_vf()
192
def _refresh_data(self):
193
if not self.is_locked():
195
if self.is_in_write_group():
196
raise IsInWriteGroupError(self)
197
# Create a new transaction to force all knits to see the scope change.
198
# This is safe because we're outside a write group.
199
self.control_files._finish_transaction()
200
if self.is_write_locked():
201
self.control_files._set_write_transaction()
203
self.control_files._set_read_transaction()
259
205
@needs_write_lock
260
206
def reconcile(self, other=None, thorough=False):
263
209
reconciler = KnitReconciler(self, thorough=thorough)
264
210
reconciler.reconcile()
265
211
return reconciler
267
def revision_parents(self, revision_id):
268
return self._get_revision_vf().get_parents(revision_id)
270
213
def _make_parents_provider(self):
271
return _KnitParentsProvider(self._get_revision_vf())
273
def _find_inconsistent_revision_parents(self):
274
"""Find revisions with different parent lists in the revision object
275
and in the index graph.
277
:returns: an iterator yielding tuples of (revison-id, parents-in-index,
278
parents-in-revision).
280
assert self.is_locked()
281
vf = self._get_revision_vf()
282
for index_version in vf.versions():
283
parents_according_to_index = tuple(vf.get_parents_with_ghosts(
285
revision = self.get_revision(index_version)
286
parents_according_to_revision = tuple(revision.parent_ids)
287
if parents_according_to_index != parents_according_to_revision:
288
yield (index_version, parents_according_to_index,
289
parents_according_to_revision)
291
def _check_for_inconsistent_revision_parents(self):
292
inconsistencies = list(self._find_inconsistent_revision_parents())
294
raise errors.BzrCheckError(
295
"Revision knit has inconsistent parents.")
297
def revision_graph_can_have_wrong_parents(self):
298
# The revision.kndx could potentially claim a revision has a different
299
# parent to the revision text.
303
class RepositoryFormatKnit(MetaDirRepositoryFormat):
304
"""Bzr repository knit format (generalized).
214
return _KnitsParentsProvider(self.revisions)
217
class RepositoryFormatKnit(MetaDirVersionedFileRepositoryFormat):
218
"""Bzr repository knit format (generalized).
306
220
This repository format has:
307
221
- knits for file texts and inventory
323
237
_commit_builder_class = None
324
238
# Set this attribute in derived clases to control the _serializer that the
325
239
# repository objects will have passed to their constructor.
326
_serializer = xml5.serializer_v5
241
def _serializer(self):
242
return xml5.serializer_v5
327
243
# Knit based repositories handle ghosts reasonably well.
328
244
supports_ghosts = True
330
def _get_control_store(self, repo_transport, control_files):
331
"""Return the control store for this repository."""
332
return VersionedFileStore(
335
file_mode=control_files._file_mode,
336
versionedfile_class=knit.KnitVersionedFile,
337
versionedfile_kwargs={'factory':knit.KnitPlainFactory()},
340
def _get_revision_store(self, repo_transport, control_files):
341
"""See RepositoryFormat._get_revision_store()."""
342
versioned_file_store = VersionedFileStore(
344
file_mode=control_files._file_mode,
347
versionedfile_class=knit.KnitVersionedFile,
348
versionedfile_kwargs={'delta':False,
349
'factory':knit.KnitPlainFactory(),
353
return KnitRevisionStore(versioned_file_store)
355
def _get_text_store(self, transport, control_files):
356
"""See RepositoryFormat._get_text_store()."""
357
return self._get_versioned_file_store('knits',
360
versionedfile_class=knit.KnitVersionedFile,
361
versionedfile_kwargs={
362
'create_parent_dir':True,
364
'dir_mode':control_files._dir_mode,
245
# External lookups are not supported in this format.
246
supports_external_lookups = False
248
supports_chks = False
249
_fetch_order = 'topological'
250
_fetch_uses_deltas = True
252
supports_funky_characters = True
253
# The revision.kndx could potentially claim a revision has a different
254
# parent to the revision text.
255
revision_graph_can_have_wrong_parents = True
257
def _get_inventories(self, repo_transport, repo, name='inventory'):
258
mapper = versionedfile.ConstantMapper(name)
259
index = _mod_knit._KndxIndex(repo_transport, mapper,
260
repo.get_transaction, repo.is_write_locked, repo.is_locked)
261
access = _mod_knit._KnitKeyAccess(repo_transport, mapper)
262
return _mod_knit.KnitVersionedFiles(index, access, annotated=False)
264
def _get_revisions(self, repo_transport, repo):
265
mapper = versionedfile.ConstantMapper('revisions')
266
index = _mod_knit._KndxIndex(repo_transport, mapper,
267
repo.get_transaction, repo.is_write_locked, repo.is_locked)
268
access = _mod_knit._KnitKeyAccess(repo_transport, mapper)
269
return _mod_knit.KnitVersionedFiles(index, access, max_delta_chain=0,
272
def _get_signatures(self, repo_transport, repo):
273
mapper = versionedfile.ConstantMapper('signatures')
274
index = _mod_knit._KndxIndex(repo_transport, mapper,
275
repo.get_transaction, repo.is_write_locked, repo.is_locked)
276
access = _mod_knit._KnitKeyAccess(repo_transport, mapper)
277
return _mod_knit.KnitVersionedFiles(index, access, max_delta_chain=0,
280
def _get_texts(self, repo_transport, repo):
281
mapper = versionedfile.HashEscapedPrefixMapper()
282
base_transport = repo_transport.clone('knits')
283
index = _mod_knit._KndxIndex(base_transport, mapper,
284
repo.get_transaction, repo.is_write_locked, repo.is_locked)
285
access = _mod_knit._KnitKeyAccess(base_transport, mapper)
286
return _mod_knit.KnitVersionedFiles(index, access, max_delta_chain=200,
368
289
def initialize(self, a_bzrdir, shared=False):
369
290
"""Create a knit format 1 repository.
373
294
:param shared: If true the repository will be initialized as a shared
376
mutter('creating repository in %s.', a_bzrdir.transport.base)
297
trace.mutter('creating repository in %s.', a_bzrdir.transport.base)
379
300
utf8_files = [('format', self.get_format_string())]
381
302
self._upload_blank_content(a_bzrdir, dirs, files, utf8_files, shared)
382
303
repo_transport = a_bzrdir.get_repository_transport(None)
383
304
control_files = lockable_files.LockableFiles(repo_transport,
384
305
'lock', lockdir.LockDir)
385
control_store = self._get_control_store(repo_transport, control_files)
386
306
transaction = transactions.WriteTransaction()
387
# trigger a write of the inventory store.
388
control_store.get_weave_or_empty('inventory', transaction)
389
_revision_store = self._get_revision_store(repo_transport, control_files)
307
result = self.open(a_bzrdir=a_bzrdir, _found=True)
390
309
# the revision id here is irrelevant: it will not be stored, and cannot
392
_revision_store.has_revision_id('A', transaction)
393
_revision_store.get_signature_file(transaction)
394
return self.open(a_bzrdir=a_bzrdir, _found=True)
310
# already exist, we do this to create files on disk for older clients.
311
result.inventories.get_parent_map([('A',)])
312
result.revisions.get_parent_map([('A',)])
313
result.signatures.get_parent_map([('A',)])
315
self._run_post_repo_init_hooks(result, a_bzrdir, shared)
396
318
def open(self, a_bzrdir, _found=False, _override_transport=None):
397
319
"""See RepositoryFormat.open().
399
321
:param _override_transport: INTERNAL USE ONLY. Allows opening the
400
322
repository at a slightly different url
401
323
than normal. I.e. during 'upgrade'.
404
format = RepositoryFormat.find_format(a_bzrdir)
405
assert format.__class__ == self.__class__
326
format = RepositoryFormatMetaDir.find_format(a_bzrdir)
406
327
if _override_transport is not None:
407
328
repo_transport = _override_transport
409
330
repo_transport = a_bzrdir.get_repository_transport(None)
410
331
control_files = lockable_files.LockableFiles(repo_transport,
411
332
'lock', lockdir.LockDir)
412
text_store = self._get_text_store(repo_transport, control_files)
413
control_store = self._get_control_store(repo_transport, control_files)
414
_revision_store = self._get_revision_store(repo_transport, control_files)
415
return self.repository_class(_format=self,
333
repo = self.repository_class(_format=self,
416
334
a_bzrdir=a_bzrdir,
417
335
control_files=control_files,
418
_revision_store=_revision_store,
419
control_store=control_store,
420
text_store=text_store,
421
336
_commit_builder_class=self._commit_builder_class,
422
337
_serializer=self._serializer)
338
repo.revisions = self._get_revisions(repo_transport, repo)
339
repo.signatures = self._get_signatures(repo_transport, repo)
340
repo.inventories = self._get_inventories(repo_transport, repo)
341
repo.texts = self._get_texts(repo_transport, repo)
342
repo.chk_bytes = None
343
repo._transport = repo_transport
425
347
class RepositoryFormatKnit1(RepositoryFormatKnit):
476
398
repository_class = KnitRepository
477
_commit_builder_class = RootCommitBuilder
399
_commit_builder_class = VersionedFileRootCommitBuilder
478
400
rich_root_data = True
479
402
supports_tree_reference = True
480
_serializer = xml7.serializer_v7
404
def _serializer(self):
405
return xml7.serializer_v7
482
407
def _get_matching_bzrdir(self):
483
return bzrdir.format_registry.make_bzrdir('dirstate-with-subtree')
408
return controldir.format_registry.make_bzrdir('dirstate-with-subtree')
485
410
def _ignore_setting_bzrdir(self, format):
488
413
_matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
490
def check_conversion_target(self, target_format):
491
if not target_format.rich_root_data:
492
raise errors.BadConversionTarget(
493
'Does not support rich root data.', target_format)
494
if not getattr(target_format, 'supports_tree_reference', False):
495
raise errors.BadConversionTarget(
496
'Does not support nested trees', target_format)
498
def get_format_string(self):
416
def get_format_string(cls):
499
417
"""See RepositoryFormat.get_format_string()."""
500
418
return "Bazaar Knit Repository Format 3 (bzr 0.15)\n"
548
464
return "Knit repository format 4"
551
def _get_stream_as_bytes(knit, required_versions):
552
"""Generate a serialised data stream.
554
The format is a bencoding of a list. The first element of the list is a
555
string of the format signature, then each subsequent element is a list
556
corresponding to a record. Those lists contain:
563
:returns: a bencoded list.
565
knit_stream = knit.get_data_stream(required_versions)
566
format_signature, data_list, callable = knit_stream
568
data.append(format_signature)
569
for version, options, length, parents in data_list:
570
data.append([version, options, parents, callable(length)])
571
return bencode.bencode(data)
467
class InterKnitRepo(InterSameDataRepository):
468
"""Optimised code paths between Knit based repositories."""
471
def _get_repo_format_to_test(self):
472
return RepositoryFormatKnit1()
475
def is_compatible(source, target):
476
"""Be compatible with known Knit formats.
478
We don't test for the stores being of specific types because that
479
could lead to confusing results, and there is no need to be
483
are_knits = (isinstance(source._format, RepositoryFormatKnit) and
484
isinstance(target._format, RepositoryFormatKnit))
485
except AttributeError:
487
return are_knits and InterRepository._same_model(source, target)
490
def search_missing_revision_ids(self,
491
find_ghosts=True, revision_ids=None, if_present_ids=None,
493
"""See InterRepository.search_missing_revision_ids()."""
494
source_ids_set = self._present_source_revisions_for(
495
revision_ids, if_present_ids)
496
# source_ids is the worst possible case we may need to pull.
497
# now we want to filter source_ids against what we actually
498
# have in target, but don't try to check for existence where we know
499
# we do not have a revision as that would be pointless.
500
target_ids = set(self.target.all_revision_ids())
501
possibly_present_revisions = target_ids.intersection(source_ids_set)
502
actually_present_revisions = set(
503
self.target._eliminate_revisions_not_present(possibly_present_revisions))
504
required_revisions = source_ids_set.difference(actually_present_revisions)
505
if revision_ids is not None:
506
# we used get_ancestry to determine source_ids then we are assured all
507
# revisions referenced are present as they are installed in topological order.
508
# and the tip revision was validated by get_ancestry.
509
result_set = required_revisions
511
# if we just grabbed the possibly available ids, then
512
# we only have an estimate of whats available and need to validate
513
# that against the revision records.
515
self.source._eliminate_revisions_not_present(required_revisions))
516
if limit is not None:
517
topo_ordered = self.source.get_graph().iter_topo_order(result_set)
518
result_set = set(itertools.islice(topo_ordered, limit))
519
return self.source.revision_ids_to_search_result(result_set)
522
InterRepository.register_optimiser(InterKnitRepo)
added
removed
bzrlib/repofmt/knitrepo.py
