1
# Copyright (C) 2005-2011 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
from bzrlib.lazy_import import lazy_import
18
lazy_import(globals(), """
37
revision as _mod_revision,
42
from bzrlib.bundle import serializer
43
from bzrlib.recordcounter import RecordCounter
44
from bzrlib.revisiontree import RevisionTree
45
from bzrlib.store.versioned import VersionedFileStore
46
from bzrlib.testament import Testament
55
from bzrlib.decorators import needs_read_lock, needs_write_lock, only_raises
56
from bzrlib.inter import InterObject
57
from bzrlib.inventory import (
63
from bzrlib.lock import _RelockDebugMixin, LogicalLockResult
64
from bzrlib.trace import (
65
log_exception_quietly, note, mutter, mutter_callsite, warning)
68
# Old formats display a warning, but only once
69
_deprecation_warning_done = False
72
class IsInWriteGroupError(errors.InternalBzrError):
74
_fmt = "May not refresh_data of repo %(repo)s while in a write group."
76
def __init__(self, repo):
77
errors.InternalBzrError.__init__(self, repo=repo)
80
class CommitBuilder(object):
81
"""Provides an interface to build up a commit.
83
This allows describing a tree to be committed without needing to
84
know the internals of the format of the repository.
87
# all clients should supply tree roots.
88
record_root_entry = True
89
# the default CommitBuilder does not manage trees whose root is versioned.
90
_versioned_root = False
91
# this commit builder supports the record_entry_contents interface
92
supports_record_entry_contents = True
94
def __init__(self, repository, parents, config, timestamp=None,
95
timezone=None, committer=None, revprops=None,
97
"""Initiate a CommitBuilder.
99
:param repository: Repository to commit to.
100
:param parents: Revision ids of the parents of the new revision.
101
:param timestamp: Optional timestamp recorded for commit.
102
:param timezone: Optional timezone for timestamp.
103
:param committer: Optional committer to set for commit.
104
:param revprops: Optional dictionary of revision properties.
105
:param revision_id: Optional revision id.
107
self._config = config
109
if committer is None:
110
self._committer = self._config.username()
111
elif not isinstance(committer, unicode):
112
self._committer = committer.decode() # throw if non-ascii
114
self._committer = committer
116
self.new_inventory = Inventory(None)
117
self._new_revision_id = revision_id
118
self.parents = parents
119
self.repository = repository
122
if revprops is not None:
123
self._validate_revprops(revprops)
124
self._revprops.update(revprops)
126
if timestamp is None:
127
timestamp = time.time()
128
# Restrict resolution to 1ms
129
self._timestamp = round(timestamp, 3)
132
self._timezone = osutils.local_time_offset()
134
self._timezone = int(timezone)
136
self._generate_revision_if_needed()
137
self.__heads = graph.HeadsCache(repository.get_graph()).heads
138
self._basis_delta = []
139
# API compatibility, older code that used CommitBuilder did not call
140
# .record_delete(), which means the delta that is computed would not be
141
# valid. Callers that will call record_delete() should call
142
# .will_record_deletes() to indicate that.
143
self._recording_deletes = False
144
# memo'd check for no-op commits.
145
self._any_changes = False
147
def any_changes(self):
148
"""Return True if any entries were changed.
150
This includes merge-only changes. It is the core for the --unchanged
153
:return: True if any changes have occured.
155
return self._any_changes
157
def _validate_unicode_text(self, text, context):
158
"""Verify things like commit messages don't have bogus characters."""
160
raise ValueError('Invalid value for %s: %r' % (context, text))
162
def _validate_revprops(self, revprops):
163
for key, value in revprops.iteritems():
164
# We know that the XML serializers do not round trip '\r'
165
# correctly, so refuse to accept them
166
if not isinstance(value, basestring):
167
raise ValueError('revision property (%s) is not a valid'
168
' (unicode) string: %r' % (key, value))
169
self._validate_unicode_text(value,
170
'revision property (%s)' % (key,))
172
def _ensure_fallback_inventories(self):
173
"""Ensure that appropriate inventories are available.
175
This only applies to repositories that are stacked, and is about
176
enusring the stacking invariants. Namely, that for any revision that is
177
present, we either have all of the file content, or we have the parent
178
inventory and the delta file content.
180
if not self.repository._fallback_repositories:
182
if not self.repository._format.supports_chks:
183
raise errors.BzrError("Cannot commit directly to a stacked branch"
184
" in pre-2a formats. See "
185
"https://bugs.launchpad.net/bzr/+bug/375013 for details.")
186
# This is a stacked repo, we need to make sure we have the parent
187
# inventories for the parents.
188
parent_keys = [(p,) for p in self.parents]
189
parent_map = self.repository.inventories._index.get_parent_map(parent_keys)
190
missing_parent_keys = set([pk for pk in parent_keys
191
if pk not in parent_map])
192
fallback_repos = list(reversed(self.repository._fallback_repositories))
193
missing_keys = [('inventories', pk[0])
194
for pk in missing_parent_keys]
196
while missing_keys and fallback_repos:
197
fallback_repo = fallback_repos.pop()
198
source = fallback_repo._get_source(self.repository._format)
199
sink = self.repository._get_sink()
200
stream = source.get_stream_for_missing_keys(missing_keys)
201
missing_keys = sink.insert_stream_without_locking(stream,
202
self.repository._format)
204
raise errors.BzrError('Unable to fill in parent inventories for a'
207
def commit(self, message):
208
"""Make the actual commit.
210
:return: The revision id of the recorded revision.
212
self._validate_unicode_text(message, 'commit message')
213
rev = _mod_revision.Revision(
214
timestamp=self._timestamp,
215
timezone=self._timezone,
216
committer=self._committer,
218
inventory_sha1=self.inv_sha1,
219
revision_id=self._new_revision_id,
220
properties=self._revprops)
221
rev.parent_ids = self.parents
222
self.repository.add_revision(self._new_revision_id, rev,
223
self.new_inventory, self._config)
224
self._ensure_fallback_inventories()
225
self.repository.commit_write_group()
226
return self._new_revision_id
229
"""Abort the commit that is being built.
231
self.repository.abort_write_group()
233
def revision_tree(self):
234
"""Return the tree that was just committed.
236
After calling commit() this can be called to get a RevisionTree
237
representing the newly committed tree. This is preferred to
238
calling Repository.revision_tree() because that may require
239
deserializing the inventory, while we already have a copy in
242
if self.new_inventory is None:
243
self.new_inventory = self.repository.get_inventory(
244
self._new_revision_id)
245
return RevisionTree(self.repository, self.new_inventory,
246
self._new_revision_id)
248
def finish_inventory(self):
249
"""Tell the builder that the inventory is finished.
251
:return: The inventory id in the repository, which can be used with
252
repository.get_inventory.
254
if self.new_inventory is None:
255
# an inventory delta was accumulated without creating a new
257
basis_id = self.basis_delta_revision
258
# We ignore the 'inventory' returned by add_inventory_by_delta
259
# because self.new_inventory is used to hint to the rest of the
260
# system what code path was taken
261
self.inv_sha1, _ = self.repository.add_inventory_by_delta(
262
basis_id, self._basis_delta, self._new_revision_id,
265
if self.new_inventory.root is None:
266
raise AssertionError('Root entry should be supplied to'
267
' record_entry_contents, as of bzr 0.10.')
268
self.new_inventory.add(InventoryDirectory(ROOT_ID, '', None))
269
self.new_inventory.revision_id = self._new_revision_id
270
self.inv_sha1 = self.repository.add_inventory(
271
self._new_revision_id,
275
return self._new_revision_id
277
def _gen_revision_id(self):
278
"""Return new revision-id."""
279
return generate_ids.gen_revision_id(self._committer, self._timestamp)
281
def _generate_revision_if_needed(self):
282
"""Create a revision id if None was supplied.
284
If the repository can not support user-specified revision ids
285
they should override this function and raise CannotSetRevisionId
286
if _new_revision_id is not None.
288
:raises: CannotSetRevisionId
290
if self._new_revision_id is None:
291
self._new_revision_id = self._gen_revision_id()
292
self.random_revid = True
294
self.random_revid = False
296
def _heads(self, file_id, revision_ids):
297
"""Calculate the graph heads for revision_ids in the graph of file_id.
299
This can use either a per-file graph or a global revision graph as we
300
have an identity relationship between the two graphs.
302
return self.__heads(revision_ids)
304
def _check_root(self, ie, parent_invs, tree):
305
"""Helper for record_entry_contents.
307
:param ie: An entry being added.
308
:param parent_invs: The inventories of the parent revisions of the
310
:param tree: The tree that is being committed.
312
# In this revision format, root entries have no knit or weave When
313
# serializing out to disk and back in root.revision is always
315
ie.revision = self._new_revision_id
317
def _require_root_change(self, tree):
318
"""Enforce an appropriate root object change.
320
This is called once when record_iter_changes is called, if and only if
321
the root was not in the delta calculated by record_iter_changes.
323
:param tree: The tree which is being committed.
325
if len(self.parents) == 0:
326
raise errors.RootMissing()
327
entry = entry_factory['directory'](tree.path2id(''), '',
329
entry.revision = self._new_revision_id
330
self._basis_delta.append(('', '', entry.file_id, entry))
332
def _get_delta(self, ie, basis_inv, path):
333
"""Get a delta against the basis inventory for ie."""
334
if ie.file_id not in basis_inv:
336
result = (None, path, ie.file_id, ie)
337
self._basis_delta.append(result)
339
elif ie != basis_inv[ie.file_id]:
341
# TODO: avoid tis id2path call.
342
result = (basis_inv.id2path(ie.file_id), path, ie.file_id, ie)
343
self._basis_delta.append(result)
349
def get_basis_delta(self):
350
"""Return the complete inventory delta versus the basis inventory.
352
This has been built up with the calls to record_delete and
353
record_entry_contents. The client must have already called
354
will_record_deletes() to indicate that they will be generating a
357
:return: An inventory delta, suitable for use with apply_delta, or
358
Repository.add_inventory_by_delta, etc.
360
if not self._recording_deletes:
361
raise AssertionError("recording deletes not activated.")
362
return self._basis_delta
364
def record_delete(self, path, file_id):
365
"""Record that a delete occured against a basis tree.
367
This is an optional API - when used it adds items to the basis_delta
368
being accumulated by the commit builder. It cannot be called unless the
369
method will_record_deletes() has been called to inform the builder that
370
a delta is being supplied.
372
:param path: The path of the thing deleted.
373
:param file_id: The file id that was deleted.
375
if not self._recording_deletes:
376
raise AssertionError("recording deletes not activated.")
377
delta = (path, None, file_id, None)
378
self._basis_delta.append(delta)
379
self._any_changes = True
382
def will_record_deletes(self):
383
"""Tell the commit builder that deletes are being notified.
385
This enables the accumulation of an inventory delta; for the resulting
386
commit to be valid, deletes against the basis MUST be recorded via
387
builder.record_delete().
389
self._recording_deletes = True
391
basis_id = self.parents[0]
393
basis_id = _mod_revision.NULL_REVISION
394
self.basis_delta_revision = basis_id
396
def record_entry_contents(self, ie, parent_invs, path, tree,
398
"""Record the content of ie from tree into the commit if needed.
400
Side effect: sets ie.revision when unchanged
402
:param ie: An inventory entry present in the commit.
403
:param parent_invs: The inventories of the parent revisions of the
405
:param path: The path the entry is at in the tree.
406
:param tree: The tree which contains this entry and should be used to
408
:param content_summary: Summary data from the tree about the paths
409
content - stat, length, exec, sha/link target. This is only
410
accessed when the entry has a revision of None - that is when it is
411
a candidate to commit.
412
:return: A tuple (change_delta, version_recorded, fs_hash).
413
change_delta is an inventory_delta change for this entry against
414
the basis tree of the commit, or None if no change occured against
416
version_recorded is True if a new version of the entry has been
417
recorded. For instance, committing a merge where a file was only
418
changed on the other side will return (delta, False).
419
fs_hash is either None, or the hash details for the path (currently
420
a tuple of the contents sha1 and the statvalue returned by
421
tree.get_file_with_stat()).
423
if self.new_inventory.root is None:
424
if ie.parent_id is not None:
425
raise errors.RootMissing()
426
self._check_root(ie, parent_invs, tree)
427
if ie.revision is None:
428
kind = content_summary[0]
430
# ie is carried over from a prior commit
432
# XXX: repository specific check for nested tree support goes here - if
433
# the repo doesn't want nested trees we skip it ?
434
if (kind == 'tree-reference' and
435
not self.repository._format.supports_tree_reference):
436
# mismatch between commit builder logic and repository:
437
# this needs the entry creation pushed down into the builder.
438
raise NotImplementedError('Missing repository subtree support.')
439
self.new_inventory.add(ie)
441
# TODO: slow, take it out of the inner loop.
443
basis_inv = parent_invs[0]
445
basis_inv = Inventory(root_id=None)
447
# ie.revision is always None if the InventoryEntry is considered
448
# for committing. We may record the previous parents revision if the
449
# content is actually unchanged against a sole head.
450
if ie.revision is not None:
451
if not self._versioned_root and path == '':
452
# repositories that do not version the root set the root's
453
# revision to the new commit even when no change occurs (more
454
# specifically, they do not record a revision on the root; and
455
# the rev id is assigned to the root during deserialisation -
456
# this masks when a change may have occurred against the basis.
457
# To match this we always issue a delta, because the revision
458
# of the root will always be changing.
459
if ie.file_id in basis_inv:
460
delta = (basis_inv.id2path(ie.file_id), path,
464
delta = (None, path, ie.file_id, ie)
465
self._basis_delta.append(delta)
466
return delta, False, None
468
# we don't need to commit this, because the caller already
469
# determined that an existing revision of this file is
470
# appropriate. If it's not being considered for committing then
471
# it and all its parents to the root must be unaltered so
472
# no-change against the basis.
473
if ie.revision == self._new_revision_id:
474
raise AssertionError("Impossible situation, a skipped "
475
"inventory entry (%r) claims to be modified in this "
476
"commit (%r).", (ie, self._new_revision_id))
477
return None, False, None
478
# XXX: Friction: parent_candidates should return a list not a dict
479
# so that we don't have to walk the inventories again.
480
parent_candiate_entries = ie.parent_candidates(parent_invs)
481
head_set = self._heads(ie.file_id, parent_candiate_entries.keys())
483
for inv in parent_invs:
484
if ie.file_id in inv:
485
old_rev = inv[ie.file_id].revision
486
if old_rev in head_set:
487
heads.append(inv[ie.file_id].revision)
488
head_set.remove(inv[ie.file_id].revision)
491
# now we check to see if we need to write a new record to the
493
# We write a new entry unless there is one head to the ancestors, and
494
# the kind-derived content is unchanged.
496
# Cheapest check first: no ancestors, or more the one head in the
497
# ancestors, we write a new node.
501
# There is a single head, look it up for comparison
502
parent_entry = parent_candiate_entries[heads[0]]
503
# if the non-content specific data has changed, we'll be writing a
505
if (parent_entry.parent_id != ie.parent_id or
506
parent_entry.name != ie.name):
508
# now we need to do content specific checks:
510
# if the kind changed the content obviously has
511
if kind != parent_entry.kind:
513
# Stat cache fingerprint feedback for the caller - None as we usually
514
# don't generate one.
517
if content_summary[2] is None:
518
raise ValueError("Files must not have executable = None")
520
# We can't trust a check of the file length because of content
522
if (# if the exec bit has changed we have to store:
523
parent_entry.executable != content_summary[2]):
525
elif parent_entry.text_sha1 == content_summary[3]:
526
# all meta and content is unchanged (using a hash cache
527
# hit to check the sha)
528
ie.revision = parent_entry.revision
529
ie.text_size = parent_entry.text_size
530
ie.text_sha1 = parent_entry.text_sha1
531
ie.executable = parent_entry.executable
532
return self._get_delta(ie, basis_inv, path), False, None
534
# Either there is only a hash change(no hash cache entry,
535
# or same size content change), or there is no change on
537
# Provide the parent's hash to the store layer, so that the
538
# content is unchanged we will not store a new node.
539
nostore_sha = parent_entry.text_sha1
541
# We want to record a new node regardless of the presence or
542
# absence of a content change in the file.
544
ie.executable = content_summary[2]
545
file_obj, stat_value = tree.get_file_with_stat(ie.file_id, path)
547
text = file_obj.read()
551
ie.text_sha1, ie.text_size = self._add_text_to_weave(
552
ie.file_id, text, heads, nostore_sha)
553
# Let the caller know we generated a stat fingerprint.
554
fingerprint = (ie.text_sha1, stat_value)
555
except errors.ExistingContent:
556
# Turns out that the file content was unchanged, and we were
557
# only going to store a new node if it was changed. Carry over
559
ie.revision = parent_entry.revision
560
ie.text_size = parent_entry.text_size
561
ie.text_sha1 = parent_entry.text_sha1
562
ie.executable = parent_entry.executable
563
return self._get_delta(ie, basis_inv, path), False, None
564
elif kind == 'directory':
566
# all data is meta here, nothing specific to directory, so
568
ie.revision = parent_entry.revision
569
return self._get_delta(ie, basis_inv, path), False, None
570
self._add_text_to_weave(ie.file_id, '', heads, None)
571
elif kind == 'symlink':
572
current_link_target = content_summary[3]
574
# symlink target is not generic metadata, check if it has
576
if current_link_target != parent_entry.symlink_target:
579
# unchanged, carry over.
580
ie.revision = parent_entry.revision
581
ie.symlink_target = parent_entry.symlink_target
582
return self._get_delta(ie, basis_inv, path), False, None
583
ie.symlink_target = current_link_target
584
self._add_text_to_weave(ie.file_id, '', heads, None)
585
elif kind == 'tree-reference':
587
if content_summary[3] != parent_entry.reference_revision:
590
# unchanged, carry over.
591
ie.reference_revision = parent_entry.reference_revision
592
ie.revision = parent_entry.revision
593
return self._get_delta(ie, basis_inv, path), False, None
594
ie.reference_revision = content_summary[3]
595
if ie.reference_revision is None:
596
raise AssertionError("invalid content_summary for nested tree: %r"
597
% (content_summary,))
598
self._add_text_to_weave(ie.file_id, '', heads, None)
600
raise NotImplementedError('unknown kind')
601
ie.revision = self._new_revision_id
602
self._any_changes = True
603
return self._get_delta(ie, basis_inv, path), True, fingerprint
605
def record_iter_changes(self, tree, basis_revision_id, iter_changes,
606
_entry_factory=entry_factory):
607
"""Record a new tree via iter_changes.
609
:param tree: The tree to obtain text contents from for changed objects.
610
:param basis_revision_id: The revision id of the tree the iter_changes
611
has been generated against. Currently assumed to be the same
612
as self.parents[0] - if it is not, errors may occur.
613
:param iter_changes: An iter_changes iterator with the changes to apply
614
to basis_revision_id. The iterator must not include any items with
615
a current kind of None - missing items must be either filtered out
616
or errored-on beefore record_iter_changes sees the item.
617
:param _entry_factory: Private method to bind entry_factory locally for
619
:return: A generator of (file_id, relpath, fs_hash) tuples for use with
622
# Create an inventory delta based on deltas between all the parents and
623
# deltas between all the parent inventories. We use inventory delta's
624
# between the inventory objects because iter_changes masks
625
# last-changed-field only changes.
627
# file_id -> change map, change is fileid, paths, changed, versioneds,
628
# parents, names, kinds, executables
630
# {file_id -> revision_id -> inventory entry, for entries in parent
631
# trees that are not parents[0]
635
revtrees = list(self.repository.revision_trees(self.parents))
636
except errors.NoSuchRevision:
637
# one or more ghosts, slow path.
639
for revision_id in self.parents:
641
revtrees.append(self.repository.revision_tree(revision_id))
642
except errors.NoSuchRevision:
644
basis_revision_id = _mod_revision.NULL_REVISION
646
revtrees.append(self.repository.revision_tree(
647
_mod_revision.NULL_REVISION))
648
# The basis inventory from a repository
650
basis_inv = revtrees[0].inventory
652
basis_inv = self.repository.revision_tree(
653
_mod_revision.NULL_REVISION).inventory
654
if len(self.parents) > 0:
655
if basis_revision_id != self.parents[0] and not ghost_basis:
657
"arbitrary basis parents not yet supported with merges")
658
for revtree in revtrees[1:]:
659
for change in revtree.inventory._make_delta(basis_inv):
660
if change[1] is None:
661
# Not present in this parent.
663
if change[2] not in merged_ids:
664
if change[0] is not None:
665
basis_entry = basis_inv[change[2]]
666
merged_ids[change[2]] = [
668
basis_entry.revision,
671
parent_entries[change[2]] = {
673
basis_entry.revision:basis_entry,
675
change[3].revision:change[3],
678
merged_ids[change[2]] = [change[3].revision]
679
parent_entries[change[2]] = {change[3].revision:change[3]}
681
merged_ids[change[2]].append(change[3].revision)
682
parent_entries[change[2]][change[3].revision] = change[3]
685
# Setup the changes from the tree:
686
# changes maps file_id -> (change, [parent revision_ids])
688
for change in iter_changes:
689
# This probably looks up in basis_inv way to much.
690
if change[1][0] is not None:
691
head_candidate = [basis_inv[change[0]].revision]
694
changes[change[0]] = change, merged_ids.get(change[0],
696
unchanged_merged = set(merged_ids) - set(changes)
697
# Extend the changes dict with synthetic changes to record merges of
699
for file_id in unchanged_merged:
700
# Record a merged version of these items that did not change vs the
701
# basis. This can be either identical parallel changes, or a revert
702
# of a specific file after a merge. The recorded content will be
703
# that of the current tree (which is the same as the basis), but
704
# the per-file graph will reflect a merge.
705
# NB:XXX: We are reconstructing path information we had, this
706
# should be preserved instead.
707
# inv delta change: (file_id, (path_in_source, path_in_target),
708
# changed_content, versioned, parent, name, kind,
711
basis_entry = basis_inv[file_id]
712
except errors.NoSuchId:
713
# a change from basis->some_parents but file_id isn't in basis
714
# so was new in the merge, which means it must have changed
715
# from basis -> current, and as it hasn't the add was reverted
716
# by the user. So we discard this change.
720
(basis_inv.id2path(file_id), tree.id2path(file_id)),
722
(basis_entry.parent_id, basis_entry.parent_id),
723
(basis_entry.name, basis_entry.name),
724
(basis_entry.kind, basis_entry.kind),
725
(basis_entry.executable, basis_entry.executable))
726
changes[file_id] = (change, merged_ids[file_id])
727
# changes contains tuples with the change and a set of inventory
728
# candidates for the file.
730
# old_path, new_path, file_id, new_inventory_entry
731
seen_root = False # Is the root in the basis delta?
732
inv_delta = self._basis_delta
733
modified_rev = self._new_revision_id
734
for change, head_candidates in changes.values():
735
if change[3][1]: # versioned in target.
736
# Several things may be happening here:
737
# We may have a fork in the per-file graph
738
# - record a change with the content from tree
739
# We may have a change against < all trees
740
# - carry over the tree that hasn't changed
741
# We may have a change against all trees
742
# - record the change with the content from tree
745
entry = _entry_factory[kind](file_id, change[5][1],
747
head_set = self._heads(change[0], set(head_candidates))
750
for head_candidate in head_candidates:
751
if head_candidate in head_set:
752
heads.append(head_candidate)
753
head_set.remove(head_candidate)
756
# Could be a carry-over situation:
757
parent_entry_revs = parent_entries.get(file_id, None)
758
if parent_entry_revs:
759
parent_entry = parent_entry_revs.get(heads[0], None)
762
if parent_entry is None:
763
# The parent iter_changes was called against is the one
764
# that is the per-file head, so any change is relevant
765
# iter_changes is valid.
766
carry_over_possible = False
768
# could be a carry over situation
769
# A change against the basis may just indicate a merge,
770
# we need to check the content against the source of the
771
# merge to determine if it was changed after the merge
773
if (parent_entry.kind != entry.kind or
774
parent_entry.parent_id != entry.parent_id or
775
parent_entry.name != entry.name):
776
# Metadata common to all entries has changed
777
# against per-file parent
778
carry_over_possible = False
780
carry_over_possible = True
781
# per-type checks for changes against the parent_entry
784
# Cannot be a carry-over situation
785
carry_over_possible = False
786
# Populate the entry in the delta
788
# XXX: There is still a small race here: If someone reverts the content of a file
789
# after iter_changes examines and decides it has changed,
790
# we will unconditionally record a new version even if some
791
# other process reverts it while commit is running (with
792
# the revert happening after iter_changes did its
795
entry.executable = True
797
entry.executable = False
798
if (carry_over_possible and
799
parent_entry.executable == entry.executable):
800
# Check the file length, content hash after reading
802
nostore_sha = parent_entry.text_sha1
805
file_obj, stat_value = tree.get_file_with_stat(file_id, change[1][1])
807
text = file_obj.read()
811
entry.text_sha1, entry.text_size = self._add_text_to_weave(
812
file_id, text, heads, nostore_sha)
813
yield file_id, change[1][1], (entry.text_sha1, stat_value)
814
except errors.ExistingContent:
815
# No content change against a carry_over parent
816
# Perhaps this should also yield a fs hash update?
818
entry.text_size = parent_entry.text_size
819
entry.text_sha1 = parent_entry.text_sha1
820
elif kind == 'symlink':
822
entry.symlink_target = tree.get_symlink_target(file_id)
823
if (carry_over_possible and
824
parent_entry.symlink_target == entry.symlink_target):
827
self._add_text_to_weave(change[0], '', heads, None)
828
elif kind == 'directory':
829
if carry_over_possible:
832
# Nothing to set on the entry.
833
# XXX: split into the Root and nonRoot versions.
834
if change[1][1] != '' or self.repository.supports_rich_root():
835
self._add_text_to_weave(change[0], '', heads, None)
836
elif kind == 'tree-reference':
837
if not self.repository._format.supports_tree_reference:
838
# This isn't quite sane as an error, but we shouldn't
839
# ever see this code path in practice: tree's don't
840
# permit references when the repo doesn't support tree
842
raise errors.UnsupportedOperation(tree.add_reference,
844
reference_revision = tree.get_reference_revision(change[0])
845
entry.reference_revision = reference_revision
846
if (carry_over_possible and
847
parent_entry.reference_revision == reference_revision):
850
self._add_text_to_weave(change[0], '', heads, None)
852
raise AssertionError('unknown kind %r' % kind)
854
entry.revision = modified_rev
856
entry.revision = parent_entry.revision
859
new_path = change[1][1]
860
inv_delta.append((change[1][0], new_path, change[0], entry))
863
self.new_inventory = None
865
# This should perhaps be guarded by a check that the basis we
866
# commit against is the basis for the commit and if not do a delta
868
self._any_changes = True
870
# housekeeping root entry changes do not affect no-change commits.
871
self._require_root_change(tree)
872
self.basis_delta_revision = basis_revision_id
874
def _add_text_to_weave(self, file_id, new_text, parents, nostore_sha):
875
parent_keys = tuple([(file_id, parent) for parent in parents])
876
return self.repository.texts._add_text(
877
(file_id, self._new_revision_id), parent_keys, new_text,
878
nostore_sha=nostore_sha, random_id=self.random_revid)[0:2]
881
class RootCommitBuilder(CommitBuilder):
882
"""This commitbuilder actually records the root id"""
884
# the root entry gets versioned properly by this builder.
885
_versioned_root = True
887
def _check_root(self, ie, parent_invs, tree):
888
"""Helper for record_entry_contents.
890
:param ie: An entry being added.
891
:param parent_invs: The inventories of the parent revisions of the
893
:param tree: The tree that is being committed.
896
def _require_root_change(self, tree):
897
"""Enforce an appropriate root object change.
899
This is called once when record_iter_changes is called, if and only if
900
the root was not in the delta calculated by record_iter_changes.
902
:param tree: The tree which is being committed.
904
# versioned roots do not change unless the tree found a change.
907
class RepositoryWriteLockResult(LogicalLockResult):
908
"""The result of write locking a repository.
910
:ivar repository_token: The token obtained from the underlying lock, or
912
:ivar unlock: A callable which will unlock the lock.
915
def __init__(self, unlock, repository_token):
916
LogicalLockResult.__init__(self, unlock)
917
self.repository_token = repository_token
920
return "RepositoryWriteLockResult(%s, %s)" % (self.repository_token,
924
######################################################################
928
class Repository(_RelockDebugMixin, controldir.ControlComponent):
929
"""Repository holding history for one or more branches.
931
The repository holds and retrieves historical information including
932
revisions and file history. It's normally accessed only by the Branch,
933
which views a particular line of development through that history.
935
The Repository builds on top of some byte storage facilies (the revisions,
936
signatures, inventories, texts and chk_bytes attributes) and a Transport,
937
which respectively provide byte storage and a means to access the (possibly
940
The byte storage facilities are addressed via tuples, which we refer to
941
as 'keys' throughout the code base. Revision_keys, inventory_keys and
942
signature_keys are all 1-tuples: (revision_id,). text_keys are two-tuples:
943
(file_id, revision_id). chk_bytes uses CHK keys - a 1-tuple with a single
944
byte string made up of a hash identifier and a hash value.
945
We use this interface because it allows low friction with the underlying
946
code that implements disk indices, network encoding and other parts of
949
:ivar revisions: A bzrlib.versionedfile.VersionedFiles instance containing
950
the serialised revisions for the repository. This can be used to obtain
951
revision graph information or to access raw serialised revisions.
952
The result of trying to insert data into the repository via this store
953
is undefined: it should be considered read-only except for implementors
955
:ivar signatures: A bzrlib.versionedfile.VersionedFiles instance containing
956
the serialised signatures for the repository. This can be used to
957
obtain access to raw serialised signatures. The result of trying to
958
insert data into the repository via this store is undefined: it should
959
be considered read-only except for implementors of repositories.
960
:ivar inventories: A bzrlib.versionedfile.VersionedFiles instance containing
961
the serialised inventories for the repository. This can be used to
962
obtain unserialised inventories. The result of trying to insert data
963
into the repository via this store is undefined: it should be
964
considered read-only except for implementors of repositories.
965
:ivar texts: A bzrlib.versionedfile.VersionedFiles instance containing the
966
texts of files and directories for the repository. This can be used to
967
obtain file texts or file graphs. Note that Repository.iter_file_bytes
968
is usually a better interface for accessing file texts.
969
The result of trying to insert data into the repository via this store
970
is undefined: it should be considered read-only except for implementors
972
:ivar chk_bytes: A bzrlib.versionedfile.VersionedFiles instance containing
973
any data the repository chooses to store or have indexed by its hash.
974
The result of trying to insert data into the repository via this store
975
is undefined: it should be considered read-only except for implementors
977
:ivar _transport: Transport for file access to repository, typically
978
pointing to .bzr/repository.
981
# What class to use for a CommitBuilder. Often it's simpler to change this
982
# in a Repository class subclass rather than to override
983
# get_commit_builder.
984
_commit_builder_class = CommitBuilder
986
def abort_write_group(self, suppress_errors=False):
987
"""Commit the contents accrued within the current write group.
989
:param suppress_errors: if true, abort_write_group will catch and log
990
unexpected errors that happen during the abort, rather than
991
allowing them to propagate. Defaults to False.
993
:seealso: start_write_group.
995
if self._write_group is not self.get_transaction():
996
# has an unlock or relock occured ?
999
'(suppressed) mismatched lock context and write group. %r, %r',
1000
self._write_group, self.get_transaction())
1002
raise errors.BzrError(
1003
'mismatched lock context and write group. %r, %r' %
1004
(self._write_group, self.get_transaction()))
1006
self._abort_write_group()
1007
except Exception, exc:
1008
self._write_group = None
1009
if not suppress_errors:
1011
mutter('abort_write_group failed')
1012
log_exception_quietly()
1013
note('bzr: ERROR (ignored): %s', exc)
1014
self._write_group = None
1016
def _abort_write_group(self):
1017
"""Template method for per-repository write group cleanup.
1019
This is called during abort before the write group is considered to be
1020
finished and should cleanup any internal state accrued during the write
1021
group. There is no requirement that data handed to the repository be
1022
*not* made available - this is not a rollback - but neither should any
1023
attempt be made to ensure that data added is fully commited. Abort is
1024
invoked when an error has occured so futher disk or network operations
1025
may not be possible or may error and if possible should not be
1029
def add_fallback_repository(self, repository):
1030
"""Add a repository to use for looking up data not held locally.
1032
:param repository: A repository.
1034
if not self._format.supports_external_lookups:
1035
raise errors.UnstackableRepositoryFormat(self._format, self.base)
1036
if self.is_locked():
1037
# This repository will call fallback.unlock() when we transition to
1038
# the unlocked state, so we make sure to increment the lock count
1039
repository.lock_read()
1040
self._check_fallback_repository(repository)
1041
self._fallback_repositories.append(repository)
1042
self.texts.add_fallback_versioned_files(repository.texts)
1043
self.inventories.add_fallback_versioned_files(repository.inventories)
1044
self.revisions.add_fallback_versioned_files(repository.revisions)
1045
self.signatures.add_fallback_versioned_files(repository.signatures)
1046
if self.chk_bytes is not None:
1047
self.chk_bytes.add_fallback_versioned_files(repository.chk_bytes)
1049
def _check_fallback_repository(self, repository):
1050
"""Check that this repository can fallback to repository safely.
1052
Raise an error if not.
1054
:param repository: A repository to fallback to.
1056
return InterRepository._assert_same_model(self, repository)
1058
def add_inventory(self, revision_id, inv, parents):
1059
"""Add the inventory inv to the repository as revision_id.
1061
:param parents: The revision ids of the parents that revision_id
1062
is known to have and are in the repository already.
1064
:returns: The validator(which is a sha1 digest, though what is sha'd is
1065
repository format specific) of the serialized inventory.
1067
if not self.is_in_write_group():
1068
raise AssertionError("%r not in write group" % (self,))
1069
_mod_revision.check_not_reserved_id(revision_id)
1070
if not (inv.revision_id is None or inv.revision_id == revision_id):
1071
raise AssertionError(
1072
"Mismatch between inventory revision"
1073
" id and insertion revid (%r, %r)"
1074
% (inv.revision_id, revision_id))
1075
if inv.root is None:
1076
raise errors.RootMissing()
1077
return self._add_inventory_checked(revision_id, inv, parents)
1079
def _add_inventory_checked(self, revision_id, inv, parents):
1080
"""Add inv to the repository after checking the inputs.
1082
This function can be overridden to allow different inventory styles.
1084
:seealso: add_inventory, for the contract.
1086
inv_lines = self._serializer.write_inventory_to_lines(inv)
1087
return self._inventory_add_lines(revision_id, parents,
1088
inv_lines, check_content=False)
1090
def add_inventory_by_delta(self, basis_revision_id, delta, new_revision_id,
1091
parents, basis_inv=None, propagate_caches=False):
1092
"""Add a new inventory expressed as a delta against another revision.
1094
See the inventory developers documentation for the theory behind
1097
:param basis_revision_id: The inventory id the delta was created
1098
against. (This does not have to be a direct parent.)
1099
:param delta: The inventory delta (see Inventory.apply_delta for
1101
:param new_revision_id: The revision id that the inventory is being
1103
:param parents: The revision ids of the parents that revision_id is
1104
known to have and are in the repository already. These are supplied
1105
for repositories that depend on the inventory graph for revision
1106
graph access, as well as for those that pun ancestry with delta
1108
:param basis_inv: The basis inventory if it is already known,
1110
:param propagate_caches: If True, the caches for this inventory are
1111
copied to and updated for the result if possible.
1113
:returns: (validator, new_inv)
1114
The validator(which is a sha1 digest, though what is sha'd is
1115
repository format specific) of the serialized inventory, and the
1116
resulting inventory.
1118
if not self.is_in_write_group():
1119
raise AssertionError("%r not in write group" % (self,))
1120
_mod_revision.check_not_reserved_id(new_revision_id)
1121
basis_tree = self.revision_tree(basis_revision_id)
1122
basis_tree.lock_read()
1124
# Note that this mutates the inventory of basis_tree, which not all
1125
# inventory implementations may support: A better idiom would be to
1126
# return a new inventory, but as there is no revision tree cache in
1127
# repository this is safe for now - RBC 20081013
1128
if basis_inv is None:
1129
basis_inv = basis_tree.inventory
1130
basis_inv.apply_delta(delta)
1131
basis_inv.revision_id = new_revision_id
1132
return (self.add_inventory(new_revision_id, basis_inv, parents),
1137
def _inventory_add_lines(self, revision_id, parents, lines,
1138
check_content=True):
1139
"""Store lines in inv_vf and return the sha1 of the inventory."""
1140
parents = [(parent,) for parent in parents]
1141
result = self.inventories.add_lines((revision_id,), parents, lines,
1142
check_content=check_content)[0]
1143
self.inventories._access.flush()
1146
def add_revision(self, revision_id, rev, inv=None, config=None):
1147
"""Add rev to the revision store as revision_id.
1149
:param revision_id: the revision id to use.
1150
:param rev: The revision object.
1151
:param inv: The inventory for the revision. if None, it will be looked
1152
up in the inventory storer
1153
:param config: If None no digital signature will be created.
1154
If supplied its signature_needed method will be used
1155
to determine if a signature should be made.
1157
# TODO: jam 20070210 Shouldn't we check rev.revision_id and
1159
_mod_revision.check_not_reserved_id(revision_id)
1160
if config is not None and config.signature_needed():
1162
inv = self.get_inventory(revision_id)
1163
plaintext = Testament(rev, inv).as_short_text()
1164
self.store_revision_signature(
1165
gpg.GPGStrategy(config), plaintext, revision_id)
1166
# check inventory present
1167
if not self.inventories.get_parent_map([(revision_id,)]):
1169
raise errors.WeaveRevisionNotPresent(revision_id,
1172
# yes, this is not suitable for adding with ghosts.
1173
rev.inventory_sha1 = self.add_inventory(revision_id, inv,
1176
key = (revision_id,)
1177
rev.inventory_sha1 = self.inventories.get_sha1s([key])[key]
1178
self._add_revision(rev)
1180
def _add_revision(self, revision):
1181
text = self._serializer.write_revision_to_string(revision)
1182
key = (revision.revision_id,)
1183
parents = tuple((parent,) for parent in revision.parent_ids)
1184
self.revisions.add_lines(key, parents, osutils.split_lines(text))
1186
def all_revision_ids(self):
1187
"""Returns a list of all the revision ids in the repository.
1189
This is conceptually deprecated because code should generally work on
1190
the graph reachable from a particular revision, and ignore any other
1191
revisions that might be present. There is no direct replacement
1194
if 'evil' in debug.debug_flags:
1195
mutter_callsite(2, "all_revision_ids is linear with history.")
1196
return self._all_revision_ids()
1198
def _all_revision_ids(self):
1199
"""Returns a list of all the revision ids in the repository.
1201
These are in as much topological order as the underlying store can
1204
raise NotImplementedError(self._all_revision_ids)
1206
def break_lock(self):
1207
"""Break a lock if one is present from another instance.
1209
Uses the ui factory to ask for confirmation if the lock may be from
1212
self.control_files.break_lock()
1215
def _eliminate_revisions_not_present(self, revision_ids):
1216
"""Check every revision id in revision_ids to see if we have it.
1218
Returns a set of the present revisions.
1221
graph = self.get_graph()
1222
parent_map = graph.get_parent_map(revision_ids)
1223
# The old API returned a list, should this actually be a set?
1224
return parent_map.keys()
1226
def _check_inventories(self, checker):
1227
"""Check the inventories found from the revision scan.
1229
This is responsible for verifying the sha1 of inventories and
1230
creating a pending_keys set that covers data referenced by inventories.
1232
bar = ui.ui_factory.nested_progress_bar()
1234
self._do_check_inventories(checker, bar)
1238
def _do_check_inventories(self, checker, bar):
1239
"""Helper for _check_inventories."""
1241
keys = {'chk_bytes':set(), 'inventories':set(), 'texts':set()}
1242
kinds = ['chk_bytes', 'texts']
1243
count = len(checker.pending_keys)
1244
bar.update("inventories", 0, 2)
1245
current_keys = checker.pending_keys
1246
checker.pending_keys = {}
1247
# Accumulate current checks.
1248
for key in current_keys:
1249
if key[0] != 'inventories' and key[0] not in kinds:
1250
checker._report_items.append('unknown key type %r' % (key,))
1251
keys[key[0]].add(key[1:])
1252
if keys['inventories']:
1253
# NB: output order *should* be roughly sorted - topo or
1254
# inverse topo depending on repository - either way decent
1255
# to just delta against. However, pre-CHK formats didn't
1256
# try to optimise inventory layout on disk. As such the
1257
# pre-CHK code path does not use inventory deltas.
1259
for record in self.inventories.check(keys=keys['inventories']):
1260
if record.storage_kind == 'absent':
1261
checker._report_items.append(
1262
'Missing inventory {%s}' % (record.key,))
1264
last_object = self._check_record('inventories', record,
1265
checker, last_object,
1266
current_keys[('inventories',) + record.key])
1267
del keys['inventories']
1270
bar.update("texts", 1)
1271
while (checker.pending_keys or keys['chk_bytes']
1273
# Something to check.
1274
current_keys = checker.pending_keys
1275
checker.pending_keys = {}
1276
# Accumulate current checks.
1277
for key in current_keys:
1278
if key[0] not in kinds:
1279
checker._report_items.append('unknown key type %r' % (key,))
1280
keys[key[0]].add(key[1:])
1281
# Check the outermost kind only - inventories || chk_bytes || texts
1285
for record in getattr(self, kind).check(keys=keys[kind]):
1286
if record.storage_kind == 'absent':
1287
checker._report_items.append(
1288
'Missing %s {%s}' % (kind, record.key,))
1290
last_object = self._check_record(kind, record,
1291
checker, last_object, current_keys[(kind,) + record.key])
1295
def _check_record(self, kind, record, checker, last_object, item_data):
1296
"""Check a single text from this repository."""
1297
if kind == 'inventories':
1298
rev_id = record.key[0]
1299
inv = self._deserialise_inventory(rev_id,
1300
record.get_bytes_as('fulltext'))
1301
if last_object is not None:
1302
delta = inv._make_delta(last_object)
1303
for old_path, path, file_id, ie in delta:
1306
ie.check(checker, rev_id, inv)
1308
for path, ie in inv.iter_entries():
1309
ie.check(checker, rev_id, inv)
1310
if self._format.fast_deltas:
1312
elif kind == 'chk_bytes':
1313
# No code written to check chk_bytes for this repo format.
1314
checker._report_items.append(
1315
'unsupported key type chk_bytes for %s' % (record.key,))
1316
elif kind == 'texts':
1317
self._check_text(record, checker, item_data)
1319
checker._report_items.append(
1320
'unknown key type %s for %s' % (kind, record.key))
1322
def _check_text(self, record, checker, item_data):
1323
"""Check a single text."""
1324
# Check it is extractable.
1325
# TODO: check length.
1326
if record.storage_kind == 'chunked':
1327
chunks = record.get_bytes_as(record.storage_kind)
1328
sha1 = osutils.sha_strings(chunks)
1329
length = sum(map(len, chunks))
1331
content = record.get_bytes_as('fulltext')
1332
sha1 = osutils.sha_string(content)
1333
length = len(content)
1334
if item_data and sha1 != item_data[1]:
1335
checker._report_items.append(
1336
'sha1 mismatch: %s has sha1 %s expected %s referenced by %s' %
1337
(record.key, sha1, item_data[1], item_data[2]))
1340
def create(a_bzrdir):
1341
"""Construct the current default format repository in a_bzrdir."""
1342
return RepositoryFormat.get_default_format().initialize(a_bzrdir)
1344
def __init__(self, _format, a_bzrdir, control_files):
1345
"""instantiate a Repository.
1347
:param _format: The format of the repository on disk.
1348
:param a_bzrdir: The BzrDir of the repository.
1350
# In the future we will have a single api for all stores for
1351
# getting file texts, inventories and revisions, then
1352
# this construct will accept instances of those things.
1353
super(Repository, self).__init__()
1354
self._format = _format
1355
# the following are part of the public API for Repository:
1356
self.bzrdir = a_bzrdir
1357
self.control_files = control_files
1358
self._transport = control_files._transport
1359
self.base = self._transport.base
1361
self._reconcile_does_inventory_gc = True
1362
self._reconcile_fixes_text_parents = False
1363
self._reconcile_backsup_inventory = True
1364
self._write_group = None
1365
# Additional places to query for data.
1366
self._fallback_repositories = []
1367
# An InventoryEntry cache, used during deserialization
1368
self._inventory_entry_cache = fifo_cache.FIFOCache(10*1024)
1369
# Is it safe to return inventory entries directly from the entry cache,
1370
# rather copying them?
1371
self._safe_to_return_from_cache = False
1374
def user_transport(self):
1375
return self.bzrdir.user_transport
1378
def control_transport(self):
1379
return self._transport
1382
if self._fallback_repositories:
1383
return '%s(%r, fallback_repositories=%r)' % (
1384
self.__class__.__name__,
1386
self._fallback_repositories)
1388
return '%s(%r)' % (self.__class__.__name__,
1391
def _has_same_fallbacks(self, other_repo):
1392
"""Returns true if the repositories have the same fallbacks."""
1393
my_fb = self._fallback_repositories
1394
other_fb = other_repo._fallback_repositories
1395
if len(my_fb) != len(other_fb):
1397
for f, g in zip(my_fb, other_fb):
1398
if not f.has_same_location(g):
1402
def has_same_location(self, other):
1403
"""Returns a boolean indicating if this repository is at the same
1404
location as another repository.
1406
This might return False even when two repository objects are accessing
1407
the same physical repository via different URLs.
1409
if self.__class__ is not other.__class__:
1411
return (self._transport.base == other._transport.base)
1413
def is_in_write_group(self):
1414
"""Return True if there is an open write group.
1416
:seealso: start_write_group.
1418
return self._write_group is not None
1420
def is_locked(self):
1421
return self.control_files.is_locked()
1423
def is_write_locked(self):
1424
"""Return True if this object is write locked."""
1425
return self.is_locked() and self.control_files._lock_mode == 'w'
1427
def lock_write(self, token=None):
1428
"""Lock this repository for writing.
1430
This causes caching within the repository obejct to start accumlating
1431
data during reads, and allows a 'write_group' to be obtained. Write
1432
groups must be used for actual data insertion.
1434
A token should be passed in if you know that you have locked the object
1435
some other way, and need to synchronise this object's state with that
1438
XXX: this docstring is duplicated in many places, e.g. lockable_files.py
1440
:param token: if this is already locked, then lock_write will fail
1441
unless the token matches the existing lock.
1442
:returns: a token if this instance supports tokens, otherwise None.
1443
:raises TokenLockingNotSupported: when a token is given but this
1444
instance doesn't support using token locks.
1445
:raises MismatchedToken: if the specified token doesn't match the token
1446
of the existing lock.
1447
:seealso: start_write_group.
1448
:return: A RepositoryWriteLockResult.
1450
locked = self.is_locked()
1451
token = self.control_files.lock_write(token=token)
1453
self._warn_if_deprecated()
1454
self._note_lock('w')
1455
for repo in self._fallback_repositories:
1456
# Writes don't affect fallback repos
1458
self._refresh_data()
1459
return RepositoryWriteLockResult(self.unlock, token)
1461
def lock_read(self):
1462
"""Lock the repository for read operations.
1464
:return: An object with an unlock method which will release the lock
1467
locked = self.is_locked()
1468
self.control_files.lock_read()
1470
self._warn_if_deprecated()
1471
self._note_lock('r')
1472
for repo in self._fallback_repositories:
1474
self._refresh_data()
1475
return LogicalLockResult(self.unlock)
1477
def get_physical_lock_status(self):
1478
return self.control_files.get_physical_lock_status()
1480
def leave_lock_in_place(self):
1481
"""Tell this repository not to release the physical lock when this
1484
If lock_write doesn't return a token, then this method is not supported.
1486
self.control_files.leave_in_place()
1488
def dont_leave_lock_in_place(self):
1489
"""Tell this repository to release the physical lock when this
1490
object is unlocked, even if it didn't originally acquire it.
1492
If lock_write doesn't return a token, then this method is not supported.
1494
self.control_files.dont_leave_in_place()
1497
def gather_stats(self, revid=None, committers=None):
1498
"""Gather statistics from a revision id.
1500
:param revid: The revision id to gather statistics from, if None, then
1501
no revision specific statistics are gathered.
1502
:param committers: Optional parameter controlling whether to grab
1503
a count of committers from the revision specific statistics.
1504
:return: A dictionary of statistics. Currently this contains:
1505
committers: The number of committers if requested.
1506
firstrev: A tuple with timestamp, timezone for the penultimate left
1507
most ancestor of revid, if revid is not the NULL_REVISION.
1508
latestrev: A tuple with timestamp, timezone for revid, if revid is
1509
not the NULL_REVISION.
1510
revisions: The total revision count in the repository.
1511
size: An estimate disk size of the repository in bytes.
1514
if revid and committers:
1515
result['committers'] = 0
1516
if revid and revid != _mod_revision.NULL_REVISION:
1518
all_committers = set()
1519
revisions = self.get_ancestry(revid)
1520
# pop the leading None
1522
first_revision = None
1524
# ignore the revisions in the middle - just grab first and last
1525
revisions = revisions[0], revisions[-1]
1526
for revision in self.get_revisions(revisions):
1527
if not first_revision:
1528
first_revision = revision
1530
all_committers.add(revision.committer)
1531
last_revision = revision
1533
result['committers'] = len(all_committers)
1534
result['firstrev'] = (first_revision.timestamp,
1535
first_revision.timezone)
1536
result['latestrev'] = (last_revision.timestamp,
1537
last_revision.timezone)
1539
# now gather global repository information
1540
# XXX: This is available for many repos regardless of listability.
1541
if self.user_transport.listable():
1542
# XXX: do we want to __define len__() ?
1543
# Maybe the versionedfiles object should provide a different
1544
# method to get the number of keys.
1545
result['revisions'] = len(self.revisions.keys())
1546
# result['size'] = t
1549
def find_branches(self, using=False):
1550
"""Find branches underneath this repository.
1552
This will include branches inside other branches.
1554
:param using: If True, list only branches using this repository.
1556
if using and not self.is_shared():
1557
return self.bzrdir.list_branches()
1558
class Evaluator(object):
1561
self.first_call = True
1563
def __call__(self, bzrdir):
1564
# On the first call, the parameter is always the bzrdir
1565
# containing the current repo.
1566
if not self.first_call:
1568
repository = bzrdir.open_repository()
1569
except errors.NoRepositoryPresent:
1572
return False, ([], repository)
1573
self.first_call = False
1574
value = (bzrdir.list_branches(), None)
1578
for branches, repository in bzrdir.BzrDir.find_bzrdirs(
1579
self.user_transport, evaluate=Evaluator()):
1580
if branches is not None:
1581
ret.extend(branches)
1582
if not using and repository is not None:
1583
ret.extend(repository.find_branches())
1587
def search_missing_revision_ids(self, other,
1588
revision_id=symbol_versioning.DEPRECATED_PARAMETER,
1589
find_ghosts=True, revision_ids=None, if_present_ids=None):
1590
"""Return the revision ids that other has that this does not.
1592
These are returned in topological order.
1594
revision_id: only return revision ids included by revision_id.
1596
if symbol_versioning.deprecated_passed(revision_id):
1597
symbol_versioning.warn(
1598
'search_missing_revision_ids(revision_id=...) was '
1599
'deprecated in 2.4. Use revision_ids=[...] instead.',
1600
DeprecationWarning, stacklevel=3)
1601
if revision_ids is not None:
1602
raise AssertionError(
1603
'revision_ids is mutually exclusive with revision_id')
1604
if revision_id is not None:
1605
revision_ids = [revision_id]
1606
return InterRepository.get(other, self).search_missing_revision_ids(
1607
find_ghosts=find_ghosts, revision_ids=revision_ids,
1608
if_present_ids=if_present_ids)
1612
"""Open the repository rooted at base.
1614
For instance, if the repository is at URL/.bzr/repository,
1615
Repository.open(URL) -> a Repository instance.
1617
control = bzrdir.BzrDir.open(base)
1618
return control.open_repository()
1620
def copy_content_into(self, destination, revision_id=None):
1621
"""Make a complete copy of the content in self into destination.
1623
This is a destructive operation! Do not use it on existing
1626
return InterRepository.get(self, destination).copy_content(revision_id)
1628
def commit_write_group(self):
1629
"""Commit the contents accrued within the current write group.
1631
:seealso: start_write_group.
1633
:return: it may return an opaque hint that can be passed to 'pack'.
1635
if self._write_group is not self.get_transaction():
1636
# has an unlock or relock occured ?
1637
raise errors.BzrError('mismatched lock context %r and '
1639
(self.get_transaction(), self._write_group))
1640
result = self._commit_write_group()
1641
self._write_group = None
1644
def _commit_write_group(self):
1645
"""Template method for per-repository write group cleanup.
1647
This is called before the write group is considered to be
1648
finished and should ensure that all data handed to the repository
1649
for writing during the write group is safely committed (to the
1650
extent possible considering file system caching etc).
1653
def suspend_write_group(self):
1654
raise errors.UnsuspendableWriteGroup(self)
1656
def get_missing_parent_inventories(self, check_for_missing_texts=True):
1657
"""Return the keys of missing inventory parents for revisions added in
1660
A revision is not complete if the inventory delta for that revision
1661
cannot be calculated. Therefore if the parent inventories of a
1662
revision are not present, the revision is incomplete, and e.g. cannot
1663
be streamed by a smart server. This method finds missing inventory
1664
parents for revisions added in this write group.
1666
if not self._format.supports_external_lookups:
1667
# This is only an issue for stacked repositories
1669
if not self.is_in_write_group():
1670
raise AssertionError('not in a write group')
1672
# XXX: We assume that every added revision already has its
1673
# corresponding inventory, so we only check for parent inventories that
1674
# might be missing, rather than all inventories.
1675
parents = set(self.revisions._index.get_missing_parents())
1676
parents.discard(_mod_revision.NULL_REVISION)
1677
unstacked_inventories = self.inventories._index
1678
present_inventories = unstacked_inventories.get_parent_map(
1679
key[-1:] for key in parents)
1680
parents.difference_update(present_inventories)
1681
if len(parents) == 0:
1682
# No missing parent inventories.
1684
if not check_for_missing_texts:
1685
return set(('inventories', rev_id) for (rev_id,) in parents)
1686
# Ok, now we have a list of missing inventories. But these only matter
1687
# if the inventories that reference them are missing some texts they
1688
# appear to introduce.
1689
# XXX: Texts referenced by all added inventories need to be present,
1690
# but at the moment we're only checking for texts referenced by
1691
# inventories at the graph's edge.
1692
key_deps = self.revisions._index._key_dependencies
1693
key_deps.satisfy_refs_for_keys(present_inventories)
1694
referrers = frozenset(r[0] for r in key_deps.get_referrers())
1695
file_ids = self.fileids_altered_by_revision_ids(referrers)
1696
missing_texts = set()
1697
for file_id, version_ids in file_ids.iteritems():
1698
missing_texts.update(
1699
(file_id, version_id) for version_id in version_ids)
1700
present_texts = self.texts.get_parent_map(missing_texts)
1701
missing_texts.difference_update(present_texts)
1702
if not missing_texts:
1703
# No texts are missing, so all revisions and their deltas are
1706
# Alternatively the text versions could be returned as the missing
1707
# keys, but this is likely to be less data.
1708
missing_keys = set(('inventories', rev_id) for (rev_id,) in parents)
1711
def refresh_data(self):
1712
"""Re-read any data needed to synchronise with disk.
1714
This method is intended to be called after another repository instance
1715
(such as one used by a smart server) has inserted data into the
1716
repository. On all repositories this will work outside of write groups.
1717
Some repository formats (pack and newer for bzrlib native formats)
1718
support refresh_data inside write groups. If called inside a write
1719
group on a repository that does not support refreshing in a write group
1720
IsInWriteGroupError will be raised.
1722
self._refresh_data()
1724
def resume_write_group(self, tokens):
1725
if not self.is_write_locked():
1726
raise errors.NotWriteLocked(self)
1727
if self._write_group:
1728
raise errors.BzrError('already in a write group')
1729
self._resume_write_group(tokens)
1730
# so we can detect unlock/relock - the write group is now entered.
1731
self._write_group = self.get_transaction()
1733
def _resume_write_group(self, tokens):
1734
raise errors.UnsuspendableWriteGroup(self)
1736
def fetch(self, source, revision_id=None, find_ghosts=False,
1738
"""Fetch the content required to construct revision_id from source.
1740
If revision_id is None and fetch_spec is None, then all content is
1743
fetch() may not be used when the repository is in a write group -
1744
either finish the current write group before using fetch, or use
1745
fetch before starting the write group.
1747
:param find_ghosts: Find and copy revisions in the source that are
1748
ghosts in the target (and not reachable directly by walking out to
1749
the first-present revision in target from revision_id).
1750
:param revision_id: If specified, all the content needed for this
1751
revision ID will be copied to the target. Fetch will determine for
1752
itself which content needs to be copied.
1753
:param fetch_spec: If specified, a SearchResult or
1754
PendingAncestryResult that describes which revisions to copy. This
1755
allows copying multiple heads at once. Mutually exclusive with
1758
if fetch_spec is not None and revision_id is not None:
1759
raise AssertionError(
1760
"fetch_spec and revision_id are mutually exclusive.")
1761
if self.is_in_write_group():
1762
raise errors.InternalBzrError(
1763
"May not fetch while in a write group.")
1764
# fast path same-url fetch operations
1765
# TODO: lift out to somewhere common with RemoteRepository
1766
# <https://bugs.launchpad.net/bzr/+bug/401646>
1767
if (self.has_same_location(source)
1768
and fetch_spec is None
1769
and self._has_same_fallbacks(source)):
1770
# check that last_revision is in 'from' and then return a
1772
if (revision_id is not None and
1773
not _mod_revision.is_null(revision_id)):
1774
self.get_revision(revision_id)
1776
inter = InterRepository.get(source, self)
1777
return inter.fetch(revision_id=revision_id,
1778
find_ghosts=find_ghosts, fetch_spec=fetch_spec)
1780
def create_bundle(self, target, base, fileobj, format=None):
1781
return serializer.write_bundle(self, target, base, fileobj, format)
1783
def get_commit_builder(self, branch, parents, config, timestamp=None,
1784
timezone=None, committer=None, revprops=None,
1786
"""Obtain a CommitBuilder for this repository.
1788
:param branch: Branch to commit to.
1789
:param parents: Revision ids of the parents of the new revision.
1790
:param config: Configuration to use.
1791
:param timestamp: Optional timestamp recorded for commit.
1792
:param timezone: Optional timezone for timestamp.
1793
:param committer: Optional committer to set for commit.
1794
:param revprops: Optional dictionary of revision properties.
1795
:param revision_id: Optional revision id.
1797
if self._fallback_repositories and not self._format.supports_chks:
1798
raise errors.BzrError("Cannot commit directly to a stacked branch"
1799
" in pre-2a formats. See "
1800
"https://bugs.launchpad.net/bzr/+bug/375013 for details.")
1801
result = self._commit_builder_class(self, parents, config,
1802
timestamp, timezone, committer, revprops, revision_id)
1803
self.start_write_group()
1806
@only_raises(errors.LockNotHeld, errors.LockBroken)
1808
if (self.control_files._lock_count == 1 and
1809
self.control_files._lock_mode == 'w'):
1810
if self._write_group is not None:
1811
self.abort_write_group()
1812
self.control_files.unlock()
1813
raise errors.BzrError(
1814
'Must end write groups before releasing write locks.')
1815
self.control_files.unlock()
1816
if self.control_files._lock_count == 0:
1817
self._inventory_entry_cache.clear()
1818
for repo in self._fallback_repositories:
1822
def clone(self, a_bzrdir, revision_id=None):
1823
"""Clone this repository into a_bzrdir using the current format.
1825
Currently no check is made that the format of this repository and
1826
the bzrdir format are compatible. FIXME RBC 20060201.
1828
:return: The newly created destination repository.
1830
# TODO: deprecate after 0.16; cloning this with all its settings is
1831
# probably not very useful -- mbp 20070423
1832
dest_repo = self._create_sprouting_repo(a_bzrdir, shared=self.is_shared())
1833
self.copy_content_into(dest_repo, revision_id)
1836
def start_write_group(self):
1837
"""Start a write group in the repository.
1839
Write groups are used by repositories which do not have a 1:1 mapping
1840
between file ids and backend store to manage the insertion of data from
1841
both fetch and commit operations.
1843
A write lock is required around the start_write_group/commit_write_group
1844
for the support of lock-requiring repository formats.
1846
One can only insert data into a repository inside a write group.
1850
if not self.is_write_locked():
1851
raise errors.NotWriteLocked(self)
1852
if self._write_group:
1853
raise errors.BzrError('already in a write group')
1854
self._start_write_group()
1855
# so we can detect unlock/relock - the write group is now entered.
1856
self._write_group = self.get_transaction()
1858
def _start_write_group(self):
1859
"""Template method for per-repository write group startup.
1861
This is called before the write group is considered to be
1866
def sprout(self, to_bzrdir, revision_id=None):
1867
"""Create a descendent repository for new development.
1869
Unlike clone, this does not copy the settings of the repository.
1871
dest_repo = self._create_sprouting_repo(to_bzrdir, shared=False)
1872
dest_repo.fetch(self, revision_id=revision_id)
1875
def _create_sprouting_repo(self, a_bzrdir, shared):
1876
if not isinstance(a_bzrdir._format, self.bzrdir._format.__class__):
1877
# use target default format.
1878
dest_repo = a_bzrdir.create_repository()
1880
# Most control formats need the repository to be specifically
1881
# created, but on some old all-in-one formats it's not needed
1883
dest_repo = self._format.initialize(a_bzrdir, shared=shared)
1884
except errors.UninitializableFormat:
1885
dest_repo = a_bzrdir.open_repository()
1888
def _get_sink(self):
1889
"""Return a sink for streaming into this repository."""
1890
return StreamSink(self)
1892
def _get_source(self, to_format):
1893
"""Return a source for streaming from this repository."""
1894
return StreamSource(self, to_format)
1897
def has_revision(self, revision_id):
1898
"""True if this repository has a copy of the revision."""
1899
return revision_id in self.has_revisions((revision_id,))
1902
def has_revisions(self, revision_ids):
1903
"""Probe to find out the presence of multiple revisions.
1905
:param revision_ids: An iterable of revision_ids.
1906
:return: A set of the revision_ids that were present.
1908
parent_map = self.revisions.get_parent_map(
1909
[(rev_id,) for rev_id in revision_ids])
1911
if _mod_revision.NULL_REVISION in revision_ids:
1912
result.add(_mod_revision.NULL_REVISION)
1913
result.update([key[0] for key in parent_map])
1917
def get_revision(self, revision_id):
1918
"""Return the Revision object for a named revision."""
1919
return self.get_revisions([revision_id])[0]
1922
def get_revision_reconcile(self, revision_id):
1923
"""'reconcile' helper routine that allows access to a revision always.
1925
This variant of get_revision does not cross check the weave graph
1926
against the revision one as get_revision does: but it should only
1927
be used by reconcile, or reconcile-alike commands that are correcting
1928
or testing the revision graph.
1930
return self._get_revisions([revision_id])[0]
1933
def get_revisions(self, revision_ids):
1934
"""Get many revisions at once.
1936
Repositories that need to check data on every revision read should
1937
subclass this method.
1939
return self._get_revisions(revision_ids)
1942
def _get_revisions(self, revision_ids):
1943
"""Core work logic to get many revisions without sanity checks."""
1945
for revid, rev in self._iter_revisions(revision_ids):
1947
raise errors.NoSuchRevision(self, revid)
1949
return [revs[revid] for revid in revision_ids]
1951
def _iter_revisions(self, revision_ids):
1952
"""Iterate over revision objects.
1954
:param revision_ids: An iterable of revisions to examine. None may be
1955
passed to request all revisions known to the repository. Note that
1956
not all repositories can find unreferenced revisions; for those
1957
repositories only referenced ones will be returned.
1958
:return: An iterator of (revid, revision) tuples. Absent revisions (
1959
those asked for but not available) are returned as (revid, None).
1961
if revision_ids is None:
1962
revision_ids = self.all_revision_ids()
1964
for rev_id in revision_ids:
1965
if not rev_id or not isinstance(rev_id, basestring):
1966
raise errors.InvalidRevisionId(revision_id=rev_id, branch=self)
1967
keys = [(key,) for key in revision_ids]
1968
stream = self.revisions.get_record_stream(keys, 'unordered', True)
1969
for record in stream:
1970
revid = record.key[0]
1971
if record.storage_kind == 'absent':
1974
text = record.get_bytes_as('fulltext')
1975
rev = self._serializer.read_revision_from_string(text)
1978
def get_deltas_for_revisions(self, revisions, specific_fileids=None):
1979
"""Produce a generator of revision deltas.
1981
Note that the input is a sequence of REVISIONS, not revision_ids.
1982
Trees will be held in memory until the generator exits.
1983
Each delta is relative to the revision's lefthand predecessor.
1985
:param specific_fileids: if not None, the result is filtered
1986
so that only those file-ids, their parents and their
1987
children are included.
1989
# Get the revision-ids of interest
1990
required_trees = set()
1991
for revision in revisions:
1992
required_trees.add(revision.revision_id)
1993
required_trees.update(revision.parent_ids[:1])
1995
# Get the matching filtered trees. Note that it's more
1996
# efficient to pass filtered trees to changes_from() rather
1997
# than doing the filtering afterwards. changes_from() could
1998
# arguably do the filtering itself but it's path-based, not
1999
# file-id based, so filtering before or afterwards is
2001
if specific_fileids is None:
2002
trees = dict((t.get_revision_id(), t) for
2003
t in self.revision_trees(required_trees))
2005
trees = dict((t.get_revision_id(), t) for
2006
t in self._filtered_revision_trees(required_trees,
2009
# Calculate the deltas
2010
for revision in revisions:
2011
if not revision.parent_ids:
2012
old_tree = self.revision_tree(_mod_revision.NULL_REVISION)
2014
old_tree = trees[revision.parent_ids[0]]
2015
yield trees[revision.revision_id].changes_from(old_tree)
2018
def get_revision_delta(self, revision_id, specific_fileids=None):
2019
"""Return the delta for one revision.
2021
The delta is relative to the left-hand predecessor of the
2024
:param specific_fileids: if not None, the result is filtered
2025
so that only those file-ids, their parents and their
2026
children are included.
2028
r = self.get_revision(revision_id)
2029
return list(self.get_deltas_for_revisions([r],
2030
specific_fileids=specific_fileids))[0]
2033
def store_revision_signature(self, gpg_strategy, plaintext, revision_id):
2034
signature = gpg_strategy.sign(plaintext)
2035
self.add_signature_text(revision_id, signature)
2038
def add_signature_text(self, revision_id, signature):
2039
self.signatures.add_lines((revision_id,), (),
2040
osutils.split_lines(signature))
2042
def find_text_key_references(self):
2043
"""Find the text key references within the repository.
2045
:return: A dictionary mapping text keys ((fileid, revision_id) tuples)
2046
to whether they were referred to by the inventory of the
2047
revision_id that they contain. The inventory texts from all present
2048
revision ids are assessed to generate this report.
2050
revision_keys = self.revisions.keys()
2051
w = self.inventories
2052
pb = ui.ui_factory.nested_progress_bar()
2054
return self._serializer._find_text_key_references(
2055
w.iter_lines_added_or_present_in_keys(revision_keys, pb=pb))
2059
def _inventory_xml_lines_for_keys(self, keys):
2060
"""Get a line iterator of the sort needed for findind references.
2062
Not relevant for non-xml inventory repositories.
2064
Ghosts in revision_keys are ignored.
2066
:param revision_keys: The revision keys for the inventories to inspect.
2067
:return: An iterator over (inventory line, revid) for the fulltexts of
2068
all of the xml inventories specified by revision_keys.
2070
stream = self.inventories.get_record_stream(keys, 'unordered', True)
2071
for record in stream:
2072
if record.storage_kind != 'absent':
2073
chunks = record.get_bytes_as('chunked')
2074
revid = record.key[-1]
2075
lines = osutils.chunks_to_lines(chunks)
2079
def _find_file_ids_from_xml_inventory_lines(self, line_iterator,
2081
"""Helper routine for fileids_altered_by_revision_ids.
2083
This performs the translation of xml lines to revision ids.
2085
:param line_iterator: An iterator of lines, origin_version_id
2086
:param revision_keys: The revision ids to filter for. This should be a
2087
set or other type which supports efficient __contains__ lookups, as
2088
the revision key from each parsed line will be looked up in the
2089
revision_keys filter.
2090
:return: a dictionary mapping altered file-ids to an iterable of
2091
revision_ids. Each altered file-ids has the exact revision_ids that
2092
altered it listed explicitly.
2094
seen = set(self._serializer._find_text_key_references(
2095
line_iterator).iterkeys())
2096
parent_keys = self._find_parent_keys_of_revisions(revision_keys)
2097
parent_seen = set(self._serializer._find_text_key_references(
2098
self._inventory_xml_lines_for_keys(parent_keys)))
2099
new_keys = seen - parent_seen
2101
setdefault = result.setdefault
2102
for key in new_keys:
2103
setdefault(key[0], set()).add(key[-1])
2106
def _find_parent_ids_of_revisions(self, revision_ids):
2107
"""Find all parent ids that are mentioned in the revision graph.
2109
:return: set of revisions that are parents of revision_ids which are
2110
not part of revision_ids themselves
2112
parent_map = self.get_parent_map(revision_ids)
2114
map(parent_ids.update, parent_map.itervalues())
2115
parent_ids.difference_update(revision_ids)
2116
parent_ids.discard(_mod_revision.NULL_REVISION)
2119
def _find_parent_keys_of_revisions(self, revision_keys):
2120
"""Similar to _find_parent_ids_of_revisions, but used with keys.
2122
:param revision_keys: An iterable of revision_keys.
2123
:return: The parents of all revision_keys that are not already in
2126
parent_map = self.revisions.get_parent_map(revision_keys)
2128
map(parent_keys.update, parent_map.itervalues())
2129
parent_keys.difference_update(revision_keys)
2130
parent_keys.discard(_mod_revision.NULL_REVISION)
2133
def fileids_altered_by_revision_ids(self, revision_ids, _inv_weave=None):
2134
"""Find the file ids and versions affected by revisions.
2136
:param revisions: an iterable containing revision ids.
2137
:param _inv_weave: The inventory weave from this repository or None.
2138
If None, the inventory weave will be opened automatically.
2139
:return: a dictionary mapping altered file-ids to an iterable of
2140
revision_ids. Each altered file-ids has the exact revision_ids that
2141
altered it listed explicitly.
2143
selected_keys = set((revid,) for revid in revision_ids)
2144
w = _inv_weave or self.inventories
2145
return self._find_file_ids_from_xml_inventory_lines(
2146
w.iter_lines_added_or_present_in_keys(
2147
selected_keys, pb=None),
2150
def iter_files_bytes(self, desired_files):
2151
"""Iterate through file versions.
2153
Files will not necessarily be returned in the order they occur in
2154
desired_files. No specific order is guaranteed.
2156
Yields pairs of identifier, bytes_iterator. identifier is an opaque
2157
value supplied by the caller as part of desired_files. It should
2158
uniquely identify the file version in the caller's context. (Examples:
2159
an index number or a TreeTransform trans_id.)
2161
bytes_iterator is an iterable of bytestrings for the file. The
2162
kind of iterable and length of the bytestrings are unspecified, but for
2163
this implementation, it is a list of bytes produced by
2164
VersionedFile.get_record_stream().
2166
:param desired_files: a list of (file_id, revision_id, identifier)
2170
for file_id, revision_id, callable_data in desired_files:
2171
text_keys[(file_id, revision_id)] = callable_data
2172
for record in self.texts.get_record_stream(text_keys, 'unordered', True):
2173
if record.storage_kind == 'absent':
2174
raise errors.RevisionNotPresent(record.key, self)
2175
yield text_keys[record.key], record.get_bytes_as('chunked')
2177
def _generate_text_key_index(self, text_key_references=None,
2179
"""Generate a new text key index for the repository.
2181
This is an expensive function that will take considerable time to run.
2183
:return: A dict mapping text keys ((file_id, revision_id) tuples) to a
2184
list of parents, also text keys. When a given key has no parents,
2185
the parents list will be [NULL_REVISION].
2187
# All revisions, to find inventory parents.
2188
if ancestors is None:
2189
graph = self.get_graph()
2190
ancestors = graph.get_parent_map(self.all_revision_ids())
2191
if text_key_references is None:
2192
text_key_references = self.find_text_key_references()
2193
pb = ui.ui_factory.nested_progress_bar()
2195
return self._do_generate_text_key_index(ancestors,
2196
text_key_references, pb)
2200
def _do_generate_text_key_index(self, ancestors, text_key_references, pb):
2201
"""Helper for _generate_text_key_index to avoid deep nesting."""
2202
revision_order = tsort.topo_sort(ancestors)
2203
invalid_keys = set()
2205
for revision_id in revision_order:
2206
revision_keys[revision_id] = set()
2207
text_count = len(text_key_references)
2208
# a cache of the text keys to allow reuse; costs a dict of all the
2209
# keys, but saves a 2-tuple for every child of a given key.
2211
for text_key, valid in text_key_references.iteritems():
2213
invalid_keys.add(text_key)
2215
revision_keys[text_key[1]].add(text_key)
2216
text_key_cache[text_key] = text_key
2217
del text_key_references
2219
text_graph = graph.Graph(graph.DictParentsProvider(text_index))
2220
NULL_REVISION = _mod_revision.NULL_REVISION
2221
# Set a cache with a size of 10 - this suffices for bzr.dev but may be
2222
# too small for large or very branchy trees. However, for 55K path
2223
# trees, it would be easy to use too much memory trivially. Ideally we
2224
# could gauge this by looking at available real memory etc, but this is
2225
# always a tricky proposition.
2226
inventory_cache = lru_cache.LRUCache(10)
2227
batch_size = 10 # should be ~150MB on a 55K path tree
2228
batch_count = len(revision_order) / batch_size + 1
2230
pb.update("Calculating text parents", processed_texts, text_count)
2231
for offset in xrange(batch_count):
2232
to_query = revision_order[offset * batch_size:(offset + 1) *
2236
for revision_id in to_query:
2237
parent_ids = ancestors[revision_id]
2238
for text_key in revision_keys[revision_id]:
2239
pb.update("Calculating text parents", processed_texts)
2240
processed_texts += 1
2241
candidate_parents = []
2242
for parent_id in parent_ids:
2243
parent_text_key = (text_key[0], parent_id)
2245
check_parent = parent_text_key not in \
2246
revision_keys[parent_id]
2248
# the parent parent_id is a ghost:
2249
check_parent = False
2250
# truncate the derived graph against this ghost.
2251
parent_text_key = None
2253
# look at the parent commit details inventories to
2254
# determine possible candidates in the per file graph.
2257
inv = inventory_cache[parent_id]
2259
inv = self.revision_tree(parent_id).inventory
2260
inventory_cache[parent_id] = inv
2262
parent_entry = inv[text_key[0]]
2263
except (KeyError, errors.NoSuchId):
2265
if parent_entry is not None:
2267
text_key[0], parent_entry.revision)
2269
parent_text_key = None
2270
if parent_text_key is not None:
2271
candidate_parents.append(
2272
text_key_cache[parent_text_key])
2273
parent_heads = text_graph.heads(candidate_parents)
2274
new_parents = list(parent_heads)
2275
new_parents.sort(key=lambda x:candidate_parents.index(x))
2276
if new_parents == []:
2277
new_parents = [NULL_REVISION]
2278
text_index[text_key] = new_parents
2280
for text_key in invalid_keys:
2281
text_index[text_key] = [NULL_REVISION]
2284
def item_keys_introduced_by(self, revision_ids, _files_pb=None):
2285
"""Get an iterable listing the keys of all the data introduced by a set
2288
The keys will be ordered so that the corresponding items can be safely
2289
fetched and inserted in that order.
2291
:returns: An iterable producing tuples of (knit-kind, file-id,
2292
versions). knit-kind is one of 'file', 'inventory', 'signatures',
2293
'revisions'. file-id is None unless knit-kind is 'file'.
2295
for result in self._find_file_keys_to_fetch(revision_ids, _files_pb):
2298
for result in self._find_non_file_keys_to_fetch(revision_ids):
2301
def _find_file_keys_to_fetch(self, revision_ids, pb):
2302
# XXX: it's a bit weird to control the inventory weave caching in this
2303
# generator. Ideally the caching would be done in fetch.py I think. Or
2304
# maybe this generator should explicitly have the contract that it
2305
# should not be iterated until the previously yielded item has been
2307
inv_w = self.inventories
2309
# file ids that changed
2310
file_ids = self.fileids_altered_by_revision_ids(revision_ids, inv_w)
2312
num_file_ids = len(file_ids)
2313
for file_id, altered_versions in file_ids.iteritems():
2315
pb.update("Fetch texts", count, num_file_ids)
2317
yield ("file", file_id, altered_versions)
2319
def _find_non_file_keys_to_fetch(self, revision_ids):
2321
yield ("inventory", None, revision_ids)
2324
# XXX: Note ATM no callers actually pay attention to this return
2325
# instead they just use the list of revision ids and ignore
2326
# missing sigs. Consider removing this work entirely
2327
revisions_with_signatures = set(self.signatures.get_parent_map(
2328
[(r,) for r in revision_ids]))
2329
revisions_with_signatures = set(
2330
[r for (r,) in revisions_with_signatures])
2331
revisions_with_signatures.intersection_update(revision_ids)
2332
yield ("signatures", None, revisions_with_signatures)
2335
yield ("revisions", None, revision_ids)
2338
def get_inventory(self, revision_id):
2339
"""Get Inventory object by revision id."""
2340
return self.iter_inventories([revision_id]).next()
2342
def iter_inventories(self, revision_ids, ordering=None):
2343
"""Get many inventories by revision_ids.
2345
This will buffer some or all of the texts used in constructing the
2346
inventories in memory, but will only parse a single inventory at a
2349
:param revision_ids: The expected revision ids of the inventories.
2350
:param ordering: optional ordering, e.g. 'topological'. If not
2351
specified, the order of revision_ids will be preserved (by
2352
buffering if necessary).
2353
:return: An iterator of inventories.
2355
if ((None in revision_ids)
2356
or (_mod_revision.NULL_REVISION in revision_ids)):
2357
raise ValueError('cannot get null revision inventory')
2358
return self._iter_inventories(revision_ids, ordering)
2360
def _iter_inventories(self, revision_ids, ordering):
2361
"""single-document based inventory iteration."""
2362
inv_xmls = self._iter_inventory_xmls(revision_ids, ordering)
2363
for text, revision_id in inv_xmls:
2364
yield self._deserialise_inventory(revision_id, text)
2366
def _iter_inventory_xmls(self, revision_ids, ordering):
2367
if ordering is None:
2368
order_as_requested = True
2369
ordering = 'unordered'
2371
order_as_requested = False
2372
keys = [(revision_id,) for revision_id in revision_ids]
2375
if order_as_requested:
2376
key_iter = iter(keys)
2377
next_key = key_iter.next()
2378
stream = self.inventories.get_record_stream(keys, ordering, True)
2380
for record in stream:
2381
if record.storage_kind != 'absent':
2382
chunks = record.get_bytes_as('chunked')
2383
if order_as_requested:
2384
text_chunks[record.key] = chunks
2386
yield ''.join(chunks), record.key[-1]
2388
raise errors.NoSuchRevision(self, record.key)
2389
if order_as_requested:
2390
# Yield as many results as we can while preserving order.
2391
while next_key in text_chunks:
2392
chunks = text_chunks.pop(next_key)
2393
yield ''.join(chunks), next_key[-1]
2395
next_key = key_iter.next()
2396
except StopIteration:
2397
# We still want to fully consume the get_record_stream,
2398
# just in case it is not actually finished at this point
2402
def _deserialise_inventory(self, revision_id, xml):
2403
"""Transform the xml into an inventory object.
2405
:param revision_id: The expected revision id of the inventory.
2406
:param xml: A serialised inventory.
2408
result = self._serializer.read_inventory_from_string(xml, revision_id,
2409
entry_cache=self._inventory_entry_cache,
2410
return_from_cache=self._safe_to_return_from_cache)
2411
if result.revision_id != revision_id:
2412
raise AssertionError('revision id mismatch %s != %s' % (
2413
result.revision_id, revision_id))
2416
def get_serializer_format(self):
2417
return self._serializer.format_num
2420
def _get_inventory_xml(self, revision_id):
2421
"""Get serialized inventory as a string."""
2422
texts = self._iter_inventory_xmls([revision_id], 'unordered')
2424
text, revision_id = texts.next()
2425
except StopIteration:
2426
raise errors.HistoryMissing(self, 'inventory', revision_id)
2429
def get_rev_id_for_revno(self, revno, known_pair):
2430
"""Return the revision id of a revno, given a later (revno, revid)
2431
pair in the same history.
2433
:return: if found (True, revid). If the available history ran out
2434
before reaching the revno, then this returns
2435
(False, (closest_revno, closest_revid)).
2437
known_revno, known_revid = known_pair
2438
partial_history = [known_revid]
2439
distance_from_known = known_revno - revno
2440
if distance_from_known < 0:
2442
'requested revno (%d) is later than given known revno (%d)'
2443
% (revno, known_revno))
2446
self, partial_history, stop_index=distance_from_known)
2447
except errors.RevisionNotPresent, err:
2448
if err.revision_id == known_revid:
2449
# The start revision (known_revid) wasn't found.
2451
# This is a stacked repository with no fallbacks, or a there's a
2452
# left-hand ghost. Either way, even though the revision named in
2453
# the error isn't in this repo, we know it's the next step in this
2454
# left-hand history.
2455
partial_history.append(err.revision_id)
2456
if len(partial_history) <= distance_from_known:
2457
# Didn't find enough history to get a revid for the revno.
2458
earliest_revno = known_revno - len(partial_history) + 1
2459
return (False, (earliest_revno, partial_history[-1]))
2460
if len(partial_history) - 1 > distance_from_known:
2461
raise AssertionError('_iter_for_revno returned too much history')
2462
return (True, partial_history[-1])
2464
def iter_reverse_revision_history(self, revision_id):
2465
"""Iterate backwards through revision ids in the lefthand history
2467
:param revision_id: The revision id to start with. All its lefthand
2468
ancestors will be traversed.
2470
graph = self.get_graph()
2471
stop_revisions = (None, _mod_revision.NULL_REVISION)
2472
return graph.iter_lefthand_ancestry(revision_id, stop_revisions)
2474
def is_shared(self):
2475
"""Return True if this repository is flagged as a shared repository."""
2476
raise NotImplementedError(self.is_shared)
2479
def reconcile(self, other=None, thorough=False):
2480
"""Reconcile this repository."""
2481
from bzrlib.reconcile import RepoReconciler
2482
reconciler = RepoReconciler(self, thorough=thorough)
2483
reconciler.reconcile()
2486
def _refresh_data(self):
2487
"""Helper called from lock_* to ensure coherency with disk.
2489
The default implementation does nothing; it is however possible
2490
for repositories to maintain loaded indices across multiple locks
2491
by checking inside their implementation of this method to see
2492
whether their indices are still valid. This depends of course on
2493
the disk format being validatable in this manner. This method is
2494
also called by the refresh_data() public interface to cause a refresh
2495
to occur while in a write lock so that data inserted by a smart server
2496
push operation is visible on the client's instance of the physical
2501
def revision_tree(self, revision_id):
2502
"""Return Tree for a revision on this branch.
2504
`revision_id` may be NULL_REVISION for the empty tree revision.
2506
revision_id = _mod_revision.ensure_null(revision_id)
2507
# TODO: refactor this to use an existing revision object
2508
# so we don't need to read it in twice.
2509
if revision_id == _mod_revision.NULL_REVISION:
2510
return RevisionTree(self, Inventory(root_id=None),
2511
_mod_revision.NULL_REVISION)
2513
inv = self.get_inventory(revision_id)
2514
return RevisionTree(self, inv, revision_id)
2516
def revision_trees(self, revision_ids):
2517
"""Return Trees for revisions in this repository.
2519
:param revision_ids: a sequence of revision-ids;
2520
a revision-id may not be None or 'null:'
2522
inventories = self.iter_inventories(revision_ids)
2523
for inv in inventories:
2524
yield RevisionTree(self, inv, inv.revision_id)
2526
def _filtered_revision_trees(self, revision_ids, file_ids):
2527
"""Return Tree for a revision on this branch with only some files.
2529
:param revision_ids: a sequence of revision-ids;
2530
a revision-id may not be None or 'null:'
2531
:param file_ids: if not None, the result is filtered
2532
so that only those file-ids, their parents and their
2533
children are included.
2535
inventories = self.iter_inventories(revision_ids)
2536
for inv in inventories:
2537
# Should we introduce a FilteredRevisionTree class rather
2538
# than pre-filter the inventory here?
2539
filtered_inv = inv.filter(file_ids)
2540
yield RevisionTree(self, filtered_inv, filtered_inv.revision_id)
2543
def get_ancestry(self, revision_id, topo_sorted=True):
2544
"""Return a list of revision-ids integrated by a revision.
2546
The first element of the list is always None, indicating the origin
2547
revision. This might change when we have history horizons, or
2548
perhaps we should have a new API.
2550
This is topologically sorted.
2552
if _mod_revision.is_null(revision_id):
2554
if not self.has_revision(revision_id):
2555
raise errors.NoSuchRevision(self, revision_id)
2556
graph = self.get_graph()
2558
search = graph._make_breadth_first_searcher([revision_id])
2561
found, ghosts = search.next_with_ghosts()
2562
except StopIteration:
2565
if _mod_revision.NULL_REVISION in keys:
2566
keys.remove(_mod_revision.NULL_REVISION)
2568
parent_map = graph.get_parent_map(keys)
2569
keys = tsort.topo_sort(parent_map)
2570
return [None] + list(keys)
2572
def pack(self, hint=None, clean_obsolete_packs=False):
2573
"""Compress the data within the repository.
2575
This operation only makes sense for some repository types. For other
2576
types it should be a no-op that just returns.
2578
This stub method does not require a lock, but subclasses should use
2579
@needs_write_lock as this is a long running call it's reasonable to
2580
implicitly lock for the user.
2582
:param hint: If not supplied, the whole repository is packed.
2583
If supplied, the repository may use the hint parameter as a
2584
hint for the parts of the repository to pack. A hint can be
2585
obtained from the result of commit_write_group(). Out of
2586
date hints are simply ignored, because concurrent operations
2587
can obsolete them rapidly.
2589
:param clean_obsolete_packs: Clean obsolete packs immediately after
2593
def get_transaction(self):
2594
return self.control_files.get_transaction()
2596
def get_parent_map(self, revision_ids):
2597
"""See graph.StackedParentsProvider.get_parent_map"""
2598
# revisions index works in keys; this just works in revisions
2599
# therefore wrap and unwrap
2602
for revision_id in revision_ids:
2603
if revision_id == _mod_revision.NULL_REVISION:
2604
result[revision_id] = ()
2605
elif revision_id is None:
2606
raise ValueError('get_parent_map(None) is not valid')
2608
query_keys.append((revision_id ,))
2609
for ((revision_id,), parent_keys) in \
2610
self.revisions.get_parent_map(query_keys).iteritems():
2612
result[revision_id] = tuple([parent_revid
2613
for (parent_revid,) in parent_keys])
2615
result[revision_id] = (_mod_revision.NULL_REVISION,)
2618
def _make_parents_provider(self):
2622
def get_known_graph_ancestry(self, revision_ids):
2623
"""Return the known graph for a set of revision ids and their ancestors.
2625
st = static_tuple.StaticTuple
2626
revision_keys = [st(r_id).intern() for r_id in revision_ids]
2627
known_graph = self.revisions.get_known_graph_ancestry(revision_keys)
2628
return graph.GraphThunkIdsToKeys(known_graph)
2630
def get_graph(self, other_repository=None):
2631
"""Return the graph walker for this repository format"""
2632
parents_provider = self._make_parents_provider()
2633
if (other_repository is not None and
2634
not self.has_same_location(other_repository)):
2635
parents_provider = graph.StackedParentsProvider(
2636
[parents_provider, other_repository._make_parents_provider()])
2637
return graph.Graph(parents_provider)
2639
def _get_versioned_file_checker(self, text_key_references=None,
2641
"""Return an object suitable for checking versioned files.
2643
:param text_key_references: if non-None, an already built
2644
dictionary mapping text keys ((fileid, revision_id) tuples)
2645
to whether they were referred to by the inventory of the
2646
revision_id that they contain. If None, this will be
2648
:param ancestors: Optional result from
2649
self.get_graph().get_parent_map(self.all_revision_ids()) if already
2652
return _VersionedFileChecker(self,
2653
text_key_references=text_key_references, ancestors=ancestors)
2655
def revision_ids_to_search_result(self, result_set):
2656
"""Convert a set of revision ids to a graph SearchResult."""
2657
result_parents = set()
2658
for parents in self.get_graph().get_parent_map(
2659
result_set).itervalues():
2660
result_parents.update(parents)
2661
included_keys = result_set.intersection(result_parents)
2662
start_keys = result_set.difference(included_keys)
2663
exclude_keys = result_parents.difference(result_set)
2664
result = graph.SearchResult(start_keys, exclude_keys,
2665
len(result_set), result_set)
2669
def set_make_working_trees(self, new_value):
2670
"""Set the policy flag for making working trees when creating branches.
2672
This only applies to branches that use this repository.
2674
The default is 'True'.
2675
:param new_value: True to restore the default, False to disable making
2678
raise NotImplementedError(self.set_make_working_trees)
2680
def make_working_trees(self):
2681
"""Returns the policy for making working trees on new branches."""
2682
raise NotImplementedError(self.make_working_trees)
2685
def sign_revision(self, revision_id, gpg_strategy):
2686
plaintext = Testament.from_revision(self, revision_id).as_short_text()
2687
self.store_revision_signature(gpg_strategy, plaintext, revision_id)
2690
def has_signature_for_revision_id(self, revision_id):
2691
"""Query for a revision signature for revision_id in the repository."""
2692
if not self.has_revision(revision_id):
2693
raise errors.NoSuchRevision(self, revision_id)
2694
sig_present = (1 == len(
2695
self.signatures.get_parent_map([(revision_id,)])))
2699
def get_signature_text(self, revision_id):
2700
"""Return the text for a signature."""
2701
stream = self.signatures.get_record_stream([(revision_id,)],
2703
record = stream.next()
2704
if record.storage_kind == 'absent':
2705
raise errors.NoSuchRevision(self, revision_id)
2706
return record.get_bytes_as('fulltext')
2709
def check(self, revision_ids=None, callback_refs=None, check_repo=True):
2710
"""Check consistency of all history of given revision_ids.
2712
Different repository implementations should override _check().
2714
:param revision_ids: A non-empty list of revision_ids whose ancestry
2715
will be checked. Typically the last revision_id of a branch.
2716
:param callback_refs: A dict of check-refs to resolve and callback
2717
the check/_check method on the items listed as wanting the ref.
2719
:param check_repo: If False do not check the repository contents, just
2720
calculate the data callback_refs requires and call them back.
2722
return self._check(revision_ids, callback_refs=callback_refs,
2723
check_repo=check_repo)
2725
def _check(self, revision_ids, callback_refs, check_repo):
2726
result = check.Check(self, check_repo=check_repo)
2727
result.check(callback_refs)
2730
def _warn_if_deprecated(self, branch=None):
2731
if not self._format.is_deprecated():
2733
global _deprecation_warning_done
2734
if _deprecation_warning_done:
2738
conf = config.GlobalConfig()
2740
conf = branch.get_config()
2741
if conf.suppress_warning('format_deprecation'):
2743
warning("Format %s for %s is deprecated -"
2744
" please use 'bzr upgrade' to get better performance"
2745
% (self._format, self.bzrdir.transport.base))
2747
_deprecation_warning_done = True
2749
def supports_rich_root(self):
2750
return self._format.rich_root_data
2752
def _check_ascii_revisionid(self, revision_id, method):
2753
"""Private helper for ascii-only repositories."""
2754
# weave repositories refuse to store revisionids that are non-ascii.
2755
if revision_id is not None:
2756
# weaves require ascii revision ids.
2757
if isinstance(revision_id, unicode):
2759
revision_id.encode('ascii')
2760
except UnicodeEncodeError:
2761
raise errors.NonAsciiRevisionId(method, self)
2764
revision_id.decode('ascii')
2765
except UnicodeDecodeError:
2766
raise errors.NonAsciiRevisionId(method, self)
2768
def revision_graph_can_have_wrong_parents(self):
2769
"""Is it possible for this repository to have a revision graph with
2772
If True, then this repository must also implement
2773
_find_inconsistent_revision_parents so that check and reconcile can
2774
check for inconsistencies before proceeding with other checks that may
2775
depend on the revision index being consistent.
2777
raise NotImplementedError(self.revision_graph_can_have_wrong_parents)
2780
def install_revision(repository, rev, revision_tree):
2781
"""Install all revision data into a repository."""
2782
install_revisions(repository, [(rev, revision_tree, None)])
2785
def install_revisions(repository, iterable, num_revisions=None, pb=None):
2786
"""Install all revision data into a repository.
2788
Accepts an iterable of revision, tree, signature tuples. The signature
2791
repository.start_write_group()
2793
inventory_cache = lru_cache.LRUCache(10)
2794
for n, (revision, revision_tree, signature) in enumerate(iterable):
2795
_install_revision(repository, revision, revision_tree, signature,
2798
pb.update('Transferring revisions', n + 1, num_revisions)
2800
repository.abort_write_group()
2803
repository.commit_write_group()
2806
def _install_revision(repository, rev, revision_tree, signature,
2808
"""Install all revision data into a repository."""
2809
present_parents = []
2811
for p_id in rev.parent_ids:
2812
if repository.has_revision(p_id):
2813
present_parents.append(p_id)
2814
parent_trees[p_id] = repository.revision_tree(p_id)
2816
parent_trees[p_id] = repository.revision_tree(
2817
_mod_revision.NULL_REVISION)
2819
inv = revision_tree.inventory
2820
entries = inv.iter_entries()
2821
# backwards compatibility hack: skip the root id.
2822
if not repository.supports_rich_root():
2823
path, root = entries.next()
2824
if root.revision != rev.revision_id:
2825
raise errors.IncompatibleRevision(repr(repository))
2827
for path, ie in entries:
2828
text_keys[(ie.file_id, ie.revision)] = ie
2829
text_parent_map = repository.texts.get_parent_map(text_keys)
2830
missing_texts = set(text_keys) - set(text_parent_map)
2831
# Add the texts that are not already present
2832
for text_key in missing_texts:
2833
ie = text_keys[text_key]
2835
# FIXME: TODO: The following loop overlaps/duplicates that done by
2836
# commit to determine parents. There is a latent/real bug here where
2837
# the parents inserted are not those commit would do - in particular
2838
# they are not filtered by heads(). RBC, AB
2839
for revision, tree in parent_trees.iteritems():
2840
if ie.file_id not in tree:
2842
parent_id = tree.inventory[ie.file_id].revision
2843
if parent_id in text_parents:
2845
text_parents.append((ie.file_id, parent_id))
2846
lines = revision_tree.get_file(ie.file_id).readlines()
2847
repository.texts.add_lines(text_key, text_parents, lines)
2849
# install the inventory
2850
if repository._format._commit_inv_deltas and len(rev.parent_ids):
2851
# Cache this inventory
2852
inventory_cache[rev.revision_id] = inv
2854
basis_inv = inventory_cache[rev.parent_ids[0]]
2856
repository.add_inventory(rev.revision_id, inv, present_parents)
2858
delta = inv._make_delta(basis_inv)
2859
repository.add_inventory_by_delta(rev.parent_ids[0], delta,
2860
rev.revision_id, present_parents)
2862
repository.add_inventory(rev.revision_id, inv, present_parents)
2863
except errors.RevisionAlreadyPresent:
2865
if signature is not None:
2866
repository.add_signature_text(rev.revision_id, signature)
2867
repository.add_revision(rev.revision_id, rev, inv)
2870
class MetaDirRepository(Repository):
2871
"""Repositories in the new meta-dir layout.
2873
:ivar _transport: Transport for access to repository control files,
2874
typically pointing to .bzr/repository.
2877
def __init__(self, _format, a_bzrdir, control_files):
2878
super(MetaDirRepository, self).__init__(_format, a_bzrdir, control_files)
2879
self._transport = control_files._transport
2881
def is_shared(self):
2882
"""Return True if this repository is flagged as a shared repository."""
2883
return self._transport.has('shared-storage')
2886
def set_make_working_trees(self, new_value):
2887
"""Set the policy flag for making working trees when creating branches.
2889
This only applies to branches that use this repository.
2891
The default is 'True'.
2892
:param new_value: True to restore the default, False to disable making
2897
self._transport.delete('no-working-trees')
2898
except errors.NoSuchFile:
2901
self._transport.put_bytes('no-working-trees', '',
2902
mode=self.bzrdir._get_file_mode())
2904
def make_working_trees(self):
2905
"""Returns the policy for making working trees on new branches."""
2906
return not self._transport.has('no-working-trees')
2909
class MetaDirVersionedFileRepository(MetaDirRepository):
2910
"""Repositories in a meta-dir, that work via versioned file objects."""
2912
def __init__(self, _format, a_bzrdir, control_files):
2913
super(MetaDirVersionedFileRepository, self).__init__(_format, a_bzrdir,
2917
class RepositoryFormatRegistry(controldir.ControlComponentFormatRegistry):
2918
"""Repository format registry."""
2920
def get_default(self):
2921
"""Return the current default format."""
2922
from bzrlib import bzrdir
2923
return bzrdir.format_registry.make_bzrdir('default').repository_format
2926
network_format_registry = registry.FormatRegistry()
2927
"""Registry of formats indexed by their network name.
2929
The network name for a repository format is an identifier that can be used when
2930
referring to formats with smart server operations. See
2931
RepositoryFormat.network_name() for more detail.
2935
format_registry = RepositoryFormatRegistry(network_format_registry)
2936
"""Registry of formats, indexed by their BzrDirMetaFormat format string.
2938
This can contain either format instances themselves, or classes/factories that
2939
can be called to obtain one.
2943
#####################################################################
2944
# Repository Formats
2946
class RepositoryFormat(controldir.ControlComponentFormat):
2947
"""A repository format.
2949
Formats provide four things:
2950
* An initialization routine to construct repository data on disk.
2951
* a optional format string which is used when the BzrDir supports
2953
* an open routine which returns a Repository instance.
2954
* A network name for referring to the format in smart server RPC
2957
There is one and only one Format subclass for each on-disk format. But
2958
there can be one Repository subclass that is used for several different
2959
formats. The _format attribute on a Repository instance can be used to
2960
determine the disk format.
2962
Formats are placed in a registry by their format string for reference
2963
during opening. These should be subclasses of RepositoryFormat for
2966
Once a format is deprecated, just deprecate the initialize and open
2967
methods on the format class. Do not deprecate the object, as the
2968
object may be created even when a repository instance hasn't been
2971
Common instance attributes:
2972
_matchingbzrdir - the bzrdir format that the repository format was
2973
originally written to work with. This can be used if manually
2974
constructing a bzrdir and repository, or more commonly for test suite
2978
# Set to True or False in derived classes. True indicates that the format
2979
# supports ghosts gracefully.
2980
supports_ghosts = None
2981
# Can this repository be given external locations to lookup additional
2982
# data. Set to True or False in derived classes.
2983
supports_external_lookups = None
2984
# Does this format support CHK bytestring lookups. Set to True or False in
2986
supports_chks = None
2987
# Should commit add an inventory, or an inventory delta to the repository.
2988
_commit_inv_deltas = True
2989
# What order should fetch operations request streams in?
2990
# The default is unordered as that is the cheapest for an origin to
2992
_fetch_order = 'unordered'
2993
# Does this repository format use deltas that can be fetched as-deltas ?
2994
# (E.g. knits, where the knit deltas can be transplanted intact.
2995
# We default to False, which will ensure that enough data to get
2996
# a full text out of any fetch stream will be grabbed.
2997
_fetch_uses_deltas = False
2998
# Should fetch trigger a reconcile after the fetch? Only needed for
2999
# some repository formats that can suffer internal inconsistencies.
3000
_fetch_reconcile = False
3001
# Does this format have < O(tree_size) delta generation. Used to hint what
3002
# code path for commit, amongst other things.
3004
# Does doing a pack operation compress data? Useful for the pack UI command
3005
# (so if there is one pack, the operation can still proceed because it may
3006
# help), and for fetching when data won't have come from the same
3008
pack_compresses = False
3009
# Does the repository inventory storage understand references to trees?
3010
supports_tree_reference = None
3011
# Is the format experimental ?
3012
experimental = False
3013
# Does this repository format escape funky characters, or does it create files with
3014
# similar names as the versioned files in its contents on disk ?
3015
supports_funky_characters = None
3016
# Does this repository format support leaving locks?
3017
supports_leaving_lock = None
3018
# Does this format support the full VersionedFiles interface?
3019
supports_full_versioned_files = None
3022
return "%s()" % self.__class__.__name__
3024
def __eq__(self, other):
3025
# format objects are generally stateless
3026
return isinstance(other, self.__class__)
3028
def __ne__(self, other):
3029
return not self == other
3032
def find_format(klass, a_bzrdir):
3033
"""Return the format for the repository object in a_bzrdir.
3035
This is used by bzr native formats that have a "format" file in
3036
the repository. Other methods may be used by different types of
3040
transport = a_bzrdir.get_repository_transport(None)
3041
format_string = transport.get_bytes("format")
3042
return format_registry.get(format_string)
3043
except errors.NoSuchFile:
3044
raise errors.NoRepositoryPresent(a_bzrdir)
3046
raise errors.UnknownFormatError(format=format_string,
3050
@symbol_versioning.deprecated_method(symbol_versioning.deprecated_in((2, 4, 0)))
3051
def register_format(klass, format):
3052
format_registry.register(format)
3055
@symbol_versioning.deprecated_method(symbol_versioning.deprecated_in((2, 4, 0)))
3056
def unregister_format(klass, format):
3057
format_registry.remove(format)
3060
@symbol_versioning.deprecated_method(symbol_versioning.deprecated_in((2, 4, 0)))
3061
def get_default_format(klass):
3062
"""Return the current default format."""
3063
return format_registry.get_default()
3065
def get_format_string(self):
3066
"""Return the ASCII format string that identifies this format.
3068
Note that in pre format ?? repositories the format string is
3069
not permitted nor written to disk.
3071
raise NotImplementedError(self.get_format_string)
3073
def get_format_description(self):
3074
"""Return the short description for this format."""
3075
raise NotImplementedError(self.get_format_description)
3077
# TODO: this shouldn't be in the base class, it's specific to things that
3078
# use weaves or knits -- mbp 20070207
3079
def _get_versioned_file_store(self,
3084
versionedfile_class=None,
3085
versionedfile_kwargs={},
3087
if versionedfile_class is None:
3088
versionedfile_class = self._versionedfile_class
3089
weave_transport = control_files._transport.clone(name)
3090
dir_mode = control_files._dir_mode
3091
file_mode = control_files._file_mode
3092
return VersionedFileStore(weave_transport, prefixed=prefixed,
3094
file_mode=file_mode,
3095
versionedfile_class=versionedfile_class,
3096
versionedfile_kwargs=versionedfile_kwargs,
3099
def initialize(self, a_bzrdir, shared=False):
3100
"""Initialize a repository of this format in a_bzrdir.
3102
:param a_bzrdir: The bzrdir to put the new repository in it.
3103
:param shared: The repository should be initialized as a sharable one.
3104
:returns: The new repository object.
3106
This may raise UninitializableFormat if shared repository are not
3107
compatible the a_bzrdir.
3109
raise NotImplementedError(self.initialize)
3111
def is_supported(self):
3112
"""Is this format supported?
3114
Supported formats must be initializable and openable.
3115
Unsupported formats may not support initialization or committing or
3116
some other features depending on the reason for not being supported.
3120
def is_deprecated(self):
3121
"""Is this format deprecated?
3123
Deprecated formats may trigger a user-visible warning recommending
3124
the user to upgrade. They are still fully supported.
3128
def network_name(self):
3129
"""A simple byte string uniquely identifying this format for RPC calls.
3131
MetaDir repository formats use their disk format string to identify the
3132
repository over the wire. All in one formats such as bzr < 0.8, and
3133
foreign formats like svn/git and hg should use some marker which is
3134
unique and immutable.
3136
raise NotImplementedError(self.network_name)
3138
def check_conversion_target(self, target_format):
3139
if self.rich_root_data and not target_format.rich_root_data:
3140
raise errors.BadConversionTarget(
3141
'Does not support rich root data.', target_format,
3143
if (self.supports_tree_reference and
3144
not getattr(target_format, 'supports_tree_reference', False)):
3145
raise errors.BadConversionTarget(
3146
'Does not support nested trees', target_format,
3149
def open(self, a_bzrdir, _found=False):
3150
"""Return an instance of this format for the bzrdir a_bzrdir.
3152
_found is a private parameter, do not use it.
3154
raise NotImplementedError(self.open)
3156
def _run_post_repo_init_hooks(self, repository, a_bzrdir, shared):
3157
from bzrlib.bzrdir import BzrDir, RepoInitHookParams
3158
hooks = BzrDir.hooks['post_repo_init']
3161
params = RepoInitHookParams(repository, self, a_bzrdir, shared)
3166
class MetaDirRepositoryFormat(RepositoryFormat):
3167
"""Common base class for the new repositories using the metadir layout."""
3169
rich_root_data = False
3170
supports_tree_reference = False
3171
supports_external_lookups = False
3172
supports_leaving_lock = True
3175
def _matchingbzrdir(self):
3176
matching = bzrdir.BzrDirMetaFormat1()
3177
matching.repository_format = self
3181
super(MetaDirRepositoryFormat, self).__init__()
3183
def _create_control_files(self, a_bzrdir):
3184
"""Create the required files and the initial control_files object."""
3185
# FIXME: RBC 20060125 don't peek under the covers
3186
# NB: no need to escape relative paths that are url safe.
3187
repository_transport = a_bzrdir.get_repository_transport(self)
3188
control_files = lockable_files.LockableFiles(repository_transport,
3189
'lock', lockdir.LockDir)
3190
control_files.create_lock()
3191
return control_files
3193
def _upload_blank_content(self, a_bzrdir, dirs, files, utf8_files, shared):
3194
"""Upload the initial blank content."""
3195
control_files = self._create_control_files(a_bzrdir)
3196
control_files.lock_write()
3197
transport = control_files._transport
3199
utf8_files += [('shared-storage', '')]
3201
transport.mkdir_multi(dirs, mode=a_bzrdir._get_dir_mode())
3202
for (filename, content_stream) in files:
3203
transport.put_file(filename, content_stream,
3204
mode=a_bzrdir._get_file_mode())
3205
for (filename, content_bytes) in utf8_files:
3206
transport.put_bytes_non_atomic(filename, content_bytes,
3207
mode=a_bzrdir._get_file_mode())
3209
control_files.unlock()
3211
def network_name(self):
3212
"""Metadir formats have matching disk and network format strings."""
3213
return self.get_format_string()
3216
# formats which have no format string are not discoverable or independently
3217
# creatable on disk, so are not registered in format_registry. They're
3218
# all in bzrlib.repofmt.knitreponow. When an instance of one of these is
3219
# needed, it's constructed directly by the BzrDir. Non-native formats where
3220
# the repository is not separately opened are similar.
3222
format_registry.register_lazy(
3223
'Bazaar-NG Knit Repository Format 1',
3224
'bzrlib.repofmt.knitrepo',
3225
'RepositoryFormatKnit1',
3228
format_registry.register_lazy(
3229
'Bazaar Knit Repository Format 3 (bzr 0.15)\n',
3230
'bzrlib.repofmt.knitrepo',
3231
'RepositoryFormatKnit3',
3234
format_registry.register_lazy(
3235
'Bazaar Knit Repository Format 4 (bzr 1.0)\n',
3236
'bzrlib.repofmt.knitrepo',
3237
'RepositoryFormatKnit4',
3240
# Pack-based formats. There is one format for pre-subtrees, and one for
3241
# post-subtrees to allow ease of testing.
3242
# NOTE: These are experimental in 0.92. Stable in 1.0 and above
3243
format_registry.register_lazy(
3244
'Bazaar pack repository format 1 (needs bzr 0.92)\n',
3245
'bzrlib.repofmt.pack_repo',
3246
'RepositoryFormatKnitPack1',
3248
format_registry.register_lazy(
3249
'Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n',
3250
'bzrlib.repofmt.pack_repo',
3251
'RepositoryFormatKnitPack3',
3253
format_registry.register_lazy(
3254
'Bazaar pack repository format 1 with rich root (needs bzr 1.0)\n',
3255
'bzrlib.repofmt.pack_repo',
3256
'RepositoryFormatKnitPack4',
3258
format_registry.register_lazy(
3259
'Bazaar RepositoryFormatKnitPack5 (bzr 1.6)\n',
3260
'bzrlib.repofmt.pack_repo',
3261
'RepositoryFormatKnitPack5',
3263
format_registry.register_lazy(
3264
'Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6.1)\n',
3265
'bzrlib.repofmt.pack_repo',
3266
'RepositoryFormatKnitPack5RichRoot',
3268
format_registry.register_lazy(
3269
'Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6)\n',
3270
'bzrlib.repofmt.pack_repo',
3271
'RepositoryFormatKnitPack5RichRootBroken',
3273
format_registry.register_lazy(
3274
'Bazaar RepositoryFormatKnitPack6 (bzr 1.9)\n',
3275
'bzrlib.repofmt.pack_repo',
3276
'RepositoryFormatKnitPack6',
3278
format_registry.register_lazy(
3279
'Bazaar RepositoryFormatKnitPack6RichRoot (bzr 1.9)\n',
3280
'bzrlib.repofmt.pack_repo',
3281
'RepositoryFormatKnitPack6RichRoot',
3283
format_registry.register_lazy(
3284
'Bazaar repository format 2a (needs bzr 1.16 or later)\n',
3285
'bzrlib.repofmt.groupcompress_repo',
3286
'RepositoryFormat2a',
3289
# Development formats.
3290
# Check their docstrings to see if/when they are obsolete.
3291
format_registry.register_lazy(
3292
("Bazaar development format 2 with subtree support "
3293
"(needs bzr.dev from before 1.8)\n"),
3294
'bzrlib.repofmt.pack_repo',
3295
'RepositoryFormatPackDevelopment2Subtree',
3297
format_registry.register_lazy(
3298
'Bazaar development format 8\n',
3299
'bzrlib.repofmt.groupcompress_repo',
3300
'RepositoryFormat2aSubtree',
3304
class InterRepository(InterObject):
3305
"""This class represents operations taking place between two repositories.
3307
Its instances have methods like copy_content and fetch, and contain
3308
references to the source and target repositories these operations can be
3311
Often we will provide convenience methods on 'repository' which carry out
3312
operations with another repository - they will always forward to
3313
InterRepository.get(other).method_name(parameters).
3316
_walk_to_common_revisions_batch_size = 50
3318
"""The available optimised InterRepository types."""
3321
def copy_content(self, revision_id=None):
3322
"""Make a complete copy of the content in self into destination.
3324
This is a destructive operation! Do not use it on existing
3327
:param revision_id: Only copy the content needed to construct
3328
revision_id and its parents.
3331
self.target.set_make_working_trees(self.source.make_working_trees())
3332
except NotImplementedError:
3334
self.target.fetch(self.source, revision_id=revision_id)
3337
def fetch(self, revision_id=None, find_ghosts=False,
3339
"""Fetch the content required to construct revision_id.
3341
The content is copied from self.source to self.target.
3343
:param revision_id: if None all content is copied, if NULL_REVISION no
3347
ui.ui_factory.warn_experimental_format_fetch(self)
3348
from bzrlib.fetch import RepoFetcher
3349
# See <https://launchpad.net/bugs/456077> asking for a warning here
3350
if self.source._format.network_name() != self.target._format.network_name():
3351
ui.ui_factory.show_user_warning('cross_format_fetch',
3352
from_format=self.source._format,
3353
to_format=self.target._format)
3354
f = RepoFetcher(to_repository=self.target,
3355
from_repository=self.source,
3356
last_revision=revision_id,
3357
fetch_spec=fetch_spec,
3358
find_ghosts=find_ghosts)
3360
def _walk_to_common_revisions(self, revision_ids, if_present_ids=None):
3361
"""Walk out from revision_ids in source to revisions target has.
3363
:param revision_ids: The start point for the search.
3364
:return: A set of revision ids.
3366
target_graph = self.target.get_graph()
3367
revision_ids = frozenset(revision_ids)
3369
all_wanted_revs = revision_ids.union(if_present_ids)
3371
all_wanted_revs = revision_ids
3372
missing_revs = set()
3373
source_graph = self.source.get_graph()
3374
# ensure we don't pay silly lookup costs.
3375
searcher = source_graph._make_breadth_first_searcher(all_wanted_revs)
3376
null_set = frozenset([_mod_revision.NULL_REVISION])
3377
searcher_exhausted = False
3381
# Iterate the searcher until we have enough next_revs
3382
while len(next_revs) < self._walk_to_common_revisions_batch_size:
3384
next_revs_part, ghosts_part = searcher.next_with_ghosts()
3385
next_revs.update(next_revs_part)
3386
ghosts.update(ghosts_part)
3387
except StopIteration:
3388
searcher_exhausted = True
3390
# If there are ghosts in the source graph, and the caller asked for
3391
# them, make sure that they are present in the target.
3392
# We don't care about other ghosts as we can't fetch them and
3393
# haven't been asked to.
3394
ghosts_to_check = set(revision_ids.intersection(ghosts))
3395
revs_to_get = set(next_revs).union(ghosts_to_check)
3397
have_revs = set(target_graph.get_parent_map(revs_to_get))
3398
# we always have NULL_REVISION present.
3399
have_revs = have_revs.union(null_set)
3400
# Check if the target is missing any ghosts we need.
3401
ghosts_to_check.difference_update(have_revs)
3403
# One of the caller's revision_ids is a ghost in both the
3404
# source and the target.
3405
raise errors.NoSuchRevision(
3406
self.source, ghosts_to_check.pop())
3407
missing_revs.update(next_revs - have_revs)
3408
# Because we may have walked past the original stop point, make
3409
# sure everything is stopped
3410
stop_revs = searcher.find_seen_ancestors(have_revs)
3411
searcher.stop_searching_any(stop_revs)
3412
if searcher_exhausted:
3414
return searcher.get_result()
3417
def search_missing_revision_ids(self,
3418
revision_id=symbol_versioning.DEPRECATED_PARAMETER,
3419
find_ghosts=True, revision_ids=None, if_present_ids=None):
3420
"""Return the revision ids that source has that target does not.
3422
:param revision_id: only return revision ids included by this
3424
:param revision_ids: return revision ids included by these
3425
revision_ids. NoSuchRevision will be raised if any of these
3426
revisions are not present.
3427
:param if_present_ids: like revision_ids, but will not cause
3428
NoSuchRevision if any of these are absent, instead they will simply
3429
not be in the result. This is useful for e.g. finding revisions
3430
to fetch for tags, which may reference absent revisions.
3431
:param find_ghosts: If True find missing revisions in deep history
3432
rather than just finding the surface difference.
3433
:return: A bzrlib.graph.SearchResult.
3435
if symbol_versioning.deprecated_passed(revision_id):
3436
symbol_versioning.warn(
3437
'search_missing_revision_ids(revision_id=...) was '
3438
'deprecated in 2.4. Use revision_ids=[...] instead.',
3439
DeprecationWarning, stacklevel=2)
3440
if revision_ids is not None:
3441
raise AssertionError(
3442
'revision_ids is mutually exclusive with revision_id')
3443
if revision_id is not None:
3444
revision_ids = [revision_id]
3446
# stop searching at found target revisions.
3447
if not find_ghosts and (revision_ids is not None or if_present_ids is
3449
return self._walk_to_common_revisions(revision_ids,
3450
if_present_ids=if_present_ids)
3451
# generic, possibly worst case, slow code path.
3452
target_ids = set(self.target.all_revision_ids())
3453
source_ids = self._present_source_revisions_for(
3454
revision_ids, if_present_ids)
3455
result_set = set(source_ids).difference(target_ids)
3456
return self.source.revision_ids_to_search_result(result_set)
3458
def _present_source_revisions_for(self, revision_ids, if_present_ids=None):
3459
"""Returns set of all revisions in ancestry of revision_ids present in
3462
:param revision_ids: if None, all revisions in source are returned.
3463
:param if_present_ids: like revision_ids, but if any/all of these are
3464
absent no error is raised.
3466
if revision_ids is not None or if_present_ids is not None:
3467
# First, ensure all specified revisions exist. Callers expect
3468
# NoSuchRevision when they pass absent revision_ids here.
3469
if revision_ids is None:
3470
revision_ids = set()
3471
if if_present_ids is None:
3472
if_present_ids = set()
3473
revision_ids = set(revision_ids)
3474
if_present_ids = set(if_present_ids)
3475
all_wanted_ids = revision_ids.union(if_present_ids)
3476
graph = self.source.get_graph()
3477
present_revs = set(graph.get_parent_map(all_wanted_ids))
3478
missing = revision_ids.difference(present_revs)
3480
raise errors.NoSuchRevision(self.source, missing.pop())
3481
found_ids = all_wanted_ids.intersection(present_revs)
3482
source_ids = [rev_id for (rev_id, parents) in
3483
graph.iter_ancestry(found_ids)
3484
if rev_id != _mod_revision.NULL_REVISION
3485
and parents is not None]
3487
source_ids = self.source.all_revision_ids()
3488
return set(source_ids)
3491
def _same_model(source, target):
3492
"""True if source and target have the same data representation.
3494
Note: this is always called on the base class; overriding it in a
3495
subclass will have no effect.
3498
InterRepository._assert_same_model(source, target)
3500
except errors.IncompatibleRepositories, e:
3504
def _assert_same_model(source, target):
3505
"""Raise an exception if two repositories do not use the same model.
3507
if source.supports_rich_root() != target.supports_rich_root():
3508
raise errors.IncompatibleRepositories(source, target,
3509
"different rich-root support")
3510
if source._serializer != target._serializer:
3511
raise errors.IncompatibleRepositories(source, target,
3512
"different serializers")
3515
class InterSameDataRepository(InterRepository):
3516
"""Code for converting between repositories that represent the same data.
3518
Data format and model must match for this to work.
3522
def _get_repo_format_to_test(self):
3523
"""Repository format for testing with.
3525
InterSameData can pull from subtree to subtree and from non-subtree to
3526
non-subtree, so we test this with the richest repository format.
3528
from bzrlib.repofmt import knitrepo
3529
return knitrepo.RepositoryFormatKnit3()
3532
def is_compatible(source, target):
3533
return InterRepository._same_model(source, target)
3536
class InterDifferingSerializer(InterRepository):
3539
def _get_repo_format_to_test(self):
3543
def is_compatible(source, target):
3544
"""Be compatible with Knit2 source and Knit3 target"""
3545
# This is redundant with format.check_conversion_target(), however that
3546
# raises an exception, and we just want to say "False" as in we won't
3547
# support converting between these formats.
3548
if 'IDS_never' in debug.debug_flags:
3550
if source.supports_rich_root() and not target.supports_rich_root():
3552
if (source._format.supports_tree_reference
3553
and not target._format.supports_tree_reference):
3555
if target._fallback_repositories and target._format.supports_chks:
3556
# IDS doesn't know how to copy CHKs for the parent inventories it
3557
# adds to stacked repos.
3559
if 'IDS_always' in debug.debug_flags:
3561
# Only use this code path for local source and target. IDS does far
3562
# too much IO (both bandwidth and roundtrips) over a network.
3563
if not source.bzrdir.transport.base.startswith('file:///'):
3565
if not target.bzrdir.transport.base.startswith('file:///'):
3569
def _get_trees(self, revision_ids, cache):
3571
for rev_id in revision_ids:
3573
possible_trees.append((rev_id, cache[rev_id]))
3575
# Not cached, but inventory might be present anyway.
3577
tree = self.source.revision_tree(rev_id)
3578
except errors.NoSuchRevision:
3579
# Nope, parent is ghost.
3582
cache[rev_id] = tree
3583
possible_trees.append((rev_id, tree))
3584
return possible_trees
3586
def _get_delta_for_revision(self, tree, parent_ids, possible_trees):
3587
"""Get the best delta and base for this revision.
3589
:return: (basis_id, delta)
3592
# Generate deltas against each tree, to find the shortest.
3593
texts_possibly_new_in_tree = set()
3594
for basis_id, basis_tree in possible_trees:
3595
delta = tree.inventory._make_delta(basis_tree.inventory)
3596
for old_path, new_path, file_id, new_entry in delta:
3597
if new_path is None:
3598
# This file_id isn't present in the new rev, so we don't
3602
# Rich roots are handled elsewhere...
3604
kind = new_entry.kind
3605
if kind != 'directory' and kind != 'file':
3606
# No text record associated with this inventory entry.
3608
# This is a directory or file that has changed somehow.
3609
texts_possibly_new_in_tree.add((file_id, new_entry.revision))
3610
deltas.append((len(delta), basis_id, delta))
3612
return deltas[0][1:]
3614
def _fetch_parent_invs_for_stacking(self, parent_map, cache):
3615
"""Find all parent revisions that are absent, but for which the
3616
inventory is present, and copy those inventories.
3618
This is necessary to preserve correctness when the source is stacked
3619
without fallbacks configured. (Note that in cases like upgrade the
3620
source may be not have _fallback_repositories even though it is
3624
for parents in parent_map.values():
3625
parent_revs.update(parents)
3626
present_parents = self.source.get_parent_map(parent_revs)
3627
absent_parents = set(parent_revs).difference(present_parents)
3628
parent_invs_keys_for_stacking = self.source.inventories.get_parent_map(
3629
(rev_id,) for rev_id in absent_parents)
3630
parent_inv_ids = [key[-1] for key in parent_invs_keys_for_stacking]
3631
for parent_tree in self.source.revision_trees(parent_inv_ids):
3632
current_revision_id = parent_tree.get_revision_id()
3633
parents_parents_keys = parent_invs_keys_for_stacking[
3634
(current_revision_id,)]
3635
parents_parents = [key[-1] for key in parents_parents_keys]
3636
basis_id = _mod_revision.NULL_REVISION
3637
basis_tree = self.source.revision_tree(basis_id)
3638
delta = parent_tree.inventory._make_delta(basis_tree.inventory)
3639
self.target.add_inventory_by_delta(
3640
basis_id, delta, current_revision_id, parents_parents)
3641
cache[current_revision_id] = parent_tree
3643
def _fetch_batch(self, revision_ids, basis_id, cache):
3644
"""Fetch across a few revisions.
3646
:param revision_ids: The revisions to copy
3647
:param basis_id: The revision_id of a tree that must be in cache, used
3648
as a basis for delta when no other base is available
3649
:param cache: A cache of RevisionTrees that we can use.
3650
:return: The revision_id of the last converted tree. The RevisionTree
3651
for it will be in cache
3653
# Walk though all revisions; get inventory deltas, copy referenced
3654
# texts that delta references, insert the delta, revision and
3656
root_keys_to_create = set()
3659
pending_revisions = []
3660
parent_map = self.source.get_parent_map(revision_ids)
3661
self._fetch_parent_invs_for_stacking(parent_map, cache)
3662
self.source._safe_to_return_from_cache = True
3663
for tree in self.source.revision_trees(revision_ids):
3664
# Find a inventory delta for this revision.
3665
# Find text entries that need to be copied, too.
3666
current_revision_id = tree.get_revision_id()
3667
parent_ids = parent_map.get(current_revision_id, ())
3668
parent_trees = self._get_trees(parent_ids, cache)
3669
possible_trees = list(parent_trees)
3670
if len(possible_trees) == 0:
3671
# There either aren't any parents, or the parents are ghosts,
3672
# so just use the last converted tree.
3673
possible_trees.append((basis_id, cache[basis_id]))
3674
basis_id, delta = self._get_delta_for_revision(tree, parent_ids,
3676
revision = self.source.get_revision(current_revision_id)
3677
pending_deltas.append((basis_id, delta,
3678
current_revision_id, revision.parent_ids))
3679
if self._converting_to_rich_root:
3680
self._revision_id_to_root_id[current_revision_id] = \
3682
# Determine which texts are in present in this revision but not in
3683
# any of the available parents.
3684
texts_possibly_new_in_tree = set()
3685
for old_path, new_path, file_id, entry in delta:
3686
if new_path is None:
3687
# This file_id isn't present in the new rev
3691
if not self.target.supports_rich_root():
3692
# The target doesn't support rich root, so we don't
3695
if self._converting_to_rich_root:
3696
# This can't be copied normally, we have to insert
3698
root_keys_to_create.add((file_id, entry.revision))
3701
texts_possibly_new_in_tree.add((file_id, entry.revision))
3702
for basis_id, basis_tree in possible_trees:
3703
basis_inv = basis_tree.inventory
3704
for file_key in list(texts_possibly_new_in_tree):
3705
file_id, file_revision = file_key
3707
entry = basis_inv[file_id]
3708
except errors.NoSuchId:
3710
if entry.revision == file_revision:
3711
texts_possibly_new_in_tree.remove(file_key)
3712
text_keys.update(texts_possibly_new_in_tree)
3713
pending_revisions.append(revision)
3714
cache[current_revision_id] = tree
3715
basis_id = current_revision_id
3716
self.source._safe_to_return_from_cache = False
3718
from_texts = self.source.texts
3719
to_texts = self.target.texts
3720
if root_keys_to_create:
3721
root_stream = _mod_fetch._new_root_data_stream(
3722
root_keys_to_create, self._revision_id_to_root_id, parent_map,
3724
to_texts.insert_record_stream(root_stream)
3725
to_texts.insert_record_stream(from_texts.get_record_stream(
3726
text_keys, self.target._format._fetch_order,
3727
not self.target._format._fetch_uses_deltas))
3728
# insert inventory deltas
3729
for delta in pending_deltas:
3730
self.target.add_inventory_by_delta(*delta)
3731
if self.target._fallback_repositories:
3732
# Make sure this stacked repository has all the parent inventories
3733
# for the new revisions that we are about to insert. We do this
3734
# before adding the revisions so that no revision is added until
3735
# all the inventories it may depend on are added.
3736
# Note that this is overzealous, as we may have fetched these in an
3739
revision_ids = set()
3740
for revision in pending_revisions:
3741
revision_ids.add(revision.revision_id)
3742
parent_ids.update(revision.parent_ids)
3743
parent_ids.difference_update(revision_ids)
3744
parent_ids.discard(_mod_revision.NULL_REVISION)
3745
parent_map = self.source.get_parent_map(parent_ids)
3746
# we iterate over parent_map and not parent_ids because we don't
3747
# want to try copying any revision which is a ghost
3748
for parent_tree in self.source.revision_trees(parent_map):
3749
current_revision_id = parent_tree.get_revision_id()
3750
parents_parents = parent_map[current_revision_id]
3751
possible_trees = self._get_trees(parents_parents, cache)
3752
if len(possible_trees) == 0:
3753
# There either aren't any parents, or the parents are
3754
# ghosts, so just use the last converted tree.
3755
possible_trees.append((basis_id, cache[basis_id]))
3756
basis_id, delta = self._get_delta_for_revision(parent_tree,
3757
parents_parents, possible_trees)
3758
self.target.add_inventory_by_delta(
3759
basis_id, delta, current_revision_id, parents_parents)
3760
# insert signatures and revisions
3761
for revision in pending_revisions:
3763
signature = self.source.get_signature_text(
3764
revision.revision_id)
3765
self.target.add_signature_text(revision.revision_id,
3767
except errors.NoSuchRevision:
3769
self.target.add_revision(revision.revision_id, revision)
3772
def _fetch_all_revisions(self, revision_ids, pb):
3773
"""Fetch everything for the list of revisions.
3775
:param revision_ids: The list of revisions to fetch. Must be in
3777
:param pb: A ProgressTask
3780
basis_id, basis_tree = self._get_basis(revision_ids[0])
3782
cache = lru_cache.LRUCache(100)
3783
cache[basis_id] = basis_tree
3784
del basis_tree # We don't want to hang on to it here
3788
for offset in range(0, len(revision_ids), batch_size):
3789
self.target.start_write_group()
3791
pb.update('Transferring revisions', offset,
3793
batch = revision_ids[offset:offset+batch_size]
3794
basis_id = self._fetch_batch(batch, basis_id, cache)
3796
self.source._safe_to_return_from_cache = False
3797
self.target.abort_write_group()
3800
hint = self.target.commit_write_group()
3803
if hints and self.target._format.pack_compresses:
3804
self.target.pack(hint=hints)
3805
pb.update('Transferring revisions', len(revision_ids),
3809
def fetch(self, revision_id=None, find_ghosts=False,
3811
"""See InterRepository.fetch()."""
3812
if fetch_spec is not None:
3813
revision_ids = fetch_spec.get_keys()
3816
ui.ui_factory.warn_experimental_format_fetch(self)
3817
if (not self.source.supports_rich_root()
3818
and self.target.supports_rich_root()):
3819
self._converting_to_rich_root = True
3820
self._revision_id_to_root_id = {}
3822
self._converting_to_rich_root = False
3823
# See <https://launchpad.net/bugs/456077> asking for a warning here
3824
if self.source._format.network_name() != self.target._format.network_name():
3825
ui.ui_factory.show_user_warning('cross_format_fetch',
3826
from_format=self.source._format,
3827
to_format=self.target._format)
3828
if revision_ids is None:
3830
search_revision_ids = [revision_id]
3832
search_revision_ids = None
3833
revision_ids = self.target.search_missing_revision_ids(self.source,
3834
revision_ids=search_revision_ids,
3835
find_ghosts=find_ghosts).get_keys()
3836
if not revision_ids:
3838
revision_ids = tsort.topo_sort(
3839
self.source.get_graph().get_parent_map(revision_ids))
3840
if not revision_ids:
3842
# Walk though all revisions; get inventory deltas, copy referenced
3843
# texts that delta references, insert the delta, revision and
3845
pb = ui.ui_factory.nested_progress_bar()
3847
self._fetch_all_revisions(revision_ids, pb)
3850
return len(revision_ids), 0
3852
def _get_basis(self, first_revision_id):
3853
"""Get a revision and tree which exists in the target.
3855
This assumes that first_revision_id is selected for transmission
3856
because all other ancestors are already present. If we can't find an
3857
ancestor we fall back to NULL_REVISION since we know that is safe.
3859
:return: (basis_id, basis_tree)
3861
first_rev = self.source.get_revision(first_revision_id)
3863
basis_id = first_rev.parent_ids[0]
3864
# only valid as a basis if the target has it
3865
self.target.get_revision(basis_id)
3866
# Try to get a basis tree - if it's a ghost it will hit the
3867
# NoSuchRevision case.
3868
basis_tree = self.source.revision_tree(basis_id)
3869
except (IndexError, errors.NoSuchRevision):
3870
basis_id = _mod_revision.NULL_REVISION
3871
basis_tree = self.source.revision_tree(basis_id)
3872
return basis_id, basis_tree
3875
InterRepository.register_optimiser(InterDifferingSerializer)
3876
InterRepository.register_optimiser(InterSameDataRepository)
3879
class CopyConverter(object):
3880
"""A repository conversion tool which just performs a copy of the content.
3882
This is slow but quite reliable.
3885
def __init__(self, target_format):
3886
"""Create a CopyConverter.
3888
:param target_format: The format the resulting repository should be.
3890
self.target_format = target_format
3892
def convert(self, repo, pb):
3893
"""Perform the conversion of to_convert, giving feedback via pb.
3895
:param to_convert: The disk object to convert.
3896
:param pb: a progress bar to use for progress information.
3898
pb = ui.ui_factory.nested_progress_bar()
3901
# this is only useful with metadir layouts - separated repo content.
3902
# trigger an assertion if not such
3903
repo._format.get_format_string()
3904
self.repo_dir = repo.bzrdir
3905
pb.update('Moving repository to repository.backup')
3906
self.repo_dir.transport.move('repository', 'repository.backup')
3907
backup_transport = self.repo_dir.transport.clone('repository.backup')
3908
repo._format.check_conversion_target(self.target_format)
3909
self.source_repo = repo._format.open(self.repo_dir,
3911
_override_transport=backup_transport)
3912
pb.update('Creating new repository')
3913
converted = self.target_format.initialize(self.repo_dir,
3914
self.source_repo.is_shared())
3915
converted.lock_write()
3917
pb.update('Copying content')
3918
self.source_repo.copy_content_into(converted)
3921
pb.update('Deleting old repository content')
3922
self.repo_dir.transport.delete_tree('repository.backup')
3923
ui.ui_factory.note('repository converted')
3927
class _VersionedFileChecker(object):
3929
def __init__(self, repository, text_key_references=None, ancestors=None):
3930
self.repository = repository
3931
self.text_index = self.repository._generate_text_key_index(
3932
text_key_references=text_key_references, ancestors=ancestors)
3934
def calculate_file_version_parents(self, text_key):
3935
"""Calculate the correct parents for a file version according to
3938
parent_keys = self.text_index[text_key]
3939
if parent_keys == [_mod_revision.NULL_REVISION]:
3941
return tuple(parent_keys)
3943
def check_file_version_parents(self, texts, progress_bar=None):
3944
"""Check the parents stored in a versioned file are correct.
3946
It also detects file versions that are not referenced by their
3947
corresponding revision's inventory.
3949
:returns: A tuple of (wrong_parents, dangling_file_versions).
3950
wrong_parents is a dict mapping {revision_id: (stored_parents,
3951
correct_parents)} for each revision_id where the stored parents
3952
are not correct. dangling_file_versions is a set of (file_id,
3953
revision_id) tuples for versions that are present in this versioned
3954
file, but not used by the corresponding inventory.
3956
local_progress = None
3957
if progress_bar is None:
3958
local_progress = ui.ui_factory.nested_progress_bar()
3959
progress_bar = local_progress
3961
return self._check_file_version_parents(texts, progress_bar)
3964
local_progress.finished()
3966
def _check_file_version_parents(self, texts, progress_bar):
3967
"""See check_file_version_parents."""
3969
self.file_ids = set([file_id for file_id, _ in
3970
self.text_index.iterkeys()])
3971
# text keys is now grouped by file_id
3972
n_versions = len(self.text_index)
3973
progress_bar.update('loading text store', 0, n_versions)
3974
parent_map = self.repository.texts.get_parent_map(self.text_index)
3975
# On unlistable transports this could well be empty/error...
3976
text_keys = self.repository.texts.keys()
3977
unused_keys = frozenset(text_keys) - set(self.text_index)
3978
for num, key in enumerate(self.text_index.iterkeys()):
3979
progress_bar.update('checking text graph', num, n_versions)
3980
correct_parents = self.calculate_file_version_parents(key)
3982
knit_parents = parent_map[key]
3983
except errors.RevisionNotPresent:
3986
if correct_parents != knit_parents:
3987
wrong_parents[key] = (knit_parents, correct_parents)
3988
return wrong_parents, unused_keys
3991
def _strip_NULL_ghosts(revision_graph):
3992
"""Also don't use this. more compatibility code for unmigrated clients."""
3993
# Filter ghosts, and null:
3994
if _mod_revision.NULL_REVISION in revision_graph:
3995
del revision_graph[_mod_revision.NULL_REVISION]
3996
for key, parents in revision_graph.items():
3997
revision_graph[key] = tuple(parent for parent in parents if parent
3999
return revision_graph
4002
class StreamSink(object):
4003
"""An object that can insert a stream into a repository.
4005
This interface handles the complexity of reserialising inventories and
4006
revisions from different formats, and allows unidirectional insertion into
4007
stacked repositories without looking for the missing basis parents
4011
def __init__(self, target_repo):
4012
self.target_repo = target_repo
4014
def insert_stream(self, stream, src_format, resume_tokens):
4015
"""Insert a stream's content into the target repository.
4017
:param src_format: a bzr repository format.
4019
:return: a list of resume tokens and an iterable of keys additional
4020
items required before the insertion can be completed.
4022
self.target_repo.lock_write()
4025
self.target_repo.resume_write_group(resume_tokens)
4028
self.target_repo.start_write_group()
4031
# locked_insert_stream performs a commit|suspend.
4032
missing_keys = self.insert_stream_without_locking(stream,
4033
src_format, is_resume)
4035
# suspend the write group and tell the caller what we is
4036
# missing. We know we can suspend or else we would not have
4037
# entered this code path. (All repositories that can handle
4038
# missing keys can handle suspending a write group).
4039
write_group_tokens = self.target_repo.suspend_write_group()
4040
return write_group_tokens, missing_keys
4041
hint = self.target_repo.commit_write_group()
4042
to_serializer = self.target_repo._format._serializer
4043
src_serializer = src_format._serializer
4044
if (to_serializer != src_serializer and
4045
self.target_repo._format.pack_compresses):
4046
self.target_repo.pack(hint=hint)
4049
self.target_repo.abort_write_group(suppress_errors=True)
4052
self.target_repo.unlock()
4054
def insert_stream_without_locking(self, stream, src_format,
4056
"""Insert a stream's content into the target repository.
4058
This assumes that you already have a locked repository and an active
4061
:param src_format: a bzr repository format.
4062
:param is_resume: Passed down to get_missing_parent_inventories to
4063
indicate if we should be checking for missing texts at the same
4066
:return: A set of keys that are missing.
4068
if not self.target_repo.is_write_locked():
4069
raise errors.ObjectNotLocked(self)
4070
if not self.target_repo.is_in_write_group():
4071
raise errors.BzrError('you must already be in a write group')
4072
to_serializer = self.target_repo._format._serializer
4073
src_serializer = src_format._serializer
4075
if to_serializer == src_serializer:
4076
# If serializers match and the target is a pack repository, set the
4077
# write cache size on the new pack. This avoids poor performance
4078
# on transports where append is unbuffered (such as
4079
# RemoteTransport). This is safe to do because nothing should read
4080
# back from the target repository while a stream with matching
4081
# serialization is being inserted.
4082
# The exception is that a delta record from the source that should
4083
# be a fulltext may need to be expanded by the target (see
4084
# test_fetch_revisions_with_deltas_into_pack); but we take care to
4085
# explicitly flush any buffered writes first in that rare case.
4087
new_pack = self.target_repo._pack_collection._new_pack
4088
except AttributeError:
4089
# Not a pack repository
4092
new_pack.set_write_cache_size(1024*1024)
4093
for substream_type, substream in stream:
4094
if 'stream' in debug.debug_flags:
4095
mutter('inserting substream: %s', substream_type)
4096
if substream_type == 'texts':
4097
self.target_repo.texts.insert_record_stream(substream)
4098
elif substream_type == 'inventories':
4099
if src_serializer == to_serializer:
4100
self.target_repo.inventories.insert_record_stream(
4103
self._extract_and_insert_inventories(
4104
substream, src_serializer)
4105
elif substream_type == 'inventory-deltas':
4106
self._extract_and_insert_inventory_deltas(
4107
substream, src_serializer)
4108
elif substream_type == 'chk_bytes':
4109
# XXX: This doesn't support conversions, as it assumes the
4110
# conversion was done in the fetch code.
4111
self.target_repo.chk_bytes.insert_record_stream(substream)
4112
elif substream_type == 'revisions':
4113
# This may fallback to extract-and-insert more often than
4114
# required if the serializers are different only in terms of
4116
if src_serializer == to_serializer:
4117
self.target_repo.revisions.insert_record_stream(substream)
4119
self._extract_and_insert_revisions(substream,
4121
elif substream_type == 'signatures':
4122
self.target_repo.signatures.insert_record_stream(substream)
4124
raise AssertionError('kaboom! %s' % (substream_type,))
4125
# Done inserting data, and the missing_keys calculations will try to
4126
# read back from the inserted data, so flush the writes to the new pack
4127
# (if this is pack format).
4128
if new_pack is not None:
4129
new_pack._write_data('', flush=True)
4130
# Find all the new revisions (including ones from resume_tokens)
4131
missing_keys = self.target_repo.get_missing_parent_inventories(
4132
check_for_missing_texts=is_resume)
4134
for prefix, versioned_file in (
4135
('texts', self.target_repo.texts),
4136
('inventories', self.target_repo.inventories),
4137
('revisions', self.target_repo.revisions),
4138
('signatures', self.target_repo.signatures),
4139
('chk_bytes', self.target_repo.chk_bytes),
4141
if versioned_file is None:
4143
# TODO: key is often going to be a StaticTuple object
4144
# I don't believe we can define a method by which
4145
# (prefix,) + StaticTuple will work, though we could
4146
# define a StaticTuple.sq_concat that would allow you to
4147
# pass in either a tuple or a StaticTuple as the second
4148
# object, so instead we could have:
4149
# StaticTuple(prefix) + key here...
4150
missing_keys.update((prefix,) + key for key in
4151
versioned_file.get_missing_compression_parent_keys())
4152
except NotImplementedError:
4153
# cannot even attempt suspending, and missing would have failed
4154
# during stream insertion.
4155
missing_keys = set()
4158
def _extract_and_insert_inventory_deltas(self, substream, serializer):
4159
target_rich_root = self.target_repo._format.rich_root_data
4160
target_tree_refs = self.target_repo._format.supports_tree_reference
4161
for record in substream:
4162
# Insert the delta directly
4163
inventory_delta_bytes = record.get_bytes_as('fulltext')
4164
deserialiser = inventory_delta.InventoryDeltaDeserializer()
4166
parse_result = deserialiser.parse_text_bytes(
4167
inventory_delta_bytes)
4168
except inventory_delta.IncompatibleInventoryDelta, err:
4169
mutter("Incompatible delta: %s", err.msg)
4170
raise errors.IncompatibleRevision(self.target_repo._format)
4171
basis_id, new_id, rich_root, tree_refs, inv_delta = parse_result
4172
revision_id = new_id
4173
parents = [key[0] for key in record.parents]
4174
self.target_repo.add_inventory_by_delta(
4175
basis_id, inv_delta, revision_id, parents)
4177
def _extract_and_insert_inventories(self, substream, serializer,
4179
"""Generate a new inventory versionedfile in target, converting data.
4181
The inventory is retrieved from the source, (deserializing it), and
4182
stored in the target (reserializing it in a different format).
4184
target_rich_root = self.target_repo._format.rich_root_data
4185
target_tree_refs = self.target_repo._format.supports_tree_reference
4186
for record in substream:
4187
# It's not a delta, so it must be a fulltext in the source
4188
# serializer's format.
4189
bytes = record.get_bytes_as('fulltext')
4190
revision_id = record.key[0]
4191
inv = serializer.read_inventory_from_string(bytes, revision_id)
4192
parents = [key[0] for key in record.parents]
4193
self.target_repo.add_inventory(revision_id, inv, parents)
4194
# No need to keep holding this full inv in memory when the rest of
4195
# the substream is likely to be all deltas.
4198
def _extract_and_insert_revisions(self, substream, serializer):
4199
for record in substream:
4200
bytes = record.get_bytes_as('fulltext')
4201
revision_id = record.key[0]
4202
rev = serializer.read_revision_from_string(bytes)
4203
if rev.revision_id != revision_id:
4204
raise AssertionError('wtf: %s != %s' % (rev, revision_id))
4205
self.target_repo.add_revision(revision_id, rev)
4208
if self.target_repo._format._fetch_reconcile:
4209
self.target_repo.reconcile()
4212
class StreamSource(object):
4213
"""A source of a stream for fetching between repositories."""
4215
def __init__(self, from_repository, to_format):
4216
"""Create a StreamSource streaming from from_repository."""
4217
self.from_repository = from_repository
4218
self.to_format = to_format
4219
self._record_counter = RecordCounter()
4221
def delta_on_metadata(self):
4222
"""Return True if delta's are permitted on metadata streams.
4224
That is on revisions and signatures.
4226
src_serializer = self.from_repository._format._serializer
4227
target_serializer = self.to_format._serializer
4228
return (self.to_format._fetch_uses_deltas and
4229
src_serializer == target_serializer)
4231
def _fetch_revision_texts(self, revs):
4232
# fetch signatures first and then the revision texts
4233
# may need to be a InterRevisionStore call here.
4234
from_sf = self.from_repository.signatures
4235
# A missing signature is just skipped.
4236
keys = [(rev_id,) for rev_id in revs]
4237
signatures = versionedfile.filter_absent(from_sf.get_record_stream(
4239
self.to_format._fetch_order,
4240
not self.to_format._fetch_uses_deltas))
4241
# If a revision has a delta, this is actually expanded inside the
4242
# insert_record_stream code now, which is an alternate fix for
4244
from_rf = self.from_repository.revisions
4245
revisions = from_rf.get_record_stream(
4247
self.to_format._fetch_order,
4248
not self.delta_on_metadata())
4249
return [('signatures', signatures), ('revisions', revisions)]
4251
def _generate_root_texts(self, revs):
4252
"""This will be called by get_stream between fetching weave texts and
4253
fetching the inventory weave.
4255
if self._rich_root_upgrade():
4256
return _mod_fetch.Inter1and2Helper(
4257
self.from_repository).generate_root_texts(revs)
4261
def get_stream(self, search):
4263
revs = search.get_keys()
4264
graph = self.from_repository.get_graph()
4265
revs = tsort.topo_sort(graph.get_parent_map(revs))
4266
data_to_fetch = self.from_repository.item_keys_introduced_by(revs)
4268
for knit_kind, file_id, revisions in data_to_fetch:
4269
if knit_kind != phase:
4271
# Make a new progress bar for this phase
4272
if knit_kind == "file":
4273
# Accumulate file texts
4274
text_keys.extend([(file_id, revision) for revision in
4276
elif knit_kind == "inventory":
4277
# Now copy the file texts.
4278
from_texts = self.from_repository.texts
4279
yield ('texts', from_texts.get_record_stream(
4280
text_keys, self.to_format._fetch_order,
4281
not self.to_format._fetch_uses_deltas))
4282
# Cause an error if a text occurs after we have done the
4285
# Before we process the inventory we generate the root
4286
# texts (if necessary) so that the inventories references
4288
for _ in self._generate_root_texts(revs):
4290
# we fetch only the referenced inventories because we do not
4291
# know for unselected inventories whether all their required
4292
# texts are present in the other repository - it could be
4294
for info in self._get_inventory_stream(revs):
4296
elif knit_kind == "signatures":
4297
# Nothing to do here; this will be taken care of when
4298
# _fetch_revision_texts happens.
4300
elif knit_kind == "revisions":
4301
for record in self._fetch_revision_texts(revs):
4304
raise AssertionError("Unknown knit kind %r" % knit_kind)
4306
def get_stream_for_missing_keys(self, missing_keys):
4307
# missing keys can only occur when we are byte copying and not
4308
# translating (because translation means we don't send
4309
# unreconstructable deltas ever).
4311
keys['texts'] = set()
4312
keys['revisions'] = set()
4313
keys['inventories'] = set()
4314
keys['chk_bytes'] = set()
4315
keys['signatures'] = set()
4316
for key in missing_keys:
4317
keys[key[0]].add(key[1:])
4318
if len(keys['revisions']):
4319
# If we allowed copying revisions at this point, we could end up
4320
# copying a revision without copying its required texts: a
4321
# violation of the requirements for repository integrity.
4322
raise AssertionError(
4323
'cannot copy revisions to fill in missing deltas %s' % (
4324
keys['revisions'],))
4325
for substream_kind, keys in keys.iteritems():
4326
vf = getattr(self.from_repository, substream_kind)
4327
if vf is None and keys:
4328
raise AssertionError(
4329
"cannot fill in keys for a versioned file we don't"
4330
" have: %s needs %s" % (substream_kind, keys))
4332
# No need to stream something we don't have
4334
if substream_kind == 'inventories':
4335
# Some missing keys are genuinely ghosts, filter those out.
4336
present = self.from_repository.inventories.get_parent_map(keys)
4337
revs = [key[0] for key in present]
4338
# Get the inventory stream more-or-less as we do for the
4339
# original stream; there's no reason to assume that records
4340
# direct from the source will be suitable for the sink. (Think
4341
# e.g. 2a -> 1.9-rich-root).
4342
for info in self._get_inventory_stream(revs, missing=True):
4346
# Ask for full texts always so that we don't need more round trips
4347
# after this stream.
4348
# Some of the missing keys are genuinely ghosts, so filter absent
4349
# records. The Sink is responsible for doing another check to
4350
# ensure that ghosts don't introduce missing data for future
4352
stream = versionedfile.filter_absent(vf.get_record_stream(keys,
4353
self.to_format._fetch_order, True))
4354
yield substream_kind, stream
4356
def inventory_fetch_order(self):
4357
if self._rich_root_upgrade():
4358
return 'topological'
4360
return self.to_format._fetch_order
4362
def _rich_root_upgrade(self):
4363
return (not self.from_repository._format.rich_root_data and
4364
self.to_format.rich_root_data)
4366
def _get_inventory_stream(self, revision_ids, missing=False):
4367
from_format = self.from_repository._format
4368
if (from_format.supports_chks and self.to_format.supports_chks and
4369
from_format.network_name() == self.to_format.network_name()):
4370
raise AssertionError(
4371
"this case should be handled by GroupCHKStreamSource")
4372
elif 'forceinvdeltas' in debug.debug_flags:
4373
return self._get_convertable_inventory_stream(revision_ids,
4374
delta_versus_null=missing)
4375
elif from_format.network_name() == self.to_format.network_name():
4377
return self._get_simple_inventory_stream(revision_ids,
4379
elif (not from_format.supports_chks and not self.to_format.supports_chks
4380
and from_format._serializer == self.to_format._serializer):
4381
# Essentially the same format.
4382
return self._get_simple_inventory_stream(revision_ids,
4385
# Any time we switch serializations, we want to use an
4386
# inventory-delta based approach.
4387
return self._get_convertable_inventory_stream(revision_ids,
4388
delta_versus_null=missing)
4390
def _get_simple_inventory_stream(self, revision_ids, missing=False):
4391
# NB: This currently reopens the inventory weave in source;
4392
# using a single stream interface instead would avoid this.
4393
from_weave = self.from_repository.inventories
4395
delta_closure = True
4397
delta_closure = not self.delta_on_metadata()
4398
yield ('inventories', from_weave.get_record_stream(
4399
[(rev_id,) for rev_id in revision_ids],
4400
self.inventory_fetch_order(), delta_closure))
4402
def _get_convertable_inventory_stream(self, revision_ids,
4403
delta_versus_null=False):
4404
# The two formats are sufficiently different that there is no fast
4405
# path, so we need to send just inventorydeltas, which any
4406
# sufficiently modern client can insert into any repository.
4407
# The StreamSink code expects to be able to
4408
# convert on the target, so we need to put bytes-on-the-wire that can
4409
# be converted. That means inventory deltas (if the remote is <1.19,
4410
# RemoteStreamSink will fallback to VFS to insert the deltas).
4411
yield ('inventory-deltas',
4412
self._stream_invs_as_deltas(revision_ids,
4413
delta_versus_null=delta_versus_null))
4415
def _stream_invs_as_deltas(self, revision_ids, delta_versus_null=False):
4416
"""Return a stream of inventory-deltas for the given rev ids.
4418
:param revision_ids: The list of inventories to transmit
4419
:param delta_versus_null: Don't try to find a minimal delta for this
4420
entry, instead compute the delta versus the NULL_REVISION. This
4421
effectively streams a complete inventory. Used for stuff like
4422
filling in missing parents, etc.
4424
from_repo = self.from_repository
4425
revision_keys = [(rev_id,) for rev_id in revision_ids]
4426
parent_map = from_repo.inventories.get_parent_map(revision_keys)
4427
# XXX: possibly repos could implement a more efficient iter_inv_deltas
4429
inventories = self.from_repository.iter_inventories(
4430
revision_ids, 'topological')
4431
format = from_repo._format
4432
invs_sent_so_far = set([_mod_revision.NULL_REVISION])
4433
inventory_cache = lru_cache.LRUCache(50)
4434
null_inventory = from_repo.revision_tree(
4435
_mod_revision.NULL_REVISION).inventory
4436
# XXX: ideally the rich-root/tree-refs flags would be per-revision, not
4437
# per-repo (e.g. streaming a non-rich-root revision out of a rich-root
4438
# repo back into a non-rich-root repo ought to be allowed)
4439
serializer = inventory_delta.InventoryDeltaSerializer(
4440
versioned_root=format.rich_root_data,
4441
tree_references=format.supports_tree_reference)
4442
for inv in inventories:
4443
key = (inv.revision_id,)
4444
parent_keys = parent_map.get(key, ())
4446
if not delta_versus_null and parent_keys:
4447
# The caller did not ask for complete inventories and we have
4448
# some parents that we can delta against. Make a delta against
4449
# each parent so that we can find the smallest.
4450
parent_ids = [parent_key[0] for parent_key in parent_keys]
4451
for parent_id in parent_ids:
4452
if parent_id not in invs_sent_so_far:
4453
# We don't know that the remote side has this basis, so
4456
if parent_id == _mod_revision.NULL_REVISION:
4457
parent_inv = null_inventory
4459
parent_inv = inventory_cache.get(parent_id, None)
4460
if parent_inv is None:
4461
parent_inv = from_repo.get_inventory(parent_id)
4462
candidate_delta = inv._make_delta(parent_inv)
4463
if (delta is None or
4464
len(delta) > len(candidate_delta)):
4465
delta = candidate_delta
4466
basis_id = parent_id
4468
# Either none of the parents ended up being suitable, or we
4469
# were asked to delta against NULL
4470
basis_id = _mod_revision.NULL_REVISION
4471
delta = inv._make_delta(null_inventory)
4472
invs_sent_so_far.add(inv.revision_id)
4473
inventory_cache[inv.revision_id] = inv
4474
delta_serialized = ''.join(
4475
serializer.delta_to_lines(basis_id, key[-1], delta))
4476
yield versionedfile.FulltextContentFactory(
4477
key, parent_keys, None, delta_serialized)
4480
def _iter_for_revno(repo, partial_history_cache, stop_index=None,
4481
stop_revision=None):
4482
"""Extend the partial history to include a given index
4484
If a stop_index is supplied, stop when that index has been reached.
4485
If a stop_revision is supplied, stop when that revision is
4486
encountered. Otherwise, stop when the beginning of history is
4489
:param stop_index: The index which should be present. When it is
4490
present, history extension will stop.
4491
:param stop_revision: The revision id which should be present. When
4492
it is encountered, history extension will stop.
4494
start_revision = partial_history_cache[-1]
4495
iterator = repo.iter_reverse_revision_history(start_revision)
4497
#skip the last revision in the list
4500
if (stop_index is not None and
4501
len(partial_history_cache) > stop_index):
4503
if partial_history_cache[-1] == stop_revision:
4505
revision_id = iterator.next()
4506
partial_history_cache.append(revision_id)
4507
except StopIteration:
added
removed
bzrlib/repository.py
