« back to all changes in this revision
Viewing changes to bzrlib/repository.py
- browse files at revision 5651.2.1
- compare with another revision
- download diff
- download tarball
- view history from revision 5651.2.1
- Committer: Eric Siegerman
- Date: 2011-02-08 23:06:34 UTC
- mto: This revision was merged to the branch mainline in revision 5656.
- Revision ID: pub08@davor.org-20110208230634-u7ek4qdxikw0wu4w
Fix traceback attempting to "bzr dump-btree --raw btree-with-0-rows".
- files added:
- bzrlib/deprecated_graph.py
- bzrlib/doc_generate/builders
- bzrlib/doc_generate/writers
- bzrlib/tests/doc_generate
- bzrlib/tests/doc_generate/builders
- bzrlib/tests/doc_generate/writers
- bzrlib/util/effbot
- bzrlib/util/effbot/org
- bzrlib/util/elementtree
- files removed:
- bzrlib/branchfmt
- bzrlib/plugins/changelog_merge
- bzrlib/plugins/changelog_merge/tests
- bzrlib/plugins/po_merge
- bzrlib/plugins/po_merge/tests
- bzrlib/plugins/weave_fmt
- bzrlib/plugins/weave_fmt/tests
- bzrlib/tests/per_repository_vf
- po
- files renamed:
- bzrlib/plugins/weave_fmt/repository.py => bzrlib/repofmt/weaverepo.py
- bzrlib/commit_signature_commands.py => bzrlib/sign_my_commits.py
- bzrlib/tests/per_repository_vf/helpers.py => bzrlib/tests/per_repository/helpers.py
- bzrlib/tests/per_repository_vf/test__generate_text_key_index.py => bzrlib/tests/per_repository/test__generate_text_key_index.py
- bzrlib/tests/per_repository_vf/test_add_inventory_by_delta.py => bzrlib/tests/per_repository/test_add_inventory_by_delta.py
- bzrlib/tests/per_repository_vf/test_check_reconcile.py => bzrlib/tests/per_repository/test_check_reconcile.py
- bzrlib/tests/per_repository_vf/test_find_text_key_references.py => bzrlib/tests/per_repository/test_find_text_key_references.py
- bzrlib/tests/per_repository/test_locking.py => bzrlib/tests/per_repository/test_is_write_locked.py
- bzrlib/tests/per_repository_vf/test_merge_directive.py => bzrlib/tests/per_repository/test_merge_directive.py
- bzrlib/plugins/weave_fmt/xml4.py => bzrlib/xml4.py
- files modified:
- Makefile
added
removed
bzrlib/repository.py
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
17
from __future__ import absolute_import
18
19
17
from bzrlib.lazy_import import lazy_import
20
18
lazy_import(globals(), """
21
import itertools
19
import cStringIO
20
import re
22
21
import time
23
22
24
23
from bzrlib import (
24
bzrdir,
25
check,
26
chk_map,
25
27
config,
26
28
controldir,
27
29
debug,
30
fetch as _mod_fetch,
31
fifo_cache,
28
32
generate_ids,
33
gpg,
29
34
graph,
35
inventory,
36
inventory_delta,
37
lazy_regex,
30
38
lockable_files,
31
39
lockdir,
40
lru_cache,
32
41
osutils,
42
pyutils,
33
43
revision as _mod_revision,
34
testament as _mod_testament,
44
static_tuple,
45
trace,
35
46
tsort,
36
gpg,
47
versionedfile,
37
48
)
38
49
from bzrlib.bundle import serializer
39
from bzrlib.i18n import gettext
50
from bzrlib.revisiontree import RevisionTree
51
from bzrlib.store.versioned import VersionedFileStore
52
from bzrlib.testament import Testament
40
53
""")
41
54
42
55
from bzrlib import (
43
bzrdir,
44
56
errors,
45
57
registry,
46
58
symbol_versioning,
48
60
)
49
61
from bzrlib.decorators import needs_read_lock, needs_write_lock, only_raises
50
62
from bzrlib.inter import InterObject
63
from bzrlib.inventory import (
64
Inventory,
65
InventoryDirectory,
66
ROOT_ID,
67
entry_factory,
68
)
69
from bzrlib.recordcounter import RecordCounter
51
70
from bzrlib.lock import _RelockDebugMixin, LogicalLockResult
52
71
from bzrlib.trace import (
53
72
log_exception_quietly, note, mutter, mutter_callsite, warning)
74
93
75
94
# all clients should supply tree roots.
76
95
record_root_entry = True
77
# whether this commit builder supports the record_entry_contents interface
78
supports_record_entry_contents = False
79
# whether this commit builder will automatically update the branch that is
80
# being committed to
81
updates_branch = False
96
# the default CommitBuilder does not manage trees whose root is versioned.
97
_versioned_root = False
82
98
83
def __init__(self, repository, parents, config_stack, timestamp=None,
99
def __init__(self, repository, parents, config, timestamp=None,
84
100
timezone=None, committer=None, revprops=None,
85
revision_id=None, lossy=False):
101
revision_id=None):
86
102
"""Initiate a CommitBuilder.
87
103
88
104
:param repository: Repository to commit to.
89
105
:param parents: Revision ids of the parents of the new revision.
106
:param config: Configuration to use.
90
107
:param timestamp: Optional timestamp recorded for commit.
91
108
:param timezone: Optional timezone for timestamp.
92
109
:param committer: Optional committer to set for commit.
93
110
:param revprops: Optional dictionary of revision properties.
94
111
:param revision_id: Optional revision id.
95
:param lossy: Whether to discard data that can not be natively
96
represented, when pushing to a foreign VCS
97
112
"""
98
self._config_stack = config_stack
99
self._lossy = lossy
113
self._config = config
100
114
101
115
if committer is None:
102
self._committer = self._config_stack.get('email')
116
self._committer = self._config.username()
103
117
elif not isinstance(committer, unicode):
104
118
self._committer = committer.decode() # throw if non-ascii
105
119
else:
106
120
self._committer = committer
107
121
122
self.new_inventory = Inventory(None)
108
123
self._new_revision_id = revision_id
109
124
self.parents = parents
110
125
self.repository = repository
125
140
self._timezone = int(timezone)
126
141
127
142
self._generate_revision_if_needed()
143
self.__heads = graph.HeadsCache(repository.get_graph()).heads
144
self._basis_delta = []
145
# API compatibility, older code that used CommitBuilder did not call
146
# .record_delete(), which means the delta that is computed would not be
147
# valid. Callers that will call record_delete() should call
148
# .will_record_deletes() to indicate that.
149
self._recording_deletes = False
150
# memo'd check for no-op commits.
151
self._any_changes = False
128
152
129
153
def any_changes(self):
130
154
"""Return True if any entries were changed.
131
155
132
156
This includes merge-only changes. It is the core for the --unchanged
133
157
detection in commit.
134
158
135
159
:return: True if any changes have occured.
136
160
"""
137
raise NotImplementedError(self.any_changes)
161
return self._any_changes
138
162
139
163
def _validate_unicode_text(self, text, context):
140
164
"""Verify things like commit messages don't have bogus characters."""
151
175
self._validate_unicode_text(value,
152
176
'revision property (%s)' % (key,))
153
177
178
def _ensure_fallback_inventories(self):
179
"""Ensure that appropriate inventories are available.
180
181
This only applies to repositories that are stacked, and is about
182
enusring the stacking invariants. Namely, that for any revision that is
183
present, we either have all of the file content, or we have the parent
184
inventory and the delta file content.
185
"""
186
if not self.repository._fallback_repositories:
187
return
188
if not self.repository._format.supports_chks:
189
raise errors.BzrError("Cannot commit directly to a stacked branch"
190
" in pre-2a formats. See "
191
"https://bugs.launchpad.net/bzr/+bug/375013 for details.")
192
# This is a stacked repo, we need to make sure we have the parent
193
# inventories for the parents.
194
parent_keys = [(p,) for p in self.parents]
195
parent_map = self.repository.inventories._index.get_parent_map(parent_keys)
196
missing_parent_keys = set([pk for pk in parent_keys
197
if pk not in parent_map])
198
fallback_repos = list(reversed(self.repository._fallback_repositories))
199
missing_keys = [('inventories', pk[0])
200
for pk in missing_parent_keys]
201
resume_tokens = []
202
while missing_keys and fallback_repos:
203
fallback_repo = fallback_repos.pop()
204
source = fallback_repo._get_source(self.repository._format)
205
sink = self.repository._get_sink()
206
stream = source.get_stream_for_missing_keys(missing_keys)
207
missing_keys = sink.insert_stream_without_locking(stream,
208
self.repository._format)
209
if missing_keys:
210
raise errors.BzrError('Unable to fill in parent inventories for a'
211
' stacked branch')
212
154
213
def commit(self, message):
155
214
"""Make the actual commit.
156
215
157
216
:return: The revision id of the recorded revision.
158
217
"""
159
raise NotImplementedError(self.commit)
218
self._validate_unicode_text(message, 'commit message')
219
rev = _mod_revision.Revision(
220
timestamp=self._timestamp,
221
timezone=self._timezone,
222
committer=self._committer,
223
message=message,
224
inventory_sha1=self.inv_sha1,
225
revision_id=self._new_revision_id,
226
properties=self._revprops)
227
rev.parent_ids = self.parents
228
self.repository.add_revision(self._new_revision_id, rev,
229
self.new_inventory, self._config)
230
self._ensure_fallback_inventories()
231
self.repository.commit_write_group()
232
return self._new_revision_id
160
233
161
234
def abort(self):
162
235
"""Abort the commit that is being built.
163
236
"""
164
raise NotImplementedError(self.abort)
237
self.repository.abort_write_group()
165
238
166
239
def revision_tree(self):
167
240
"""Return the tree that was just committed.
168
241
169
After calling commit() this can be called to get a
170
RevisionTree representing the newly committed tree. This is
171
preferred to calling Repository.revision_tree() because that may
172
require deserializing the inventory, while we already have a copy in
242
After calling commit() this can be called to get a RevisionTree
243
representing the newly committed tree. This is preferred to
244
calling Repository.revision_tree() because that may require
245
deserializing the inventory, while we already have a copy in
173
246
memory.
174
247
"""
175
raise NotImplementedError(self.revision_tree)
248
if self.new_inventory is None:
249
self.new_inventory = self.repository.get_inventory(
250
self._new_revision_id)
251
return RevisionTree(self.repository, self.new_inventory,
252
self._new_revision_id)
176
253
177
254
def finish_inventory(self):
178
255
"""Tell the builder that the inventory is finished.
180
257
:return: The inventory id in the repository, which can be used with
181
258
repository.get_inventory.
182
259
"""
183
raise NotImplementedError(self.finish_inventory)
260
if self.new_inventory is None:
261
# an inventory delta was accumulated without creating a new
262
# inventory.
263
basis_id = self.basis_delta_revision
264
# We ignore the 'inventory' returned by add_inventory_by_delta
265
# because self.new_inventory is used to hint to the rest of the
266
# system what code path was taken
267
self.inv_sha1, _ = self.repository.add_inventory_by_delta(
268
basis_id, self._basis_delta, self._new_revision_id,
269
self.parents)
270
else:
271
if self.new_inventory.root is None:
272
raise AssertionError('Root entry should be supplied to'
273
' record_entry_contents, as of bzr 0.10.')
274
self.new_inventory.add(InventoryDirectory(ROOT_ID, '', None))
275
self.new_inventory.revision_id = self._new_revision_id
276
self.inv_sha1 = self.repository.add_inventory(
277
self._new_revision_id,
278
self.new_inventory,
279
self.parents
280
)
281
return self._new_revision_id
184
282
185
283
def _gen_revision_id(self):
186
284
"""Return new revision-id."""
201
299
else:
202
300
self.random_revid = False
203
301
302
def _heads(self, file_id, revision_ids):
303
"""Calculate the graph heads for revision_ids in the graph of file_id.
304
305
This can use either a per-file graph or a global revision graph as we
306
have an identity relationship between the two graphs.
307
"""
308
return self.__heads(revision_ids)
309
310
def _check_root(self, ie, parent_invs, tree):
311
"""Helper for record_entry_contents.
312
313
:param ie: An entry being added.
314
:param parent_invs: The inventories of the parent revisions of the
315
commit.
316
:param tree: The tree that is being committed.
317
"""
318
# In this revision format, root entries have no knit or weave When
319
# serializing out to disk and back in root.revision is always
320
# _new_revision_id
321
ie.revision = self._new_revision_id
322
323
def _require_root_change(self, tree):
324
"""Enforce an appropriate root object change.
325
326
This is called once when record_iter_changes is called, if and only if
327
the root was not in the delta calculated by record_iter_changes.
328
329
:param tree: The tree which is being committed.
330
"""
331
if len(self.parents) == 0:
332
raise errors.RootMissing()
333
entry = entry_factory['directory'](tree.path2id(''), '',
334
None)
335
entry.revision = self._new_revision_id
336
self._basis_delta.append(('', '', entry.file_id, entry))
337
338
def _get_delta(self, ie, basis_inv, path):
339
"""Get a delta against the basis inventory for ie."""
340
if ie.file_id not in basis_inv:
341
# add
342
result = (None, path, ie.file_id, ie)
343
self._basis_delta.append(result)
344
return result
345
elif ie != basis_inv[ie.file_id]:
346
# common but altered
347
# TODO: avoid tis id2path call.
348
result = (basis_inv.id2path(ie.file_id), path, ie.file_id, ie)
349
self._basis_delta.append(result)
350
return result
351
else:
352
# common, unaltered
353
return None
354
355
def get_basis_delta(self):
356
"""Return the complete inventory delta versus the basis inventory.
357
358
This has been built up with the calls to record_delete and
359
record_entry_contents. The client must have already called
360
will_record_deletes() to indicate that they will be generating a
361
complete delta.
362
363
:return: An inventory delta, suitable for use with apply_delta, or
364
Repository.add_inventory_by_delta, etc.
365
"""
366
if not self._recording_deletes:
367
raise AssertionError("recording deletes not activated.")
368
return self._basis_delta
369
370
def record_delete(self, path, file_id):
371
"""Record that a delete occured against a basis tree.
372
373
This is an optional API - when used it adds items to the basis_delta
374
being accumulated by the commit builder. It cannot be called unless the
375
method will_record_deletes() has been called to inform the builder that
376
a delta is being supplied.
377
378
:param path: The path of the thing deleted.
379
:param file_id: The file id that was deleted.
380
"""
381
if not self._recording_deletes:
382
raise AssertionError("recording deletes not activated.")
383
delta = (path, None, file_id, None)
384
self._basis_delta.append(delta)
385
self._any_changes = True
386
return delta
387
204
388
def will_record_deletes(self):
205
389
"""Tell the commit builder that deletes are being notified.
206
390
208
392
commit to be valid, deletes against the basis MUST be recorded via
209
393
builder.record_delete().
210
394
"""
211
raise NotImplementedError(self.will_record_deletes)
212
213
def record_iter_changes(self, tree, basis_revision_id, iter_changes):
395
self._recording_deletes = True
396
try:
397
basis_id = self.parents[0]
398
except IndexError:
399
basis_id = _mod_revision.NULL_REVISION
400
self.basis_delta_revision = basis_id
401
402
def record_entry_contents(self, ie, parent_invs, path, tree,
403
content_summary):
404
"""Record the content of ie from tree into the commit if needed.
405
406
Side effect: sets ie.revision when unchanged
407
408
:param ie: An inventory entry present in the commit.
409
:param parent_invs: The inventories of the parent revisions of the
410
commit.
411
:param path: The path the entry is at in the tree.
412
:param tree: The tree which contains this entry and should be used to
413
obtain content.
414
:param content_summary: Summary data from the tree about the paths
415
content - stat, length, exec, sha/link target. This is only
416
accessed when the entry has a revision of None - that is when it is
417
a candidate to commit.
418
:return: A tuple (change_delta, version_recorded, fs_hash).
419
change_delta is an inventory_delta change for this entry against
420
the basis tree of the commit, or None if no change occured against
421
the basis tree.
422
version_recorded is True if a new version of the entry has been
423
recorded. For instance, committing a merge where a file was only
424
changed on the other side will return (delta, False).
425
fs_hash is either None, or the hash details for the path (currently
426
a tuple of the contents sha1 and the statvalue returned by
427
tree.get_file_with_stat()).
428
"""
429
if self.new_inventory.root is None:
430
if ie.parent_id is not None:
431
raise errors.RootMissing()
432
self._check_root(ie, parent_invs, tree)
433
if ie.revision is None:
434
kind = content_summary[0]
435
else:
436
# ie is carried over from a prior commit
437
kind = ie.kind
438
# XXX: repository specific check for nested tree support goes here - if
439
# the repo doesn't want nested trees we skip it ?
440
if (kind == 'tree-reference' and
441
not self.repository._format.supports_tree_reference):
442
# mismatch between commit builder logic and repository:
443
# this needs the entry creation pushed down into the builder.
444
raise NotImplementedError('Missing repository subtree support.')
445
self.new_inventory.add(ie)
446
447
# TODO: slow, take it out of the inner loop.
448
try:
449
basis_inv = parent_invs[0]
450
except IndexError:
451
basis_inv = Inventory(root_id=None)
452
453
# ie.revision is always None if the InventoryEntry is considered
454
# for committing. We may record the previous parents revision if the
455
# content is actually unchanged against a sole head.
456
if ie.revision is not None:
457
if not self._versioned_root and path == '':
458
# repositories that do not version the root set the root's
459
# revision to the new commit even when no change occurs (more
460
# specifically, they do not record a revision on the root; and
461
# the rev id is assigned to the root during deserialisation -
462
# this masks when a change may have occurred against the basis.
463
# To match this we always issue a delta, because the revision
464
# of the root will always be changing.
465
if ie.file_id in basis_inv:
466
delta = (basis_inv.id2path(ie.file_id), path,
467
ie.file_id, ie)
468
else:
469
# add
470
delta = (None, path, ie.file_id, ie)
471
self._basis_delta.append(delta)
472
return delta, False, None
473
else:
474
# we don't need to commit this, because the caller already
475
# determined that an existing revision of this file is
476
# appropriate. If it's not being considered for committing then
477
# it and all its parents to the root must be unaltered so
478
# no-change against the basis.
479
if ie.revision == self._new_revision_id:
480
raise AssertionError("Impossible situation, a skipped "
481
"inventory entry (%r) claims to be modified in this "
482
"commit (%r).", (ie, self._new_revision_id))
483
return None, False, None
484
# XXX: Friction: parent_candidates should return a list not a dict
485
# so that we don't have to walk the inventories again.
486
parent_candiate_entries = ie.parent_candidates(parent_invs)
487
head_set = self._heads(ie.file_id, parent_candiate_entries.keys())
488
heads = []
489
for inv in parent_invs:
490
if ie.file_id in inv:
491
old_rev = inv[ie.file_id].revision
492
if old_rev in head_set:
493
heads.append(inv[ie.file_id].revision)
494
head_set.remove(inv[ie.file_id].revision)
495
496
store = False
497
# now we check to see if we need to write a new record to the
498
# file-graph.
499
# We write a new entry unless there is one head to the ancestors, and
500
# the kind-derived content is unchanged.
501
502
# Cheapest check first: no ancestors, or more the one head in the
503
# ancestors, we write a new node.
504
if len(heads) != 1:
505
store = True
506
if not store:
507
# There is a single head, look it up for comparison
508
parent_entry = parent_candiate_entries[heads[0]]
509
# if the non-content specific data has changed, we'll be writing a
510
# node:
511
if (parent_entry.parent_id != ie.parent_id or
512
parent_entry.name != ie.name):
513
store = True
514
# now we need to do content specific checks:
515
if not store:
516
# if the kind changed the content obviously has
517
if kind != parent_entry.kind:
518
store = True
519
# Stat cache fingerprint feedback for the caller - None as we usually
520
# don't generate one.
521
fingerprint = None
522
if kind == 'file':
523
if content_summary[2] is None:
524
raise ValueError("Files must not have executable = None")
525
if not store:
526
# We can't trust a check of the file length because of content
527
# filtering...
528
if (# if the exec bit has changed we have to store:
529
parent_entry.executable != content_summary[2]):
530
store = True
531
elif parent_entry.text_sha1 == content_summary[3]:
532
# all meta and content is unchanged (using a hash cache
533
# hit to check the sha)
534
ie.revision = parent_entry.revision
535
ie.text_size = parent_entry.text_size
536
ie.text_sha1 = parent_entry.text_sha1
537
ie.executable = parent_entry.executable
538
return self._get_delta(ie, basis_inv, path), False, None
539
else:
540
# Either there is only a hash change(no hash cache entry,
541
# or same size content change), or there is no change on
542
# this file at all.
543
# Provide the parent's hash to the store layer, so that the
544
# content is unchanged we will not store a new node.
545
nostore_sha = parent_entry.text_sha1
546
if store:
547
# We want to record a new node regardless of the presence or
548
# absence of a content change in the file.
549
nostore_sha = None
550
ie.executable = content_summary[2]
551
file_obj, stat_value = tree.get_file_with_stat(ie.file_id, path)
552
try:
553
text = file_obj.read()
554
finally:
555
file_obj.close()
556
try:
557
ie.text_sha1, ie.text_size = self._add_text_to_weave(
558
ie.file_id, text, heads, nostore_sha)
559
# Let the caller know we generated a stat fingerprint.
560
fingerprint = (ie.text_sha1, stat_value)
561
except errors.ExistingContent:
562
# Turns out that the file content was unchanged, and we were
563
# only going to store a new node if it was changed. Carry over
564
# the entry.
565
ie.revision = parent_entry.revision
566
ie.text_size = parent_entry.text_size
567
ie.text_sha1 = parent_entry.text_sha1
568
ie.executable = parent_entry.executable
569
return self._get_delta(ie, basis_inv, path), False, None
570
elif kind == 'directory':
571
if not store:
572
# all data is meta here, nothing specific to directory, so
573
# carry over:
574
ie.revision = parent_entry.revision
575
return self._get_delta(ie, basis_inv, path), False, None
576
self._add_text_to_weave(ie.file_id, '', heads, None)
577
elif kind == 'symlink':
578
current_link_target = content_summary[3]
579
if not store:
580
# symlink target is not generic metadata, check if it has
581
# changed.
582
if current_link_target != parent_entry.symlink_target:
583
store = True
584
if not store:
585
# unchanged, carry over.
586
ie.revision = parent_entry.revision
587
ie.symlink_target = parent_entry.symlink_target
588
return self._get_delta(ie, basis_inv, path), False, None
589
ie.symlink_target = current_link_target
590
self._add_text_to_weave(ie.file_id, '', heads, None)
591
elif kind == 'tree-reference':
592
if not store:
593
if content_summary[3] != parent_entry.reference_revision:
594
store = True
595
if not store:
596
# unchanged, carry over.
597
ie.reference_revision = parent_entry.reference_revision
598
ie.revision = parent_entry.revision
599
return self._get_delta(ie, basis_inv, path), False, None
600
ie.reference_revision = content_summary[3]
601
if ie.reference_revision is None:
602
raise AssertionError("invalid content_summary for nested tree: %r"
603
% (content_summary,))
604
self._add_text_to_weave(ie.file_id, '', heads, None)
605
else:
606
raise NotImplementedError('unknown kind')
607
ie.revision = self._new_revision_id
608
self._any_changes = True
609
return self._get_delta(ie, basis_inv, path), True, fingerprint
610
611
def record_iter_changes(self, tree, basis_revision_id, iter_changes,
612
_entry_factory=entry_factory):
214
613
"""Record a new tree via iter_changes.
215
614
216
615
:param tree: The tree to obtain text contents from for changed objects.
221
620
to basis_revision_id. The iterator must not include any items with
222
621
a current kind of None - missing items must be either filtered out
223
622
or errored-on beefore record_iter_changes sees the item.
623
:param _entry_factory: Private method to bind entry_factory locally for
624
performance.
224
625
:return: A generator of (file_id, relpath, fs_hash) tuples for use with
225
626
tree._observed_sha1.
226
627
"""
227
raise NotImplementedError(self.record_iter_changes)
628
# Create an inventory delta based on deltas between all the parents and
629
# deltas between all the parent inventories. We use inventory delta's
630
# between the inventory objects because iter_changes masks
631
# last-changed-field only changes.
632
# Working data:
633
# file_id -> change map, change is fileid, paths, changed, versioneds,
634
# parents, names, kinds, executables
635
merged_ids = {}
636
# {file_id -> revision_id -> inventory entry, for entries in parent
637
# trees that are not parents[0]
638
parent_entries = {}
639
ghost_basis = False
640
try:
641
revtrees = list(self.repository.revision_trees(self.parents))
642
except errors.NoSuchRevision:
643
# one or more ghosts, slow path.
644
revtrees = []
645
for revision_id in self.parents:
646
try:
647
revtrees.append(self.repository.revision_tree(revision_id))
648
except errors.NoSuchRevision:
649
if not revtrees:
650
basis_revision_id = _mod_revision.NULL_REVISION
651
ghost_basis = True
652
revtrees.append(self.repository.revision_tree(
653
_mod_revision.NULL_REVISION))
654
# The basis inventory from a repository
655
if revtrees:
656
basis_inv = revtrees[0].inventory
657
else:
658
basis_inv = self.repository.revision_tree(
659
_mod_revision.NULL_REVISION).inventory
660
if len(self.parents) > 0:
661
if basis_revision_id != self.parents[0] and not ghost_basis:
662
raise Exception(
663
"arbitrary basis parents not yet supported with merges")
664
for revtree in revtrees[1:]:
665
for change in revtree.inventory._make_delta(basis_inv):
666
if change[1] is None:
667
# Not present in this parent.
668
continue
669
if change[2] not in merged_ids:
670
if change[0] is not None:
671
basis_entry = basis_inv[change[2]]
672
merged_ids[change[2]] = [
673
# basis revid
674
basis_entry.revision,
675
# new tree revid
676
change[3].revision]
677
parent_entries[change[2]] = {
678
# basis parent
679
basis_entry.revision:basis_entry,
680
# this parent
681
change[3].revision:change[3],
682
}
683
else:
684
merged_ids[change[2]] = [change[3].revision]
685
parent_entries[change[2]] = {change[3].revision:change[3]}
686
else:
687
merged_ids[change[2]].append(change[3].revision)
688
parent_entries[change[2]][change[3].revision] = change[3]
689
else:
690
merged_ids = {}
691
# Setup the changes from the tree:
692
# changes maps file_id -> (change, [parent revision_ids])
693
changes= {}
694
for change in iter_changes:
695
# This probably looks up in basis_inv way to much.
696
if change[1][0] is not None:
697
head_candidate = [basis_inv[change[0]].revision]
698
else:
699
head_candidate = []
700
changes[change[0]] = change, merged_ids.get(change[0],
701
head_candidate)
702
unchanged_merged = set(merged_ids) - set(changes)
703
# Extend the changes dict with synthetic changes to record merges of
704
# texts.
705
for file_id in unchanged_merged:
706
# Record a merged version of these items that did not change vs the
707
# basis. This can be either identical parallel changes, or a revert
708
# of a specific file after a merge. The recorded content will be
709
# that of the current tree (which is the same as the basis), but
710
# the per-file graph will reflect a merge.
711
# NB:XXX: We are reconstructing path information we had, this
712
# should be preserved instead.
713
# inv delta change: (file_id, (path_in_source, path_in_target),
714
# changed_content, versioned, parent, name, kind,
715
# executable)
716
try:
717
basis_entry = basis_inv[file_id]
718
except errors.NoSuchId:
719
# a change from basis->some_parents but file_id isn't in basis
720
# so was new in the merge, which means it must have changed
721
# from basis -> current, and as it hasn't the add was reverted
722
# by the user. So we discard this change.
723
pass
724
else:
725
change = (file_id,
726
(basis_inv.id2path(file_id), tree.id2path(file_id)),
727
False, (True, True),
728
(basis_entry.parent_id, basis_entry.parent_id),
729
(basis_entry.name, basis_entry.name),
730
(basis_entry.kind, basis_entry.kind),
731
(basis_entry.executable, basis_entry.executable))
732
changes[file_id] = (change, merged_ids[file_id])
733
# changes contains tuples with the change and a set of inventory
734
# candidates for the file.
735
# inv delta is:
736
# old_path, new_path, file_id, new_inventory_entry
737
seen_root = False # Is the root in the basis delta?
738
inv_delta = self._basis_delta
739
modified_rev = self._new_revision_id
740
for change, head_candidates in changes.values():
741
if change[3][1]: # versioned in target.
742
# Several things may be happening here:
743
# We may have a fork in the per-file graph
744
# - record a change with the content from tree
745
# We may have a change against < all trees
746
# - carry over the tree that hasn't changed
747
# We may have a change against all trees
748
# - record the change with the content from tree
749
kind = change[6][1]
750
file_id = change[0]
751
entry = _entry_factory[kind](file_id, change[5][1],
752
change[4][1])
753
head_set = self._heads(change[0], set(head_candidates))
754
heads = []
755
# Preserve ordering.
756
for head_candidate in head_candidates:
757
if head_candidate in head_set:
758
heads.append(head_candidate)
759
head_set.remove(head_candidate)
760
carried_over = False
761
if len(heads) == 1:
762
# Could be a carry-over situation:
763
parent_entry_revs = parent_entries.get(file_id, None)
764
if parent_entry_revs:
765
parent_entry = parent_entry_revs.get(heads[0], None)
766
else:
767
parent_entry = None
768
if parent_entry is None:
769
# The parent iter_changes was called against is the one
770
# that is the per-file head, so any change is relevant
771
# iter_changes is valid.
772
carry_over_possible = False
773
else:
774
# could be a carry over situation
775
# A change against the basis may just indicate a merge,
776
# we need to check the content against the source of the
777
# merge to determine if it was changed after the merge
778
# or carried over.
779
if (parent_entry.kind != entry.kind or
780
parent_entry.parent_id != entry.parent_id or
781
parent_entry.name != entry.name):
782
# Metadata common to all entries has changed
783
# against per-file parent
784
carry_over_possible = False
785
else:
786
carry_over_possible = True
787
# per-type checks for changes against the parent_entry
788
# are done below.
789
else:
790
# Cannot be a carry-over situation
791
carry_over_possible = False
792
# Populate the entry in the delta
793
if kind == 'file':
794
# XXX: There is still a small race here: If someone reverts the content of a file
795
# after iter_changes examines and decides it has changed,
796
# we will unconditionally record a new version even if some
797
# other process reverts it while commit is running (with
798
# the revert happening after iter_changes did its
799
# examination).
800
if change[7][1]:
801
entry.executable = True
802
else:
803
entry.executable = False
804
if (carry_over_possible and
805
parent_entry.executable == entry.executable):
806
# Check the file length, content hash after reading
807
# the file.
808
nostore_sha = parent_entry.text_sha1
809
else:
810
nostore_sha = None
811
file_obj, stat_value = tree.get_file_with_stat(file_id, change[1][1])
812
try:
813
text = file_obj.read()
814
finally:
815
file_obj.close()
816
try:
817
entry.text_sha1, entry.text_size = self._add_text_to_weave(
818
file_id, text, heads, nostore_sha)
819
yield file_id, change[1][1], (entry.text_sha1, stat_value)
820
except errors.ExistingContent:
821
# No content change against a carry_over parent
822
# Perhaps this should also yield a fs hash update?
823
carried_over = True
824
entry.text_size = parent_entry.text_size
825
entry.text_sha1 = parent_entry.text_sha1
826
elif kind == 'symlink':
827
# Wants a path hint?
828
entry.symlink_target = tree.get_symlink_target(file_id)
829
if (carry_over_possible and
830
parent_entry.symlink_target == entry.symlink_target):
831
carried_over = True
832
else:
833
self._add_text_to_weave(change[0], '', heads, None)
834
elif kind == 'directory':
835
if carry_over_possible:
836
carried_over = True
837
else:
838
# Nothing to set on the entry.
839
# XXX: split into the Root and nonRoot versions.
840
if change[1][1] != '' or self.repository.supports_rich_root():
841
self._add_text_to_weave(change[0], '', heads, None)
842
elif kind == 'tree-reference':
843
if not self.repository._format.supports_tree_reference:
844
# This isn't quite sane as an error, but we shouldn't
845
# ever see this code path in practice: tree's don't
846
# permit references when the repo doesn't support tree
847
# references.
848
raise errors.UnsupportedOperation(tree.add_reference,
849
self.repository)
850
reference_revision = tree.get_reference_revision(change[0])
851
entry.reference_revision = reference_revision
852
if (carry_over_possible and
853
parent_entry.reference_revision == reference_revision):
854
carried_over = True
855
else:
856
self._add_text_to_weave(change[0], '', heads, None)
857
else:
858
raise AssertionError('unknown kind %r' % kind)
859
if not carried_over:
860
entry.revision = modified_rev
861
else:
862
entry.revision = parent_entry.revision
863
else:
864
entry = None
865
new_path = change[1][1]
866
inv_delta.append((change[1][0], new_path, change[0], entry))
867
if new_path == '':
868
seen_root = True
869
self.new_inventory = None
870
if len(inv_delta):
871
# This should perhaps be guarded by a check that the basis we
872
# commit against is the basis for the commit and if not do a delta
873
# against the basis.
874
self._any_changes = True
875
if not seen_root:
876
# housekeeping root entry changes do not affect no-change commits.
877
self._require_root_change(tree)
878
self.basis_delta_revision = basis_revision_id
879
880
def _add_text_to_weave(self, file_id, new_text, parents, nostore_sha):
881
parent_keys = tuple([(file_id, parent) for parent in parents])
882
return self.repository.texts._add_text(
883
(file_id, self._new_revision_id), parent_keys, new_text,
884
nostore_sha=nostore_sha, random_id=self.random_revid)[0:2]
885
886
887
class RootCommitBuilder(CommitBuilder):
888
"""This commitbuilder actually records the root id"""
889
890
# the root entry gets versioned properly by this builder.
891
_versioned_root = True
892
893
def _check_root(self, ie, parent_invs, tree):
894
"""Helper for record_entry_contents.
895
896
:param ie: An entry being added.
897
:param parent_invs: The inventories of the parent revisions of the
898
commit.
899
:param tree: The tree that is being committed.
900
"""
901
902
def _require_root_change(self, tree):
903
"""Enforce an appropriate root object change.
904
905
This is called once when record_iter_changes is called, if and only if
906
the root was not in the delta calculated by record_iter_changes.
907
908
:param tree: The tree which is being committed.
909
"""
910
# versioned roots do not change unless the tree found a change.
228
911
229
912
230
913
class RepositoryWriteLockResult(LogicalLockResult):
255
938
revisions and file history. It's normally accessed only by the Branch,
256
939
which views a particular line of development through that history.
257
940
258
See VersionedFileRepository in bzrlib.vf_repository for the
259
base class for most Bazaar repositories.
941
The Repository builds on top of some byte storage facilies (the revisions,
942
signatures, inventories, texts and chk_bytes attributes) and a Transport,
943
which respectively provide byte storage and a means to access the (possibly
944
remote) disk.
945
946
The byte storage facilities are addressed via tuples, which we refer to
947
as 'keys' throughout the code base. Revision_keys, inventory_keys and
948
signature_keys are all 1-tuples: (revision_id,). text_keys are two-tuples:
949
(file_id, revision_id). chk_bytes uses CHK keys - a 1-tuple with a single
950
byte string made up of a hash identifier and a hash value.
951
We use this interface because it allows low friction with the underlying
952
code that implements disk indices, network encoding and other parts of
953
bzrlib.
954
955
:ivar revisions: A bzrlib.versionedfile.VersionedFiles instance containing
956
the serialised revisions for the repository. This can be used to obtain
957
revision graph information or to access raw serialised revisions.
958
The result of trying to insert data into the repository via this store
959
is undefined: it should be considered read-only except for implementors
960
of repositories.
961
:ivar signatures: A bzrlib.versionedfile.VersionedFiles instance containing
962
the serialised signatures for the repository. This can be used to
963
obtain access to raw serialised signatures. The result of trying to
964
insert data into the repository via this store is undefined: it should
965
be considered read-only except for implementors of repositories.
966
:ivar inventories: A bzrlib.versionedfile.VersionedFiles instance containing
967
the serialised inventories for the repository. This can be used to
968
obtain unserialised inventories. The result of trying to insert data
969
into the repository via this store is undefined: it should be
970
considered read-only except for implementors of repositories.
971
:ivar texts: A bzrlib.versionedfile.VersionedFiles instance containing the
972
texts of files and directories for the repository. This can be used to
973
obtain file texts or file graphs. Note that Repository.iter_file_bytes
974
is usually a better interface for accessing file texts.
975
The result of trying to insert data into the repository via this store
976
is undefined: it should be considered read-only except for implementors
977
of repositories.
978
:ivar chk_bytes: A bzrlib.versionedfile.VersionedFiles instance containing
979
any data the repository chooses to store or have indexed by its hash.
980
The result of trying to insert data into the repository via this store
981
is undefined: it should be considered read-only except for implementors
982
of repositories.
983
:ivar _transport: Transport for file access to repository, typically
984
pointing to .bzr/repository.
260
985
"""
261
986
987
# What class to use for a CommitBuilder. Often it's simpler to change this
988
# in a Repository class subclass rather than to override
989
# get_commit_builder.
990
_commit_builder_class = CommitBuilder
991
# The search regex used by xml based repositories to determine what things
992
# where changed in a single commit.
993
_file_ids_altered_regex = lazy_regex.lazy_compile(
994
r'file_id="(?P<file_id>[^"]+)"'
995
r'.* revision="(?P<revision_id>[^"]+)"'
996
)
997
262
998
def abort_write_group(self, suppress_errors=False):
263
999
"""Commit the contents accrued within the current write group.
264
1000
286
1022
raise
287
1023
mutter('abort_write_group failed')
288
1024
log_exception_quietly()
289
note(gettext('bzr: ERROR (ignored): %s'), exc)
1025
note('bzr: ERROR (ignored): %s', exc)
290
1026
self._write_group = None
291
1027
292
1028
def _abort_write_group(self):
307
1043
308
1044
:param repository: A repository.
309
1045
"""
310
raise NotImplementedError(self.add_fallback_repository)
1046
if not self._format.supports_external_lookups:
1047
raise errors.UnstackableRepositoryFormat(self._format, self.base)
1048
if self.is_locked():
1049
# This repository will call fallback.unlock() when we transition to
1050
# the unlocked state, so we make sure to increment the lock count
1051
repository.lock_read()
1052
self._check_fallback_repository(repository)
1053
self._fallback_repositories.append(repository)
1054
self.texts.add_fallback_versioned_files(repository.texts)
1055
self.inventories.add_fallback_versioned_files(repository.inventories)
1056
self.revisions.add_fallback_versioned_files(repository.revisions)
1057
self.signatures.add_fallback_versioned_files(repository.signatures)
1058
if self.chk_bytes is not None:
1059
self.chk_bytes.add_fallback_versioned_files(repository.chk_bytes)
311
1060
312
1061
def _check_fallback_repository(self, repository):
313
1062
"""Check that this repository can fallback to repository safely.
318
1067
"""
319
1068
return InterRepository._assert_same_model(self, repository)
320
1069
1070
def add_inventory(self, revision_id, inv, parents):
1071
"""Add the inventory inv to the repository as revision_id.
1072
1073
:param parents: The revision ids of the parents that revision_id
1074
is known to have and are in the repository already.
1075
1076
:returns: The validator(which is a sha1 digest, though what is sha'd is
1077
repository format specific) of the serialized inventory.
1078
"""
1079
if not self.is_in_write_group():
1080
raise AssertionError("%r not in write group" % (self,))
1081
_mod_revision.check_not_reserved_id(revision_id)
1082
if not (inv.revision_id is None or inv.revision_id == revision_id):
1083
raise AssertionError(
1084
"Mismatch between inventory revision"
1085
" id and insertion revid (%r, %r)"
1086
% (inv.revision_id, revision_id))
1087
if inv.root is None:
1088
raise errors.RootMissing()
1089
return self._add_inventory_checked(revision_id, inv, parents)
1090
1091
def _add_inventory_checked(self, revision_id, inv, parents):
1092
"""Add inv to the repository after checking the inputs.
1093
1094
This function can be overridden to allow different inventory styles.
1095
1096
:seealso: add_inventory, for the contract.
1097
"""
1098
inv_lines = self._serializer.write_inventory_to_lines(inv)
1099
return self._inventory_add_lines(revision_id, parents,
1100
inv_lines, check_content=False)
1101
1102
def add_inventory_by_delta(self, basis_revision_id, delta, new_revision_id,
1103
parents, basis_inv=None, propagate_caches=False):
1104
"""Add a new inventory expressed as a delta against another revision.
1105
1106
See the inventory developers documentation for the theory behind
1107
inventory deltas.
1108
1109
:param basis_revision_id: The inventory id the delta was created
1110
against. (This does not have to be a direct parent.)
1111
:param delta: The inventory delta (see Inventory.apply_delta for
1112
details).
1113
:param new_revision_id: The revision id that the inventory is being
1114
added for.
1115
:param parents: The revision ids of the parents that revision_id is
1116
known to have and are in the repository already. These are supplied
1117
for repositories that depend on the inventory graph for revision
1118
graph access, as well as for those that pun ancestry with delta
1119
compression.
1120
:param basis_inv: The basis inventory if it is already known,
1121
otherwise None.
1122
:param propagate_caches: If True, the caches for this inventory are
1123
copied to and updated for the result if possible.
1124
1125
:returns: (validator, new_inv)
1126
The validator(which is a sha1 digest, though what is sha'd is
1127
repository format specific) of the serialized inventory, and the
1128
resulting inventory.
1129
"""
1130
if not self.is_in_write_group():
1131
raise AssertionError("%r not in write group" % (self,))
1132
_mod_revision.check_not_reserved_id(new_revision_id)
1133
basis_tree = self.revision_tree(basis_revision_id)
1134
basis_tree.lock_read()
1135
try:
1136
# Note that this mutates the inventory of basis_tree, which not all
1137
# inventory implementations may support: A better idiom would be to
1138
# return a new inventory, but as there is no revision tree cache in
1139
# repository this is safe for now - RBC 20081013
1140
if basis_inv is None:
1141
basis_inv = basis_tree.inventory
1142
basis_inv.apply_delta(delta)
1143
basis_inv.revision_id = new_revision_id
1144
return (self.add_inventory(new_revision_id, basis_inv, parents),
1145
basis_inv)
1146
finally:
1147
basis_tree.unlock()
1148
1149
def _inventory_add_lines(self, revision_id, parents, lines,
1150
check_content=True):
1151
"""Store lines in inv_vf and return the sha1 of the inventory."""
1152
parents = [(parent,) for parent in parents]
1153
result = self.inventories.add_lines((revision_id,), parents, lines,
1154
check_content=check_content)[0]
1155
self.inventories._access.flush()
1156
return result
1157
1158
def add_revision(self, revision_id, rev, inv=None, config=None):
1159
"""Add rev to the revision store as revision_id.
1160
1161
:param revision_id: the revision id to use.
1162
:param rev: The revision object.
1163
:param inv: The inventory for the revision. if None, it will be looked
1164
up in the inventory storer
1165
:param config: If None no digital signature will be created.
1166
If supplied its signature_needed method will be used
1167
to determine if a signature should be made.
1168
"""
1169
# TODO: jam 20070210 Shouldn't we check rev.revision_id and
1170
# rev.parent_ids?
1171
_mod_revision.check_not_reserved_id(revision_id)
1172
if config is not None and config.signature_needed():
1173
if inv is None:
1174
inv = self.get_inventory(revision_id)
1175
plaintext = Testament(rev, inv).as_short_text()
1176
self.store_revision_signature(
1177
gpg.GPGStrategy(config), plaintext, revision_id)
1178
# check inventory present
1179
if not self.inventories.get_parent_map([(revision_id,)]):
1180
if inv is None:
1181
raise errors.WeaveRevisionNotPresent(revision_id,
1182
self.inventories)
1183
else:
1184
# yes, this is not suitable for adding with ghosts.
1185
rev.inventory_sha1 = self.add_inventory(revision_id, inv,
1186
rev.parent_ids)
1187
else:
1188
key = (revision_id,)
1189
rev.inventory_sha1 = self.inventories.get_sha1s([key])[key]
1190
self._add_revision(rev)
1191
1192
def _add_revision(self, revision):
1193
text = self._serializer.write_revision_to_string(revision)
1194
key = (revision.revision_id,)
1195
parents = tuple((parent,) for parent in revision.parent_ids)
1196
self.revisions.add_lines(key, parents, osutils.split_lines(text))
1197
321
1198
def all_revision_ids(self):
322
1199
"""Returns a list of all the revision ids in the repository.
323
1200
346
1223
"""
347
1224
self.control_files.break_lock()
348
1225
1226
@needs_read_lock
1227
def _eliminate_revisions_not_present(self, revision_ids):
1228
"""Check every revision id in revision_ids to see if we have it.
1229
1230
Returns a set of the present revisions.
1231
"""
1232
result = []
1233
graph = self.get_graph()
1234
parent_map = graph.get_parent_map(revision_ids)
1235
# The old API returned a list, should this actually be a set?
1236
return parent_map.keys()
1237
1238
def _check_inventories(self, checker):
1239
"""Check the inventories found from the revision scan.
1240
1241
This is responsible for verifying the sha1 of inventories and
1242
creating a pending_keys set that covers data referenced by inventories.
1243
"""
1244
bar = ui.ui_factory.nested_progress_bar()
1245
try:
1246
self._do_check_inventories(checker, bar)
1247
finally:
1248
bar.finished()
1249
1250
def _do_check_inventories(self, checker, bar):
1251
"""Helper for _check_inventories."""
1252
revno = 0
1253
keys = {'chk_bytes':set(), 'inventories':set(), 'texts':set()}
1254
kinds = ['chk_bytes', 'texts']
1255
count = len(checker.pending_keys)
1256
bar.update("inventories", 0, 2)
1257
current_keys = checker.pending_keys
1258
checker.pending_keys = {}
1259
# Accumulate current checks.
1260
for key in current_keys:
1261
if key[0] != 'inventories' and key[0] not in kinds:
1262
checker._report_items.append('unknown key type %r' % (key,))
1263
keys[key[0]].add(key[1:])
1264
if keys['inventories']:
1265
# NB: output order *should* be roughly sorted - topo or
1266
# inverse topo depending on repository - either way decent
1267
# to just delta against. However, pre-CHK formats didn't
1268
# try to optimise inventory layout on disk. As such the
1269
# pre-CHK code path does not use inventory deltas.
1270
last_object = None
1271
for record in self.inventories.check(keys=keys['inventories']):
1272
if record.storage_kind == 'absent':
1273
checker._report_items.append(
1274
'Missing inventory {%s}' % (record.key,))
1275
else:
1276
last_object = self._check_record('inventories', record,
1277
checker, last_object,
1278
current_keys[('inventories',) + record.key])
1279
del keys['inventories']
1280
else:
1281
return
1282
bar.update("texts", 1)
1283
while (checker.pending_keys or keys['chk_bytes']
1284
or keys['texts']):
1285
# Something to check.
1286
current_keys = checker.pending_keys
1287
checker.pending_keys = {}
1288
# Accumulate current checks.
1289
for key in current_keys:
1290
if key[0] not in kinds:
1291
checker._report_items.append('unknown key type %r' % (key,))
1292
keys[key[0]].add(key[1:])
1293
# Check the outermost kind only - inventories || chk_bytes || texts
1294
for kind in kinds:
1295
if keys[kind]:
1296
last_object = None
1297
for record in getattr(self, kind).check(keys=keys[kind]):
1298
if record.storage_kind == 'absent':
1299
checker._report_items.append(
1300
'Missing %s {%s}' % (kind, record.key,))
1301
else:
1302
last_object = self._check_record(kind, record,
1303
checker, last_object, current_keys[(kind,) + record.key])
1304
keys[kind] = set()
1305
break
1306
1307
def _check_record(self, kind, record, checker, last_object, item_data):
1308
"""Check a single text from this repository."""
1309
if kind == 'inventories':
1310
rev_id = record.key[0]
1311
inv = self._deserialise_inventory(rev_id,
1312
record.get_bytes_as('fulltext'))
1313
if last_object is not None:
1314
delta = inv._make_delta(last_object)
1315
for old_path, path, file_id, ie in delta:
1316
if ie is None:
1317
continue
1318
ie.check(checker, rev_id, inv)
1319
else:
1320
for path, ie in inv.iter_entries():
1321
ie.check(checker, rev_id, inv)
1322
if self._format.fast_deltas:
1323
return inv
1324
elif kind == 'chk_bytes':
1325
# No code written to check chk_bytes for this repo format.
1326
checker._report_items.append(
1327
'unsupported key type chk_bytes for %s' % (record.key,))
1328
elif kind == 'texts':
1329
self._check_text(record, checker, item_data)
1330
else:
1331
checker._report_items.append(
1332
'unknown key type %s for %s' % (kind, record.key))
1333
1334
def _check_text(self, record, checker, item_data):
1335
"""Check a single text."""
1336
# Check it is extractable.
1337
# TODO: check length.
1338
if record.storage_kind == 'chunked':
1339
chunks = record.get_bytes_as(record.storage_kind)
1340
sha1 = osutils.sha_strings(chunks)
1341
length = sum(map(len, chunks))
1342
else:
1343
content = record.get_bytes_as('fulltext')
1344
sha1 = osutils.sha_string(content)
1345
length = len(content)
1346
if item_data and sha1 != item_data[1]:
1347
checker._report_items.append(
1348
'sha1 mismatch: %s has sha1 %s expected %s referenced by %s' %
1349
(record.key, sha1, item_data[1], item_data[2]))
1350
349
1351
@staticmethod
350
def create(controldir):
351
"""Construct the current default format repository in controldir."""
352
return RepositoryFormat.get_default_format().initialize(controldir)
1352
def create(a_bzrdir):
1353
"""Construct the current default format repository in a_bzrdir."""
1354
return RepositoryFormat.get_default_format().initialize(a_bzrdir)
353
1355
354
def __init__(self, _format, controldir, control_files):
1356
def __init__(self, _format, a_bzrdir, control_files):
355
1357
"""instantiate a Repository.
356
1358
357
1359
:param _format: The format of the repository on disk.
358
:param controldir: The ControlDir of the repository.
359
:param control_files: Control files to use for locking, etc.
1360
:param a_bzrdir: The BzrDir of the repository.
360
1361
"""
361
1362
# In the future we will have a single api for all stores for
362
1363
# getting file texts, inventories and revisions, then
364
1365
super(Repository, self).__init__()
365
1366
self._format = _format
366
1367
# the following are part of the public API for Repository:
367
self.bzrdir = controldir
1368
self.bzrdir = a_bzrdir
368
1369
self.control_files = control_files
1370
self._transport = control_files._transport
1371
self.base = self._transport.base
369
1372
# for tests
1373
self._reconcile_does_inventory_gc = True
1374
self._reconcile_fixes_text_parents = False
1375
self._reconcile_backsup_inventory = True
370
1376
self._write_group = None
371
1377
# Additional places to query for data.
372
1378
self._fallback_repositories = []
1379
# An InventoryEntry cache, used during deserialization
1380
self._inventory_entry_cache = fifo_cache.FIFOCache(10*1024)
1381
# Is it safe to return inventory entries directly from the entry cache,
1382
# rather copying them?
1383
self._safe_to_return_from_cache = False
373
1384
374
1385
@property
375
1386
def user_transport(self):
409
1420
"""
410
1421
if self.__class__ is not other.__class__:
411
1422
return False
412
return (self.control_url == other.control_url)
1423
return (self._transport.base == other._transport.base)
413
1424
414
1425
def is_in_write_group(self):
415
1426
"""Return True if there is an open write group.
515
1526
if revid and committers:
516
1527
result['committers'] = 0
517
1528
if revid and revid != _mod_revision.NULL_REVISION:
518
graph = self.get_graph()
519
1529
if committers:
520
1530
all_committers = set()
521
revisions = [r for (r, p) in graph.iter_ancestry([revid])
522
if r != _mod_revision.NULL_REVISION]
523
last_revision = None
1531
revisions = self.get_ancestry(revid)
1532
# pop the leading None
1533
revisions.pop(0)
1534
first_revision = None
524
1535
if not committers:
525
1536
# ignore the revisions in the middle - just grab first and last
526
1537
revisions = revisions[0], revisions[-1]
527
1538
for revision in self.get_revisions(revisions):
528
if not last_revision:
529
last_revision = revision
1539
if not first_revision:
1540
first_revision = revision
530
1541
if committers:
531
1542
all_committers.add(revision.committer)
532
first_revision = revision
1543
last_revision = revision
533
1544
if committers:
534
1545
result['committers'] = len(all_committers)
535
1546
result['firstrev'] = (first_revision.timestamp,
536
1547
first_revision.timezone)
537
1548
result['latestrev'] = (last_revision.timestamp,
538
1549
last_revision.timezone)
1550
1551
# now gather global repository information
1552
# XXX: This is available for many repos regardless of listability.
1553
if self.user_transport.listable():
1554
# XXX: do we want to __define len__() ?
1555
# Maybe the versionedfiles object should provide a different
1556
# method to get the number of keys.
1557
result['revisions'] = len(self.revisions.keys())
1558
# result['size'] = t
539
1559
return result
540
1560
541
1561
def find_branches(self, using=False):
552
1572
def __init__(self):
553
1573
self.first_call = True
554
1574
555
def __call__(self, controldir):
556
# On the first call, the parameter is always the controldir
1575
def __call__(self, bzrdir):
1576
# On the first call, the parameter is always the bzrdir
557
1577
# containing the current repo.
558
1578
if not self.first_call:
559
1579
try:
560
repository = controldir.open_repository()
1580
repository = bzrdir.open_repository()
561
1581
except errors.NoRepositoryPresent:
562
1582
pass
563
1583
else:
564
1584
return False, ([], repository)
565
1585
self.first_call = False
566
value = (controldir.list_branches(), None)
1586
value = (bzrdir.list_branches(), None)
567
1587
return True, value
568
1588
569
1589
ret = []
570
for branches, repository in controldir.ControlDir.find_bzrdirs(
1590
for branches, repository in bzrdir.BzrDir.find_bzrdirs(
571
1591
self.user_transport, evaluate=Evaluator()):
572
1592
if branches is not None:
573
1593
ret.extend(branches)
578
1598
@needs_read_lock
579
1599
def search_missing_revision_ids(self, other,
580
1600
revision_id=symbol_versioning.DEPRECATED_PARAMETER,
581
find_ghosts=True, revision_ids=None, if_present_ids=None,
582
limit=None):
1601
find_ghosts=True, revision_ids=None, if_present_ids=None):
583
1602
"""Return the revision ids that other has that this does not.
584
1603
585
1604
These are returned in topological order.
598
1617
revision_ids = [revision_id]
599
1618
return InterRepository.get(other, self).search_missing_revision_ids(
600
1619
find_ghosts=find_ghosts, revision_ids=revision_ids,
601
if_present_ids=if_present_ids, limit=limit)
1620
if_present_ids=if_present_ids)
602
1621
603
1622
@staticmethod
604
1623
def open(base):
607
1626
For instance, if the repository is at URL/.bzr/repository,
608
1627
Repository.open(URL) -> a Repository instance.
609
1628
"""
610
control = controldir.ControlDir.open(base)
1629
control = bzrdir.BzrDir.open(base)
611
1630
return control.open_repository()
612
1631
613
1632
def copy_content_into(self, destination, revision_id=None):
644
1663
"""
645
1664
646
1665
def suspend_write_group(self):
647
"""Suspend a write group.
648
649
:raise UnsuspendableWriteGroup: If the write group can not be
650
suspended.
651
:return: List of tokens
1666
raise errors.UnsuspendableWriteGroup(self)
1667
1668
def get_missing_parent_inventories(self, check_for_missing_texts=True):
1669
"""Return the keys of missing inventory parents for revisions added in
1670
this write group.
1671
1672
A revision is not complete if the inventory delta for that revision
1673
cannot be calculated. Therefore if the parent inventories of a
1674
revision are not present, the revision is incomplete, and e.g. cannot
1675
be streamed by a smart server. This method finds missing inventory
1676
parents for revisions added in this write group.
652
1677
"""
653
raise errors.UnsuspendableWriteGroup(self)
1678
if not self._format.supports_external_lookups:
1679
# This is only an issue for stacked repositories
1680
return set()
1681
if not self.is_in_write_group():
1682
raise AssertionError('not in a write group')
1683
1684
# XXX: We assume that every added revision already has its
1685
# corresponding inventory, so we only check for parent inventories that
1686
# might be missing, rather than all inventories.
1687
parents = set(self.revisions._index.get_missing_parents())
1688
parents.discard(_mod_revision.NULL_REVISION)
1689
unstacked_inventories = self.inventories._index
1690
present_inventories = unstacked_inventories.get_parent_map(
1691
key[-1:] for key in parents)
1692
parents.difference_update(present_inventories)
1693
if len(parents) == 0:
1694
# No missing parent inventories.
1695
return set()
1696
if not check_for_missing_texts:
1697
return set(('inventories', rev_id) for (rev_id,) in parents)
1698
# Ok, now we have a list of missing inventories. But these only matter
1699
# if the inventories that reference them are missing some texts they
1700
# appear to introduce.
1701
# XXX: Texts referenced by all added inventories need to be present,
1702
# but at the moment we're only checking for texts referenced by
1703
# inventories at the graph's edge.
1704
key_deps = self.revisions._index._key_dependencies
1705
key_deps.satisfy_refs_for_keys(present_inventories)
1706
referrers = frozenset(r[0] for r in key_deps.get_referrers())
1707
file_ids = self.fileids_altered_by_revision_ids(referrers)
1708
missing_texts = set()
1709
for file_id, version_ids in file_ids.iteritems():
1710
missing_texts.update(
1711
(file_id, version_id) for version_id in version_ids)
1712
present_texts = self.texts.get_parent_map(missing_texts)
1713
missing_texts.difference_update(present_texts)
1714
if not missing_texts:
1715
# No texts are missing, so all revisions and their deltas are
1716
# reconstructable.
1717
return set()
1718
# Alternatively the text versions could be returned as the missing
1719
# keys, but this is likely to be less data.
1720
missing_keys = set(('inventories', rev_id) for (rev_id,) in parents)
1721
return missing_keys
654
1722
655
1723
def refresh_data(self):
656
1724
"""Re-read any data needed to synchronise with disk.
677
1745
def _resume_write_group(self, tokens):
678
1746
raise errors.UnsuspendableWriteGroup(self)
679
1747
680
def fetch(self, source, revision_id=None, find_ghosts=False):
1748
def fetch(self, source, revision_id=None, pb=None, find_ghosts=False,
1749
fetch_spec=None):
681
1750
"""Fetch the content required to construct revision_id from source.
682
1751
683
If revision_id is None, then all content is copied.
1752
If revision_id is None and fetch_spec is None, then all content is
1753
copied.
684
1754
685
1755
fetch() may not be used when the repository is in a write group -
686
1756
either finish the current write group before using fetch, or use
692
1762
:param revision_id: If specified, all the content needed for this
693
1763
revision ID will be copied to the target. Fetch will determine for
694
1764
itself which content needs to be copied.
1765
:param fetch_spec: If specified, a SearchResult or
1766
PendingAncestryResult that describes which revisions to copy. This
1767
allows copying multiple heads at once. Mutually exclusive with
1768
revision_id.
695
1769
"""
1770
if fetch_spec is not None and revision_id is not None:
1771
raise AssertionError(
1772
"fetch_spec and revision_id are mutually exclusive.")
696
1773
if self.is_in_write_group():
697
1774
raise errors.InternalBzrError(
698
1775
"May not fetch while in a write group.")
700
1777
# TODO: lift out to somewhere common with RemoteRepository
701
1778
# <https://bugs.launchpad.net/bzr/+bug/401646>
702
1779
if (self.has_same_location(source)
1780
and fetch_spec is None
703
1781
and self._has_same_fallbacks(source)):
704
1782
# check that last_revision is in 'from' and then return a
705
1783
# no-operation.
708
1786
self.get_revision(revision_id)
709
1787
return 0, []
710
1788
inter = InterRepository.get(source, self)
711
return inter.fetch(revision_id=revision_id, find_ghosts=find_ghosts)
1789
return inter.fetch(revision_id=revision_id, pb=pb,
1790
find_ghosts=find_ghosts, fetch_spec=fetch_spec)
712
1791
713
1792
def create_bundle(self, target, base, fileobj, format=None):
714
1793
return serializer.write_bundle(self, target, base, fileobj, format)
715
1794
716
def get_commit_builder(self, branch, parents, config_stack, timestamp=None,
1795
def get_commit_builder(self, branch, parents, config, timestamp=None,
717
1796
timezone=None, committer=None, revprops=None,
718
revision_id=None, lossy=False):
1797
revision_id=None):
719
1798
"""Obtain a CommitBuilder for this repository.
720
1799
721
1800
:param branch: Branch to commit to.
722
1801
:param parents: Revision ids of the parents of the new revision.
723
:param config_stack: Configuration stack to use.
1802
:param config: Configuration to use.
724
1803
:param timestamp: Optional timestamp recorded for commit.
725
1804
:param timezone: Optional timezone for timestamp.
726
1805
:param committer: Optional committer to set for commit.
727
1806
:param revprops: Optional dictionary of revision properties.
728
1807
:param revision_id: Optional revision id.
729
:param lossy: Whether to discard data that can not be natively
730
represented, when pushing to a foreign VCS
731
1808
"""
732
raise NotImplementedError(self.get_commit_builder)
1809
if self._fallback_repositories and not self._format.supports_chks:
1810
raise errors.BzrError("Cannot commit directly to a stacked branch"
1811
" in pre-2a formats. See "
1812
"https://bugs.launchpad.net/bzr/+bug/375013 for details.")
1813
result = self._commit_builder_class(self, parents, config,
1814
timestamp, timezone, committer, revprops, revision_id)
1815
self.start_write_group()
1816
return result
733
1817
734
1818
@only_raises(errors.LockNotHeld, errors.LockBroken)
735
1819
def unlock(self):
742
1826
'Must end write groups before releasing write locks.')
743
1827
self.control_files.unlock()
744
1828
if self.control_files._lock_count == 0:
1829
self._inventory_entry_cache.clear()
745
1830
for repo in self._fallback_repositories:
746
1831
repo.unlock()
747
1832
748
1833
@needs_read_lock
749
def clone(self, controldir, revision_id=None):
750
"""Clone this repository into controldir using the current format.
1834
def clone(self, a_bzrdir, revision_id=None):
1835
"""Clone this repository into a_bzrdir using the current format.
751
1836
752
1837
Currently no check is made that the format of this repository and
753
1838
the bzrdir format are compatible. FIXME RBC 20060201.
756
1841
"""
757
1842
# TODO: deprecate after 0.16; cloning this with all its settings is
758
1843
# probably not very useful -- mbp 20070423
759
dest_repo = self._create_sprouting_repo(
760
controldir, shared=self.is_shared())
1844
dest_repo = self._create_sprouting_repo(a_bzrdir, shared=self.is_shared())
761
1845
self.copy_content_into(dest_repo, revision_id)
762
1846
return dest_repo
763
1847
813
1897
dest_repo = a_bzrdir.open_repository()
814
1898
return dest_repo
815
1899
1900
def _get_sink(self):
1901
"""Return a sink for streaming into this repository."""
1902
return StreamSink(self)
1903
1904
def _get_source(self, to_format):
1905
"""Return a source for streaming from this repository."""
1906
return StreamSource(self, to_format)
1907
816
1908
@needs_read_lock
817
1909
def has_revision(self, revision_id):
818
1910
"""True if this repository has a copy of the revision."""
825
1917
:param revision_ids: An iterable of revision_ids.
826
1918
:return: A set of the revision_ids that were present.
827
1919
"""
828
raise NotImplementedError(self.has_revisions)
1920
parent_map = self.revisions.get_parent_map(
1921
[(rev_id,) for rev_id in revision_ids])
1922
result = set()
1923
if _mod_revision.NULL_REVISION in revision_ids:
1924
result.add(_mod_revision.NULL_REVISION)
1925
result.update([key[0] for key in parent_map])
1926
return result
829
1927
830
1928
@needs_read_lock
831
1929
def get_revision(self, revision_id):
832
1930
"""Return the Revision object for a named revision."""
833
1931
return self.get_revisions([revision_id])[0]
834
1932
1933
@needs_read_lock
835
1934
def get_revision_reconcile(self, revision_id):
836
1935
"""'reconcile' helper routine that allows access to a revision always.
837
1936
840
1939
be used by reconcile, or reconcile-alike commands that are correcting
841
1940
or testing the revision graph.
842
1941
"""
843
raise NotImplementedError(self.get_revision_reconcile)
1942
return self._get_revisions([revision_id])[0]
844
1943
1944
@needs_read_lock
845
1945
def get_revisions(self, revision_ids):
846
1946
"""Get many revisions at once.
847
1947
848
1948
Repositories that need to check data on every revision read should
849
1949
subclass this method.
850
1950
"""
851
raise NotImplementedError(self.get_revisions)
1951
return self._get_revisions(revision_ids)
1952
1953
@needs_read_lock
1954
def _get_revisions(self, revision_ids):
1955
"""Core work logic to get many revisions without sanity checks."""
1956
revs = {}
1957
for revid, rev in self._iter_revisions(revision_ids):
1958
if rev is None:
1959
raise errors.NoSuchRevision(self, revid)
1960
revs[revid] = rev
1961
return [revs[revid] for revid in revision_ids]
1962
1963
def _iter_revisions(self, revision_ids):
1964
"""Iterate over revision objects.
1965
1966
:param revision_ids: An iterable of revisions to examine. None may be
1967
passed to request all revisions known to the repository. Note that
1968
not all repositories can find unreferenced revisions; for those
1969
repositories only referenced ones will be returned.
1970
:return: An iterator of (revid, revision) tuples. Absent revisions (
1971
those asked for but not available) are returned as (revid, None).
1972
"""
1973
if revision_ids is None:
1974
revision_ids = self.all_revision_ids()
1975
else:
1976
for rev_id in revision_ids:
1977
if not rev_id or not isinstance(rev_id, basestring):
1978
raise errors.InvalidRevisionId(revision_id=rev_id, branch=self)
1979
keys = [(key,) for key in revision_ids]
1980
stream = self.revisions.get_record_stream(keys, 'unordered', True)
1981
for record in stream:
1982
revid = record.key[0]
1983
if record.storage_kind == 'absent':
1984
yield (revid, None)
1985
else:
1986
text = record.get_bytes_as('fulltext')
1987
rev = self._serializer.read_revision_from_string(text)
1988
yield (revid, rev)
852
1989
853
1990
def get_deltas_for_revisions(self, revisions, specific_fileids=None):
854
1991
"""Produce a generator of revision deltas.
909
2046
signature = gpg_strategy.sign(plaintext)
910
2047
self.add_signature_text(revision_id, signature)
911
2048
2049
@needs_write_lock
912
2050
def add_signature_text(self, revision_id, signature):
913
"""Store a signature text for a revision.
914
915
:param revision_id: Revision id of the revision
916
:param signature: Signature text.
917
"""
918
raise NotImplementedError(self.add_signature_text)
2051
self.signatures.add_lines((revision_id,), (),
2052
osutils.split_lines(signature))
2053
2054
def find_text_key_references(self):
2055
"""Find the text key references within the repository.
2056
2057
:return: A dictionary mapping text keys ((fileid, revision_id) tuples)
2058
to whether they were referred to by the inventory of the
2059
revision_id that they contain. The inventory texts from all present
2060
revision ids are assessed to generate this report.
2061
"""
2062
revision_keys = self.revisions.keys()
2063
w = self.inventories
2064
pb = ui.ui_factory.nested_progress_bar()
2065
try:
2066
return self._find_text_key_references_from_xml_inventory_lines(
2067
w.iter_lines_added_or_present_in_keys(revision_keys, pb=pb))
2068
finally:
2069
pb.finished()
2070
2071
def _find_text_key_references_from_xml_inventory_lines(self,
2072
line_iterator):
2073
"""Core routine for extracting references to texts from inventories.
2074
2075
This performs the translation of xml lines to revision ids.
2076
2077
:param line_iterator: An iterator of lines, origin_version_id
2078
:return: A dictionary mapping text keys ((fileid, revision_id) tuples)
2079
to whether they were referred to by the inventory of the
2080
revision_id that they contain. Note that if that revision_id was
2081
not part of the line_iterator's output then False will be given -
2082
even though it may actually refer to that key.
2083
"""
2084
if not self._serializer.support_altered_by_hack:
2085
raise AssertionError(
2086
"_find_text_key_references_from_xml_inventory_lines only "
2087
"supported for branches which store inventory as unnested xml"
2088
", not on %r" % self)
2089
result = {}
2090
2091
# this code needs to read every new line in every inventory for the
2092
# inventories [revision_ids]. Seeing a line twice is ok. Seeing a line
2093
# not present in one of those inventories is unnecessary but not
2094
# harmful because we are filtering by the revision id marker in the
2095
# inventory lines : we only select file ids altered in one of those
2096
# revisions. We don't need to see all lines in the inventory because
2097
# only those added in an inventory in rev X can contain a revision=X
2098
# line.
2099
unescape_revid_cache = {}
2100
unescape_fileid_cache = {}
2101
2102
# jam 20061218 In a big fetch, this handles hundreds of thousands
2103
# of lines, so it has had a lot of inlining and optimizing done.
2104
# Sorry that it is a little bit messy.
2105
# Move several functions to be local variables, since this is a long
2106
# running loop.
2107
search = self._file_ids_altered_regex.search
2108
unescape = _unescape_xml
2109
setdefault = result.setdefault
2110
for line, line_key in line_iterator:
2111
match = search(line)
2112
if match is None:
2113
continue
2114
# One call to match.group() returning multiple items is quite a
2115
# bit faster than 2 calls to match.group() each returning 1
2116
file_id, revision_id = match.group('file_id', 'revision_id')
2117
2118
# Inlining the cache lookups helps a lot when you make 170,000
2119
# lines and 350k ids, versus 8.4 unique ids.
2120
# Using a cache helps in 2 ways:
2121
# 1) Avoids unnecessary decoding calls
2122
# 2) Re-uses cached strings, which helps in future set and
2123
# equality checks.
2124
# (2) is enough that removing encoding entirely along with
2125
# the cache (so we are using plain strings) results in no
2126
# performance improvement.
2127
try:
2128
revision_id = unescape_revid_cache[revision_id]
2129
except KeyError:
2130
unescaped = unescape(revision_id)
2131
unescape_revid_cache[revision_id] = unescaped
2132
revision_id = unescaped
2133
2134
# Note that unconditionally unescaping means that we deserialise
2135
# every fileid, which for general 'pull' is not great, but we don't
2136
# really want to have some many fulltexts that this matters anyway.
2137
# RBC 20071114.
2138
try:
2139
file_id = unescape_fileid_cache[file_id]
2140
except KeyError:
2141
unescaped = unescape(file_id)
2142
unescape_fileid_cache[file_id] = unescaped
2143
file_id = unescaped
2144
2145
key = (file_id, revision_id)
2146
setdefault(key, False)
2147
if revision_id == line_key[-1]:
2148
result[key] = True
2149
return result
2150
2151
def _inventory_xml_lines_for_keys(self, keys):
2152
"""Get a line iterator of the sort needed for findind references.
2153
2154
Not relevant for non-xml inventory repositories.
2155
2156
Ghosts in revision_keys are ignored.
2157
2158
:param revision_keys: The revision keys for the inventories to inspect.
2159
:return: An iterator over (inventory line, revid) for the fulltexts of
2160
all of the xml inventories specified by revision_keys.
2161
"""
2162
stream = self.inventories.get_record_stream(keys, 'unordered', True)
2163
for record in stream:
2164
if record.storage_kind != 'absent':
2165
chunks = record.get_bytes_as('chunked')
2166
revid = record.key[-1]
2167
lines = osutils.chunks_to_lines(chunks)
2168
for line in lines:
2169
yield line, revid
2170
2171
def _find_file_ids_from_xml_inventory_lines(self, line_iterator,
2172
revision_keys):
2173
"""Helper routine for fileids_altered_by_revision_ids.
2174
2175
This performs the translation of xml lines to revision ids.
2176
2177
:param line_iterator: An iterator of lines, origin_version_id
2178
:param revision_keys: The revision ids to filter for. This should be a
2179
set or other type which supports efficient __contains__ lookups, as
2180
the revision key from each parsed line will be looked up in the
2181
revision_keys filter.
2182
:return: a dictionary mapping altered file-ids to an iterable of
2183
revision_ids. Each altered file-ids has the exact revision_ids that
2184
altered it listed explicitly.
2185
"""
2186
seen = set(self._find_text_key_references_from_xml_inventory_lines(
2187
line_iterator).iterkeys())
2188
parent_keys = self._find_parent_keys_of_revisions(revision_keys)
2189
parent_seen = set(self._find_text_key_references_from_xml_inventory_lines(
2190
self._inventory_xml_lines_for_keys(parent_keys)))
2191
new_keys = seen - parent_seen
2192
result = {}
2193
setdefault = result.setdefault
2194
for key in new_keys:
2195
setdefault(key[0], set()).add(key[-1])
2196
return result
919
2197
920
2198
def _find_parent_ids_of_revisions(self, revision_ids):
921
2199
"""Find all parent ids that are mentioned in the revision graph.
930
2208
parent_ids.discard(_mod_revision.NULL_REVISION)
931
2209
return parent_ids
932
2210
2211
def _find_parent_keys_of_revisions(self, revision_keys):
2212
"""Similar to _find_parent_ids_of_revisions, but used with keys.
2213
2214
:param revision_keys: An iterable of revision_keys.
2215
:return: The parents of all revision_keys that are not already in
2216
revision_keys
2217
"""
2218
parent_map = self.revisions.get_parent_map(revision_keys)
2219
parent_keys = set()
2220
map(parent_keys.update, parent_map.itervalues())
2221
parent_keys.difference_update(revision_keys)
2222
parent_keys.discard(_mod_revision.NULL_REVISION)
2223
return parent_keys
2224
2225
def fileids_altered_by_revision_ids(self, revision_ids, _inv_weave=None):
2226
"""Find the file ids and versions affected by revisions.
2227
2228
:param revisions: an iterable containing revision ids.
2229
:param _inv_weave: The inventory weave from this repository or None.
2230
If None, the inventory weave will be opened automatically.
2231
:return: a dictionary mapping altered file-ids to an iterable of
2232
revision_ids. Each altered file-ids has the exact revision_ids that
2233
altered it listed explicitly.
2234
"""
2235
selected_keys = set((revid,) for revid in revision_ids)
2236
w = _inv_weave or self.inventories
2237
return self._find_file_ids_from_xml_inventory_lines(
2238
w.iter_lines_added_or_present_in_keys(
2239
selected_keys, pb=None),
2240
selected_keys)
2241
933
2242
def iter_files_bytes(self, desired_files):
934
2243
"""Iterate through file versions.
935
2244
941
2250
uniquely identify the file version in the caller's context. (Examples:
942
2251
an index number or a TreeTransform trans_id.)
943
2252
2253
bytes_iterator is an iterable of bytestrings for the file. The
2254
kind of iterable and length of the bytestrings are unspecified, but for
2255
this implementation, it is a list of bytes produced by
2256
VersionedFile.get_record_stream().
2257
944
2258
:param desired_files: a list of (file_id, revision_id, identifier)
945
2259
triples
946
2260
"""
947
raise NotImplementedError(self.iter_files_bytes)
2261
text_keys = {}
2262
for file_id, revision_id, callable_data in desired_files:
2263
text_keys[(file_id, revision_id)] = callable_data
2264
for record in self.texts.get_record_stream(text_keys, 'unordered', True):
2265
if record.storage_kind == 'absent':
2266
raise errors.RevisionNotPresent(record.key, self)
2267
yield text_keys[record.key], record.get_bytes_as('chunked')
2268
2269
def _generate_text_key_index(self, text_key_references=None,
2270
ancestors=None):
2271
"""Generate a new text key index for the repository.
2272
2273
This is an expensive function that will take considerable time to run.
2274
2275
:return: A dict mapping text keys ((file_id, revision_id) tuples) to a
2276
list of parents, also text keys. When a given key has no parents,
2277
the parents list will be [NULL_REVISION].
2278
"""
2279
# All revisions, to find inventory parents.
2280
if ancestors is None:
2281
graph = self.get_graph()
2282
ancestors = graph.get_parent_map(self.all_revision_ids())
2283
if text_key_references is None:
2284
text_key_references = self.find_text_key_references()
2285
pb = ui.ui_factory.nested_progress_bar()
2286
try:
2287
return self._do_generate_text_key_index(ancestors,
2288
text_key_references, pb)
2289
finally:
2290
pb.finished()
2291
2292
def _do_generate_text_key_index(self, ancestors, text_key_references, pb):
2293
"""Helper for _generate_text_key_index to avoid deep nesting."""
2294
revision_order = tsort.topo_sort(ancestors)
2295
invalid_keys = set()
2296
revision_keys = {}
2297
for revision_id in revision_order:
2298
revision_keys[revision_id] = set()
2299
text_count = len(text_key_references)
2300
# a cache of the text keys to allow reuse; costs a dict of all the
2301
# keys, but saves a 2-tuple for every child of a given key.
2302
text_key_cache = {}
2303
for text_key, valid in text_key_references.iteritems():
2304
if not valid:
2305
invalid_keys.add(text_key)
2306
else:
2307
revision_keys[text_key[1]].add(text_key)
2308
text_key_cache[text_key] = text_key
2309
del text_key_references
2310
text_index = {}
2311
text_graph = graph.Graph(graph.DictParentsProvider(text_index))
2312
NULL_REVISION = _mod_revision.NULL_REVISION
2313
# Set a cache with a size of 10 - this suffices for bzr.dev but may be
2314
# too small for large or very branchy trees. However, for 55K path
2315
# trees, it would be easy to use too much memory trivially. Ideally we
2316
# could gauge this by looking at available real memory etc, but this is
2317
# always a tricky proposition.
2318
inventory_cache = lru_cache.LRUCache(10)
2319
batch_size = 10 # should be ~150MB on a 55K path tree
2320
batch_count = len(revision_order) / batch_size + 1
2321
processed_texts = 0
2322
pb.update("Calculating text parents", processed_texts, text_count)
2323
for offset in xrange(batch_count):
2324
to_query = revision_order[offset * batch_size:(offset + 1) *
2325
batch_size]
2326
if not to_query:
2327
break
2328
for revision_id in to_query:
2329
parent_ids = ancestors[revision_id]
2330
for text_key in revision_keys[revision_id]:
2331
pb.update("Calculating text parents", processed_texts)
2332
processed_texts += 1
2333
candidate_parents = []
2334
for parent_id in parent_ids:
2335
parent_text_key = (text_key[0], parent_id)
2336
try:
2337
check_parent = parent_text_key not in \
2338
revision_keys[parent_id]
2339
except KeyError:
2340
# the parent parent_id is a ghost:
2341
check_parent = False
2342
# truncate the derived graph against this ghost.
2343
parent_text_key = None
2344
if check_parent:
2345
# look at the parent commit details inventories to
2346
# determine possible candidates in the per file graph.
2347
# TODO: cache here.
2348
try:
2349
inv = inventory_cache[parent_id]
2350
except KeyError:
2351
inv = self.revision_tree(parent_id).inventory
2352
inventory_cache[parent_id] = inv
2353
try:
2354
parent_entry = inv[text_key[0]]
2355
except (KeyError, errors.NoSuchId):
2356
parent_entry = None
2357
if parent_entry is not None:
2358
parent_text_key = (
2359
text_key[0], parent_entry.revision)
2360
else:
2361
parent_text_key = None
2362
if parent_text_key is not None:
2363
candidate_parents.append(
2364
text_key_cache[parent_text_key])
2365
parent_heads = text_graph.heads(candidate_parents)
2366
new_parents = list(parent_heads)
2367
new_parents.sort(key=lambda x:candidate_parents.index(x))
2368
if new_parents == []:
2369
new_parents = [NULL_REVISION]
2370
text_index[text_key] = new_parents
2371
2372
for text_key in invalid_keys:
2373
text_index[text_key] = [NULL_REVISION]
2374
return text_index
2375
2376
def item_keys_introduced_by(self, revision_ids, _files_pb=None):
2377
"""Get an iterable listing the keys of all the data introduced by a set
2378
of revision IDs.
2379
2380
The keys will be ordered so that the corresponding items can be safely
2381
fetched and inserted in that order.
2382
2383
:returns: An iterable producing tuples of (knit-kind, file-id,
2384
versions). knit-kind is one of 'file', 'inventory', 'signatures',
2385
'revisions'. file-id is None unless knit-kind is 'file'.
2386
"""
2387
for result in self._find_file_keys_to_fetch(revision_ids, _files_pb):
2388
yield result
2389
del _files_pb
2390
for result in self._find_non_file_keys_to_fetch(revision_ids):
2391
yield result
2392
2393
def _find_file_keys_to_fetch(self, revision_ids, pb):
2394
# XXX: it's a bit weird to control the inventory weave caching in this
2395
# generator. Ideally the caching would be done in fetch.py I think. Or
2396
# maybe this generator should explicitly have the contract that it
2397
# should not be iterated until the previously yielded item has been
2398
# processed?
2399
inv_w = self.inventories
2400
2401
# file ids that changed
2402
file_ids = self.fileids_altered_by_revision_ids(revision_ids, inv_w)
2403
count = 0
2404
num_file_ids = len(file_ids)
2405
for file_id, altered_versions in file_ids.iteritems():
2406
if pb is not None:
2407
pb.update("Fetch texts", count, num_file_ids)
2408
count += 1
2409
yield ("file", file_id, altered_versions)
2410
2411
def _find_non_file_keys_to_fetch(self, revision_ids):
2412
# inventory
2413
yield ("inventory", None, revision_ids)
2414
2415
# signatures
2416
# XXX: Note ATM no callers actually pay attention to this return
2417
# instead they just use the list of revision ids and ignore
2418
# missing sigs. Consider removing this work entirely
2419
revisions_with_signatures = set(self.signatures.get_parent_map(
2420
[(r,) for r in revision_ids]))
2421
revisions_with_signatures = set(
2422
[r for (r,) in revisions_with_signatures])
2423
revisions_with_signatures.intersection_update(revision_ids)
2424
yield ("signatures", None, revisions_with_signatures)
2425
2426
# revisions
2427
yield ("revisions", None, revision_ids)
2428
2429
@needs_read_lock
2430
def get_inventory(self, revision_id):
2431
"""Get Inventory object by revision id."""
2432
return self.iter_inventories([revision_id]).next()
2433
2434
def iter_inventories(self, revision_ids, ordering=None):
2435
"""Get many inventories by revision_ids.
2436
2437
This will buffer some or all of the texts used in constructing the
2438
inventories in memory, but will only parse a single inventory at a
2439
time.
2440
2441
:param revision_ids: The expected revision ids of the inventories.
2442
:param ordering: optional ordering, e.g. 'topological'. If not
2443
specified, the order of revision_ids will be preserved (by
2444
buffering if necessary).
2445
:return: An iterator of inventories.
2446
"""
2447
if ((None in revision_ids)
2448
or (_mod_revision.NULL_REVISION in revision_ids)):
2449
raise ValueError('cannot get null revision inventory')
2450
return self._iter_inventories(revision_ids, ordering)
2451
2452
def _iter_inventories(self, revision_ids, ordering):
2453
"""single-document based inventory iteration."""
2454
inv_xmls = self._iter_inventory_xmls(revision_ids, ordering)
2455
for text, revision_id in inv_xmls:
2456
yield self._deserialise_inventory(revision_id, text)
2457
2458
def _iter_inventory_xmls(self, revision_ids, ordering):
2459
if ordering is None:
2460
order_as_requested = True
2461
ordering = 'unordered'
2462
else:
2463
order_as_requested = False
2464
keys = [(revision_id,) for revision_id in revision_ids]
2465
if not keys:
2466
return
2467
if order_as_requested:
2468
key_iter = iter(keys)
2469
next_key = key_iter.next()
2470
stream = self.inventories.get_record_stream(keys, ordering, True)
2471
text_chunks = {}
2472
for record in stream:
2473
if record.storage_kind != 'absent':
2474
chunks = record.get_bytes_as('chunked')
2475
if order_as_requested:
2476
text_chunks[record.key] = chunks
2477
else:
2478
yield ''.join(chunks), record.key[-1]
2479
else:
2480
raise errors.NoSuchRevision(self, record.key)
2481
if order_as_requested:
2482
# Yield as many results as we can while preserving order.
2483
while next_key in text_chunks:
2484
chunks = text_chunks.pop(next_key)
2485
yield ''.join(chunks), next_key[-1]
2486
try:
2487
next_key = key_iter.next()
2488
except StopIteration:
2489
# We still want to fully consume the get_record_stream,
2490
# just in case it is not actually finished at this point
2491
next_key = None
2492
break
2493
2494
def _deserialise_inventory(self, revision_id, xml):
2495
"""Transform the xml into an inventory object.
2496
2497
:param revision_id: The expected revision id of the inventory.
2498
:param xml: A serialised inventory.
2499
"""
2500
result = self._serializer.read_inventory_from_string(xml, revision_id,
2501
entry_cache=self._inventory_entry_cache,
2502
return_from_cache=self._safe_to_return_from_cache)
2503
if result.revision_id != revision_id:
2504
raise AssertionError('revision id mismatch %s != %s' % (
2505
result.revision_id, revision_id))
2506
return result
2507
2508
def get_serializer_format(self):
2509
return self._serializer.format_num
2510
2511
@needs_read_lock
2512
def _get_inventory_xml(self, revision_id):
2513
"""Get serialized inventory as a string."""
2514
texts = self._iter_inventory_xmls([revision_id], 'unordered')
2515
try:
2516
text, revision_id = texts.next()
2517
except StopIteration:
2518
raise errors.HistoryMissing(self, 'inventory', revision_id)
2519
return text
948
2520
949
2521
def get_rev_id_for_revno(self, revno, known_pair):
950
2522
"""Return the revision id of a revno, given a later (revno, revid)
981
2553
raise AssertionError('_iter_for_revno returned too much history')
982
2554
return (True, partial_history[-1])
983
2555
2556
def iter_reverse_revision_history(self, revision_id):
2557
"""Iterate backwards through revision ids in the lefthand history
2558
2559
:param revision_id: The revision id to start with. All its lefthand
2560
ancestors will be traversed.
2561
"""
2562
graph = self.get_graph()
2563
stop_revisions = (None, _mod_revision.NULL_REVISION)
2564
return graph.iter_lefthand_ancestry(revision_id, stop_revisions)
2565
984
2566
def is_shared(self):
985
2567
"""Return True if this repository is flagged as a shared repository."""
986
2568
raise NotImplementedError(self.is_shared)
1013
2595
1014
2596
`revision_id` may be NULL_REVISION for the empty tree revision.
1015
2597
"""
1016
raise NotImplementedError(self.revision_tree)
2598
revision_id = _mod_revision.ensure_null(revision_id)
2599
# TODO: refactor this to use an existing revision object
2600
# so we don't need to read it in twice.
2601
if revision_id == _mod_revision.NULL_REVISION:
2602
return RevisionTree(self, Inventory(root_id=None),
2603
_mod_revision.NULL_REVISION)
2604
else:
2605
inv = self.get_inventory(revision_id)
2606
return RevisionTree(self, inv, revision_id)
1017
2607
1018
2608
def revision_trees(self, revision_ids):
1019
2609
"""Return Trees for revisions in this repository.
1021
2611
:param revision_ids: a sequence of revision-ids;
1022
2612
a revision-id may not be None or 'null:'
1023
2613
"""
1024
raise NotImplementedError(self.revision_trees)
2614
inventories = self.iter_inventories(revision_ids)
2615
for inv in inventories:
2616
yield RevisionTree(self, inv, inv.revision_id)
2617
2618
def _filtered_revision_trees(self, revision_ids, file_ids):
2619
"""Return Tree for a revision on this branch with only some files.
2620
2621
:param revision_ids: a sequence of revision-ids;
2622
a revision-id may not be None or 'null:'
2623
:param file_ids: if not None, the result is filtered
2624
so that only those file-ids, their parents and their
2625
children are included.
2626
"""
2627
inventories = self.iter_inventories(revision_ids)
2628
for inv in inventories:
2629
# Should we introduce a FilteredRevisionTree class rather
2630
# than pre-filter the inventory here?
2631
filtered_inv = inv.filter(file_ids)
2632
yield RevisionTree(self, filtered_inv, filtered_inv.revision_id)
2633
2634
@needs_read_lock
2635
def get_ancestry(self, revision_id, topo_sorted=True):
2636
"""Return a list of revision-ids integrated by a revision.
2637
2638
The first element of the list is always None, indicating the origin
2639
revision. This might change when we have history horizons, or
2640
perhaps we should have a new API.
2641
2642
This is topologically sorted.
2643
"""
2644
if _mod_revision.is_null(revision_id):
2645
return [None]
2646
if not self.has_revision(revision_id):
2647
raise errors.NoSuchRevision(self, revision_id)
2648
graph = self.get_graph()
2649
keys = set()
2650
search = graph._make_breadth_first_searcher([revision_id])
2651
while True:
2652
try:
2653
found, ghosts = search.next_with_ghosts()
2654
except StopIteration:
2655
break
2656
keys.update(found)
2657
if _mod_revision.NULL_REVISION in keys:
2658
keys.remove(_mod_revision.NULL_REVISION)
2659
if topo_sorted:
2660
parent_map = graph.get_parent_map(keys)
2661
keys = tsort.topo_sort(parent_map)
2662
return [None] + list(keys)
1025
2663
1026
2664
def pack(self, hint=None, clean_obsolete_packs=False):
1027
2665
"""Compress the data within the repository.
1049
2687
1050
2688
def get_parent_map(self, revision_ids):
1051
2689
"""See graph.StackedParentsProvider.get_parent_map"""
1052
raise NotImplementedError(self.get_parent_map)
1053
1054
def _get_parent_map_no_fallbacks(self, revision_ids):
1055
"""Same as Repository.get_parent_map except doesn't query fallbacks."""
1056
2690
# revisions index works in keys; this just works in revisions
1057
2691
# therefore wrap and unwrap
1058
2692
query_keys = []
1064
2698
raise ValueError('get_parent_map(None) is not valid')
1065
2699
else:
1066
2700
query_keys.append((revision_id ,))
1067
vf = self.revisions.without_fallbacks()
1068
2701
for ((revision_id,), parent_keys) in \
1069
vf.get_parent_map(query_keys).iteritems():
2702
self.revisions.get_parent_map(query_keys).iteritems():
1070
2703
if parent_keys:
1071
2704
result[revision_id] = tuple([parent_revid
1072
2705
for (parent_revid,) in parent_keys])
1075
2708
return result
1076
2709
1077
2710
def _make_parents_provider(self):
1078
if not self._format.supports_external_lookups:
1079
return self
1080
return graph.StackedParentsProvider(_LazyListJoin(
1081
[self._make_parents_provider_unstacked()],
1082
self._fallback_repositories))
1083
1084
def _make_parents_provider_unstacked(self):
1085
return graph.CallableToParentsProviderAdapter(
1086
self._get_parent_map_no_fallbacks)
2711
return self
1087
2712
1088
2713
@needs_read_lock
1089
2714
def get_known_graph_ancestry(self, revision_ids):
1090
2715
"""Return the known graph for a set of revision ids and their ancestors.
1091
2716
"""
1092
raise NotImplementedError(self.get_known_graph_ancestry)
1093
1094
def get_file_graph(self):
1095
"""Return the graph walker for files."""
1096
raise NotImplementedError(self.get_file_graph)
2717
st = static_tuple.StaticTuple
2718
revision_keys = [st(r_id).intern() for r_id in revision_ids]
2719
known_graph = self.revisions.get_known_graph_ancestry(revision_keys)
2720
return graph.GraphThunkIdsToKeys(known_graph)
1097
2721
1098
2722
def get_graph(self, other_repository=None):
1099
2723
"""Return the graph walker for this repository format"""
1104
2728
[parents_provider, other_repository._make_parents_provider()])
1105
2729
return graph.Graph(parents_provider)
1106
2730
2731
def _get_versioned_file_checker(self, text_key_references=None,
2732
ancestors=None):
2733
"""Return an object suitable for checking versioned files.
2734
2735
:param text_key_references: if non-None, an already built
2736
dictionary mapping text keys ((fileid, revision_id) tuples)
2737
to whether they were referred to by the inventory of the
2738
revision_id that they contain. If None, this will be
2739
calculated.
2740
:param ancestors: Optional result from
2741
self.get_graph().get_parent_map(self.all_revision_ids()) if already
2742
available.
2743
"""
2744
return _VersionedFileChecker(self,
2745
text_key_references=text_key_references, ancestors=ancestors)
2746
2747
def revision_ids_to_search_result(self, result_set):
2748
"""Convert a set of revision ids to a graph SearchResult."""
2749
result_parents = set()
2750
for parents in self.get_graph().get_parent_map(
2751
result_set).itervalues():
2752
result_parents.update(parents)
2753
included_keys = result_set.intersection(result_parents)
2754
start_keys = result_set.difference(included_keys)
2755
exclude_keys = result_parents.difference(result_set)
2756
result = graph.SearchResult(start_keys, exclude_keys,
2757
len(result_set), result_set)
2758
return result
2759
1107
2760
@needs_write_lock
1108
2761
def set_make_working_trees(self, new_value):
1109
2762
"""Set the policy flag for making working trees when creating branches.
1122
2775
1123
2776
@needs_write_lock
1124
2777
def sign_revision(self, revision_id, gpg_strategy):
1125
testament = _mod_testament.Testament.from_revision(self, revision_id)
1126
plaintext = testament.as_short_text()
2778
plaintext = Testament.from_revision(self, revision_id).as_short_text()
1127
2779
self.store_revision_signature(gpg_strategy, plaintext, revision_id)
1128
2780
1129
2781
@needs_read_lock
1130
def verify_revision_signature(self, revision_id, gpg_strategy):
1131
"""Verify the signature on a revision.
1132
1133
:param revision_id: the revision to verify
1134
:gpg_strategy: the GPGStrategy object to used
1135
1136
:return: gpg.SIGNATURE_VALID or a failed SIGNATURE_ value
1137
"""
1138
if not self.has_signature_for_revision_id(revision_id):
1139
return gpg.SIGNATURE_NOT_SIGNED, None
1140
signature = self.get_signature_text(revision_id)
1141
1142
testament = _mod_testament.Testament.from_revision(self, revision_id)
1143
plaintext = testament.as_short_text()
1144
1145
return gpg_strategy.verify(signature, plaintext)
1146
1147
@needs_read_lock
1148
def verify_revision_signatures(self, revision_ids, gpg_strategy):
1149
"""Verify revision signatures for a number of revisions.
1150
1151
:param revision_id: the revision to verify
1152
:gpg_strategy: the GPGStrategy object to used
1153
:return: Iterator over tuples with revision id, result and keys
1154
"""
1155
for revid in revision_ids:
1156
(result, key) = self.verify_revision_signature(revid, gpg_strategy)
1157
yield revid, result, key
1158
1159
2782
def has_signature_for_revision_id(self, revision_id):
1160
2783
"""Query for a revision signature for revision_id in the repository."""
1161
raise NotImplementedError(self.has_signature_for_revision_id)
2784
if not self.has_revision(revision_id):
2785
raise errors.NoSuchRevision(self, revision_id)
2786
sig_present = (1 == len(
2787
self.signatures.get_parent_map([(revision_id,)])))
2788
return sig_present
1162
2789
2790
@needs_read_lock
1163
2791
def get_signature_text(self, revision_id):
1164
2792
"""Return the text for a signature."""
1165
raise NotImplementedError(self.get_signature_text)
2793
stream = self.signatures.get_record_stream([(revision_id,)],
2794
'unordered', True)
2795
record = stream.next()
2796
if record.storage_kind == 'absent':
2797
raise errors.NoSuchRevision(self, revision_id)
2798
return record.get_bytes_as('fulltext')
1166
2799
2800
@needs_read_lock
1167
2801
def check(self, revision_ids=None, callback_refs=None, check_repo=True):
1168
2802
"""Check consistency of all history of given revision_ids.
1169
2803
1177
2811
:param check_repo: If False do not check the repository contents, just
1178
2812
calculate the data callback_refs requires and call them back.
1179
2813
"""
1180
return self._check(revision_ids=revision_ids, callback_refs=callback_refs,
2814
return self._check(revision_ids, callback_refs=callback_refs,
1181
2815
check_repo=check_repo)
1182
2816
1183
def _check(self, revision_ids=None, callback_refs=None, check_repo=True):
1184
raise NotImplementedError(self.check)
2817
def _check(self, revision_ids, callback_refs, check_repo):
2818
result = check.Check(self, check_repo=check_repo)
2819
result.check(callback_refs)
2820
return result
1185
2821
1186
2822
def _warn_if_deprecated(self, branch=None):
1187
if not self._format.is_deprecated():
1188
return
1189
2823
global _deprecation_warning_done
1190
2824
if _deprecation_warning_done:
1191
2825
return
1192
2826
try:
1193
2827
if branch is None:
1194
conf = config.GlobalStack()
2828
conf = config.GlobalConfig()
1195
2829
else:
1196
conf = branch.get_config_stack()
1197
if 'format_deprecation' in conf.get('suppress_warnings'):
2830
conf = branch.get_config()
2831
if conf.suppress_warning('format_deprecation'):
1198
2832
return
1199
2833
warning("Format %s for %s is deprecated -"
1200
2834
" please use 'bzr upgrade' to get better performance"
1221
2855
except UnicodeDecodeError:
1222
2856
raise errors.NonAsciiRevisionId(method, self)
1223
2857
2858
def revision_graph_can_have_wrong_parents(self):
2859
"""Is it possible for this repository to have a revision graph with
2860
incorrect parents?
2861
2862
If True, then this repository must also implement
2863
_find_inconsistent_revision_parents so that check and reconcile can
2864
check for inconsistencies before proceeding with other checks that may
2865
depend on the revision index being consistent.
2866
"""
2867
raise NotImplementedError(self.revision_graph_can_have_wrong_parents)
2868
2869
2870
def install_revision(repository, rev, revision_tree):
2871
"""Install all revision data into a repository."""
2872
install_revisions(repository, [(rev, revision_tree, None)])
2873
2874
2875
def install_revisions(repository, iterable, num_revisions=None, pb=None):
2876
"""Install all revision data into a repository.
2877
2878
Accepts an iterable of revision, tree, signature tuples. The signature
2879
may be None.
2880
"""
2881
repository.start_write_group()
2882
try:
2883
inventory_cache = lru_cache.LRUCache(10)
2884
for n, (revision, revision_tree, signature) in enumerate(iterable):
2885
_install_revision(repository, revision, revision_tree, signature,
2886
inventory_cache)
2887
if pb is not None:
2888
pb.update('Transferring revisions', n + 1, num_revisions)
2889
except:
2890
repository.abort_write_group()
2891
raise
2892
else:
2893
repository.commit_write_group()
2894
2895
2896
def _install_revision(repository, rev, revision_tree, signature,
2897
inventory_cache):
2898
"""Install all revision data into a repository."""
2899
present_parents = []
2900
parent_trees = {}
2901
for p_id in rev.parent_ids:
2902
if repository.has_revision(p_id):
2903
present_parents.append(p_id)
2904
parent_trees[p_id] = repository.revision_tree(p_id)
2905
else:
2906
parent_trees[p_id] = repository.revision_tree(
2907
_mod_revision.NULL_REVISION)
2908
2909
inv = revision_tree.inventory
2910
entries = inv.iter_entries()
2911
# backwards compatibility hack: skip the root id.
2912
if not repository.supports_rich_root():
2913
path, root = entries.next()
2914
if root.revision != rev.revision_id:
2915
raise errors.IncompatibleRevision(repr(repository))
2916
text_keys = {}
2917
for path, ie in entries:
2918
text_keys[(ie.file_id, ie.revision)] = ie
2919
text_parent_map = repository.texts.get_parent_map(text_keys)
2920
missing_texts = set(text_keys) - set(text_parent_map)
2921
# Add the texts that are not already present
2922
for text_key in missing_texts:
2923
ie = text_keys[text_key]
2924
text_parents = []
2925
# FIXME: TODO: The following loop overlaps/duplicates that done by
2926
# commit to determine parents. There is a latent/real bug here where
2927
# the parents inserted are not those commit would do - in particular
2928
# they are not filtered by heads(). RBC, AB
2929
for revision, tree in parent_trees.iteritems():
2930
if ie.file_id not in tree:
2931
continue
2932
parent_id = tree.inventory[ie.file_id].revision
2933
if parent_id in text_parents:
2934
continue
2935
text_parents.append((ie.file_id, parent_id))
2936
lines = revision_tree.get_file(ie.file_id).readlines()
2937
repository.texts.add_lines(text_key, text_parents, lines)
2938
try:
2939
# install the inventory
2940
if repository._format._commit_inv_deltas and len(rev.parent_ids):
2941
# Cache this inventory
2942
inventory_cache[rev.revision_id] = inv
2943
try:
2944
basis_inv = inventory_cache[rev.parent_ids[0]]
2945
except KeyError:
2946
repository.add_inventory(rev.revision_id, inv, present_parents)
2947
else:
2948
delta = inv._make_delta(basis_inv)
2949
repository.add_inventory_by_delta(rev.parent_ids[0], delta,
2950
rev.revision_id, present_parents)
2951
else:
2952
repository.add_inventory(rev.revision_id, inv, present_parents)
2953
except errors.RevisionAlreadyPresent:
2954
pass
2955
if signature is not None:
2956
repository.add_signature_text(rev.revision_id, signature)
2957
repository.add_revision(rev.revision_id, rev, inv)
2958
1224
2959
1225
2960
class MetaDirRepository(Repository):
1226
2961
"""Repositories in the new meta-dir layout.
1260
2995
"""Returns the policy for making working trees on new branches."""
1261
2996
return not self._transport.has('no-working-trees')
1262
2997
1263
@needs_write_lock
1264
def update_feature_flags(self, updated_flags):
1265
"""Update the feature flags for this branch.
1266
1267
:param updated_flags: Dictionary mapping feature names to necessities
1268
A necessity can be None to indicate the feature should be removed
1269
"""
1270
self._format._update_feature_flags(updated_flags)
1271
self.control_transport.put_bytes('format', self._format.as_string())
1272
1273
1274
class RepositoryFormatRegistry(controldir.ControlComponentFormatRegistry):
1275
"""Repository format registry."""
1276
1277
def get_default(self):
1278
"""Return the current default format."""
1279
return controldir.format_registry.make_bzrdir('default').repository_format
2998
2999
class MetaDirVersionedFileRepository(MetaDirRepository):
3000
"""Repositories in a meta-dir, that work via versioned file objects."""
3001
3002
def __init__(self, _format, a_bzrdir, control_files):
3003
super(MetaDirVersionedFileRepository, self).__init__(_format, a_bzrdir,
3004
control_files)
1280
3005
1281
3006
1282
3007
network_format_registry = registry.FormatRegistry()
1288
3013
"""
1289
3014
1290
3015
1291
format_registry = RepositoryFormatRegistry(network_format_registry)
3016
format_registry = registry.FormatRegistry(network_format_registry)
1292
3017
"""Registry of formats, indexed by their BzrDirMetaFormat format string.
1293
3018
1294
3019
This can contain either format instances themselves, or classes/factories that
1299
3024
#####################################################################
1300
3025
# Repository Formats
1301
3026
1302
class RepositoryFormat(controldir.ControlComponentFormat):
3027
class RepositoryFormat(object):
1303
3028
"""A repository format.
1304
3029
1305
3030
Formats provide four things:
1325
3050
created.
1326
3051
1327
3052
Common instance attributes:
1328
_matchingbzrdir - the controldir format that the repository format was
3053
_matchingbzrdir - the bzrdir format that the repository format was
1329
3054
originally written to work with. This can be used if manually
1330
3055
constructing a bzrdir and repository, or more commonly for test suite
1331
3056
parameterization.
1340
3065
# Does this format support CHK bytestring lookups. Set to True or False in
1341
3066
# derived classes.
1342
3067
supports_chks = None
3068
# Should commit add an inventory, or an inventory delta to the repository.
3069
_commit_inv_deltas = True
3070
# What order should fetch operations request streams in?
3071
# The default is unordered as that is the cheapest for an origin to
3072
# provide.
3073
_fetch_order = 'unordered'
3074
# Does this repository format use deltas that can be fetched as-deltas ?
3075
# (E.g. knits, where the knit deltas can be transplanted intact.
3076
# We default to False, which will ensure that enough data to get
3077
# a full text out of any fetch stream will be grabbed.
3078
_fetch_uses_deltas = False
1343
3079
# Should fetch trigger a reconcile after the fetch? Only needed for
1344
3080
# some repository formats that can suffer internal inconsistencies.
1345
3081
_fetch_reconcile = False
1351
3087
# help), and for fetching when data won't have come from the same
1352
3088
# compressor.
1353
3089
pack_compresses = False
1354
# Does the repository storage understand references to trees?
3090
# Does the repository inventory storage understand references to trees?
1355
3091
supports_tree_reference = None
1356
3092
# Is the format experimental ?
1357
3093
experimental = False
1358
# Does this repository format escape funky characters, or does it create
1359
# files with similar names as the versioned files in its contents on disk
1360
# ?
1361
supports_funky_characters = None
1362
# Does this repository format support leaving locks?
1363
supports_leaving_lock = None
1364
# Does this format support the full VersionedFiles interface?
1365
supports_full_versioned_files = None
1366
# Does this format support signing revision signatures?
1367
supports_revision_signatures = True
1368
# Can the revision graph have incorrect parents?
1369
revision_graph_can_have_wrong_parents = None
1370
# Does this format support rich root data?
1371
rich_root_data = None
1372
# Does this format support explicitly versioned directories?
1373
supports_versioned_directories = None
1374
# Can other repositories be nested into one of this format?
1375
supports_nesting_repositories = None
1376
# Is it possible for revisions to be present without being referenced
1377
# somewhere ?
1378
supports_unreferenced_revisions = None
3094
# Does this repository format escape funky characters, or does it create files with
3095
# similar names as the versioned files in its contents on disk ?
3096
supports_funky_characters = True
1379
3097
1380
3098
def __repr__(self):
1381
3099
return "%s()" % self.__class__.__name__
1387
3105
def __ne__(self, other):
1388
3106
return not self == other
1389
3107
3108
@classmethod
3109
def find_format(klass, a_bzrdir):
3110
"""Return the format for the repository object in a_bzrdir.
3111
3112
This is used by bzr native formats that have a "format" file in
3113
the repository. Other methods may be used by different types of
3114
control directory.
3115
"""
3116
try:
3117
transport = a_bzrdir.get_repository_transport(None)
3118
format_string = transport.get_bytes("format")
3119
return format_registry.get(format_string)
3120
except errors.NoSuchFile:
3121
raise errors.NoRepositoryPresent(a_bzrdir)
3122
except KeyError:
3123
raise errors.UnknownFormatError(format=format_string,
3124
kind='repository')
3125
3126
@classmethod
3127
def register_format(klass, format):
3128
format_registry.register(format.get_format_string(), format)
3129
3130
@classmethod
3131
def unregister_format(klass, format):
3132
format_registry.remove(format.get_format_string())
3133
3134
@classmethod
3135
def get_default_format(klass):
3136
"""Return the current default format."""
3137
from bzrlib import bzrdir
3138
return bzrdir.format_registry.make_bzrdir('default').repository_format
3139
3140
def get_format_string(self):
3141
"""Return the ASCII format string that identifies this format.
3142
3143
Note that in pre format ?? repositories the format string is
3144
not permitted nor written to disk.
3145
"""
3146
raise NotImplementedError(self.get_format_string)
3147
1390
3148
def get_format_description(self):
1391
3149
"""Return the short description for this format."""
1392
3150
raise NotImplementedError(self.get_format_description)
1393
3151
1394
def initialize(self, controldir, shared=False):
1395
"""Initialize a repository of this format in controldir.
1396
1397
:param controldir: The controldir to put the new repository in it.
3152
# TODO: this shouldn't be in the base class, it's specific to things that
3153
# use weaves or knits -- mbp 20070207
3154
def _get_versioned_file_store(self,
3155
name,
3156
transport,
3157
control_files,
3158
prefixed=True,
3159
versionedfile_class=None,
3160
versionedfile_kwargs={},
3161
escaped=False):
3162
if versionedfile_class is None:
3163
versionedfile_class = self._versionedfile_class
3164
weave_transport = control_files._transport.clone(name)
3165
dir_mode = control_files._dir_mode
3166
file_mode = control_files._file_mode
3167
return VersionedFileStore(weave_transport, prefixed=prefixed,
3168
dir_mode=dir_mode,
3169
file_mode=file_mode,
3170
versionedfile_class=versionedfile_class,
3171
versionedfile_kwargs=versionedfile_kwargs,
3172
escaped=escaped)
3173
3174
def initialize(self, a_bzrdir, shared=False):
3175
"""Initialize a repository of this format in a_bzrdir.
3176
3177
:param a_bzrdir: The bzrdir to put the new repository in it.
1398
3178
:param shared: The repository should be initialized as a sharable one.
1399
3179
:returns: The new repository object.
1400
3180
1401
3181
This may raise UninitializableFormat if shared repository are not
1402
compatible the controldir.
3182
compatible the a_bzrdir.
1403
3183
"""
1404
3184
raise NotImplementedError(self.initialize)
1405
3185
1412
3192
"""
1413
3193
return True
1414
3194
1415
def is_deprecated(self):
1416
"""Is this format deprecated?
1417
1418
Deprecated formats may trigger a user-visible warning recommending
1419
the user to upgrade. They are still fully supported.
1420
"""
1421
return False
1422
1423
3195
def network_name(self):
1424
3196
"""A simple byte string uniquely identifying this format for RPC calls.
1425
3197
1441
3213
'Does not support nested trees', target_format,
1442
3214
from_format=self)
1443
3215
1444
def open(self, controldir, _found=False):
1445
"""Return an instance of this format for a controldir.
3216
def open(self, a_bzrdir, _found=False):
3217
"""Return an instance of this format for the bzrdir a_bzrdir.
1446
3218
1447
3219
_found is a private parameter, do not use it.
1448
3220
"""
1449
3221
raise NotImplementedError(self.open)
1450
3222
1451
def _run_post_repo_init_hooks(self, repository, controldir, shared):
1452
from bzrlib.controldir import ControlDir, RepoInitHookParams
1453
hooks = ControlDir.hooks['post_repo_init']
3223
def _run_post_repo_init_hooks(self, repository, a_bzrdir, shared):
3224
from bzrlib.bzrdir import BzrDir, RepoInitHookParams
3225
hooks = BzrDir.hooks['post_repo_init']
1454
3226
if not hooks:
1455
3227
return
1456
params = RepoInitHookParams(repository, self, controldir, shared)
3228
params = RepoInitHookParams(repository, self, a_bzrdir, shared)
1457
3229
for hook in hooks:
1458
3230
hook(params)
1459
3231
1460
3232
1461
class RepositoryFormatMetaDir(bzrdir.BzrFormat, RepositoryFormat):
3233
class MetaDirRepositoryFormat(RepositoryFormat):
1462
3234
"""Common base class for the new repositories using the metadir layout."""
1463
3235
1464
3236
rich_root_data = False
1465
3237
supports_tree_reference = False
1466
3238
supports_external_lookups = False
1467
supports_leaving_lock = True
1468
supports_nesting_repositories = True
1469
3239
1470
3240
@property
1471
3241
def _matchingbzrdir(self):
1474
3244
return matching
1475
3245
1476
3246
def __init__(self):
1477
RepositoryFormat.__init__(self)
1478
bzrdir.BzrFormat.__init__(self)
3247
super(MetaDirRepositoryFormat, self).__init__()
1479
3248
1480
3249
def _create_control_files(self, a_bzrdir):
1481
3250
"""Create the required files and the initial control_files object."""
1505
3274
finally:
1506
3275
control_files.unlock()
1507
3276
1508
@classmethod
1509
def find_format(klass, a_bzrdir):
1510
"""Return the format for the repository object in a_bzrdir.
1511
1512
This is used by bzr native formats that have a "format" file in
1513
the repository. Other methods may be used by different types of
1514
control directory.
1515
"""
1516
try:
1517
transport = a_bzrdir.get_repository_transport(None)
1518
format_string = transport.get_bytes("format")
1519
except errors.NoSuchFile:
1520
raise errors.NoRepositoryPresent(a_bzrdir)
1521
return klass._find_format(format_registry, 'repository', format_string)
1522
1523
def check_support_status(self, allow_unsupported, recommend_upgrade=True,
1524
basedir=None):
1525
RepositoryFormat.check_support_status(self,
1526
allow_unsupported=allow_unsupported, recommend_upgrade=recommend_upgrade,
1527
basedir=basedir)
1528
bzrdir.BzrFormat.check_support_status(self, allow_unsupported=allow_unsupported,
1529
recommend_upgrade=recommend_upgrade, basedir=basedir)
1530
3277
def network_name(self):
3278
"""Metadir formats have matching disk and network format strings."""
3279
return self.get_format_string()
3280
3281
3282
# Pre-0.8 formats that don't have a disk format string (because they are
3283
# versioned by the matching control directory). We use the control directories
3284
# disk format string as a key for the network_name because they meet the
3285
# constraints (simple string, unique, immutable).
3286
network_format_registry.register_lazy(
3287
"Bazaar-NG branch, format 5\n",
3288
'bzrlib.repofmt.weaverepo',
3289
'RepositoryFormat5',
3290
)
3291
network_format_registry.register_lazy(
3292
"Bazaar-NG branch, format 6\n",
3293
'bzrlib.repofmt.weaverepo',
3294
'RepositoryFormat6',
3295
)
1531
3296
1532
3297
# formats which have no format string are not discoverable or independently
1533
3298
# creatable on disk, so are not registered in format_registry. They're
1534
# all in bzrlib.repofmt.knitreponow. When an instance of one of these is
1535
# needed, it's constructed directly by the ControlDir. Non-native formats where
3299
# all in bzrlib.repofmt.weaverepo now. When an instance of one of these is
3300
# needed, it's constructed directly by the BzrDir. Non-native formats where
1536
3301
# the repository is not separately opened are similar.
1537
3302
1538
3303
format_registry.register_lazy(
3304
'Bazaar-NG Repository format 7',
3305
'bzrlib.repofmt.weaverepo',
3306
'RepositoryFormat7'
3307
)
3308
3309
format_registry.register_lazy(
1539
3310
'Bazaar-NG Knit Repository Format 1',
1540
3311
'bzrlib.repofmt.knitrepo',
1541
3312
'RepositoryFormatKnit1',
1558
3329
# NOTE: These are experimental in 0.92. Stable in 1.0 and above
1559
3330
format_registry.register_lazy(
1560
3331
'Bazaar pack repository format 1 (needs bzr 0.92)\n',
1561
'bzrlib.repofmt.knitpack_repo',
3332
'bzrlib.repofmt.pack_repo',
1562
3333
'RepositoryFormatKnitPack1',
1563
3334
)
1564
3335
format_registry.register_lazy(
1565
3336
'Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n',
1566
'bzrlib.repofmt.knitpack_repo',
3337
'bzrlib.repofmt.pack_repo',
1567
3338
'RepositoryFormatKnitPack3',
1568
3339
)
1569
3340
format_registry.register_lazy(
1570
3341
'Bazaar pack repository format 1 with rich root (needs bzr 1.0)\n',
1571
'bzrlib.repofmt.knitpack_repo',
3342
'bzrlib.repofmt.pack_repo',
1572
3343
'RepositoryFormatKnitPack4',
1573
3344
)
1574
3345
format_registry.register_lazy(
1575
3346
'Bazaar RepositoryFormatKnitPack5 (bzr 1.6)\n',
1576
'bzrlib.repofmt.knitpack_repo',
3347
'bzrlib.repofmt.pack_repo',
1577
3348
'RepositoryFormatKnitPack5',
1578
3349
)
1579
3350
format_registry.register_lazy(
1580
3351
'Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6.1)\n',
1581
'bzrlib.repofmt.knitpack_repo',
3352
'bzrlib.repofmt.pack_repo',
1582
3353
'RepositoryFormatKnitPack5RichRoot',
1583
3354
)
1584
3355
format_registry.register_lazy(
1585
3356
'Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6)\n',
1586
'bzrlib.repofmt.knitpack_repo',
3357
'bzrlib.repofmt.pack_repo',
1587
3358
'RepositoryFormatKnitPack5RichRootBroken',
1588
3359
)
1589
3360
format_registry.register_lazy(
1590
3361
'Bazaar RepositoryFormatKnitPack6 (bzr 1.9)\n',
1591
'bzrlib.repofmt.knitpack_repo',
3362
'bzrlib.repofmt.pack_repo',
1592
3363
'RepositoryFormatKnitPack6',
1593
3364
)
1594
3365
format_registry.register_lazy(
1595
3366
'Bazaar RepositoryFormatKnitPack6RichRoot (bzr 1.9)\n',
1596
'bzrlib.repofmt.knitpack_repo',
3367
'bzrlib.repofmt.pack_repo',
1597
3368
'RepositoryFormatKnitPack6RichRoot',
1598
3369
)
1599
3370
format_registry.register_lazy(
1607
3378
format_registry.register_lazy(
1608
3379
("Bazaar development format 2 with subtree support "
1609
3380
"(needs bzr.dev from before 1.8)\n"),
1610
'bzrlib.repofmt.knitpack_repo',
3381
'bzrlib.repofmt.pack_repo',
1611
3382
'RepositoryFormatPackDevelopment2Subtree',
1612
3383
)
1613
3384
format_registry.register_lazy(
1629
3400
InterRepository.get(other).method_name(parameters).
1630
3401
"""
1631
3402
3403
_walk_to_common_revisions_batch_size = 50
1632
3404
_optimisers = []
1633
3405
"""The available optimised InterRepository types."""
1634
3406
1649
3421
self.target.fetch(self.source, revision_id=revision_id)
1650
3422
1651
3423
@needs_write_lock
1652
def fetch(self, revision_id=None, find_ghosts=False):
3424
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
3425
fetch_spec=None):
1653
3426
"""Fetch the content required to construct revision_id.
1654
3427
1655
3428
The content is copied from self.source to self.target.
1656
3429
1657
3430
:param revision_id: if None all content is copied, if NULL_REVISION no
1658
3431
content is copied.
3432
:param pb: ignored.
1659
3433
:return: None.
1660
3434
"""
1661
raise NotImplementedError(self.fetch)
3435
ui.ui_factory.warn_experimental_format_fetch(self)
3436
from bzrlib.fetch import RepoFetcher
3437
# See <https://launchpad.net/bugs/456077> asking for a warning here
3438
if self.source._format.network_name() != self.target._format.network_name():
3439
ui.ui_factory.show_user_warning('cross_format_fetch',
3440
from_format=self.source._format,
3441
to_format=self.target._format)
3442
f = RepoFetcher(to_repository=self.target,
3443
from_repository=self.source,
3444
last_revision=revision_id,
3445
fetch_spec=fetch_spec,
3446
find_ghosts=find_ghosts)
3447
3448
def _walk_to_common_revisions(self, revision_ids, if_present_ids=None):
3449
"""Walk out from revision_ids in source to revisions target has.
3450
3451
:param revision_ids: The start point for the search.
3452
:return: A set of revision ids.
3453
"""
3454
target_graph = self.target.get_graph()
3455
revision_ids = frozenset(revision_ids)
3456
if if_present_ids:
3457
all_wanted_revs = revision_ids.union(if_present_ids)
3458
else:
3459
all_wanted_revs = revision_ids
3460
missing_revs = set()
3461
source_graph = self.source.get_graph()
3462
# ensure we don't pay silly lookup costs.
3463
searcher = source_graph._make_breadth_first_searcher(all_wanted_revs)
3464
null_set = frozenset([_mod_revision.NULL_REVISION])
3465
searcher_exhausted = False
3466
while True:
3467
next_revs = set()
3468
ghosts = set()
3469
# Iterate the searcher until we have enough next_revs
3470
while len(next_revs) < self._walk_to_common_revisions_batch_size:
3471
try:
3472
next_revs_part, ghosts_part = searcher.next_with_ghosts()
3473
next_revs.update(next_revs_part)
3474
ghosts.update(ghosts_part)
3475
except StopIteration:
3476
searcher_exhausted = True
3477
break
3478
# If there are ghosts in the source graph, and the caller asked for
3479
# them, make sure that they are present in the target.
3480
# We don't care about other ghosts as we can't fetch them and
3481
# haven't been asked to.
3482
ghosts_to_check = set(revision_ids.intersection(ghosts))
3483
revs_to_get = set(next_revs).union(ghosts_to_check)
3484
if revs_to_get:
3485
have_revs = set(target_graph.get_parent_map(revs_to_get))
3486
# we always have NULL_REVISION present.
3487
have_revs = have_revs.union(null_set)
3488
# Check if the target is missing any ghosts we need.
3489
ghosts_to_check.difference_update(have_revs)
3490
if ghosts_to_check:
3491
# One of the caller's revision_ids is a ghost in both the
3492
# source and the target.
3493
raise errors.NoSuchRevision(
3494
self.source, ghosts_to_check.pop())
3495
missing_revs.update(next_revs - have_revs)
3496
# Because we may have walked past the original stop point, make
3497
# sure everything is stopped
3498
stop_revs = searcher.find_seen_ancestors(have_revs)
3499
searcher.stop_searching_any(stop_revs)
3500
if searcher_exhausted:
3501
break
3502
return searcher.get_result()
1662
3503
1663
3504
@needs_read_lock
1664
3505
def search_missing_revision_ids(self,
1665
3506
revision_id=symbol_versioning.DEPRECATED_PARAMETER,
1666
find_ghosts=True, revision_ids=None, if_present_ids=None,
1667
limit=None):
3507
find_ghosts=True, revision_ids=None, if_present_ids=None):
1668
3508
"""Return the revision ids that source has that target does not.
1669
3509
1670
3510
:param revision_id: only return revision ids included by this
1678
3518
to fetch for tags, which may reference absent revisions.
1679
3519
:param find_ghosts: If True find missing revisions in deep history
1680
3520
rather than just finding the surface difference.
1681
:param limit: Maximum number of revisions to return, topologically
1682
ordered
1683
3521
:return: A bzrlib.graph.SearchResult.
1684
3522
"""
1685
raise NotImplementedError(self.search_missing_revision_ids)
3523
if symbol_versioning.deprecated_passed(revision_id):
3524
symbol_versioning.warn(
3525
'search_missing_revision_ids(revision_id=...) was '
3526
'deprecated in 2.4. Use revision_ids=[...] instead.',
3527
DeprecationWarning, stacklevel=2)
3528
if revision_ids is not None:
3529
raise AssertionError(
3530
'revision_ids is mutually exclusive with revision_id')
3531
if revision_id is not None:
3532
revision_ids = [revision_id]
3533
del revision_id
3534
# stop searching at found target revisions.
3535
if not find_ghosts and (revision_ids is not None or if_present_ids is
3536
not None):
3537
return self._walk_to_common_revisions(revision_ids,
3538
if_present_ids=if_present_ids)
3539
# generic, possibly worst case, slow code path.
3540
target_ids = set(self.target.all_revision_ids())
3541
source_ids = self._present_source_revisions_for(
3542
revision_ids, if_present_ids)
3543
result_set = set(source_ids).difference(target_ids)
3544
return self.source.revision_ids_to_search_result(result_set)
3545
3546
def _present_source_revisions_for(self, revision_ids, if_present_ids=None):
3547
"""Returns set of all revisions in ancestry of revision_ids present in
3548
the source repo.
3549
3550
:param revision_ids: if None, all revisions in source are returned.
3551
:param if_present_ids: like revision_ids, but if any/all of these are
3552
absent no error is raised.
3553
"""
3554
if revision_ids is not None or if_present_ids is not None:
3555
# First, ensure all specified revisions exist. Callers expect
3556
# NoSuchRevision when they pass absent revision_ids here.
3557
if revision_ids is None:
3558
revision_ids = set()
3559
if if_present_ids is None:
3560
if_present_ids = set()
3561
revision_ids = set(revision_ids)
3562
if_present_ids = set(if_present_ids)
3563
all_wanted_ids = revision_ids.union(if_present_ids)
3564
graph = self.source.get_graph()
3565
present_revs = set(graph.get_parent_map(all_wanted_ids))
3566
missing = revision_ids.difference(present_revs)
3567
if missing:
3568
raise errors.NoSuchRevision(self.source, missing.pop())
3569
found_ids = all_wanted_ids.intersection(present_revs)
3570
source_ids = [rev_id for (rev_id, parents) in
3571
graph.iter_ancestry(found_ids)
3572
if rev_id != _mod_revision.NULL_REVISION
3573
and parents is not None]
3574
else:
3575
source_ids = self.source.all_revision_ids()
3576
return set(source_ids)
1686
3577
1687
3578
@staticmethod
1688
3579
def _same_model(source, target):
1709
3600
"different serializers")
1710
3601
1711
3602
3603
class InterSameDataRepository(InterRepository):
3604
"""Code for converting between repositories that represent the same data.
3605
3606
Data format and model must match for this to work.
3607
"""
3608
3609
@classmethod
3610
def _get_repo_format_to_test(self):
3611
"""Repository format for testing with.
3612
3613
InterSameData can pull from subtree to subtree and from non-subtree to
3614
non-subtree, so we test this with the richest repository format.
3615
"""
3616
from bzrlib.repofmt import knitrepo
3617
return knitrepo.RepositoryFormatKnit3()
3618
3619
@staticmethod
3620
def is_compatible(source, target):
3621
return InterRepository._same_model(source, target)
3622
3623
3624
class InterDifferingSerializer(InterRepository):
3625
3626
@classmethod
3627
def _get_repo_format_to_test(self):
3628
return None
3629
3630
@staticmethod
3631
def is_compatible(source, target):
3632
"""Be compatible with Knit2 source and Knit3 target"""
3633
# This is redundant with format.check_conversion_target(), however that
3634
# raises an exception, and we just want to say "False" as in we won't
3635
# support converting between these formats.
3636
if 'IDS_never' in debug.debug_flags:
3637
return False
3638
if source.supports_rich_root() and not target.supports_rich_root():
3639
return False
3640
if (source._format.supports_tree_reference
3641
and not target._format.supports_tree_reference):
3642
return False
3643
if target._fallback_repositories and target._format.supports_chks:
3644
# IDS doesn't know how to copy CHKs for the parent inventories it
3645
# adds to stacked repos.
3646
return False
3647
if 'IDS_always' in debug.debug_flags:
3648
return True
3649
# Only use this code path for local source and target. IDS does far
3650
# too much IO (both bandwidth and roundtrips) over a network.
3651
if not source.bzrdir.transport.base.startswith('file:///'):
3652
return False
3653
if not target.bzrdir.transport.base.startswith('file:///'):
3654
return False
3655
return True
3656
3657
def _get_trees(self, revision_ids, cache):
3658
possible_trees = []
3659
for rev_id in revision_ids:
3660
if rev_id in cache:
3661
possible_trees.append((rev_id, cache[rev_id]))
3662
else:
3663
# Not cached, but inventory might be present anyway.
3664
try:
3665
tree = self.source.revision_tree(rev_id)
3666
except errors.NoSuchRevision:
3667
# Nope, parent is ghost.
3668
pass
3669
else:
3670
cache[rev_id] = tree
3671
possible_trees.append((rev_id, tree))
3672
return possible_trees
3673
3674
def _get_delta_for_revision(self, tree, parent_ids, possible_trees):
3675
"""Get the best delta and base for this revision.
3676
3677
:return: (basis_id, delta)
3678
"""
3679
deltas = []
3680
# Generate deltas against each tree, to find the shortest.
3681
texts_possibly_new_in_tree = set()
3682
for basis_id, basis_tree in possible_trees:
3683
delta = tree.inventory._make_delta(basis_tree.inventory)
3684
for old_path, new_path, file_id, new_entry in delta:
3685
if new_path is None:
3686
# This file_id isn't present in the new rev, so we don't
3687
# care about it.
3688
continue
3689
if not new_path:
3690
# Rich roots are handled elsewhere...
3691
continue
3692
kind = new_entry.kind
3693
if kind != 'directory' and kind != 'file':
3694
# No text record associated with this inventory entry.
3695
continue
3696
# This is a directory or file that has changed somehow.
3697
texts_possibly_new_in_tree.add((file_id, new_entry.revision))
3698
deltas.append((len(delta), basis_id, delta))
3699
deltas.sort()
3700
return deltas[0][1:]
3701
3702
def _fetch_parent_invs_for_stacking(self, parent_map, cache):
3703
"""Find all parent revisions that are absent, but for which the
3704
inventory is present, and copy those inventories.
3705
3706
This is necessary to preserve correctness when the source is stacked
3707
without fallbacks configured. (Note that in cases like upgrade the
3708
source may be not have _fallback_repositories even though it is
3709
stacked.)
3710
"""
3711
parent_revs = set()
3712
for parents in parent_map.values():
3713
parent_revs.update(parents)
3714
present_parents = self.source.get_parent_map(parent_revs)
3715
absent_parents = set(parent_revs).difference(present_parents)
3716
parent_invs_keys_for_stacking = self.source.inventories.get_parent_map(
3717
(rev_id,) for rev_id in absent_parents)
3718
parent_inv_ids = [key[-1] for key in parent_invs_keys_for_stacking]
3719
for parent_tree in self.source.revision_trees(parent_inv_ids):
3720
current_revision_id = parent_tree.get_revision_id()
3721
parents_parents_keys = parent_invs_keys_for_stacking[
3722
(current_revision_id,)]
3723
parents_parents = [key[-1] for key in parents_parents_keys]
3724
basis_id = _mod_revision.NULL_REVISION
3725
basis_tree = self.source.revision_tree(basis_id)
3726
delta = parent_tree.inventory._make_delta(basis_tree.inventory)
3727
self.target.add_inventory_by_delta(
3728
basis_id, delta, current_revision_id, parents_parents)
3729
cache[current_revision_id] = parent_tree
3730
3731
def _fetch_batch(self, revision_ids, basis_id, cache):
3732
"""Fetch across a few revisions.
3733
3734
:param revision_ids: The revisions to copy
3735
:param basis_id: The revision_id of a tree that must be in cache, used
3736
as a basis for delta when no other base is available
3737
:param cache: A cache of RevisionTrees that we can use.
3738
:return: The revision_id of the last converted tree. The RevisionTree
3739
for it will be in cache
3740
"""
3741
# Walk though all revisions; get inventory deltas, copy referenced
3742
# texts that delta references, insert the delta, revision and
3743
# signature.
3744
root_keys_to_create = set()
3745
text_keys = set()
3746
pending_deltas = []
3747
pending_revisions = []
3748
parent_map = self.source.get_parent_map(revision_ids)
3749
self._fetch_parent_invs_for_stacking(parent_map, cache)
3750
self.source._safe_to_return_from_cache = True
3751
for tree in self.source.revision_trees(revision_ids):
3752
# Find a inventory delta for this revision.
3753
# Find text entries that need to be copied, too.
3754
current_revision_id = tree.get_revision_id()
3755
parent_ids = parent_map.get(current_revision_id, ())
3756
parent_trees = self._get_trees(parent_ids, cache)
3757
possible_trees = list(parent_trees)
3758
if len(possible_trees) == 0:
3759
# There either aren't any parents, or the parents are ghosts,
3760
# so just use the last converted tree.
3761
possible_trees.append((basis_id, cache[basis_id]))
3762
basis_id, delta = self._get_delta_for_revision(tree, parent_ids,
3763
possible_trees)
3764
revision = self.source.get_revision(current_revision_id)
3765
pending_deltas.append((basis_id, delta,
3766
current_revision_id, revision.parent_ids))
3767
if self._converting_to_rich_root:
3768
self._revision_id_to_root_id[current_revision_id] = \
3769
tree.get_root_id()
3770
# Determine which texts are in present in this revision but not in
3771
# any of the available parents.
3772
texts_possibly_new_in_tree = set()
3773
for old_path, new_path, file_id, entry in delta:
3774
if new_path is None:
3775
# This file_id isn't present in the new rev
3776
continue
3777
if not new_path:
3778
# This is the root
3779
if not self.target.supports_rich_root():
3780
# The target doesn't support rich root, so we don't
3781
# copy
3782
continue
3783
if self._converting_to_rich_root:
3784
# This can't be copied normally, we have to insert
3785
# it specially
3786
root_keys_to_create.add((file_id, entry.revision))
3787
continue
3788
kind = entry.kind
3789
texts_possibly_new_in_tree.add((file_id, entry.revision))
3790
for basis_id, basis_tree in possible_trees:
3791
basis_inv = basis_tree.inventory
3792
for file_key in list(texts_possibly_new_in_tree):
3793
file_id, file_revision = file_key
3794
try:
3795
entry = basis_inv[file_id]
3796
except errors.NoSuchId:
3797
continue
3798
if entry.revision == file_revision:
3799
texts_possibly_new_in_tree.remove(file_key)
3800
text_keys.update(texts_possibly_new_in_tree)
3801
pending_revisions.append(revision)
3802
cache[current_revision_id] = tree
3803
basis_id = current_revision_id
3804
self.source._safe_to_return_from_cache = False
3805
# Copy file texts
3806
from_texts = self.source.texts
3807
to_texts = self.target.texts
3808
if root_keys_to_create:
3809
root_stream = _mod_fetch._new_root_data_stream(
3810
root_keys_to_create, self._revision_id_to_root_id, parent_map,
3811
self.source)
3812
to_texts.insert_record_stream(root_stream)
3813
to_texts.insert_record_stream(from_texts.get_record_stream(
3814
text_keys, self.target._format._fetch_order,
3815
not self.target._format._fetch_uses_deltas))
3816
# insert inventory deltas
3817
for delta in pending_deltas:
3818
self.target.add_inventory_by_delta(*delta)
3819
if self.target._fallback_repositories:
3820
# Make sure this stacked repository has all the parent inventories
3821
# for the new revisions that we are about to insert. We do this
3822
# before adding the revisions so that no revision is added until
3823
# all the inventories it may depend on are added.
3824
# Note that this is overzealous, as we may have fetched these in an
3825
# earlier batch.
3826
parent_ids = set()
3827
revision_ids = set()
3828
for revision in pending_revisions:
3829
revision_ids.add(revision.revision_id)
3830
parent_ids.update(revision.parent_ids)
3831
parent_ids.difference_update(revision_ids)
3832
parent_ids.discard(_mod_revision.NULL_REVISION)
3833
parent_map = self.source.get_parent_map(parent_ids)
3834
# we iterate over parent_map and not parent_ids because we don't
3835
# want to try copying any revision which is a ghost
3836
for parent_tree in self.source.revision_trees(parent_map):
3837
current_revision_id = parent_tree.get_revision_id()
3838
parents_parents = parent_map[current_revision_id]
3839
possible_trees = self._get_trees(parents_parents, cache)
3840
if len(possible_trees) == 0:
3841
# There either aren't any parents, or the parents are
3842
# ghosts, so just use the last converted tree.
3843
possible_trees.append((basis_id, cache[basis_id]))
3844
basis_id, delta = self._get_delta_for_revision(parent_tree,
3845
parents_parents, possible_trees)
3846
self.target.add_inventory_by_delta(
3847
basis_id, delta, current_revision_id, parents_parents)
3848
# insert signatures and revisions
3849
for revision in pending_revisions:
3850
try:
3851
signature = self.source.get_signature_text(
3852
revision.revision_id)
3853
self.target.add_signature_text(revision.revision_id,
3854
signature)
3855
except errors.NoSuchRevision:
3856
pass
3857
self.target.add_revision(revision.revision_id, revision)
3858
return basis_id
3859
3860
def _fetch_all_revisions(self, revision_ids, pb):
3861
"""Fetch everything for the list of revisions.
3862
3863
:param revision_ids: The list of revisions to fetch. Must be in
3864
topological order.
3865
:param pb: A ProgressTask
3866
:return: None
3867
"""
3868
basis_id, basis_tree = self._get_basis(revision_ids[0])
3869
batch_size = 100
3870
cache = lru_cache.LRUCache(100)
3871
cache[basis_id] = basis_tree
3872
del basis_tree # We don't want to hang on to it here
3873
hints = []
3874
a_graph = None
3875
3876
for offset in range(0, len(revision_ids), batch_size):
3877
self.target.start_write_group()
3878
try:
3879
pb.update('Transferring revisions', offset,
3880
len(revision_ids))
3881
batch = revision_ids[offset:offset+batch_size]
3882
basis_id = self._fetch_batch(batch, basis_id, cache)
3883
except:
3884
self.source._safe_to_return_from_cache = False
3885
self.target.abort_write_group()
3886
raise
3887
else:
3888
hint = self.target.commit_write_group()
3889
if hint:
3890
hints.extend(hint)
3891
if hints and self.target._format.pack_compresses:
3892
self.target.pack(hint=hints)
3893
pb.update('Transferring revisions', len(revision_ids),
3894
len(revision_ids))
3895
3896
@needs_write_lock
3897
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
3898
fetch_spec=None):
3899
"""See InterRepository.fetch()."""
3900
if fetch_spec is not None:
3901
revision_ids = fetch_spec.get_keys()
3902
else:
3903
revision_ids = None
3904
ui.ui_factory.warn_experimental_format_fetch(self)
3905
if (not self.source.supports_rich_root()
3906
and self.target.supports_rich_root()):
3907
self._converting_to_rich_root = True
3908
self._revision_id_to_root_id = {}
3909
else:
3910
self._converting_to_rich_root = False
3911
# See <https://launchpad.net/bugs/456077> asking for a warning here
3912
if self.source._format.network_name() != self.target._format.network_name():
3913
ui.ui_factory.show_user_warning('cross_format_fetch',
3914
from_format=self.source._format,
3915
to_format=self.target._format)
3916
if revision_ids is None:
3917
if revision_id:
3918
search_revision_ids = [revision_id]
3919
else:
3920
search_revision_ids = None
3921
revision_ids = self.target.search_missing_revision_ids(self.source,
3922
revision_ids=search_revision_ids,
3923
find_ghosts=find_ghosts).get_keys()
3924
if not revision_ids:
3925
return 0, 0
3926
revision_ids = tsort.topo_sort(
3927
self.source.get_graph().get_parent_map(revision_ids))
3928
if not revision_ids:
3929
return 0, 0
3930
# Walk though all revisions; get inventory deltas, copy referenced
3931
# texts that delta references, insert the delta, revision and
3932
# signature.
3933
if pb is None:
3934
my_pb = ui.ui_factory.nested_progress_bar()
3935
pb = my_pb
3936
else:
3937
symbol_versioning.warn(
3938
symbol_versioning.deprecated_in((1, 14, 0))
3939
% "pb parameter to fetch()")
3940
my_pb = None
3941
try:
3942
self._fetch_all_revisions(revision_ids, pb)
3943
finally:
3944
if my_pb is not None:
3945
my_pb.finished()
3946
return len(revision_ids), 0
3947
3948
def _get_basis(self, first_revision_id):
3949
"""Get a revision and tree which exists in the target.
3950
3951
This assumes that first_revision_id is selected for transmission
3952
because all other ancestors are already present. If we can't find an
3953
ancestor we fall back to NULL_REVISION since we know that is safe.
3954
3955
:return: (basis_id, basis_tree)
3956
"""
3957
first_rev = self.source.get_revision(first_revision_id)
3958
try:
3959
basis_id = first_rev.parent_ids[0]
3960
# only valid as a basis if the target has it
3961
self.target.get_revision(basis_id)
3962
# Try to get a basis tree - if it's a ghost it will hit the
3963
# NoSuchRevision case.
3964
basis_tree = self.source.revision_tree(basis_id)
3965
except (IndexError, errors.NoSuchRevision):
3966
basis_id = _mod_revision.NULL_REVISION
3967
basis_tree = self.source.revision_tree(basis_id)
3968
return basis_id, basis_tree
3969
3970
3971
InterRepository.register_optimiser(InterDifferingSerializer)
3972
InterRepository.register_optimiser(InterSameDataRepository)
3973
3974
1712
3975
class CopyConverter(object):
1713
3976
"""A repository conversion tool which just performs a copy of the content.
1714
3977
1735
3998
# trigger an assertion if not such
1736
3999
repo._format.get_format_string()
1737
4000
self.repo_dir = repo.bzrdir
1738
pb.update(gettext('Moving repository to repository.backup'))
4001
pb.update('Moving repository to repository.backup')
1739
4002
self.repo_dir.transport.move('repository', 'repository.backup')
1740
4003
backup_transport = self.repo_dir.transport.clone('repository.backup')
1741
4004
repo._format.check_conversion_target(self.target_format)
1742
4005
self.source_repo = repo._format.open(self.repo_dir,
1743
4006
_found=True,
1744
4007
_override_transport=backup_transport)
1745
pb.update(gettext('Creating new repository'))
4008
pb.update('Creating new repository')
1746
4009
converted = self.target_format.initialize(self.repo_dir,
1747
4010
self.source_repo.is_shared())
1748
4011
converted.lock_write()
1749
4012
try:
1750
pb.update(gettext('Copying content'))
4013
pb.update('Copying content')
1751
4014
self.source_repo.copy_content_into(converted)
1752
4015
finally:
1753
4016
converted.unlock()
1754
pb.update(gettext('Deleting old repository content'))
4017
pb.update('Deleting old repository content')
1755
4018
self.repo_dir.transport.delete_tree('repository.backup')
1756
ui.ui_factory.note(gettext('repository converted'))
4019
ui.ui_factory.note('repository converted')
1757
4020
pb.finished()
1758
4021
1759
4022
4023
_unescape_map = {
4024
'apos':"'",
4025
'quot':'"',
4026
'amp':'&',
4027
'lt':'<',
4028
'gt':'>'
4029
}
4030
4031
4032
def _unescaper(match, _map=_unescape_map):
4033
code = match.group(1)
4034
try:
4035
return _map[code]
4036
except KeyError:
4037
if not code.startswith('#'):
4038
raise
4039
return unichr(int(code[1:])).encode('utf8')
4040
4041
4042
_unescape_re = None
4043
4044
4045
def _unescape_xml(data):
4046
"""Unescape predefined XML entities in a string of data."""
4047
global _unescape_re
4048
if _unescape_re is None:
4049
_unescape_re = re.compile('\&([^;]*);')
4050
return _unescape_re.sub(_unescaper, data)
4051
4052
4053
class _VersionedFileChecker(object):
4054
4055
def __init__(self, repository, text_key_references=None, ancestors=None):
4056
self.repository = repository
4057
self.text_index = self.repository._generate_text_key_index(
4058
text_key_references=text_key_references, ancestors=ancestors)
4059
4060
def calculate_file_version_parents(self, text_key):
4061
"""Calculate the correct parents for a file version according to
4062
the inventories.
4063
"""
4064
parent_keys = self.text_index[text_key]
4065
if parent_keys == [_mod_revision.NULL_REVISION]:
4066
return ()
4067
return tuple(parent_keys)
4068
4069
def check_file_version_parents(self, texts, progress_bar=None):
4070
"""Check the parents stored in a versioned file are correct.
4071
4072
It also detects file versions that are not referenced by their
4073
corresponding revision's inventory.
4074
4075
:returns: A tuple of (wrong_parents, dangling_file_versions).
4076
wrong_parents is a dict mapping {revision_id: (stored_parents,
4077
correct_parents)} for each revision_id where the stored parents
4078
are not correct. dangling_file_versions is a set of (file_id,
4079
revision_id) tuples for versions that are present in this versioned
4080
file, but not used by the corresponding inventory.
4081
"""
4082
local_progress = None
4083
if progress_bar is None:
4084
local_progress = ui.ui_factory.nested_progress_bar()
4085
progress_bar = local_progress
4086
try:
4087
return self._check_file_version_parents(texts, progress_bar)
4088
finally:
4089
if local_progress:
4090
local_progress.finished()
4091
4092
def _check_file_version_parents(self, texts, progress_bar):
4093
"""See check_file_version_parents."""
4094
wrong_parents = {}
4095
self.file_ids = set([file_id for file_id, _ in
4096
self.text_index.iterkeys()])
4097
# text keys is now grouped by file_id
4098
n_versions = len(self.text_index)
4099
progress_bar.update('loading text store', 0, n_versions)
4100
parent_map = self.repository.texts.get_parent_map(self.text_index)
4101
# On unlistable transports this could well be empty/error...
4102
text_keys = self.repository.texts.keys()
4103
unused_keys = frozenset(text_keys) - set(self.text_index)
4104
for num, key in enumerate(self.text_index.iterkeys()):
4105
progress_bar.update('checking text graph', num, n_versions)
4106
correct_parents = self.calculate_file_version_parents(key)
4107
try:
4108
knit_parents = parent_map[key]
4109
except errors.RevisionNotPresent:
4110
# Missing text!
4111
knit_parents = None
4112
if correct_parents != knit_parents:
4113
wrong_parents[key] = (knit_parents, correct_parents)
4114
return wrong_parents, unused_keys
4115
4116
4117
def _old_get_graph(repository, revision_id):
4118
"""DO NOT USE. That is all. I'm serious."""
4119
graph = repository.get_graph()
4120
revision_graph = dict(((key, value) for key, value in
4121
graph.iter_ancestry([revision_id]) if value is not None))
4122
return _strip_NULL_ghosts(revision_graph)
4123
4124
1760
4125
def _strip_NULL_ghosts(revision_graph):
1761
4126
"""Also don't use this. more compatibility code for unmigrated clients."""
1762
4127
# Filter ghosts, and null:
1768
4133
return revision_graph
1769
4134
1770
4135
4136
class StreamSink(object):
4137
"""An object that can insert a stream into a repository.
4138
4139
This interface handles the complexity of reserialising inventories and
4140
revisions from different formats, and allows unidirectional insertion into
4141
stacked repositories without looking for the missing basis parents
4142
beforehand.
4143
"""
4144
4145
def __init__(self, target_repo):
4146
self.target_repo = target_repo
4147
4148
def insert_stream(self, stream, src_format, resume_tokens):
4149
"""Insert a stream's content into the target repository.
4150
4151
:param src_format: a bzr repository format.
4152
4153
:return: a list of resume tokens and an iterable of keys additional
4154
items required before the insertion can be completed.
4155
"""
4156
self.target_repo.lock_write()
4157
try:
4158
if resume_tokens:
4159
self.target_repo.resume_write_group(resume_tokens)
4160
is_resume = True
4161
else:
4162
self.target_repo.start_write_group()
4163
is_resume = False
4164
try:
4165
# locked_insert_stream performs a commit|suspend.
4166
missing_keys = self.insert_stream_without_locking(stream,
4167
src_format, is_resume)
4168
if missing_keys:
4169
# suspend the write group and tell the caller what we is
4170
# missing. We know we can suspend or else we would not have
4171
# entered this code path. (All repositories that can handle
4172
# missing keys can handle suspending a write group).
4173
write_group_tokens = self.target_repo.suspend_write_group()
4174
return write_group_tokens, missing_keys
4175
hint = self.target_repo.commit_write_group()
4176
to_serializer = self.target_repo._format._serializer
4177
src_serializer = src_format._serializer
4178
if (to_serializer != src_serializer and
4179
self.target_repo._format.pack_compresses):
4180
self.target_repo.pack(hint=hint)
4181
return [], set()
4182
except:
4183
self.target_repo.abort_write_group(suppress_errors=True)
4184
raise
4185
finally:
4186
self.target_repo.unlock()
4187
4188
def insert_stream_without_locking(self, stream, src_format,
4189
is_resume=False):
4190
"""Insert a stream's content into the target repository.
4191
4192
This assumes that you already have a locked repository and an active
4193
write group.
4194
4195
:param src_format: a bzr repository format.
4196
:param is_resume: Passed down to get_missing_parent_inventories to
4197
indicate if we should be checking for missing texts at the same
4198
time.
4199
4200
:return: A set of keys that are missing.
4201
"""
4202
if not self.target_repo.is_write_locked():
4203
raise errors.ObjectNotLocked(self)
4204
if not self.target_repo.is_in_write_group():
4205
raise errors.BzrError('you must already be in a write group')
4206
to_serializer = self.target_repo._format._serializer
4207
src_serializer = src_format._serializer
4208
new_pack = None
4209
if to_serializer == src_serializer:
4210
# If serializers match and the target is a pack repository, set the
4211
# write cache size on the new pack. This avoids poor performance
4212
# on transports where append is unbuffered (such as
4213
# RemoteTransport). This is safe to do because nothing should read
4214
# back from the target repository while a stream with matching
4215
# serialization is being inserted.
4216
# The exception is that a delta record from the source that should
4217
# be a fulltext may need to be expanded by the target (see
4218
# test_fetch_revisions_with_deltas_into_pack); but we take care to
4219
# explicitly flush any buffered writes first in that rare case.
4220
try:
4221
new_pack = self.target_repo._pack_collection._new_pack
4222
except AttributeError:
4223
# Not a pack repository
4224
pass
4225
else:
4226
new_pack.set_write_cache_size(1024*1024)
4227
for substream_type, substream in stream:
4228
if 'stream' in debug.debug_flags:
4229
mutter('inserting substream: %s', substream_type)
4230
if substream_type == 'texts':
4231
self.target_repo.texts.insert_record_stream(substream)
4232
elif substream_type == 'inventories':
4233
if src_serializer == to_serializer:
4234
self.target_repo.inventories.insert_record_stream(
4235
substream)
4236
else:
4237
self._extract_and_insert_inventories(
4238
substream, src_serializer)
4239
elif substream_type == 'inventory-deltas':
4240
self._extract_and_insert_inventory_deltas(
4241
substream, src_serializer)
4242
elif substream_type == 'chk_bytes':
4243
# XXX: This doesn't support conversions, as it assumes the
4244
# conversion was done in the fetch code.
4245
self.target_repo.chk_bytes.insert_record_stream(substream)
4246
elif substream_type == 'revisions':
4247
# This may fallback to extract-and-insert more often than
4248
# required if the serializers are different only in terms of
4249
# the inventory.
4250
if src_serializer == to_serializer:
4251
self.target_repo.revisions.insert_record_stream(substream)
4252
else:
4253
self._extract_and_insert_revisions(substream,
4254
src_serializer)
4255
elif substream_type == 'signatures':
4256
self.target_repo.signatures.insert_record_stream(substream)
4257
else:
4258
raise AssertionError('kaboom! %s' % (substream_type,))
4259
# Done inserting data, and the missing_keys calculations will try to
4260
# read back from the inserted data, so flush the writes to the new pack
4261
# (if this is pack format).
4262
if new_pack is not None:
4263
new_pack._write_data('', flush=True)
4264
# Find all the new revisions (including ones from resume_tokens)
4265
missing_keys = self.target_repo.get_missing_parent_inventories(
4266
check_for_missing_texts=is_resume)
4267
try:
4268
for prefix, versioned_file in (
4269
('texts', self.target_repo.texts),
4270
('inventories', self.target_repo.inventories),
4271
('revisions', self.target_repo.revisions),
4272
('signatures', self.target_repo.signatures),
4273
('chk_bytes', self.target_repo.chk_bytes),
4274
):
4275
if versioned_file is None:
4276
continue
4277
# TODO: key is often going to be a StaticTuple object
4278
# I don't believe we can define a method by which
4279
# (prefix,) + StaticTuple will work, though we could
4280
# define a StaticTuple.sq_concat that would allow you to
4281
# pass in either a tuple or a StaticTuple as the second
4282
# object, so instead we could have:
4283
# StaticTuple(prefix) + key here...
4284
missing_keys.update((prefix,) + key for key in
4285
versioned_file.get_missing_compression_parent_keys())
4286
except NotImplementedError:
4287
# cannot even attempt suspending, and missing would have failed
4288
# during stream insertion.
4289
missing_keys = set()
4290
return missing_keys
4291
4292
def _extract_and_insert_inventory_deltas(self, substream, serializer):
4293
target_rich_root = self.target_repo._format.rich_root_data
4294
target_tree_refs = self.target_repo._format.supports_tree_reference
4295
for record in substream:
4296
# Insert the delta directly
4297
inventory_delta_bytes = record.get_bytes_as('fulltext')
4298
deserialiser = inventory_delta.InventoryDeltaDeserializer()
4299
try:
4300
parse_result = deserialiser.parse_text_bytes(
4301
inventory_delta_bytes)
4302
except inventory_delta.IncompatibleInventoryDelta, err:
4303
trace.mutter("Incompatible delta: %s", err.msg)
4304
raise errors.IncompatibleRevision(self.target_repo._format)
4305
basis_id, new_id, rich_root, tree_refs, inv_delta = parse_result
4306
revision_id = new_id
4307
parents = [key[0] for key in record.parents]
4308
self.target_repo.add_inventory_by_delta(
4309
basis_id, inv_delta, revision_id, parents)
4310
4311
def _extract_and_insert_inventories(self, substream, serializer,
4312
parse_delta=None):
4313
"""Generate a new inventory versionedfile in target, converting data.
4314
4315
The inventory is retrieved from the source, (deserializing it), and
4316
stored in the target (reserializing it in a different format).
4317
"""
4318
target_rich_root = self.target_repo._format.rich_root_data
4319
target_tree_refs = self.target_repo._format.supports_tree_reference
4320
for record in substream:
4321
# It's not a delta, so it must be a fulltext in the source
4322
# serializer's format.
4323
bytes = record.get_bytes_as('fulltext')
4324
revision_id = record.key[0]
4325
inv = serializer.read_inventory_from_string(bytes, revision_id)
4326
parents = [key[0] for key in record.parents]
4327
self.target_repo.add_inventory(revision_id, inv, parents)
4328
# No need to keep holding this full inv in memory when the rest of
4329
# the substream is likely to be all deltas.
4330
del inv
4331
4332
def _extract_and_insert_revisions(self, substream, serializer):
4333
for record in substream:
4334
bytes = record.get_bytes_as('fulltext')
4335
revision_id = record.key[0]
4336
rev = serializer.read_revision_from_string(bytes)
4337
if rev.revision_id != revision_id:
4338
raise AssertionError('wtf: %s != %s' % (rev, revision_id))
4339
self.target_repo.add_revision(revision_id, rev)
4340
4341
def finished(self):
4342
if self.target_repo._format._fetch_reconcile:
4343
self.target_repo.reconcile()
4344
4345
4346
class StreamSource(object):
4347
"""A source of a stream for fetching between repositories."""
4348
4349
def __init__(self, from_repository, to_format):
4350
"""Create a StreamSource streaming from from_repository."""
4351
self.from_repository = from_repository
4352
self.to_format = to_format
4353
self._record_counter = RecordCounter()
4354
4355
def delta_on_metadata(self):
4356
"""Return True if delta's are permitted on metadata streams.
4357
4358
That is on revisions and signatures.
4359
"""
4360
src_serializer = self.from_repository._format._serializer
4361
target_serializer = self.to_format._serializer
4362
return (self.to_format._fetch_uses_deltas and
4363
src_serializer == target_serializer)
4364
4365
def _fetch_revision_texts(self, revs):
4366
# fetch signatures first and then the revision texts
4367
# may need to be a InterRevisionStore call here.
4368
from_sf = self.from_repository.signatures
4369
# A missing signature is just skipped.
4370
keys = [(rev_id,) for rev_id in revs]
4371
signatures = versionedfile.filter_absent(from_sf.get_record_stream(
4372
keys,
4373
self.to_format._fetch_order,
4374
not self.to_format._fetch_uses_deltas))
4375
# If a revision has a delta, this is actually expanded inside the
4376
# insert_record_stream code now, which is an alternate fix for
4377
# bug #261339
4378
from_rf = self.from_repository.revisions
4379
revisions = from_rf.get_record_stream(
4380
keys,
4381
self.to_format._fetch_order,
4382
not self.delta_on_metadata())
4383
return [('signatures', signatures), ('revisions', revisions)]
4384
4385
def _generate_root_texts(self, revs):
4386
"""This will be called by get_stream between fetching weave texts and
4387
fetching the inventory weave.
4388
"""
4389
if self._rich_root_upgrade():
4390
return _mod_fetch.Inter1and2Helper(
4391
self.from_repository).generate_root_texts(revs)
4392
else:
4393
return []
4394
4395
def get_stream(self, search):
4396
phase = 'file'
4397
revs = search.get_keys()
4398
graph = self.from_repository.get_graph()
4399
revs = tsort.topo_sort(graph.get_parent_map(revs))
4400
data_to_fetch = self.from_repository.item_keys_introduced_by(revs)
4401
text_keys = []
4402
for knit_kind, file_id, revisions in data_to_fetch:
4403
if knit_kind != phase:
4404
phase = knit_kind
4405
# Make a new progress bar for this phase
4406
if knit_kind == "file":
4407
# Accumulate file texts
4408
text_keys.extend([(file_id, revision) for revision in
4409
revisions])
4410
elif knit_kind == "inventory":
4411
# Now copy the file texts.
4412
from_texts = self.from_repository.texts
4413
yield ('texts', from_texts.get_record_stream(
4414
text_keys, self.to_format._fetch_order,
4415
not self.to_format._fetch_uses_deltas))
4416
# Cause an error if a text occurs after we have done the
4417
# copy.
4418
text_keys = None
4419
# Before we process the inventory we generate the root
4420
# texts (if necessary) so that the inventories references
4421
# will be valid.
4422
for _ in self._generate_root_texts(revs):
4423
yield _
4424
# we fetch only the referenced inventories because we do not
4425
# know for unselected inventories whether all their required
4426
# texts are present in the other repository - it could be
4427
# corrupt.
4428
for info in self._get_inventory_stream(revs):
4429
yield info
4430
elif knit_kind == "signatures":
4431
# Nothing to do here; this will be taken care of when
4432
# _fetch_revision_texts happens.
4433
pass
4434
elif knit_kind == "revisions":
4435
for record in self._fetch_revision_texts(revs):
4436
yield record
4437
else:
4438
raise AssertionError("Unknown knit kind %r" % knit_kind)
4439
4440
def get_stream_for_missing_keys(self, missing_keys):
4441
# missing keys can only occur when we are byte copying and not
4442
# translating (because translation means we don't send
4443
# unreconstructable deltas ever).
4444
keys = {}
4445
keys['texts'] = set()
4446
keys['revisions'] = set()
4447
keys['inventories'] = set()
4448
keys['chk_bytes'] = set()
4449
keys['signatures'] = set()
4450
for key in missing_keys:
4451
keys[key[0]].add(key[1:])
4452
if len(keys['revisions']):
4453
# If we allowed copying revisions at this point, we could end up
4454
# copying a revision without copying its required texts: a
4455
# violation of the requirements for repository integrity.
4456
raise AssertionError(
4457
'cannot copy revisions to fill in missing deltas %s' % (
4458
keys['revisions'],))
4459
for substream_kind, keys in keys.iteritems():
4460
vf = getattr(self.from_repository, substream_kind)
4461
if vf is None and keys:
4462
raise AssertionError(
4463
"cannot fill in keys for a versioned file we don't"
4464
" have: %s needs %s" % (substream_kind, keys))
4465
if not keys:
4466
# No need to stream something we don't have
4467
continue
4468
if substream_kind == 'inventories':
4469
# Some missing keys are genuinely ghosts, filter those out.
4470
present = self.from_repository.inventories.get_parent_map(keys)
4471
revs = [key[0] for key in present]
4472
# Get the inventory stream more-or-less as we do for the
4473
# original stream; there's no reason to assume that records
4474
# direct from the source will be suitable for the sink. (Think
4475
# e.g. 2a -> 1.9-rich-root).
4476
for info in self._get_inventory_stream(revs, missing=True):
4477
yield info
4478
continue
4479
4480
# Ask for full texts always so that we don't need more round trips
4481
# after this stream.
4482
# Some of the missing keys are genuinely ghosts, so filter absent
4483
# records. The Sink is responsible for doing another check to
4484
# ensure that ghosts don't introduce missing data for future
4485
# fetches.
4486
stream = versionedfile.filter_absent(vf.get_record_stream(keys,
4487
self.to_format._fetch_order, True))
4488
yield substream_kind, stream
4489
4490
def inventory_fetch_order(self):
4491
if self._rich_root_upgrade():
4492
return 'topological'
4493
else:
4494
return self.to_format._fetch_order
4495
4496
def _rich_root_upgrade(self):
4497
return (not self.from_repository._format.rich_root_data and
4498
self.to_format.rich_root_data)
4499
4500
def _get_inventory_stream(self, revision_ids, missing=False):
4501
from_format = self.from_repository._format
4502
if (from_format.supports_chks and self.to_format.supports_chks and
4503
from_format.network_name() == self.to_format.network_name()):
4504
raise AssertionError(
4505
"this case should be handled by GroupCHKStreamSource")
4506
elif 'forceinvdeltas' in debug.debug_flags:
4507
return self._get_convertable_inventory_stream(revision_ids,
4508
delta_versus_null=missing)
4509
elif from_format.network_name() == self.to_format.network_name():
4510
# Same format.
4511
return self._get_simple_inventory_stream(revision_ids,
4512
missing=missing)
4513
elif (not from_format.supports_chks and not self.to_format.supports_chks
4514
and from_format._serializer == self.to_format._serializer):
4515
# Essentially the same format.
4516
return self._get_simple_inventory_stream(revision_ids,
4517
missing=missing)
4518
else:
4519
# Any time we switch serializations, we want to use an
4520
# inventory-delta based approach.
4521
return self._get_convertable_inventory_stream(revision_ids,
4522
delta_versus_null=missing)
4523
4524
def _get_simple_inventory_stream(self, revision_ids, missing=False):
4525
# NB: This currently reopens the inventory weave in source;
4526
# using a single stream interface instead would avoid this.
4527
from_weave = self.from_repository.inventories
4528
if missing:
4529
delta_closure = True
4530
else:
4531
delta_closure = not self.delta_on_metadata()
4532
yield ('inventories', from_weave.get_record_stream(
4533
[(rev_id,) for rev_id in revision_ids],
4534
self.inventory_fetch_order(), delta_closure))
4535
4536
def _get_convertable_inventory_stream(self, revision_ids,
4537
delta_versus_null=False):
4538
# The two formats are sufficiently different that there is no fast
4539
# path, so we need to send just inventorydeltas, which any
4540
# sufficiently modern client can insert into any repository.
4541
# The StreamSink code expects to be able to
4542
# convert on the target, so we need to put bytes-on-the-wire that can
4543
# be converted. That means inventory deltas (if the remote is <1.19,
4544
# RemoteStreamSink will fallback to VFS to insert the deltas).
4545
yield ('inventory-deltas',
4546
self._stream_invs_as_deltas(revision_ids,
4547
delta_versus_null=delta_versus_null))
4548
4549
def _stream_invs_as_deltas(self, revision_ids, delta_versus_null=False):
4550
"""Return a stream of inventory-deltas for the given rev ids.
4551
4552
:param revision_ids: The list of inventories to transmit
4553
:param delta_versus_null: Don't try to find a minimal delta for this
4554
entry, instead compute the delta versus the NULL_REVISION. This
4555
effectively streams a complete inventory. Used for stuff like
4556
filling in missing parents, etc.
4557
"""
4558
from_repo = self.from_repository
4559
revision_keys = [(rev_id,) for rev_id in revision_ids]
4560
parent_map = from_repo.inventories.get_parent_map(revision_keys)
4561
# XXX: possibly repos could implement a more efficient iter_inv_deltas
4562
# method...
4563
inventories = self.from_repository.iter_inventories(
4564
revision_ids, 'topological')
4565
format = from_repo._format
4566
invs_sent_so_far = set([_mod_revision.NULL_REVISION])
4567
inventory_cache = lru_cache.LRUCache(50)
4568
null_inventory = from_repo.revision_tree(
4569
_mod_revision.NULL_REVISION).inventory
4570
# XXX: ideally the rich-root/tree-refs flags would be per-revision, not
4571
# per-repo (e.g. streaming a non-rich-root revision out of a rich-root
4572
# repo back into a non-rich-root repo ought to be allowed)
4573
serializer = inventory_delta.InventoryDeltaSerializer(
4574
versioned_root=format.rich_root_data,
4575
tree_references=format.supports_tree_reference)
4576
for inv in inventories:
4577
key = (inv.revision_id,)
4578
parent_keys = parent_map.get(key, ())
4579
delta = None
4580
if not delta_versus_null and parent_keys:
4581
# The caller did not ask for complete inventories and we have
4582
# some parents that we can delta against. Make a delta against
4583
# each parent so that we can find the smallest.
4584
parent_ids = [parent_key[0] for parent_key in parent_keys]
4585
for parent_id in parent_ids:
4586
if parent_id not in invs_sent_so_far:
4587
# We don't know that the remote side has this basis, so
4588
# we can't use it.
4589
continue
4590
if parent_id == _mod_revision.NULL_REVISION:
4591
parent_inv = null_inventory
4592
else:
4593
parent_inv = inventory_cache.get(parent_id, None)
4594
if parent_inv is None:
4595
parent_inv = from_repo.get_inventory(parent_id)
4596
candidate_delta = inv._make_delta(parent_inv)
4597
if (delta is None or
4598
len(delta) > len(candidate_delta)):
4599
delta = candidate_delta
4600
basis_id = parent_id
4601
if delta is None:
4602
# Either none of the parents ended up being suitable, or we
4603
# were asked to delta against NULL
4604
basis_id = _mod_revision.NULL_REVISION
4605
delta = inv._make_delta(null_inventory)
4606
invs_sent_so_far.add(inv.revision_id)
4607
inventory_cache[inv.revision_id] = inv
4608
delta_serialized = ''.join(
4609
serializer.delta_to_lines(basis_id, key[-1], delta))
4610
yield versionedfile.FulltextContentFactory(
4611
key, parent_keys, None, delta_serialized)
4612
4613
1771
4614
def _iter_for_revno(repo, partial_history_cache, stop_index=None,
1772
4615
stop_revision=None):
1773
4616
"""Extend the partial history to include a given index
1783
4626
it is encountered, history extension will stop.
1784
4627
"""
1785
4628
start_revision = partial_history_cache[-1]
1786
graph = repo.get_graph()
1787
iterator = graph.iter_lefthand_ancestry(start_revision,
1788
(_mod_revision.NULL_REVISION,))
4629
iterator = repo.iter_reverse_revision_history(start_revision)
1789
4630
try:
1790
# skip the last revision in the list
4631
#skip the last revision in the list
1791
4632
iterator.next()
1792
4633
while True:
1793
4634
if (stop_index is not None and
1801
4642
# No more history
1802
4643
return
1803
4644
1804
1805
class _LazyListJoin(object):
1806
"""An iterable yielding the contents of many lists as one list.
1807
1808
Each iterator made from this will reflect the current contents of the lists
1809
at the time the iterator is made.
1810
1811
This is used by Repository's _make_parents_provider implementation so that
1812
it is safe to do::
1813
1814
pp = repo._make_parents_provider() # uses a list of fallback repos
1815
pp.add_fallback_repository(other_repo) # appends to that list
1816
result = pp.get_parent_map(...)
1817
# The result will include revs from other_repo
1818
"""
1819
1820
def __init__(self, *list_parts):
1821
self.list_parts = list_parts
1822
1823
def __iter__(self):
1824
full_list = []
1825
for list_part in self.list_parts:
1826
full_list.extend(list_part)
1827
return iter(full_list)
1828
1829
def __repr__(self):
1830
return "%s.%s(%s)" % (self.__module__, self.__class__.__name__,
1831
self.list_parts)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1