« back to all changes in this revision
Viewing changes to bzrlib/repository.py
- browse files at revision 6621
- compare with another revision
- download diff
- download tarball
- view history from revision 6621
- Committer: Tarmac
- Author(s): Vincent Ladeuil
- Date: 2017-01-30 14:42:05 UTC
- mfrom: (6620.1.1 trunk)
- Revision ID: tarmac-20170130144205-r8fh2xpmiuxyozpv
Merge 2.7 into trunk including fix for bug #1657238 [r=vila]
- files added:
- bzrlib/_termcolor.py
- bzrlib/branchfmt
- bzrlib/plugins/changelog_merge
- bzrlib/plugins/changelog_merge/tests
- bzrlib/plugins/grep
- bzrlib/plugins/po_merge
- bzrlib/plugins/po_merge/tests
- bzrlib/plugins/weave_fmt
- bzrlib/plugins/weave_fmt/tests
- bzrlib/tests/per_repository_vf
- po
- files removed:
- bzrlib/deprecated_graph.py
- bzrlib/doc_generate/builders
- bzrlib/doc_generate/writers
- bzrlib/tests/doc_generate
- bzrlib/tests/doc_generate/builders
- bzrlib/tests/doc_generate/writers
- bzrlib/util/effbot
- bzrlib/util/effbot/org
- bzrlib/util/elementtree
- files renamed:
- bzrlib/sign_my_commits.py => bzrlib/commit_signature_commands.py
- bzrlib/repofmt/weaverepo.py => bzrlib/plugins/weave_fmt/repository.py
- bzrlib/xml4.py => bzrlib/plugins/weave_fmt/xml4.py
- bzrlib/tests/per_repository/test_is_write_locked.py => bzrlib/tests/per_repository/test_locking.py
- bzrlib/tests/per_repository/helpers.py => bzrlib/tests/per_repository_vf/helpers.py
- bzrlib/tests/per_repository/test__generate_text_key_index.py => bzrlib/tests/per_repository_vf/test__generate_text_key_index.py
- bzrlib/tests/per_repository/test_add_inventory_by_delta.py => bzrlib/tests/per_repository_vf/test_add_inventory_by_delta.py
- bzrlib/tests/per_repository/test_check_reconcile.py => bzrlib/tests/per_repository_vf/test_check_reconcile.py
- bzrlib/tests/per_repository/test_find_text_key_references.py => bzrlib/tests/per_repository_vf/test_find_text_key_references.py
- bzrlib/tests/per_repository/test_merge_directive.py => bzrlib/tests/per_repository_vf/test_merge_directive.py
- files modified:
- .bzrignore
added
removed
bzrlib/repository.py
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
17
from __future__ import absolute_import
18
17
19
from bzrlib.lazy_import import lazy_import
18
20
lazy_import(globals(), """
19
import cStringIO
20
import re
21
import itertools
21
22
import time
22
23
23
24
from bzrlib import (
24
bzrdir,
25
check,
26
chk_map,
27
25
config,
28
26
controldir,
29
27
debug,
30
fetch as _mod_fetch,
31
fifo_cache,
32
28
generate_ids,
33
gpg,
34
29
graph,
35
inventory,
36
inventory_delta,
37
lazy_regex,
38
30
lockable_files,
39
31
lockdir,
40
lru_cache,
41
32
osutils,
42
pyutils,
43
33
revision as _mod_revision,
44
static_tuple,
45
trace,
34
testament as _mod_testament,
46
35
tsort,
47
versionedfile,
36
gpg,
48
37
)
49
38
from bzrlib.bundle import serializer
50
from bzrlib.revisiontree import RevisionTree
51
from bzrlib.store.versioned import VersionedFileStore
52
from bzrlib.testament import Testament
39
from bzrlib.i18n import gettext
53
40
""")
54
41
55
42
from bzrlib import (
43
bzrdir,
56
44
errors,
57
45
registry,
58
46
symbol_versioning,
60
48
)
61
49
from bzrlib.decorators import needs_read_lock, needs_write_lock, only_raises
62
50
from bzrlib.inter import InterObject
63
from bzrlib.inventory import (
64
Inventory,
65
InventoryDirectory,
66
ROOT_ID,
67
entry_factory,
68
)
69
from bzrlib.recordcounter import RecordCounter
70
51
from bzrlib.lock import _RelockDebugMixin, LogicalLockResult
71
52
from bzrlib.trace import (
72
53
log_exception_quietly, note, mutter, mutter_callsite, warning)
93
74
94
75
# all clients should supply tree roots.
95
76
record_root_entry = True
96
# the default CommitBuilder does not manage trees whose root is versioned.
97
_versioned_root = False
77
# whether this commit builder supports the record_entry_contents interface
78
supports_record_entry_contents = False
79
# whether this commit builder will automatically update the branch that is
80
# being committed to
81
updates_branch = False
98
82
99
def __init__(self, repository, parents, config, timestamp=None,
83
def __init__(self, repository, parents, config_stack, timestamp=None,
100
84
timezone=None, committer=None, revprops=None,
101
revision_id=None):
85
revision_id=None, lossy=False):
102
86
"""Initiate a CommitBuilder.
103
87
104
88
:param repository: Repository to commit to.
105
89
:param parents: Revision ids of the parents of the new revision.
106
:param config: Configuration to use.
107
90
:param timestamp: Optional timestamp recorded for commit.
108
91
:param timezone: Optional timezone for timestamp.
109
92
:param committer: Optional committer to set for commit.
110
93
:param revprops: Optional dictionary of revision properties.
111
94
:param revision_id: Optional revision id.
95
:param lossy: Whether to discard data that can not be natively
96
represented, when pushing to a foreign VCS
112
97
"""
113
self._config = config
98
self._config_stack = config_stack
99
self._lossy = lossy
114
100
115
101
if committer is None:
116
self._committer = self._config.username()
102
self._committer = self._config_stack.get('email')
117
103
elif not isinstance(committer, unicode):
118
104
self._committer = committer.decode() # throw if non-ascii
119
105
else:
120
106
self._committer = committer
121
107
122
self.new_inventory = Inventory(None)
123
108
self._new_revision_id = revision_id
124
109
self.parents = parents
125
110
self.repository = repository
140
125
self._timezone = int(timezone)
141
126
142
127
self._generate_revision_if_needed()
143
self.__heads = graph.HeadsCache(repository.get_graph()).heads
144
self._basis_delta = []
145
# API compatibility, older code that used CommitBuilder did not call
146
# .record_delete(), which means the delta that is computed would not be
147
# valid. Callers that will call record_delete() should call
148
# .will_record_deletes() to indicate that.
149
self._recording_deletes = False
150
# memo'd check for no-op commits.
151
self._any_changes = False
152
128
153
129
def any_changes(self):
154
130
"""Return True if any entries were changed.
155
131
156
132
This includes merge-only changes. It is the core for the --unchanged
157
133
detection in commit.
158
134
159
135
:return: True if any changes have occured.
160
136
"""
161
return self._any_changes
137
raise NotImplementedError(self.any_changes)
162
138
163
139
def _validate_unicode_text(self, text, context):
164
140
"""Verify things like commit messages don't have bogus characters."""
175
151
self._validate_unicode_text(value,
176
152
'revision property (%s)' % (key,))
177
153
178
def _ensure_fallback_inventories(self):
179
"""Ensure that appropriate inventories are available.
180
181
This only applies to repositories that are stacked, and is about
182
enusring the stacking invariants. Namely, that for any revision that is
183
present, we either have all of the file content, or we have the parent
184
inventory and the delta file content.
185
"""
186
if not self.repository._fallback_repositories:
187
return
188
if not self.repository._format.supports_chks:
189
raise errors.BzrError("Cannot commit directly to a stacked branch"
190
" in pre-2a formats. See "
191
"https://bugs.launchpad.net/bzr/+bug/375013 for details.")
192
# This is a stacked repo, we need to make sure we have the parent
193
# inventories for the parents.
194
parent_keys = [(p,) for p in self.parents]
195
parent_map = self.repository.inventories._index.get_parent_map(parent_keys)
196
missing_parent_keys = set([pk for pk in parent_keys
197
if pk not in parent_map])
198
fallback_repos = list(reversed(self.repository._fallback_repositories))
199
missing_keys = [('inventories', pk[0])
200
for pk in missing_parent_keys]
201
resume_tokens = []
202
while missing_keys and fallback_repos:
203
fallback_repo = fallback_repos.pop()
204
source = fallback_repo._get_source(self.repository._format)
205
sink = self.repository._get_sink()
206
stream = source.get_stream_for_missing_keys(missing_keys)
207
missing_keys = sink.insert_stream_without_locking(stream,
208
self.repository._format)
209
if missing_keys:
210
raise errors.BzrError('Unable to fill in parent inventories for a'
211
' stacked branch')
212
213
154
def commit(self, message):
214
155
"""Make the actual commit.
215
156
216
157
:return: The revision id of the recorded revision.
217
158
"""
218
self._validate_unicode_text(message, 'commit message')
219
rev = _mod_revision.Revision(
220
timestamp=self._timestamp,
221
timezone=self._timezone,
222
committer=self._committer,
223
message=message,
224
inventory_sha1=self.inv_sha1,
225
revision_id=self._new_revision_id,
226
properties=self._revprops)
227
rev.parent_ids = self.parents
228
self.repository.add_revision(self._new_revision_id, rev,
229
self.new_inventory, self._config)
230
self._ensure_fallback_inventories()
231
self.repository.commit_write_group()
232
return self._new_revision_id
159
raise NotImplementedError(self.commit)
233
160
234
161
def abort(self):
235
162
"""Abort the commit that is being built.
236
163
"""
237
self.repository.abort_write_group()
164
raise NotImplementedError(self.abort)
238
165
239
166
def revision_tree(self):
240
167
"""Return the tree that was just committed.
241
168
242
After calling commit() this can be called to get a RevisionTree
243
representing the newly committed tree. This is preferred to
244
calling Repository.revision_tree() because that may require
245
deserializing the inventory, while we already have a copy in
169
After calling commit() this can be called to get a
170
RevisionTree representing the newly committed tree. This is
171
preferred to calling Repository.revision_tree() because that may
172
require deserializing the inventory, while we already have a copy in
246
173
memory.
247
174
"""
248
if self.new_inventory is None:
249
self.new_inventory = self.repository.get_inventory(
250
self._new_revision_id)
251
return RevisionTree(self.repository, self.new_inventory,
252
self._new_revision_id)
175
raise NotImplementedError(self.revision_tree)
253
176
254
177
def finish_inventory(self):
255
178
"""Tell the builder that the inventory is finished.
257
180
:return: The inventory id in the repository, which can be used with
258
181
repository.get_inventory.
259
182
"""
260
if self.new_inventory is None:
261
# an inventory delta was accumulated without creating a new
262
# inventory.
263
basis_id = self.basis_delta_revision
264
# We ignore the 'inventory' returned by add_inventory_by_delta
265
# because self.new_inventory is used to hint to the rest of the
266
# system what code path was taken
267
self.inv_sha1, _ = self.repository.add_inventory_by_delta(
268
basis_id, self._basis_delta, self._new_revision_id,
269
self.parents)
270
else:
271
if self.new_inventory.root is None:
272
raise AssertionError('Root entry should be supplied to'
273
' record_entry_contents, as of bzr 0.10.')
274
self.new_inventory.add(InventoryDirectory(ROOT_ID, '', None))
275
self.new_inventory.revision_id = self._new_revision_id
276
self.inv_sha1 = self.repository.add_inventory(
277
self._new_revision_id,
278
self.new_inventory,
279
self.parents
280
)
281
return self._new_revision_id
183
raise NotImplementedError(self.finish_inventory)
282
184
283
185
def _gen_revision_id(self):
284
186
"""Return new revision-id."""
299
201
else:
300
202
self.random_revid = False
301
203
302
def _heads(self, file_id, revision_ids):
303
"""Calculate the graph heads for revision_ids in the graph of file_id.
304
305
This can use either a per-file graph or a global revision graph as we
306
have an identity relationship between the two graphs.
307
"""
308
return self.__heads(revision_ids)
309
310
def _check_root(self, ie, parent_invs, tree):
311
"""Helper for record_entry_contents.
312
313
:param ie: An entry being added.
314
:param parent_invs: The inventories of the parent revisions of the
315
commit.
316
:param tree: The tree that is being committed.
317
"""
318
# In this revision format, root entries have no knit or weave When
319
# serializing out to disk and back in root.revision is always
320
# _new_revision_id
321
ie.revision = self._new_revision_id
322
323
def _require_root_change(self, tree):
324
"""Enforce an appropriate root object change.
325
326
This is called once when record_iter_changes is called, if and only if
327
the root was not in the delta calculated by record_iter_changes.
328
329
:param tree: The tree which is being committed.
330
"""
331
if len(self.parents) == 0:
332
raise errors.RootMissing()
333
entry = entry_factory['directory'](tree.path2id(''), '',
334
None)
335
entry.revision = self._new_revision_id
336
self._basis_delta.append(('', '', entry.file_id, entry))
337
338
def _get_delta(self, ie, basis_inv, path):
339
"""Get a delta against the basis inventory for ie."""
340
if ie.file_id not in basis_inv:
341
# add
342
result = (None, path, ie.file_id, ie)
343
self._basis_delta.append(result)
344
return result
345
elif ie != basis_inv[ie.file_id]:
346
# common but altered
347
# TODO: avoid tis id2path call.
348
result = (basis_inv.id2path(ie.file_id), path, ie.file_id, ie)
349
self._basis_delta.append(result)
350
return result
351
else:
352
# common, unaltered
353
return None
354
355
def get_basis_delta(self):
356
"""Return the complete inventory delta versus the basis inventory.
357
358
This has been built up with the calls to record_delete and
359
record_entry_contents. The client must have already called
360
will_record_deletes() to indicate that they will be generating a
361
complete delta.
362
363
:return: An inventory delta, suitable for use with apply_delta, or
364
Repository.add_inventory_by_delta, etc.
365
"""
366
if not self._recording_deletes:
367
raise AssertionError("recording deletes not activated.")
368
return self._basis_delta
369
370
def record_delete(self, path, file_id):
371
"""Record that a delete occured against a basis tree.
372
373
This is an optional API - when used it adds items to the basis_delta
374
being accumulated by the commit builder. It cannot be called unless the
375
method will_record_deletes() has been called to inform the builder that
376
a delta is being supplied.
377
378
:param path: The path of the thing deleted.
379
:param file_id: The file id that was deleted.
380
"""
381
if not self._recording_deletes:
382
raise AssertionError("recording deletes not activated.")
383
delta = (path, None, file_id, None)
384
self._basis_delta.append(delta)
385
self._any_changes = True
386
return delta
387
388
204
def will_record_deletes(self):
389
205
"""Tell the commit builder that deletes are being notified.
390
206
392
208
commit to be valid, deletes against the basis MUST be recorded via
393
209
builder.record_delete().
394
210
"""
395
self._recording_deletes = True
396
try:
397
basis_id = self.parents[0]
398
except IndexError:
399
basis_id = _mod_revision.NULL_REVISION
400
self.basis_delta_revision = basis_id
401
402
def record_entry_contents(self, ie, parent_invs, path, tree,
403
content_summary):
404
"""Record the content of ie from tree into the commit if needed.
405
406
Side effect: sets ie.revision when unchanged
407
408
:param ie: An inventory entry present in the commit.
409
:param parent_invs: The inventories of the parent revisions of the
410
commit.
411
:param path: The path the entry is at in the tree.
412
:param tree: The tree which contains this entry and should be used to
413
obtain content.
414
:param content_summary: Summary data from the tree about the paths
415
content - stat, length, exec, sha/link target. This is only
416
accessed when the entry has a revision of None - that is when it is
417
a candidate to commit.
418
:return: A tuple (change_delta, version_recorded, fs_hash).
419
change_delta is an inventory_delta change for this entry against
420
the basis tree of the commit, or None if no change occured against
421
the basis tree.
422
version_recorded is True if a new version of the entry has been
423
recorded. For instance, committing a merge where a file was only
424
changed on the other side will return (delta, False).
425
fs_hash is either None, or the hash details for the path (currently
426
a tuple of the contents sha1 and the statvalue returned by
427
tree.get_file_with_stat()).
428
"""
429
if self.new_inventory.root is None:
430
if ie.parent_id is not None:
431
raise errors.RootMissing()
432
self._check_root(ie, parent_invs, tree)
433
if ie.revision is None:
434
kind = content_summary[0]
435
else:
436
# ie is carried over from a prior commit
437
kind = ie.kind
438
# XXX: repository specific check for nested tree support goes here - if
439
# the repo doesn't want nested trees we skip it ?
440
if (kind == 'tree-reference' and
441
not self.repository._format.supports_tree_reference):
442
# mismatch between commit builder logic and repository:
443
# this needs the entry creation pushed down into the builder.
444
raise NotImplementedError('Missing repository subtree support.')
445
self.new_inventory.add(ie)
446
447
# TODO: slow, take it out of the inner loop.
448
try:
449
basis_inv = parent_invs[0]
450
except IndexError:
451
basis_inv = Inventory(root_id=None)
452
453
# ie.revision is always None if the InventoryEntry is considered
454
# for committing. We may record the previous parents revision if the
455
# content is actually unchanged against a sole head.
456
if ie.revision is not None:
457
if not self._versioned_root and path == '':
458
# repositories that do not version the root set the root's
459
# revision to the new commit even when no change occurs (more
460
# specifically, they do not record a revision on the root; and
461
# the rev id is assigned to the root during deserialisation -
462
# this masks when a change may have occurred against the basis.
463
# To match this we always issue a delta, because the revision
464
# of the root will always be changing.
465
if ie.file_id in basis_inv:
466
delta = (basis_inv.id2path(ie.file_id), path,
467
ie.file_id, ie)
468
else:
469
# add
470
delta = (None, path, ie.file_id, ie)
471
self._basis_delta.append(delta)
472
return delta, False, None
473
else:
474
# we don't need to commit this, because the caller already
475
# determined that an existing revision of this file is
476
# appropriate. If it's not being considered for committing then
477
# it and all its parents to the root must be unaltered so
478
# no-change against the basis.
479
if ie.revision == self._new_revision_id:
480
raise AssertionError("Impossible situation, a skipped "
481
"inventory entry (%r) claims to be modified in this "
482
"commit (%r).", (ie, self._new_revision_id))
483
return None, False, None
484
# XXX: Friction: parent_candidates should return a list not a dict
485
# so that we don't have to walk the inventories again.
486
parent_candiate_entries = ie.parent_candidates(parent_invs)
487
head_set = self._heads(ie.file_id, parent_candiate_entries.keys())
488
heads = []
489
for inv in parent_invs:
490
if ie.file_id in inv:
491
old_rev = inv[ie.file_id].revision
492
if old_rev in head_set:
493
heads.append(inv[ie.file_id].revision)
494
head_set.remove(inv[ie.file_id].revision)
495
496
store = False
497
# now we check to see if we need to write a new record to the
498
# file-graph.
499
# We write a new entry unless there is one head to the ancestors, and
500
# the kind-derived content is unchanged.
501
502
# Cheapest check first: no ancestors, or more the one head in the
503
# ancestors, we write a new node.
504
if len(heads) != 1:
505
store = True
506
if not store:
507
# There is a single head, look it up for comparison
508
parent_entry = parent_candiate_entries[heads[0]]
509
# if the non-content specific data has changed, we'll be writing a
510
# node:
511
if (parent_entry.parent_id != ie.parent_id or
512
parent_entry.name != ie.name):
513
store = True
514
# now we need to do content specific checks:
515
if not store:
516
# if the kind changed the content obviously has
517
if kind != parent_entry.kind:
518
store = True
519
# Stat cache fingerprint feedback for the caller - None as we usually
520
# don't generate one.
521
fingerprint = None
522
if kind == 'file':
523
if content_summary[2] is None:
524
raise ValueError("Files must not have executable = None")
525
if not store:
526
# We can't trust a check of the file length because of content
527
# filtering...
528
if (# if the exec bit has changed we have to store:
529
parent_entry.executable != content_summary[2]):
530
store = True
531
elif parent_entry.text_sha1 == content_summary[3]:
532
# all meta and content is unchanged (using a hash cache
533
# hit to check the sha)
534
ie.revision = parent_entry.revision
535
ie.text_size = parent_entry.text_size
536
ie.text_sha1 = parent_entry.text_sha1
537
ie.executable = parent_entry.executable
538
return self._get_delta(ie, basis_inv, path), False, None
539
else:
540
# Either there is only a hash change(no hash cache entry,
541
# or same size content change), or there is no change on
542
# this file at all.
543
# Provide the parent's hash to the store layer, so that the
544
# content is unchanged we will not store a new node.
545
nostore_sha = parent_entry.text_sha1
546
if store:
547
# We want to record a new node regardless of the presence or
548
# absence of a content change in the file.
549
nostore_sha = None
550
ie.executable = content_summary[2]
551
file_obj, stat_value = tree.get_file_with_stat(ie.file_id, path)
552
try:
553
text = file_obj.read()
554
finally:
555
file_obj.close()
556
try:
557
ie.text_sha1, ie.text_size = self._add_text_to_weave(
558
ie.file_id, text, heads, nostore_sha)
559
# Let the caller know we generated a stat fingerprint.
560
fingerprint = (ie.text_sha1, stat_value)
561
except errors.ExistingContent:
562
# Turns out that the file content was unchanged, and we were
563
# only going to store a new node if it was changed. Carry over
564
# the entry.
565
ie.revision = parent_entry.revision
566
ie.text_size = parent_entry.text_size
567
ie.text_sha1 = parent_entry.text_sha1
568
ie.executable = parent_entry.executable
569
return self._get_delta(ie, basis_inv, path), False, None
570
elif kind == 'directory':
571
if not store:
572
# all data is meta here, nothing specific to directory, so
573
# carry over:
574
ie.revision = parent_entry.revision
575
return self._get_delta(ie, basis_inv, path), False, None
576
self._add_text_to_weave(ie.file_id, '', heads, None)
577
elif kind == 'symlink':
578
current_link_target = content_summary[3]
579
if not store:
580
# symlink target is not generic metadata, check if it has
581
# changed.
582
if current_link_target != parent_entry.symlink_target:
583
store = True
584
if not store:
585
# unchanged, carry over.
586
ie.revision = parent_entry.revision
587
ie.symlink_target = parent_entry.symlink_target
588
return self._get_delta(ie, basis_inv, path), False, None
589
ie.symlink_target = current_link_target
590
self._add_text_to_weave(ie.file_id, '', heads, None)
591
elif kind == 'tree-reference':
592
if not store:
593
if content_summary[3] != parent_entry.reference_revision:
594
store = True
595
if not store:
596
# unchanged, carry over.
597
ie.reference_revision = parent_entry.reference_revision
598
ie.revision = parent_entry.revision
599
return self._get_delta(ie, basis_inv, path), False, None
600
ie.reference_revision = content_summary[3]
601
if ie.reference_revision is None:
602
raise AssertionError("invalid content_summary for nested tree: %r"
603
% (content_summary,))
604
self._add_text_to_weave(ie.file_id, '', heads, None)
605
else:
606
raise NotImplementedError('unknown kind')
607
ie.revision = self._new_revision_id
608
self._any_changes = True
609
return self._get_delta(ie, basis_inv, path), True, fingerprint
610
611
def record_iter_changes(self, tree, basis_revision_id, iter_changes,
612
_entry_factory=entry_factory):
211
raise NotImplementedError(self.will_record_deletes)
212
213
def record_iter_changes(self, tree, basis_revision_id, iter_changes):
613
214
"""Record a new tree via iter_changes.
614
215
615
216
:param tree: The tree to obtain text contents from for changed objects.
620
221
to basis_revision_id. The iterator must not include any items with
621
222
a current kind of None - missing items must be either filtered out
622
223
or errored-on beefore record_iter_changes sees the item.
623
:param _entry_factory: Private method to bind entry_factory locally for
624
performance.
625
224
:return: A generator of (file_id, relpath, fs_hash) tuples for use with
626
225
tree._observed_sha1.
627
226
"""
628
# Create an inventory delta based on deltas between all the parents and
629
# deltas between all the parent inventories. We use inventory delta's
630
# between the inventory objects because iter_changes masks
631
# last-changed-field only changes.
632
# Working data:
633
# file_id -> change map, change is fileid, paths, changed, versioneds,
634
# parents, names, kinds, executables
635
merged_ids = {}
636
# {file_id -> revision_id -> inventory entry, for entries in parent
637
# trees that are not parents[0]
638
parent_entries = {}
639
ghost_basis = False
640
try:
641
revtrees = list(self.repository.revision_trees(self.parents))
642
except errors.NoSuchRevision:
643
# one or more ghosts, slow path.
644
revtrees = []
645
for revision_id in self.parents:
646
try:
647
revtrees.append(self.repository.revision_tree(revision_id))
648
except errors.NoSuchRevision:
649
if not revtrees:
650
basis_revision_id = _mod_revision.NULL_REVISION
651
ghost_basis = True
652
revtrees.append(self.repository.revision_tree(
653
_mod_revision.NULL_REVISION))
654
# The basis inventory from a repository
655
if revtrees:
656
basis_inv = revtrees[0].inventory
657
else:
658
basis_inv = self.repository.revision_tree(
659
_mod_revision.NULL_REVISION).inventory
660
if len(self.parents) > 0:
661
if basis_revision_id != self.parents[0] and not ghost_basis:
662
raise Exception(
663
"arbitrary basis parents not yet supported with merges")
664
for revtree in revtrees[1:]:
665
for change in revtree.inventory._make_delta(basis_inv):
666
if change[1] is None:
667
# Not present in this parent.
668
continue
669
if change[2] not in merged_ids:
670
if change[0] is not None:
671
basis_entry = basis_inv[change[2]]
672
merged_ids[change[2]] = [
673
# basis revid
674
basis_entry.revision,
675
# new tree revid
676
change[3].revision]
677
parent_entries[change[2]] = {
678
# basis parent
679
basis_entry.revision:basis_entry,
680
# this parent
681
change[3].revision:change[3],
682
}
683
else:
684
merged_ids[change[2]] = [change[3].revision]
685
parent_entries[change[2]] = {change[3].revision:change[3]}
686
else:
687
merged_ids[change[2]].append(change[3].revision)
688
parent_entries[change[2]][change[3].revision] = change[3]
689
else:
690
merged_ids = {}
691
# Setup the changes from the tree:
692
# changes maps file_id -> (change, [parent revision_ids])
693
changes= {}
694
for change in iter_changes:
695
# This probably looks up in basis_inv way to much.
696
if change[1][0] is not None:
697
head_candidate = [basis_inv[change[0]].revision]
698
else:
699
head_candidate = []
700
changes[change[0]] = change, merged_ids.get(change[0],
701
head_candidate)
702
unchanged_merged = set(merged_ids) - set(changes)
703
# Extend the changes dict with synthetic changes to record merges of
704
# texts.
705
for file_id in unchanged_merged:
706
# Record a merged version of these items that did not change vs the
707
# basis. This can be either identical parallel changes, or a revert
708
# of a specific file after a merge. The recorded content will be
709
# that of the current tree (which is the same as the basis), but
710
# the per-file graph will reflect a merge.
711
# NB:XXX: We are reconstructing path information we had, this
712
# should be preserved instead.
713
# inv delta change: (file_id, (path_in_source, path_in_target),
714
# changed_content, versioned, parent, name, kind,
715
# executable)
716
try:
717
basis_entry = basis_inv[file_id]
718
except errors.NoSuchId:
719
# a change from basis->some_parents but file_id isn't in basis
720
# so was new in the merge, which means it must have changed
721
# from basis -> current, and as it hasn't the add was reverted
722
# by the user. So we discard this change.
723
pass
724
else:
725
change = (file_id,
726
(basis_inv.id2path(file_id), tree.id2path(file_id)),
727
False, (True, True),
728
(basis_entry.parent_id, basis_entry.parent_id),
729
(basis_entry.name, basis_entry.name),
730
(basis_entry.kind, basis_entry.kind),
731
(basis_entry.executable, basis_entry.executable))
732
changes[file_id] = (change, merged_ids[file_id])
733
# changes contains tuples with the change and a set of inventory
734
# candidates for the file.
735
# inv delta is:
736
# old_path, new_path, file_id, new_inventory_entry
737
seen_root = False # Is the root in the basis delta?
738
inv_delta = self._basis_delta
739
modified_rev = self._new_revision_id
740
for change, head_candidates in changes.values():
741
if change[3][1]: # versioned in target.
742
# Several things may be happening here:
743
# We may have a fork in the per-file graph
744
# - record a change with the content from tree
745
# We may have a change against < all trees
746
# - carry over the tree that hasn't changed
747
# We may have a change against all trees
748
# - record the change with the content from tree
749
kind = change[6][1]
750
file_id = change[0]
751
entry = _entry_factory[kind](file_id, change[5][1],
752
change[4][1])
753
head_set = self._heads(change[0], set(head_candidates))
754
heads = []
755
# Preserve ordering.
756
for head_candidate in head_candidates:
757
if head_candidate in head_set:
758
heads.append(head_candidate)
759
head_set.remove(head_candidate)
760
carried_over = False
761
if len(heads) == 1:
762
# Could be a carry-over situation:
763
parent_entry_revs = parent_entries.get(file_id, None)
764
if parent_entry_revs:
765
parent_entry = parent_entry_revs.get(heads[0], None)
766
else:
767
parent_entry = None
768
if parent_entry is None:
769
# The parent iter_changes was called against is the one
770
# that is the per-file head, so any change is relevant
771
# iter_changes is valid.
772
carry_over_possible = False
773
else:
774
# could be a carry over situation
775
# A change against the basis may just indicate a merge,
776
# we need to check the content against the source of the
777
# merge to determine if it was changed after the merge
778
# or carried over.
779
if (parent_entry.kind != entry.kind or
780
parent_entry.parent_id != entry.parent_id or
781
parent_entry.name != entry.name):
782
# Metadata common to all entries has changed
783
# against per-file parent
784
carry_over_possible = False
785
else:
786
carry_over_possible = True
787
# per-type checks for changes against the parent_entry
788
# are done below.
789
else:
790
# Cannot be a carry-over situation
791
carry_over_possible = False
792
# Populate the entry in the delta
793
if kind == 'file':
794
# XXX: There is still a small race here: If someone reverts the content of a file
795
# after iter_changes examines and decides it has changed,
796
# we will unconditionally record a new version even if some
797
# other process reverts it while commit is running (with
798
# the revert happening after iter_changes did its
799
# examination).
800
if change[7][1]:
801
entry.executable = True
802
else:
803
entry.executable = False
804
if (carry_over_possible and
805
parent_entry.executable == entry.executable):
806
# Check the file length, content hash after reading
807
# the file.
808
nostore_sha = parent_entry.text_sha1
809
else:
810
nostore_sha = None
811
file_obj, stat_value = tree.get_file_with_stat(file_id, change[1][1])
812
try:
813
text = file_obj.read()
814
finally:
815
file_obj.close()
816
try:
817
entry.text_sha1, entry.text_size = self._add_text_to_weave(
818
file_id, text, heads, nostore_sha)
819
yield file_id, change[1][1], (entry.text_sha1, stat_value)
820
except errors.ExistingContent:
821
# No content change against a carry_over parent
822
# Perhaps this should also yield a fs hash update?
823
carried_over = True
824
entry.text_size = parent_entry.text_size
825
entry.text_sha1 = parent_entry.text_sha1
826
elif kind == 'symlink':
827
# Wants a path hint?
828
entry.symlink_target = tree.get_symlink_target(file_id)
829
if (carry_over_possible and
830
parent_entry.symlink_target == entry.symlink_target):
831
carried_over = True
832
else:
833
self._add_text_to_weave(change[0], '', heads, None)
834
elif kind == 'directory':
835
if carry_over_possible:
836
carried_over = True
837
else:
838
# Nothing to set on the entry.
839
# XXX: split into the Root and nonRoot versions.
840
if change[1][1] != '' or self.repository.supports_rich_root():
841
self._add_text_to_weave(change[0], '', heads, None)
842
elif kind == 'tree-reference':
843
if not self.repository._format.supports_tree_reference:
844
# This isn't quite sane as an error, but we shouldn't
845
# ever see this code path in practice: tree's don't
846
# permit references when the repo doesn't support tree
847
# references.
848
raise errors.UnsupportedOperation(tree.add_reference,
849
self.repository)
850
reference_revision = tree.get_reference_revision(change[0])
851
entry.reference_revision = reference_revision
852
if (carry_over_possible and
853
parent_entry.reference_revision == reference_revision):
854
carried_over = True
855
else:
856
self._add_text_to_weave(change[0], '', heads, None)
857
else:
858
raise AssertionError('unknown kind %r' % kind)
859
if not carried_over:
860
entry.revision = modified_rev
861
else:
862
entry.revision = parent_entry.revision
863
else:
864
entry = None
865
new_path = change[1][1]
866
inv_delta.append((change[1][0], new_path, change[0], entry))
867
if new_path == '':
868
seen_root = True
869
self.new_inventory = None
870
if len(inv_delta):
871
# This should perhaps be guarded by a check that the basis we
872
# commit against is the basis for the commit and if not do a delta
873
# against the basis.
874
self._any_changes = True
875
if not seen_root:
876
# housekeeping root entry changes do not affect no-change commits.
877
self._require_root_change(tree)
878
self.basis_delta_revision = basis_revision_id
879
880
def _add_text_to_weave(self, file_id, new_text, parents, nostore_sha):
881
parent_keys = tuple([(file_id, parent) for parent in parents])
882
return self.repository.texts._add_text(
883
(file_id, self._new_revision_id), parent_keys, new_text,
884
nostore_sha=nostore_sha, random_id=self.random_revid)[0:2]
885
886
887
class RootCommitBuilder(CommitBuilder):
888
"""This commitbuilder actually records the root id"""
889
890
# the root entry gets versioned properly by this builder.
891
_versioned_root = True
892
893
def _check_root(self, ie, parent_invs, tree):
894
"""Helper for record_entry_contents.
895
896
:param ie: An entry being added.
897
:param parent_invs: The inventories of the parent revisions of the
898
commit.
899
:param tree: The tree that is being committed.
900
"""
901
902
def _require_root_change(self, tree):
903
"""Enforce an appropriate root object change.
904
905
This is called once when record_iter_changes is called, if and only if
906
the root was not in the delta calculated by record_iter_changes.
907
908
:param tree: The tree which is being committed.
909
"""
910
# versioned roots do not change unless the tree found a change.
227
raise NotImplementedError(self.record_iter_changes)
911
228
912
229
913
230
class RepositoryWriteLockResult(LogicalLockResult):
938
255
revisions and file history. It's normally accessed only by the Branch,
939
256
which views a particular line of development through that history.
940
257
941
The Repository builds on top of some byte storage facilies (the revisions,
942
signatures, inventories, texts and chk_bytes attributes) and a Transport,
943
which respectively provide byte storage and a means to access the (possibly
944
remote) disk.
945
946
The byte storage facilities are addressed via tuples, which we refer to
947
as 'keys' throughout the code base. Revision_keys, inventory_keys and
948
signature_keys are all 1-tuples: (revision_id,). text_keys are two-tuples:
949
(file_id, revision_id). chk_bytes uses CHK keys - a 1-tuple with a single
950
byte string made up of a hash identifier and a hash value.
951
We use this interface because it allows low friction with the underlying
952
code that implements disk indices, network encoding and other parts of
953
bzrlib.
954
955
:ivar revisions: A bzrlib.versionedfile.VersionedFiles instance containing
956
the serialised revisions for the repository. This can be used to obtain
957
revision graph information or to access raw serialised revisions.
958
The result of trying to insert data into the repository via this store
959
is undefined: it should be considered read-only except for implementors
960
of repositories.
961
:ivar signatures: A bzrlib.versionedfile.VersionedFiles instance containing
962
the serialised signatures for the repository. This can be used to
963
obtain access to raw serialised signatures. The result of trying to
964
insert data into the repository via this store is undefined: it should
965
be considered read-only except for implementors of repositories.
966
:ivar inventories: A bzrlib.versionedfile.VersionedFiles instance containing
967
the serialised inventories for the repository. This can be used to
968
obtain unserialised inventories. The result of trying to insert data
969
into the repository via this store is undefined: it should be
970
considered read-only except for implementors of repositories.
971
:ivar texts: A bzrlib.versionedfile.VersionedFiles instance containing the
972
texts of files and directories for the repository. This can be used to
973
obtain file texts or file graphs. Note that Repository.iter_file_bytes
974
is usually a better interface for accessing file texts.
975
The result of trying to insert data into the repository via this store
976
is undefined: it should be considered read-only except for implementors
977
of repositories.
978
:ivar chk_bytes: A bzrlib.versionedfile.VersionedFiles instance containing
979
any data the repository chooses to store or have indexed by its hash.
980
The result of trying to insert data into the repository via this store
981
is undefined: it should be considered read-only except for implementors
982
of repositories.
983
:ivar _transport: Transport for file access to repository, typically
984
pointing to .bzr/repository.
258
See VersionedFileRepository in bzrlib.vf_repository for the
259
base class for most Bazaar repositories.
985
260
"""
986
261
987
# What class to use for a CommitBuilder. Often it's simpler to change this
988
# in a Repository class subclass rather than to override
989
# get_commit_builder.
990
_commit_builder_class = CommitBuilder
991
# The search regex used by xml based repositories to determine what things
992
# where changed in a single commit.
993
_file_ids_altered_regex = lazy_regex.lazy_compile(
994
r'file_id="(?P<file_id>[^"]+)"'
995
r'.* revision="(?P<revision_id>[^"]+)"'
996
)
997
998
262
def abort_write_group(self, suppress_errors=False):
999
263
"""Commit the contents accrued within the current write group.
1000
264
1022
286
raise
1023
287
mutter('abort_write_group failed')
1024
288
log_exception_quietly()
1025
note('bzr: ERROR (ignored): %s', exc)
289
note(gettext('bzr: ERROR (ignored): %s'), exc)
1026
290
self._write_group = None
1027
291
1028
292
def _abort_write_group(self):
1043
307
1044
308
:param repository: A repository.
1045
309
"""
1046
if not self._format.supports_external_lookups:
1047
raise errors.UnstackableRepositoryFormat(self._format, self.base)
1048
if self.is_locked():
1049
# This repository will call fallback.unlock() when we transition to
1050
# the unlocked state, so we make sure to increment the lock count
1051
repository.lock_read()
1052
self._check_fallback_repository(repository)
1053
self._fallback_repositories.append(repository)
1054
self.texts.add_fallback_versioned_files(repository.texts)
1055
self.inventories.add_fallback_versioned_files(repository.inventories)
1056
self.revisions.add_fallback_versioned_files(repository.revisions)
1057
self.signatures.add_fallback_versioned_files(repository.signatures)
1058
if self.chk_bytes is not None:
1059
self.chk_bytes.add_fallback_versioned_files(repository.chk_bytes)
310
raise NotImplementedError(self.add_fallback_repository)
1060
311
1061
312
def _check_fallback_repository(self, repository):
1062
313
"""Check that this repository can fallback to repository safely.
1067
318
"""
1068
319
return InterRepository._assert_same_model(self, repository)
1069
320
1070
def add_inventory(self, revision_id, inv, parents):
1071
"""Add the inventory inv to the repository as revision_id.
1072
1073
:param parents: The revision ids of the parents that revision_id
1074
is known to have and are in the repository already.
1075
1076
:returns: The validator(which is a sha1 digest, though what is sha'd is
1077
repository format specific) of the serialized inventory.
1078
"""
1079
if not self.is_in_write_group():
1080
raise AssertionError("%r not in write group" % (self,))
1081
_mod_revision.check_not_reserved_id(revision_id)
1082
if not (inv.revision_id is None or inv.revision_id == revision_id):
1083
raise AssertionError(
1084
"Mismatch between inventory revision"
1085
" id and insertion revid (%r, %r)"
1086
% (inv.revision_id, revision_id))
1087
if inv.root is None:
1088
raise errors.RootMissing()
1089
return self._add_inventory_checked(revision_id, inv, parents)
1090
1091
def _add_inventory_checked(self, revision_id, inv, parents):
1092
"""Add inv to the repository after checking the inputs.
1093
1094
This function can be overridden to allow different inventory styles.
1095
1096
:seealso: add_inventory, for the contract.
1097
"""
1098
inv_lines = self._serializer.write_inventory_to_lines(inv)
1099
return self._inventory_add_lines(revision_id, parents,
1100
inv_lines, check_content=False)
1101
1102
def add_inventory_by_delta(self, basis_revision_id, delta, new_revision_id,
1103
parents, basis_inv=None, propagate_caches=False):
1104
"""Add a new inventory expressed as a delta against another revision.
1105
1106
See the inventory developers documentation for the theory behind
1107
inventory deltas.
1108
1109
:param basis_revision_id: The inventory id the delta was created
1110
against. (This does not have to be a direct parent.)
1111
:param delta: The inventory delta (see Inventory.apply_delta for
1112
details).
1113
:param new_revision_id: The revision id that the inventory is being
1114
added for.
1115
:param parents: The revision ids of the parents that revision_id is
1116
known to have and are in the repository already. These are supplied
1117
for repositories that depend on the inventory graph for revision
1118
graph access, as well as for those that pun ancestry with delta
1119
compression.
1120
:param basis_inv: The basis inventory if it is already known,
1121
otherwise None.
1122
:param propagate_caches: If True, the caches for this inventory are
1123
copied to and updated for the result if possible.
1124
1125
:returns: (validator, new_inv)
1126
The validator(which is a sha1 digest, though what is sha'd is
1127
repository format specific) of the serialized inventory, and the
1128
resulting inventory.
1129
"""
1130
if not self.is_in_write_group():
1131
raise AssertionError("%r not in write group" % (self,))
1132
_mod_revision.check_not_reserved_id(new_revision_id)
1133
basis_tree = self.revision_tree(basis_revision_id)
1134
basis_tree.lock_read()
1135
try:
1136
# Note that this mutates the inventory of basis_tree, which not all
1137
# inventory implementations may support: A better idiom would be to
1138
# return a new inventory, but as there is no revision tree cache in
1139
# repository this is safe for now - RBC 20081013
1140
if basis_inv is None:
1141
basis_inv = basis_tree.inventory
1142
basis_inv.apply_delta(delta)
1143
basis_inv.revision_id = new_revision_id
1144
return (self.add_inventory(new_revision_id, basis_inv, parents),
1145
basis_inv)
1146
finally:
1147
basis_tree.unlock()
1148
1149
def _inventory_add_lines(self, revision_id, parents, lines,
1150
check_content=True):
1151
"""Store lines in inv_vf and return the sha1 of the inventory."""
1152
parents = [(parent,) for parent in parents]
1153
result = self.inventories.add_lines((revision_id,), parents, lines,
1154
check_content=check_content)[0]
1155
self.inventories._access.flush()
1156
return result
1157
1158
def add_revision(self, revision_id, rev, inv=None, config=None):
1159
"""Add rev to the revision store as revision_id.
1160
1161
:param revision_id: the revision id to use.
1162
:param rev: The revision object.
1163
:param inv: The inventory for the revision. if None, it will be looked
1164
up in the inventory storer
1165
:param config: If None no digital signature will be created.
1166
If supplied its signature_needed method will be used
1167
to determine if a signature should be made.
1168
"""
1169
# TODO: jam 20070210 Shouldn't we check rev.revision_id and
1170
# rev.parent_ids?
1171
_mod_revision.check_not_reserved_id(revision_id)
1172
if config is not None and config.signature_needed():
1173
if inv is None:
1174
inv = self.get_inventory(revision_id)
1175
plaintext = Testament(rev, inv).as_short_text()
1176
self.store_revision_signature(
1177
gpg.GPGStrategy(config), plaintext, revision_id)
1178
# check inventory present
1179
if not self.inventories.get_parent_map([(revision_id,)]):
1180
if inv is None:
1181
raise errors.WeaveRevisionNotPresent(revision_id,
1182
self.inventories)
1183
else:
1184
# yes, this is not suitable for adding with ghosts.
1185
rev.inventory_sha1 = self.add_inventory(revision_id, inv,
1186
rev.parent_ids)
1187
else:
1188
key = (revision_id,)
1189
rev.inventory_sha1 = self.inventories.get_sha1s([key])[key]
1190
self._add_revision(rev)
1191
1192
def _add_revision(self, revision):
1193
text = self._serializer.write_revision_to_string(revision)
1194
key = (revision.revision_id,)
1195
parents = tuple((parent,) for parent in revision.parent_ids)
1196
self.revisions.add_lines(key, parents, osutils.split_lines(text))
1197
1198
321
def all_revision_ids(self):
1199
322
"""Returns a list of all the revision ids in the repository.
1200
323
1223
346
"""
1224
347
self.control_files.break_lock()
1225
348
1226
@needs_read_lock
1227
def _eliminate_revisions_not_present(self, revision_ids):
1228
"""Check every revision id in revision_ids to see if we have it.
1229
1230
Returns a set of the present revisions.
1231
"""
1232
result = []
1233
graph = self.get_graph()
1234
parent_map = graph.get_parent_map(revision_ids)
1235
# The old API returned a list, should this actually be a set?
1236
return parent_map.keys()
1237
1238
def _check_inventories(self, checker):
1239
"""Check the inventories found from the revision scan.
1240
1241
This is responsible for verifying the sha1 of inventories and
1242
creating a pending_keys set that covers data referenced by inventories.
1243
"""
1244
bar = ui.ui_factory.nested_progress_bar()
1245
try:
1246
self._do_check_inventories(checker, bar)
1247
finally:
1248
bar.finished()
1249
1250
def _do_check_inventories(self, checker, bar):
1251
"""Helper for _check_inventories."""
1252
revno = 0
1253
keys = {'chk_bytes':set(), 'inventories':set(), 'texts':set()}
1254
kinds = ['chk_bytes', 'texts']
1255
count = len(checker.pending_keys)
1256
bar.update("inventories", 0, 2)
1257
current_keys = checker.pending_keys
1258
checker.pending_keys = {}
1259
# Accumulate current checks.
1260
for key in current_keys:
1261
if key[0] != 'inventories' and key[0] not in kinds:
1262
checker._report_items.append('unknown key type %r' % (key,))
1263
keys[key[0]].add(key[1:])
1264
if keys['inventories']:
1265
# NB: output order *should* be roughly sorted - topo or
1266
# inverse topo depending on repository - either way decent
1267
# to just delta against. However, pre-CHK formats didn't
1268
# try to optimise inventory layout on disk. As such the
1269
# pre-CHK code path does not use inventory deltas.
1270
last_object = None
1271
for record in self.inventories.check(keys=keys['inventories']):
1272
if record.storage_kind == 'absent':
1273
checker._report_items.append(
1274
'Missing inventory {%s}' % (record.key,))
1275
else:
1276
last_object = self._check_record('inventories', record,
1277
checker, last_object,
1278
current_keys[('inventories',) + record.key])
1279
del keys['inventories']
1280
else:
1281
return
1282
bar.update("texts", 1)
1283
while (checker.pending_keys or keys['chk_bytes']
1284
or keys['texts']):
1285
# Something to check.
1286
current_keys = checker.pending_keys
1287
checker.pending_keys = {}
1288
# Accumulate current checks.
1289
for key in current_keys:
1290
if key[0] not in kinds:
1291
checker._report_items.append('unknown key type %r' % (key,))
1292
keys[key[0]].add(key[1:])
1293
# Check the outermost kind only - inventories || chk_bytes || texts
1294
for kind in kinds:
1295
if keys[kind]:
1296
last_object = None
1297
for record in getattr(self, kind).check(keys=keys[kind]):
1298
if record.storage_kind == 'absent':
1299
checker._report_items.append(
1300
'Missing %s {%s}' % (kind, record.key,))
1301
else:
1302
last_object = self._check_record(kind, record,
1303
checker, last_object, current_keys[(kind,) + record.key])
1304
keys[kind] = set()
1305
break
1306
1307
def _check_record(self, kind, record, checker, last_object, item_data):
1308
"""Check a single text from this repository."""
1309
if kind == 'inventories':
1310
rev_id = record.key[0]
1311
inv = self._deserialise_inventory(rev_id,
1312
record.get_bytes_as('fulltext'))
1313
if last_object is not None:
1314
delta = inv._make_delta(last_object)
1315
for old_path, path, file_id, ie in delta:
1316
if ie is None:
1317
continue
1318
ie.check(checker, rev_id, inv)
1319
else:
1320
for path, ie in inv.iter_entries():
1321
ie.check(checker, rev_id, inv)
1322
if self._format.fast_deltas:
1323
return inv
1324
elif kind == 'chk_bytes':
1325
# No code written to check chk_bytes for this repo format.
1326
checker._report_items.append(
1327
'unsupported key type chk_bytes for %s' % (record.key,))
1328
elif kind == 'texts':
1329
self._check_text(record, checker, item_data)
1330
else:
1331
checker._report_items.append(
1332
'unknown key type %s for %s' % (kind, record.key))
1333
1334
def _check_text(self, record, checker, item_data):
1335
"""Check a single text."""
1336
# Check it is extractable.
1337
# TODO: check length.
1338
if record.storage_kind == 'chunked':
1339
chunks = record.get_bytes_as(record.storage_kind)
1340
sha1 = osutils.sha_strings(chunks)
1341
length = sum(map(len, chunks))
1342
else:
1343
content = record.get_bytes_as('fulltext')
1344
sha1 = osutils.sha_string(content)
1345
length = len(content)
1346
if item_data and sha1 != item_data[1]:
1347
checker._report_items.append(
1348
'sha1 mismatch: %s has sha1 %s expected %s referenced by %s' %
1349
(record.key, sha1, item_data[1], item_data[2]))
1350
1351
349
@staticmethod
1352
def create(a_bzrdir):
1353
"""Construct the current default format repository in a_bzrdir."""
1354
return RepositoryFormat.get_default_format().initialize(a_bzrdir)
350
def create(controldir):
351
"""Construct the current default format repository in controldir."""
352
return RepositoryFormat.get_default_format().initialize(controldir)
1355
353
1356
def __init__(self, _format, a_bzrdir, control_files):
354
def __init__(self, _format, controldir, control_files):
1357
355
"""instantiate a Repository.
1358
356
1359
357
:param _format: The format of the repository on disk.
1360
:param a_bzrdir: The BzrDir of the repository.
358
:param controldir: The ControlDir of the repository.
359
:param control_files: Control files to use for locking, etc.
1361
360
"""
1362
361
# In the future we will have a single api for all stores for
1363
362
# getting file texts, inventories and revisions, then
1365
364
super(Repository, self).__init__()
1366
365
self._format = _format
1367
366
# the following are part of the public API for Repository:
1368
self.bzrdir = a_bzrdir
367
self.bzrdir = controldir
1369
368
self.control_files = control_files
1370
self._transport = control_files._transport
1371
self.base = self._transport.base
1372
369
# for tests
1373
self._reconcile_does_inventory_gc = True
1374
self._reconcile_fixes_text_parents = False
1375
self._reconcile_backsup_inventory = True
1376
370
self._write_group = None
1377
371
# Additional places to query for data.
1378
372
self._fallback_repositories = []
1379
# An InventoryEntry cache, used during deserialization
1380
self._inventory_entry_cache = fifo_cache.FIFOCache(10*1024)
1381
# Is it safe to return inventory entries directly from the entry cache,
1382
# rather copying them?
1383
self._safe_to_return_from_cache = False
1384
373
1385
374
@property
1386
375
def user_transport(self):
1420
409
"""
1421
410
if self.__class__ is not other.__class__:
1422
411
return False
1423
return (self._transport.base == other._transport.base)
412
return (self.control_url == other.control_url)
1424
413
1425
414
def is_in_write_group(self):
1426
415
"""Return True if there is an open write group.
1526
515
if revid and committers:
1527
516
result['committers'] = 0
1528
517
if revid and revid != _mod_revision.NULL_REVISION:
518
graph = self.get_graph()
1529
519
if committers:
1530
520
all_committers = set()
1531
revisions = self.get_ancestry(revid)
1532
# pop the leading None
1533
revisions.pop(0)
1534
first_revision = None
521
revisions = [r for (r, p) in graph.iter_ancestry([revid])
522
if r != _mod_revision.NULL_REVISION]
523
last_revision = None
1535
524
if not committers:
1536
525
# ignore the revisions in the middle - just grab first and last
1537
526
revisions = revisions[0], revisions[-1]
1538
527
for revision in self.get_revisions(revisions):
1539
if not first_revision:
1540
first_revision = revision
528
if not last_revision:
529
last_revision = revision
1541
530
if committers:
1542
531
all_committers.add(revision.committer)
1543
last_revision = revision
532
first_revision = revision
1544
533
if committers:
1545
534
result['committers'] = len(all_committers)
1546
535
result['firstrev'] = (first_revision.timestamp,
1547
536
first_revision.timezone)
1548
537
result['latestrev'] = (last_revision.timestamp,
1549
538
last_revision.timezone)
1550
1551
# now gather global repository information
1552
# XXX: This is available for many repos regardless of listability.
1553
if self.user_transport.listable():
1554
# XXX: do we want to __define len__() ?
1555
# Maybe the versionedfiles object should provide a different
1556
# method to get the number of keys.
1557
result['revisions'] = len(self.revisions.keys())
1558
# result['size'] = t
1559
539
return result
1560
540
1561
541
def find_branches(self, using=False):
1572
552
def __init__(self):
1573
553
self.first_call = True
1574
554
1575
def __call__(self, bzrdir):
1576
# On the first call, the parameter is always the bzrdir
555
def __call__(self, controldir):
556
# On the first call, the parameter is always the controldir
1577
557
# containing the current repo.
1578
558
if not self.first_call:
1579
559
try:
1580
repository = bzrdir.open_repository()
560
repository = controldir.open_repository()
1581
561
except errors.NoRepositoryPresent:
1582
562
pass
1583
563
else:
1584
564
return False, ([], repository)
1585
565
self.first_call = False
1586
value = (bzrdir.list_branches(), None)
566
value = (controldir.list_branches(), None)
1587
567
return True, value
1588
568
1589
569
ret = []
1590
for branches, repository in bzrdir.BzrDir.find_bzrdirs(
570
for branches, repository in controldir.ControlDir.find_bzrdirs(
1591
571
self.user_transport, evaluate=Evaluator()):
1592
572
if branches is not None:
1593
573
ret.extend(branches)
1598
578
@needs_read_lock
1599
579
def search_missing_revision_ids(self, other,
1600
580
revision_id=symbol_versioning.DEPRECATED_PARAMETER,
1601
find_ghosts=True, revision_ids=None, if_present_ids=None):
581
find_ghosts=True, revision_ids=None, if_present_ids=None,
582
limit=None):
1602
583
"""Return the revision ids that other has that this does not.
1603
584
1604
585
These are returned in topological order.
1617
598
revision_ids = [revision_id]
1618
599
return InterRepository.get(other, self).search_missing_revision_ids(
1619
600
find_ghosts=find_ghosts, revision_ids=revision_ids,
1620
if_present_ids=if_present_ids)
601
if_present_ids=if_present_ids, limit=limit)
1621
602
1622
603
@staticmethod
1623
604
def open(base):
1626
607
For instance, if the repository is at URL/.bzr/repository,
1627
608
Repository.open(URL) -> a Repository instance.
1628
609
"""
1629
control = bzrdir.BzrDir.open(base)
610
control = controldir.ControlDir.open(base)
1630
611
return control.open_repository()
1631
612
1632
613
def copy_content_into(self, destination, revision_id=None):
1663
644
"""
1664
645
1665
646
def suspend_write_group(self):
647
"""Suspend a write group.
648
649
:raise UnsuspendableWriteGroup: If the write group can not be
650
suspended.
651
:return: List of tokens
652
"""
1666
653
raise errors.UnsuspendableWriteGroup(self)
1667
654
1668
def get_missing_parent_inventories(self, check_for_missing_texts=True):
1669
"""Return the keys of missing inventory parents for revisions added in
1670
this write group.
1671
1672
A revision is not complete if the inventory delta for that revision
1673
cannot be calculated. Therefore if the parent inventories of a
1674
revision are not present, the revision is incomplete, and e.g. cannot
1675
be streamed by a smart server. This method finds missing inventory
1676
parents for revisions added in this write group.
1677
"""
1678
if not self._format.supports_external_lookups:
1679
# This is only an issue for stacked repositories
1680
return set()
1681
if not self.is_in_write_group():
1682
raise AssertionError('not in a write group')
1683
1684
# XXX: We assume that every added revision already has its
1685
# corresponding inventory, so we only check for parent inventories that
1686
# might be missing, rather than all inventories.
1687
parents = set(self.revisions._index.get_missing_parents())
1688
parents.discard(_mod_revision.NULL_REVISION)
1689
unstacked_inventories = self.inventories._index
1690
present_inventories = unstacked_inventories.get_parent_map(
1691
key[-1:] for key in parents)
1692
parents.difference_update(present_inventories)
1693
if len(parents) == 0:
1694
# No missing parent inventories.
1695
return set()
1696
if not check_for_missing_texts:
1697
return set(('inventories', rev_id) for (rev_id,) in parents)
1698
# Ok, now we have a list of missing inventories. But these only matter
1699
# if the inventories that reference them are missing some texts they
1700
# appear to introduce.
1701
# XXX: Texts referenced by all added inventories need to be present,
1702
# but at the moment we're only checking for texts referenced by
1703
# inventories at the graph's edge.
1704
key_deps = self.revisions._index._key_dependencies
1705
key_deps.satisfy_refs_for_keys(present_inventories)
1706
referrers = frozenset(r[0] for r in key_deps.get_referrers())
1707
file_ids = self.fileids_altered_by_revision_ids(referrers)
1708
missing_texts = set()
1709
for file_id, version_ids in file_ids.iteritems():
1710
missing_texts.update(
1711
(file_id, version_id) for version_id in version_ids)
1712
present_texts = self.texts.get_parent_map(missing_texts)
1713
missing_texts.difference_update(present_texts)
1714
if not missing_texts:
1715
# No texts are missing, so all revisions and their deltas are
1716
# reconstructable.
1717
return set()
1718
# Alternatively the text versions could be returned as the missing
1719
# keys, but this is likely to be less data.
1720
missing_keys = set(('inventories', rev_id) for (rev_id,) in parents)
1721
return missing_keys
1722
1723
655
def refresh_data(self):
1724
656
"""Re-read any data needed to synchronise with disk.
1725
657
1745
677
def _resume_write_group(self, tokens):
1746
678
raise errors.UnsuspendableWriteGroup(self)
1747
679
1748
def fetch(self, source, revision_id=None, pb=None, find_ghosts=False,
1749
fetch_spec=None):
680
def fetch(self, source, revision_id=None, find_ghosts=False):
1750
681
"""Fetch the content required to construct revision_id from source.
1751
682
1752
If revision_id is None and fetch_spec is None, then all content is
1753
copied.
683
If revision_id is None, then all content is copied.
1754
684
1755
685
fetch() may not be used when the repository is in a write group -
1756
686
either finish the current write group before using fetch, or use
1762
692
:param revision_id: If specified, all the content needed for this
1763
693
revision ID will be copied to the target. Fetch will determine for
1764
694
itself which content needs to be copied.
1765
:param fetch_spec: If specified, a SearchResult or
1766
PendingAncestryResult that describes which revisions to copy. This
1767
allows copying multiple heads at once. Mutually exclusive with
1768
revision_id.
1769
695
"""
1770
if fetch_spec is not None and revision_id is not None:
1771
raise AssertionError(
1772
"fetch_spec and revision_id are mutually exclusive.")
1773
696
if self.is_in_write_group():
1774
697
raise errors.InternalBzrError(
1775
698
"May not fetch while in a write group.")
1777
700
# TODO: lift out to somewhere common with RemoteRepository
1778
701
# <https://bugs.launchpad.net/bzr/+bug/401646>
1779
702
if (self.has_same_location(source)
1780
and fetch_spec is None
1781
703
and self._has_same_fallbacks(source)):
1782
704
# check that last_revision is in 'from' and then return a
1783
705
# no-operation.
1786
708
self.get_revision(revision_id)
1787
709
return 0, []
1788
710
inter = InterRepository.get(source, self)
1789
return inter.fetch(revision_id=revision_id, pb=pb,
1790
find_ghosts=find_ghosts, fetch_spec=fetch_spec)
711
return inter.fetch(revision_id=revision_id, find_ghosts=find_ghosts)
1791
712
1792
713
def create_bundle(self, target, base, fileobj, format=None):
1793
714
return serializer.write_bundle(self, target, base, fileobj, format)
1794
715
1795
def get_commit_builder(self, branch, parents, config, timestamp=None,
716
def get_commit_builder(self, branch, parents, config_stack, timestamp=None,
1796
717
timezone=None, committer=None, revprops=None,
1797
revision_id=None):
718
revision_id=None, lossy=False):
1798
719
"""Obtain a CommitBuilder for this repository.
1799
720
1800
721
:param branch: Branch to commit to.
1801
722
:param parents: Revision ids of the parents of the new revision.
1802
:param config: Configuration to use.
723
:param config_stack: Configuration stack to use.
1803
724
:param timestamp: Optional timestamp recorded for commit.
1804
725
:param timezone: Optional timezone for timestamp.
1805
726
:param committer: Optional committer to set for commit.
1806
727
:param revprops: Optional dictionary of revision properties.
1807
728
:param revision_id: Optional revision id.
729
:param lossy: Whether to discard data that can not be natively
730
represented, when pushing to a foreign VCS
1808
731
"""
1809
if self._fallback_repositories and not self._format.supports_chks:
1810
raise errors.BzrError("Cannot commit directly to a stacked branch"
1811
" in pre-2a formats. See "
1812
"https://bugs.launchpad.net/bzr/+bug/375013 for details.")
1813
result = self._commit_builder_class(self, parents, config,
1814
timestamp, timezone, committer, revprops, revision_id)
1815
self.start_write_group()
1816
return result
732
raise NotImplementedError(self.get_commit_builder)
1817
733
1818
734
@only_raises(errors.LockNotHeld, errors.LockBroken)
1819
735
def unlock(self):
1826
742
'Must end write groups before releasing write locks.')
1827
743
self.control_files.unlock()
1828
744
if self.control_files._lock_count == 0:
1829
self._inventory_entry_cache.clear()
1830
745
for repo in self._fallback_repositories:
1831
746
repo.unlock()
1832
747
1833
748
@needs_read_lock
1834
def clone(self, a_bzrdir, revision_id=None):
1835
"""Clone this repository into a_bzrdir using the current format.
749
def clone(self, controldir, revision_id=None):
750
"""Clone this repository into controldir using the current format.
1836
751
1837
752
Currently no check is made that the format of this repository and
1838
753
the bzrdir format are compatible. FIXME RBC 20060201.
1841
756
"""
1842
757
# TODO: deprecate after 0.16; cloning this with all its settings is
1843
758
# probably not very useful -- mbp 20070423
1844
dest_repo = self._create_sprouting_repo(a_bzrdir, shared=self.is_shared())
759
dest_repo = self._create_sprouting_repo(
760
controldir, shared=self.is_shared())
1845
761
self.copy_content_into(dest_repo, revision_id)
1846
762
return dest_repo
1847
763
1897
813
dest_repo = a_bzrdir.open_repository()
1898
814
return dest_repo
1899
815
1900
def _get_sink(self):
1901
"""Return a sink for streaming into this repository."""
1902
return StreamSink(self)
1903
1904
def _get_source(self, to_format):
1905
"""Return a source for streaming from this repository."""
1906
return StreamSource(self, to_format)
1907
1908
816
@needs_read_lock
1909
817
def has_revision(self, revision_id):
1910
818
"""True if this repository has a copy of the revision."""
1917
825
:param revision_ids: An iterable of revision_ids.
1918
826
:return: A set of the revision_ids that were present.
1919
827
"""
1920
parent_map = self.revisions.get_parent_map(
1921
[(rev_id,) for rev_id in revision_ids])
1922
result = set()
1923
if _mod_revision.NULL_REVISION in revision_ids:
1924
result.add(_mod_revision.NULL_REVISION)
1925
result.update([key[0] for key in parent_map])
1926
return result
828
raise NotImplementedError(self.has_revisions)
1927
829
1928
830
@needs_read_lock
1929
831
def get_revision(self, revision_id):
1930
832
"""Return the Revision object for a named revision."""
1931
833
return self.get_revisions([revision_id])[0]
1932
834
1933
@needs_read_lock
1934
835
def get_revision_reconcile(self, revision_id):
1935
836
"""'reconcile' helper routine that allows access to a revision always.
1936
837
1939
840
be used by reconcile, or reconcile-alike commands that are correcting
1940
841
or testing the revision graph.
1941
842
"""
1942
return self._get_revisions([revision_id])[0]
843
raise NotImplementedError(self.get_revision_reconcile)
1943
844
1944
@needs_read_lock
1945
845
def get_revisions(self, revision_ids):
1946
846
"""Get many revisions at once.
1947
847
1948
848
Repositories that need to check data on every revision read should
1949
849
subclass this method.
1950
850
"""
1951
return self._get_revisions(revision_ids)
1952
1953
@needs_read_lock
1954
def _get_revisions(self, revision_ids):
1955
"""Core work logic to get many revisions without sanity checks."""
1956
revs = {}
1957
for revid, rev in self._iter_revisions(revision_ids):
1958
if rev is None:
1959
raise errors.NoSuchRevision(self, revid)
1960
revs[revid] = rev
1961
return [revs[revid] for revid in revision_ids]
1962
1963
def _iter_revisions(self, revision_ids):
1964
"""Iterate over revision objects.
1965
1966
:param revision_ids: An iterable of revisions to examine. None may be
1967
passed to request all revisions known to the repository. Note that
1968
not all repositories can find unreferenced revisions; for those
1969
repositories only referenced ones will be returned.
1970
:return: An iterator of (revid, revision) tuples. Absent revisions (
1971
those asked for but not available) are returned as (revid, None).
1972
"""
1973
if revision_ids is None:
1974
revision_ids = self.all_revision_ids()
1975
else:
1976
for rev_id in revision_ids:
1977
if not rev_id or not isinstance(rev_id, basestring):
1978
raise errors.InvalidRevisionId(revision_id=rev_id, branch=self)
1979
keys = [(key,) for key in revision_ids]
1980
stream = self.revisions.get_record_stream(keys, 'unordered', True)
1981
for record in stream:
1982
revid = record.key[0]
1983
if record.storage_kind == 'absent':
1984
yield (revid, None)
1985
else:
1986
text = record.get_bytes_as('fulltext')
1987
rev = self._serializer.read_revision_from_string(text)
1988
yield (revid, rev)
851
raise NotImplementedError(self.get_revisions)
1989
852
1990
853
def get_deltas_for_revisions(self, revisions, specific_fileids=None):
1991
854
"""Produce a generator of revision deltas.
2046
909
signature = gpg_strategy.sign(plaintext)
2047
910
self.add_signature_text(revision_id, signature)
2048
911
2049
@needs_write_lock
2050
912
def add_signature_text(self, revision_id, signature):
2051
self.signatures.add_lines((revision_id,), (),
2052
osutils.split_lines(signature))
2053
2054
def find_text_key_references(self):
2055
"""Find the text key references within the repository.
2056
2057
:return: A dictionary mapping text keys ((fileid, revision_id) tuples)
2058
to whether they were referred to by the inventory of the
2059
revision_id that they contain. The inventory texts from all present
2060
revision ids are assessed to generate this report.
2061
"""
2062
revision_keys = self.revisions.keys()
2063
w = self.inventories
2064
pb = ui.ui_factory.nested_progress_bar()
2065
try:
2066
return self._find_text_key_references_from_xml_inventory_lines(
2067
w.iter_lines_added_or_present_in_keys(revision_keys, pb=pb))
2068
finally:
2069
pb.finished()
2070
2071
def _find_text_key_references_from_xml_inventory_lines(self,
2072
line_iterator):
2073
"""Core routine for extracting references to texts from inventories.
2074
2075
This performs the translation of xml lines to revision ids.
2076
2077
:param line_iterator: An iterator of lines, origin_version_id
2078
:return: A dictionary mapping text keys ((fileid, revision_id) tuples)
2079
to whether they were referred to by the inventory of the
2080
revision_id that they contain. Note that if that revision_id was
2081
not part of the line_iterator's output then False will be given -
2082
even though it may actually refer to that key.
2083
"""
2084
if not self._serializer.support_altered_by_hack:
2085
raise AssertionError(
2086
"_find_text_key_references_from_xml_inventory_lines only "
2087
"supported for branches which store inventory as unnested xml"
2088
", not on %r" % self)
2089
result = {}
2090
2091
# this code needs to read every new line in every inventory for the
2092
# inventories [revision_ids]. Seeing a line twice is ok. Seeing a line
2093
# not present in one of those inventories is unnecessary but not
2094
# harmful because we are filtering by the revision id marker in the
2095
# inventory lines : we only select file ids altered in one of those
2096
# revisions. We don't need to see all lines in the inventory because
2097
# only those added in an inventory in rev X can contain a revision=X
2098
# line.
2099
unescape_revid_cache = {}
2100
unescape_fileid_cache = {}
2101
2102
# jam 20061218 In a big fetch, this handles hundreds of thousands
2103
# of lines, so it has had a lot of inlining and optimizing done.
2104
# Sorry that it is a little bit messy.
2105
# Move several functions to be local variables, since this is a long
2106
# running loop.
2107
search = self._file_ids_altered_regex.search
2108
unescape = _unescape_xml
2109
setdefault = result.setdefault
2110
for line, line_key in line_iterator:
2111
match = search(line)
2112
if match is None:
2113
continue
2114
# One call to match.group() returning multiple items is quite a
2115
# bit faster than 2 calls to match.group() each returning 1
2116
file_id, revision_id = match.group('file_id', 'revision_id')
2117
2118
# Inlining the cache lookups helps a lot when you make 170,000
2119
# lines and 350k ids, versus 8.4 unique ids.
2120
# Using a cache helps in 2 ways:
2121
# 1) Avoids unnecessary decoding calls
2122
# 2) Re-uses cached strings, which helps in future set and
2123
# equality checks.
2124
# (2) is enough that removing encoding entirely along with
2125
# the cache (so we are using plain strings) results in no
2126
# performance improvement.
2127
try:
2128
revision_id = unescape_revid_cache[revision_id]
2129
except KeyError:
2130
unescaped = unescape(revision_id)
2131
unescape_revid_cache[revision_id] = unescaped
2132
revision_id = unescaped
2133
2134
# Note that unconditionally unescaping means that we deserialise
2135
# every fileid, which for general 'pull' is not great, but we don't
2136
# really want to have some many fulltexts that this matters anyway.
2137
# RBC 20071114.
2138
try:
2139
file_id = unescape_fileid_cache[file_id]
2140
except KeyError:
2141
unescaped = unescape(file_id)
2142
unescape_fileid_cache[file_id] = unescaped
2143
file_id = unescaped
2144
2145
key = (file_id, revision_id)
2146
setdefault(key, False)
2147
if revision_id == line_key[-1]:
2148
result[key] = True
2149
return result
2150
2151
def _inventory_xml_lines_for_keys(self, keys):
2152
"""Get a line iterator of the sort needed for findind references.
2153
2154
Not relevant for non-xml inventory repositories.
2155
2156
Ghosts in revision_keys are ignored.
2157
2158
:param revision_keys: The revision keys for the inventories to inspect.
2159
:return: An iterator over (inventory line, revid) for the fulltexts of
2160
all of the xml inventories specified by revision_keys.
2161
"""
2162
stream = self.inventories.get_record_stream(keys, 'unordered', True)
2163
for record in stream:
2164
if record.storage_kind != 'absent':
2165
chunks = record.get_bytes_as('chunked')
2166
revid = record.key[-1]
2167
lines = osutils.chunks_to_lines(chunks)
2168
for line in lines:
2169
yield line, revid
2170
2171
def _find_file_ids_from_xml_inventory_lines(self, line_iterator,
2172
revision_keys):
2173
"""Helper routine for fileids_altered_by_revision_ids.
2174
2175
This performs the translation of xml lines to revision ids.
2176
2177
:param line_iterator: An iterator of lines, origin_version_id
2178
:param revision_keys: The revision ids to filter for. This should be a
2179
set or other type which supports efficient __contains__ lookups, as
2180
the revision key from each parsed line will be looked up in the
2181
revision_keys filter.
2182
:return: a dictionary mapping altered file-ids to an iterable of
2183
revision_ids. Each altered file-ids has the exact revision_ids that
2184
altered it listed explicitly.
2185
"""
2186
seen = set(self._find_text_key_references_from_xml_inventory_lines(
2187
line_iterator).iterkeys())
2188
parent_keys = self._find_parent_keys_of_revisions(revision_keys)
2189
parent_seen = set(self._find_text_key_references_from_xml_inventory_lines(
2190
self._inventory_xml_lines_for_keys(parent_keys)))
2191
new_keys = seen - parent_seen
2192
result = {}
2193
setdefault = result.setdefault
2194
for key in new_keys:
2195
setdefault(key[0], set()).add(key[-1])
2196
return result
913
"""Store a signature text for a revision.
914
915
:param revision_id: Revision id of the revision
916
:param signature: Signature text.
917
"""
918
raise NotImplementedError(self.add_signature_text)
2197
919
2198
920
def _find_parent_ids_of_revisions(self, revision_ids):
2199
921
"""Find all parent ids that are mentioned in the revision graph.
2208
930
parent_ids.discard(_mod_revision.NULL_REVISION)
2209
931
return parent_ids
2210
932
2211
def _find_parent_keys_of_revisions(self, revision_keys):
2212
"""Similar to _find_parent_ids_of_revisions, but used with keys.
2213
2214
:param revision_keys: An iterable of revision_keys.
2215
:return: The parents of all revision_keys that are not already in
2216
revision_keys
2217
"""
2218
parent_map = self.revisions.get_parent_map(revision_keys)
2219
parent_keys = set()
2220
map(parent_keys.update, parent_map.itervalues())
2221
parent_keys.difference_update(revision_keys)
2222
parent_keys.discard(_mod_revision.NULL_REVISION)
2223
return parent_keys
2224
2225
def fileids_altered_by_revision_ids(self, revision_ids, _inv_weave=None):
2226
"""Find the file ids and versions affected by revisions.
2227
2228
:param revisions: an iterable containing revision ids.
2229
:param _inv_weave: The inventory weave from this repository or None.
2230
If None, the inventory weave will be opened automatically.
2231
:return: a dictionary mapping altered file-ids to an iterable of
2232
revision_ids. Each altered file-ids has the exact revision_ids that
2233
altered it listed explicitly.
2234
"""
2235
selected_keys = set((revid,) for revid in revision_ids)
2236
w = _inv_weave or self.inventories
2237
return self._find_file_ids_from_xml_inventory_lines(
2238
w.iter_lines_added_or_present_in_keys(
2239
selected_keys, pb=None),
2240
selected_keys)
2241
2242
933
def iter_files_bytes(self, desired_files):
2243
934
"""Iterate through file versions.
2244
935
2250
941
uniquely identify the file version in the caller's context. (Examples:
2251
942
an index number or a TreeTransform trans_id.)
2252
943
2253
bytes_iterator is an iterable of bytestrings for the file. The
2254
kind of iterable and length of the bytestrings are unspecified, but for
2255
this implementation, it is a list of bytes produced by
2256
VersionedFile.get_record_stream().
2257
2258
944
:param desired_files: a list of (file_id, revision_id, identifier)
2259
945
triples
2260
946
"""
2261
text_keys = {}
2262
for file_id, revision_id, callable_data in desired_files:
2263
text_keys[(file_id, revision_id)] = callable_data
2264
for record in self.texts.get_record_stream(text_keys, 'unordered', True):
2265
if record.storage_kind == 'absent':
2266
raise errors.RevisionNotPresent(record.key, self)
2267
yield text_keys[record.key], record.get_bytes_as('chunked')
2268
2269
def _generate_text_key_index(self, text_key_references=None,
2270
ancestors=None):
2271
"""Generate a new text key index for the repository.
2272
2273
This is an expensive function that will take considerable time to run.
2274
2275
:return: A dict mapping text keys ((file_id, revision_id) tuples) to a
2276
list of parents, also text keys. When a given key has no parents,
2277
the parents list will be [NULL_REVISION].
2278
"""
2279
# All revisions, to find inventory parents.
2280
if ancestors is None:
2281
graph = self.get_graph()
2282
ancestors = graph.get_parent_map(self.all_revision_ids())
2283
if text_key_references is None:
2284
text_key_references = self.find_text_key_references()
2285
pb = ui.ui_factory.nested_progress_bar()
2286
try:
2287
return self._do_generate_text_key_index(ancestors,
2288
text_key_references, pb)
2289
finally:
2290
pb.finished()
2291
2292
def _do_generate_text_key_index(self, ancestors, text_key_references, pb):
2293
"""Helper for _generate_text_key_index to avoid deep nesting."""
2294
revision_order = tsort.topo_sort(ancestors)
2295
invalid_keys = set()
2296
revision_keys = {}
2297
for revision_id in revision_order:
2298
revision_keys[revision_id] = set()
2299
text_count = len(text_key_references)
2300
# a cache of the text keys to allow reuse; costs a dict of all the
2301
# keys, but saves a 2-tuple for every child of a given key.
2302
text_key_cache = {}
2303
for text_key, valid in text_key_references.iteritems():
2304
if not valid:
2305
invalid_keys.add(text_key)
2306
else:
2307
revision_keys[text_key[1]].add(text_key)
2308
text_key_cache[text_key] = text_key
2309
del text_key_references
2310
text_index = {}
2311
text_graph = graph.Graph(graph.DictParentsProvider(text_index))
2312
NULL_REVISION = _mod_revision.NULL_REVISION
2313
# Set a cache with a size of 10 - this suffices for bzr.dev but may be
2314
# too small for large or very branchy trees. However, for 55K path
2315
# trees, it would be easy to use too much memory trivially. Ideally we
2316
# could gauge this by looking at available real memory etc, but this is
2317
# always a tricky proposition.
2318
inventory_cache = lru_cache.LRUCache(10)
2319
batch_size = 10 # should be ~150MB on a 55K path tree
2320
batch_count = len(revision_order) / batch_size + 1
2321
processed_texts = 0
2322
pb.update("Calculating text parents", processed_texts, text_count)
2323
for offset in xrange(batch_count):
2324
to_query = revision_order[offset * batch_size:(offset + 1) *
2325
batch_size]
2326
if not to_query:
2327
break
2328
for revision_id in to_query:
2329
parent_ids = ancestors[revision_id]
2330
for text_key in revision_keys[revision_id]:
2331
pb.update("Calculating text parents", processed_texts)
2332
processed_texts += 1
2333
candidate_parents = []
2334
for parent_id in parent_ids:
2335
parent_text_key = (text_key[0], parent_id)
2336
try:
2337
check_parent = parent_text_key not in \
2338
revision_keys[parent_id]
2339
except KeyError:
2340
# the parent parent_id is a ghost:
2341
check_parent = False
2342
# truncate the derived graph against this ghost.
2343
parent_text_key = None
2344
if check_parent:
2345
# look at the parent commit details inventories to
2346
# determine possible candidates in the per file graph.
2347
# TODO: cache here.
2348
try:
2349
inv = inventory_cache[parent_id]
2350
except KeyError:
2351
inv = self.revision_tree(parent_id).inventory
2352
inventory_cache[parent_id] = inv
2353
try:
2354
parent_entry = inv[text_key[0]]
2355
except (KeyError, errors.NoSuchId):
2356
parent_entry = None
2357
if parent_entry is not None:
2358
parent_text_key = (
2359
text_key[0], parent_entry.revision)
2360
else:
2361
parent_text_key = None
2362
if parent_text_key is not None:
2363
candidate_parents.append(
2364
text_key_cache[parent_text_key])
2365
parent_heads = text_graph.heads(candidate_parents)
2366
new_parents = list(parent_heads)
2367
new_parents.sort(key=lambda x:candidate_parents.index(x))
2368
if new_parents == []:
2369
new_parents = [NULL_REVISION]
2370
text_index[text_key] = new_parents
2371
2372
for text_key in invalid_keys:
2373
text_index[text_key] = [NULL_REVISION]
2374
return text_index
2375
2376
def item_keys_introduced_by(self, revision_ids, _files_pb=None):
2377
"""Get an iterable listing the keys of all the data introduced by a set
2378
of revision IDs.
2379
2380
The keys will be ordered so that the corresponding items can be safely
2381
fetched and inserted in that order.
2382
2383
:returns: An iterable producing tuples of (knit-kind, file-id,
2384
versions). knit-kind is one of 'file', 'inventory', 'signatures',
2385
'revisions'. file-id is None unless knit-kind is 'file'.
2386
"""
2387
for result in self._find_file_keys_to_fetch(revision_ids, _files_pb):
2388
yield result
2389
del _files_pb
2390
for result in self._find_non_file_keys_to_fetch(revision_ids):
2391
yield result
2392
2393
def _find_file_keys_to_fetch(self, revision_ids, pb):
2394
# XXX: it's a bit weird to control the inventory weave caching in this
2395
# generator. Ideally the caching would be done in fetch.py I think. Or
2396
# maybe this generator should explicitly have the contract that it
2397
# should not be iterated until the previously yielded item has been
2398
# processed?
2399
inv_w = self.inventories
2400
2401
# file ids that changed
2402
file_ids = self.fileids_altered_by_revision_ids(revision_ids, inv_w)
2403
count = 0
2404
num_file_ids = len(file_ids)
2405
for file_id, altered_versions in file_ids.iteritems():
2406
if pb is not None:
2407
pb.update("Fetch texts", count, num_file_ids)
2408
count += 1
2409
yield ("file", file_id, altered_versions)
2410
2411
def _find_non_file_keys_to_fetch(self, revision_ids):
2412
# inventory
2413
yield ("inventory", None, revision_ids)
2414
2415
# signatures
2416
# XXX: Note ATM no callers actually pay attention to this return
2417
# instead they just use the list of revision ids and ignore
2418
# missing sigs. Consider removing this work entirely
2419
revisions_with_signatures = set(self.signatures.get_parent_map(
2420
[(r,) for r in revision_ids]))
2421
revisions_with_signatures = set(
2422
[r for (r,) in revisions_with_signatures])
2423
revisions_with_signatures.intersection_update(revision_ids)
2424
yield ("signatures", None, revisions_with_signatures)
2425
2426
# revisions
2427
yield ("revisions", None, revision_ids)
2428
2429
@needs_read_lock
2430
def get_inventory(self, revision_id):
2431
"""Get Inventory object by revision id."""
2432
return self.iter_inventories([revision_id]).next()
2433
2434
def iter_inventories(self, revision_ids, ordering=None):
2435
"""Get many inventories by revision_ids.
2436
2437
This will buffer some or all of the texts used in constructing the
2438
inventories in memory, but will only parse a single inventory at a
2439
time.
2440
2441
:param revision_ids: The expected revision ids of the inventories.
2442
:param ordering: optional ordering, e.g. 'topological'. If not
2443
specified, the order of revision_ids will be preserved (by
2444
buffering if necessary).
2445
:return: An iterator of inventories.
2446
"""
2447
if ((None in revision_ids)
2448
or (_mod_revision.NULL_REVISION in revision_ids)):
2449
raise ValueError('cannot get null revision inventory')
2450
return self._iter_inventories(revision_ids, ordering)
2451
2452
def _iter_inventories(self, revision_ids, ordering):
2453
"""single-document based inventory iteration."""
2454
inv_xmls = self._iter_inventory_xmls(revision_ids, ordering)
2455
for text, revision_id in inv_xmls:
2456
yield self._deserialise_inventory(revision_id, text)
2457
2458
def _iter_inventory_xmls(self, revision_ids, ordering):
2459
if ordering is None:
2460
order_as_requested = True
2461
ordering = 'unordered'
2462
else:
2463
order_as_requested = False
2464
keys = [(revision_id,) for revision_id in revision_ids]
2465
if not keys:
2466
return
2467
if order_as_requested:
2468
key_iter = iter(keys)
2469
next_key = key_iter.next()
2470
stream = self.inventories.get_record_stream(keys, ordering, True)
2471
text_chunks = {}
2472
for record in stream:
2473
if record.storage_kind != 'absent':
2474
chunks = record.get_bytes_as('chunked')
2475
if order_as_requested:
2476
text_chunks[record.key] = chunks
2477
else:
2478
yield ''.join(chunks), record.key[-1]
2479
else:
2480
raise errors.NoSuchRevision(self, record.key)
2481
if order_as_requested:
2482
# Yield as many results as we can while preserving order.
2483
while next_key in text_chunks:
2484
chunks = text_chunks.pop(next_key)
2485
yield ''.join(chunks), next_key[-1]
2486
try:
2487
next_key = key_iter.next()
2488
except StopIteration:
2489
# We still want to fully consume the get_record_stream,
2490
# just in case it is not actually finished at this point
2491
next_key = None
2492
break
2493
2494
def _deserialise_inventory(self, revision_id, xml):
2495
"""Transform the xml into an inventory object.
2496
2497
:param revision_id: The expected revision id of the inventory.
2498
:param xml: A serialised inventory.
2499
"""
2500
result = self._serializer.read_inventory_from_string(xml, revision_id,
2501
entry_cache=self._inventory_entry_cache,
2502
return_from_cache=self._safe_to_return_from_cache)
2503
if result.revision_id != revision_id:
2504
raise AssertionError('revision id mismatch %s != %s' % (
2505
result.revision_id, revision_id))
2506
return result
2507
2508
def get_serializer_format(self):
2509
return self._serializer.format_num
2510
2511
@needs_read_lock
2512
def _get_inventory_xml(self, revision_id):
2513
"""Get serialized inventory as a string."""
2514
texts = self._iter_inventory_xmls([revision_id], 'unordered')
2515
try:
2516
text, revision_id = texts.next()
2517
except StopIteration:
2518
raise errors.HistoryMissing(self, 'inventory', revision_id)
2519
return text
947
raise NotImplementedError(self.iter_files_bytes)
2520
948
2521
949
def get_rev_id_for_revno(self, revno, known_pair):
2522
950
"""Return the revision id of a revno, given a later (revno, revid)
2553
981
raise AssertionError('_iter_for_revno returned too much history')
2554
982
return (True, partial_history[-1])
2555
983
2556
def iter_reverse_revision_history(self, revision_id):
2557
"""Iterate backwards through revision ids in the lefthand history
2558
2559
:param revision_id: The revision id to start with. All its lefthand
2560
ancestors will be traversed.
2561
"""
2562
graph = self.get_graph()
2563
stop_revisions = (None, _mod_revision.NULL_REVISION)
2564
return graph.iter_lefthand_ancestry(revision_id, stop_revisions)
2565
2566
984
def is_shared(self):
2567
985
"""Return True if this repository is flagged as a shared repository."""
2568
986
raise NotImplementedError(self.is_shared)
2595
1013
2596
1014
`revision_id` may be NULL_REVISION for the empty tree revision.
2597
1015
"""
2598
revision_id = _mod_revision.ensure_null(revision_id)
2599
# TODO: refactor this to use an existing revision object
2600
# so we don't need to read it in twice.
2601
if revision_id == _mod_revision.NULL_REVISION:
2602
return RevisionTree(self, Inventory(root_id=None),
2603
_mod_revision.NULL_REVISION)
2604
else:
2605
inv = self.get_inventory(revision_id)
2606
return RevisionTree(self, inv, revision_id)
1016
raise NotImplementedError(self.revision_tree)
2607
1017
2608
1018
def revision_trees(self, revision_ids):
2609
1019
"""Return Trees for revisions in this repository.
2611
1021
:param revision_ids: a sequence of revision-ids;
2612
1022
a revision-id may not be None or 'null:'
2613
1023
"""
2614
inventories = self.iter_inventories(revision_ids)
2615
for inv in inventories:
2616
yield RevisionTree(self, inv, inv.revision_id)
2617
2618
def _filtered_revision_trees(self, revision_ids, file_ids):
2619
"""Return Tree for a revision on this branch with only some files.
2620
2621
:param revision_ids: a sequence of revision-ids;
2622
a revision-id may not be None or 'null:'
2623
:param file_ids: if not None, the result is filtered
2624
so that only those file-ids, their parents and their
2625
children are included.
2626
"""
2627
inventories = self.iter_inventories(revision_ids)
2628
for inv in inventories:
2629
# Should we introduce a FilteredRevisionTree class rather
2630
# than pre-filter the inventory here?
2631
filtered_inv = inv.filter(file_ids)
2632
yield RevisionTree(self, filtered_inv, filtered_inv.revision_id)
2633
2634
@needs_read_lock
2635
def get_ancestry(self, revision_id, topo_sorted=True):
2636
"""Return a list of revision-ids integrated by a revision.
2637
2638
The first element of the list is always None, indicating the origin
2639
revision. This might change when we have history horizons, or
2640
perhaps we should have a new API.
2641
2642
This is topologically sorted.
2643
"""
2644
if _mod_revision.is_null(revision_id):
2645
return [None]
2646
if not self.has_revision(revision_id):
2647
raise errors.NoSuchRevision(self, revision_id)
2648
graph = self.get_graph()
2649
keys = set()
2650
search = graph._make_breadth_first_searcher([revision_id])
2651
while True:
2652
try:
2653
found, ghosts = search.next_with_ghosts()
2654
except StopIteration:
2655
break
2656
keys.update(found)
2657
if _mod_revision.NULL_REVISION in keys:
2658
keys.remove(_mod_revision.NULL_REVISION)
2659
if topo_sorted:
2660
parent_map = graph.get_parent_map(keys)
2661
keys = tsort.topo_sort(parent_map)
2662
return [None] + list(keys)
1024
raise NotImplementedError(self.revision_trees)
2663
1025
2664
1026
def pack(self, hint=None, clean_obsolete_packs=False):
2665
1027
"""Compress the data within the repository.
2687
1049
2688
1050
def get_parent_map(self, revision_ids):
2689
1051
"""See graph.StackedParentsProvider.get_parent_map"""
1052
raise NotImplementedError(self.get_parent_map)
1053
1054
def _get_parent_map_no_fallbacks(self, revision_ids):
1055
"""Same as Repository.get_parent_map except doesn't query fallbacks."""
2690
1056
# revisions index works in keys; this just works in revisions
2691
1057
# therefore wrap and unwrap
2692
1058
query_keys = []
2698
1064
raise ValueError('get_parent_map(None) is not valid')
2699
1065
else:
2700
1066
query_keys.append((revision_id ,))
1067
vf = self.revisions.without_fallbacks()
2701
1068
for ((revision_id,), parent_keys) in \
2702
self.revisions.get_parent_map(query_keys).iteritems():
1069
vf.get_parent_map(query_keys).iteritems():
2703
1070
if parent_keys:
2704
1071
result[revision_id] = tuple([parent_revid
2705
1072
for (parent_revid,) in parent_keys])
2708
1075
return result
2709
1076
2710
1077
def _make_parents_provider(self):
2711
return self
1078
if not self._format.supports_external_lookups:
1079
return self
1080
return graph.StackedParentsProvider(_LazyListJoin(
1081
[self._make_parents_provider_unstacked()],
1082
self._fallback_repositories))
1083
1084
def _make_parents_provider_unstacked(self):
1085
return graph.CallableToParentsProviderAdapter(
1086
self._get_parent_map_no_fallbacks)
2712
1087
2713
1088
@needs_read_lock
2714
1089
def get_known_graph_ancestry(self, revision_ids):
2715
1090
"""Return the known graph for a set of revision ids and their ancestors.
2716
1091
"""
2717
st = static_tuple.StaticTuple
2718
revision_keys = [st(r_id).intern() for r_id in revision_ids]
2719
known_graph = self.revisions.get_known_graph_ancestry(revision_keys)
2720
return graph.GraphThunkIdsToKeys(known_graph)
1092
raise NotImplementedError(self.get_known_graph_ancestry)
1093
1094
def get_file_graph(self):
1095
"""Return the graph walker for files."""
1096
raise NotImplementedError(self.get_file_graph)
2721
1097
2722
1098
def get_graph(self, other_repository=None):
2723
1099
"""Return the graph walker for this repository format"""
2728
1104
[parents_provider, other_repository._make_parents_provider()])
2729
1105
return graph.Graph(parents_provider)
2730
1106
2731
def _get_versioned_file_checker(self, text_key_references=None,
2732
ancestors=None):
2733
"""Return an object suitable for checking versioned files.
2734
2735
:param text_key_references: if non-None, an already built
2736
dictionary mapping text keys ((fileid, revision_id) tuples)
2737
to whether they were referred to by the inventory of the
2738
revision_id that they contain. If None, this will be
2739
calculated.
2740
:param ancestors: Optional result from
2741
self.get_graph().get_parent_map(self.all_revision_ids()) if already
2742
available.
2743
"""
2744
return _VersionedFileChecker(self,
2745
text_key_references=text_key_references, ancestors=ancestors)
2746
2747
def revision_ids_to_search_result(self, result_set):
2748
"""Convert a set of revision ids to a graph SearchResult."""
2749
result_parents = set()
2750
for parents in self.get_graph().get_parent_map(
2751
result_set).itervalues():
2752
result_parents.update(parents)
2753
included_keys = result_set.intersection(result_parents)
2754
start_keys = result_set.difference(included_keys)
2755
exclude_keys = result_parents.difference(result_set)
2756
result = graph.SearchResult(start_keys, exclude_keys,
2757
len(result_set), result_set)
2758
return result
2759
2760
1107
@needs_write_lock
2761
1108
def set_make_working_trees(self, new_value):
2762
1109
"""Set the policy flag for making working trees when creating branches.
2775
1122
2776
1123
@needs_write_lock
2777
1124
def sign_revision(self, revision_id, gpg_strategy):
2778
plaintext = Testament.from_revision(self, revision_id).as_short_text()
1125
testament = _mod_testament.Testament.from_revision(self, revision_id)
1126
plaintext = testament.as_short_text()
2779
1127
self.store_revision_signature(gpg_strategy, plaintext, revision_id)
2780
1128
2781
1129
@needs_read_lock
1130
def verify_revision_signature(self, revision_id, gpg_strategy):
1131
"""Verify the signature on a revision.
1132
1133
:param revision_id: the revision to verify
1134
:gpg_strategy: the GPGStrategy object to used
1135
1136
:return: gpg.SIGNATURE_VALID or a failed SIGNATURE_ value
1137
"""
1138
if not self.has_signature_for_revision_id(revision_id):
1139
return gpg.SIGNATURE_NOT_SIGNED, None
1140
signature = self.get_signature_text(revision_id)
1141
1142
testament = _mod_testament.Testament.from_revision(self, revision_id)
1143
plaintext = testament.as_short_text()
1144
1145
return gpg_strategy.verify(signature, plaintext)
1146
1147
@needs_read_lock
1148
def verify_revision_signatures(self, revision_ids, gpg_strategy):
1149
"""Verify revision signatures for a number of revisions.
1150
1151
:param revision_id: the revision to verify
1152
:gpg_strategy: the GPGStrategy object to used
1153
:return: Iterator over tuples with revision id, result and keys
1154
"""
1155
for revid in revision_ids:
1156
(result, key) = self.verify_revision_signature(revid, gpg_strategy)
1157
yield revid, result, key
1158
2782
1159
def has_signature_for_revision_id(self, revision_id):
2783
1160
"""Query for a revision signature for revision_id in the repository."""
2784
if not self.has_revision(revision_id):
2785
raise errors.NoSuchRevision(self, revision_id)
2786
sig_present = (1 == len(
2787
self.signatures.get_parent_map([(revision_id,)])))
2788
return sig_present
1161
raise NotImplementedError(self.has_signature_for_revision_id)
2789
1162
2790
@needs_read_lock
2791
1163
def get_signature_text(self, revision_id):
2792
1164
"""Return the text for a signature."""
2793
stream = self.signatures.get_record_stream([(revision_id,)],
2794
'unordered', True)
2795
record = stream.next()
2796
if record.storage_kind == 'absent':
2797
raise errors.NoSuchRevision(self, revision_id)
2798
return record.get_bytes_as('fulltext')
1165
raise NotImplementedError(self.get_signature_text)
2799
1166
2800
@needs_read_lock
2801
1167
def check(self, revision_ids=None, callback_refs=None, check_repo=True):
2802
1168
"""Check consistency of all history of given revision_ids.
2803
1169
2811
1177
:param check_repo: If False do not check the repository contents, just
2812
1178
calculate the data callback_refs requires and call them back.
2813
1179
"""
2814
return self._check(revision_ids, callback_refs=callback_refs,
1180
return self._check(revision_ids=revision_ids, callback_refs=callback_refs,
2815
1181
check_repo=check_repo)
2816
1182
2817
def _check(self, revision_ids, callback_refs, check_repo):
2818
result = check.Check(self, check_repo=check_repo)
2819
result.check(callback_refs)
2820
return result
1183
def _check(self, revision_ids=None, callback_refs=None, check_repo=True):
1184
raise NotImplementedError(self.check)
2821
1185
2822
1186
def _warn_if_deprecated(self, branch=None):
1187
if not self._format.is_deprecated():
1188
return
2823
1189
global _deprecation_warning_done
2824
1190
if _deprecation_warning_done:
2825
1191
return
2826
1192
try:
2827
1193
if branch is None:
2828
conf = config.GlobalConfig()
1194
conf = config.GlobalStack()
2829
1195
else:
2830
conf = branch.get_config()
2831
if conf.suppress_warning('format_deprecation'):
1196
conf = branch.get_config_stack()
1197
if 'format_deprecation' in conf.get('suppress_warnings'):
2832
1198
return
2833
1199
warning("Format %s for %s is deprecated -"
2834
1200
" please use 'bzr upgrade' to get better performance"
2855
1221
except UnicodeDecodeError:
2856
1222
raise errors.NonAsciiRevisionId(method, self)
2857
1223
2858
def revision_graph_can_have_wrong_parents(self):
2859
"""Is it possible for this repository to have a revision graph with
2860
incorrect parents?
2861
2862
If True, then this repository must also implement
2863
_find_inconsistent_revision_parents so that check and reconcile can
2864
check for inconsistencies before proceeding with other checks that may
2865
depend on the revision index being consistent.
2866
"""
2867
raise NotImplementedError(self.revision_graph_can_have_wrong_parents)
2868
2869
2870
def install_revision(repository, rev, revision_tree):
2871
"""Install all revision data into a repository."""
2872
install_revisions(repository, [(rev, revision_tree, None)])
2873
2874
2875
def install_revisions(repository, iterable, num_revisions=None, pb=None):
2876
"""Install all revision data into a repository.
2877
2878
Accepts an iterable of revision, tree, signature tuples. The signature
2879
may be None.
2880
"""
2881
repository.start_write_group()
2882
try:
2883
inventory_cache = lru_cache.LRUCache(10)
2884
for n, (revision, revision_tree, signature) in enumerate(iterable):
2885
_install_revision(repository, revision, revision_tree, signature,
2886
inventory_cache)
2887
if pb is not None:
2888
pb.update('Transferring revisions', n + 1, num_revisions)
2889
except:
2890
repository.abort_write_group()
2891
raise
2892
else:
2893
repository.commit_write_group()
2894
2895
2896
def _install_revision(repository, rev, revision_tree, signature,
2897
inventory_cache):
2898
"""Install all revision data into a repository."""
2899
present_parents = []
2900
parent_trees = {}
2901
for p_id in rev.parent_ids:
2902
if repository.has_revision(p_id):
2903
present_parents.append(p_id)
2904
parent_trees[p_id] = repository.revision_tree(p_id)
2905
else:
2906
parent_trees[p_id] = repository.revision_tree(
2907
_mod_revision.NULL_REVISION)
2908
2909
inv = revision_tree.inventory
2910
entries = inv.iter_entries()
2911
# backwards compatibility hack: skip the root id.
2912
if not repository.supports_rich_root():
2913
path, root = entries.next()
2914
if root.revision != rev.revision_id:
2915
raise errors.IncompatibleRevision(repr(repository))
2916
text_keys = {}
2917
for path, ie in entries:
2918
text_keys[(ie.file_id, ie.revision)] = ie
2919
text_parent_map = repository.texts.get_parent_map(text_keys)
2920
missing_texts = set(text_keys) - set(text_parent_map)
2921
# Add the texts that are not already present
2922
for text_key in missing_texts:
2923
ie = text_keys[text_key]
2924
text_parents = []
2925
# FIXME: TODO: The following loop overlaps/duplicates that done by
2926
# commit to determine parents. There is a latent/real bug here where
2927
# the parents inserted are not those commit would do - in particular
2928
# they are not filtered by heads(). RBC, AB
2929
for revision, tree in parent_trees.iteritems():
2930
if ie.file_id not in tree:
2931
continue
2932
parent_id = tree.inventory[ie.file_id].revision
2933
if parent_id in text_parents:
2934
continue
2935
text_parents.append((ie.file_id, parent_id))
2936
lines = revision_tree.get_file(ie.file_id).readlines()
2937
repository.texts.add_lines(text_key, text_parents, lines)
2938
try:
2939
# install the inventory
2940
if repository._format._commit_inv_deltas and len(rev.parent_ids):
2941
# Cache this inventory
2942
inventory_cache[rev.revision_id] = inv
2943
try:
2944
basis_inv = inventory_cache[rev.parent_ids[0]]
2945
except KeyError:
2946
repository.add_inventory(rev.revision_id, inv, present_parents)
2947
else:
2948
delta = inv._make_delta(basis_inv)
2949
repository.add_inventory_by_delta(rev.parent_ids[0], delta,
2950
rev.revision_id, present_parents)
2951
else:
2952
repository.add_inventory(rev.revision_id, inv, present_parents)
2953
except errors.RevisionAlreadyPresent:
2954
pass
2955
if signature is not None:
2956
repository.add_signature_text(rev.revision_id, signature)
2957
repository.add_revision(rev.revision_id, rev, inv)
2958
2959
1224
2960
1225
class MetaDirRepository(Repository):
2961
1226
"""Repositories in the new meta-dir layout.
2995
1260
"""Returns the policy for making working trees on new branches."""
2996
1261
return not self._transport.has('no-working-trees')
2997
1262
2998
2999
class MetaDirVersionedFileRepository(MetaDirRepository):
3000
"""Repositories in a meta-dir, that work via versioned file objects."""
3001
3002
def __init__(self, _format, a_bzrdir, control_files):
3003
super(MetaDirVersionedFileRepository, self).__init__(_format, a_bzrdir,
3004
control_files)
3005
3006
3007
class RepositoryFormatRegistry(registry.FormatRegistry):
1263
@needs_write_lock
1264
def update_feature_flags(self, updated_flags):
1265
"""Update the feature flags for this branch.
1266
1267
:param updated_flags: Dictionary mapping feature names to necessities
1268
A necessity can be None to indicate the feature should be removed
1269
"""
1270
self._format._update_feature_flags(updated_flags)
1271
self.control_transport.put_bytes('format', self._format.as_string())
1272
1273
1274
class RepositoryFormatRegistry(controldir.ControlComponentFormatRegistry):
3008
1275
"""Repository format registry."""
3009
1276
3010
def __init__(self, other_registry=None):
3011
super(RepositoryFormatRegistry, self).__init__(other_registry)
3012
self._extra_formats = []
3013
3014
def register(self, format):
3015
"""Register a new repository format."""
3016
super(RepositoryFormatRegistry, self).register(
3017
format.get_format_string(), format)
3018
3019
def remove(self, format):
3020
"""Remove a registered repository format."""
3021
super(RepositoryFormatRegistry, self).remove(
3022
format.get_format_string())
3023
3024
def register_extra(self, format):
3025
"""Register a repository format that can not be used in a metadir.
3026
3027
This is mainly useful to allow custom repository formats, such as older
3028
Bazaar formats and foreign formats, to be tested.
3029
"""
3030
self._extra_formats.append(registry._ObjectGetter(format))
3031
3032
def remove_extra(self, format):
3033
"""Remove an extra repository format.
3034
"""
3035
self._extra_formats.remove(registry._ObjectGetter(format))
3036
3037
def register_extra_lazy(self, module_name, member_name):
3038
"""Register a repository format lazily.
3039
"""
3040
self._extra_formats.append(
3041
registry._LazyObjectGetter(module_name, member_name))
3042
3043
1277
def get_default(self):
3044
1278
"""Return the current default format."""
3045
from bzrlib import bzrdir
3046
return bzrdir.format_registry.make_bzrdir('default').repository_format
3047
3048
def _get_extra(self):
3049
result = []
3050
for getter in self._extra_formats:
3051
f = getter.get_obj()
3052
if callable(f):
3053
f = f()
3054
result.append(f)
3055
return result
3056
3057
def _get_all(self):
3058
"""Return all repository formats, even those not usable in metadirs.
3059
"""
3060
return [self.get(k) for k in self.keys()] + self._get_extra()
1279
return controldir.format_registry.make_bzrdir('default').repository_format
3061
1280
3062
1281
3063
1282
network_format_registry = registry.FormatRegistry()
3080
1299
#####################################################################
3081
1300
# Repository Formats
3082
1301
3083
class RepositoryFormat(object):
1302
class RepositoryFormat(controldir.ControlComponentFormat):
3084
1303
"""A repository format.
3085
1304
3086
1305
Formats provide four things:
3106
1325
created.
3107
1326
3108
1327
Common instance attributes:
3109
_matchingbzrdir - the bzrdir format that the repository format was
1328
_matchingbzrdir - the controldir format that the repository format was
3110
1329
originally written to work with. This can be used if manually
3111
1330
constructing a bzrdir and repository, or more commonly for test suite
3112
1331
parameterization.
3121
1340
# Does this format support CHK bytestring lookups. Set to True or False in
3122
1341
# derived classes.
3123
1342
supports_chks = None
3124
# Should commit add an inventory, or an inventory delta to the repository.
3125
_commit_inv_deltas = True
3126
# What order should fetch operations request streams in?
3127
# The default is unordered as that is the cheapest for an origin to
3128
# provide.
3129
_fetch_order = 'unordered'
3130
# Does this repository format use deltas that can be fetched as-deltas ?
3131
# (E.g. knits, where the knit deltas can be transplanted intact.
3132
# We default to False, which will ensure that enough data to get
3133
# a full text out of any fetch stream will be grabbed.
3134
_fetch_uses_deltas = False
3135
1343
# Should fetch trigger a reconcile after the fetch? Only needed for
3136
1344
# some repository formats that can suffer internal inconsistencies.
3137
1345
_fetch_reconcile = False
3143
1351
# help), and for fetching when data won't have come from the same
3144
1352
# compressor.
3145
1353
pack_compresses = False
3146
# Does the repository inventory storage understand references to trees?
1354
# Does the repository storage understand references to trees?
3147
1355
supports_tree_reference = None
3148
1356
# Is the format experimental ?
3149
1357
experimental = False
3150
# Does this repository format escape funky characters, or does it create files with
3151
# similar names as the versioned files in its contents on disk ?
3152
supports_funky_characters = True
1358
# Does this repository format escape funky characters, or does it create
1359
# files with similar names as the versioned files in its contents on disk
1360
# ?
1361
supports_funky_characters = None
1362
# Does this repository format support leaving locks?
1363
supports_leaving_lock = None
1364
# Does this format support the full VersionedFiles interface?
1365
supports_full_versioned_files = None
1366
# Does this format support signing revision signatures?
1367
supports_revision_signatures = True
1368
# Can the revision graph have incorrect parents?
1369
revision_graph_can_have_wrong_parents = None
1370
# Does this format support rich root data?
1371
rich_root_data = None
1372
# Does this format support explicitly versioned directories?
1373
supports_versioned_directories = None
1374
# Can other repositories be nested into one of this format?
1375
supports_nesting_repositories = None
1376
# Is it possible for revisions to be present without being referenced
1377
# somewhere ?
1378
supports_unreferenced_revisions = None
3153
1379
3154
1380
def __repr__(self):
3155
1381
return "%s()" % self.__class__.__name__
3161
1387
def __ne__(self, other):
3162
1388
return not self == other
3163
1389
3164
@classmethod
3165
def find_format(klass, a_bzrdir):
3166
"""Return the format for the repository object in a_bzrdir.
3167
3168
This is used by bzr native formats that have a "format" file in
3169
the repository. Other methods may be used by different types of
3170
control directory.
3171
"""
3172
try:
3173
transport = a_bzrdir.get_repository_transport(None)
3174
format_string = transport.get_bytes("format")
3175
return format_registry.get(format_string)
3176
except errors.NoSuchFile:
3177
raise errors.NoRepositoryPresent(a_bzrdir)
3178
except KeyError:
3179
raise errors.UnknownFormatError(format=format_string,
3180
kind='repository')
3181
3182
@classmethod
3183
@symbol_versioning.deprecated_method(symbol_versioning.deprecated_in((2, 4, 0)))
3184
def register_format(klass, format):
3185
format_registry.register(format)
3186
3187
@classmethod
3188
@symbol_versioning.deprecated_method(symbol_versioning.deprecated_in((2, 4, 0)))
3189
def unregister_format(klass, format):
3190
format_registry.remove(format)
3191
3192
@classmethod
3193
@symbol_versioning.deprecated_method(symbol_versioning.deprecated_in((2, 4, 0)))
3194
def get_default_format(klass):
3195
"""Return the current default format."""
3196
return format_registry.get_default()
3197
3198
def get_format_string(self):
3199
"""Return the ASCII format string that identifies this format.
3200
3201
Note that in pre format ?? repositories the format string is
3202
not permitted nor written to disk.
3203
"""
3204
raise NotImplementedError(self.get_format_string)
3205
3206
1390
def get_format_description(self):
3207
1391
"""Return the short description for this format."""
3208
1392
raise NotImplementedError(self.get_format_description)
3209
1393
3210
# TODO: this shouldn't be in the base class, it's specific to things that
3211
# use weaves or knits -- mbp 20070207
3212
def _get_versioned_file_store(self,
3213
name,
3214
transport,
3215
control_files,
3216
prefixed=True,
3217
versionedfile_class=None,
3218
versionedfile_kwargs={},
3219
escaped=False):
3220
if versionedfile_class is None:
3221
versionedfile_class = self._versionedfile_class
3222
weave_transport = control_files._transport.clone(name)
3223
dir_mode = control_files._dir_mode
3224
file_mode = control_files._file_mode
3225
return VersionedFileStore(weave_transport, prefixed=prefixed,
3226
dir_mode=dir_mode,
3227
file_mode=file_mode,
3228
versionedfile_class=versionedfile_class,
3229
versionedfile_kwargs=versionedfile_kwargs,
3230
escaped=escaped)
3231
3232
def initialize(self, a_bzrdir, shared=False):
3233
"""Initialize a repository of this format in a_bzrdir.
3234
3235
:param a_bzrdir: The bzrdir to put the new repository in it.
1394
def initialize(self, controldir, shared=False):
1395
"""Initialize a repository of this format in controldir.
1396
1397
:param controldir: The controldir to put the new repository in it.
3236
1398
:param shared: The repository should be initialized as a sharable one.
3237
1399
:returns: The new repository object.
3238
1400
3239
1401
This may raise UninitializableFormat if shared repository are not
3240
compatible the a_bzrdir.
1402
compatible the controldir.
3241
1403
"""
3242
1404
raise NotImplementedError(self.initialize)
3243
1405
3250
1412
"""
3251
1413
return True
3252
1414
1415
def is_deprecated(self):
1416
"""Is this format deprecated?
1417
1418
Deprecated formats may trigger a user-visible warning recommending
1419
the user to upgrade. They are still fully supported.
1420
"""
1421
return False
1422
3253
1423
def network_name(self):
3254
1424
"""A simple byte string uniquely identifying this format for RPC calls.
3255
1425
3271
1441
'Does not support nested trees', target_format,
3272
1442
from_format=self)
3273
1443
3274
def open(self, a_bzrdir, _found=False):
3275
"""Return an instance of this format for the bzrdir a_bzrdir.
1444
def open(self, controldir, _found=False):
1445
"""Return an instance of this format for a controldir.
3276
1446
3277
1447
_found is a private parameter, do not use it.
3278
1448
"""
3279
1449
raise NotImplementedError(self.open)
3280
1450
3281
def _run_post_repo_init_hooks(self, repository, a_bzrdir, shared):
3282
from bzrlib.bzrdir import BzrDir, RepoInitHookParams
3283
hooks = BzrDir.hooks['post_repo_init']
1451
def _run_post_repo_init_hooks(self, repository, controldir, shared):
1452
from bzrlib.controldir import ControlDir, RepoInitHookParams
1453
hooks = ControlDir.hooks['post_repo_init']
3284
1454
if not hooks:
3285
1455
return
3286
params = RepoInitHookParams(repository, self, a_bzrdir, shared)
1456
params = RepoInitHookParams(repository, self, controldir, shared)
3287
1457
for hook in hooks:
3288
1458
hook(params)
3289
1459
3290
1460
3291
class MetaDirRepositoryFormat(RepositoryFormat):
1461
class RepositoryFormatMetaDir(bzrdir.BzrFormat, RepositoryFormat):
3292
1462
"""Common base class for the new repositories using the metadir layout."""
3293
1463
3294
1464
rich_root_data = False
3295
1465
supports_tree_reference = False
3296
1466
supports_external_lookups = False
1467
supports_leaving_lock = True
1468
supports_nesting_repositories = True
3297
1469
3298
1470
@property
3299
1471
def _matchingbzrdir(self):
3302
1474
return matching
3303
1475
3304
1476
def __init__(self):
3305
super(MetaDirRepositoryFormat, self).__init__()
1477
RepositoryFormat.__init__(self)
1478
bzrdir.BzrFormat.__init__(self)
3306
1479
3307
1480
def _create_control_files(self, a_bzrdir):
3308
1481
"""Create the required files and the initial control_files object."""
3332
1505
finally:
3333
1506
control_files.unlock()
3334
1507
3335
def network_name(self):
3336
"""Metadir formats have matching disk and network format strings."""
3337
return self.get_format_string()
3338
3339
3340
# Pre-0.8 formats that don't have a disk format string (because they are
3341
# versioned by the matching control directory). We use the control directories
3342
# disk format string as a key for the network_name because they meet the
3343
# constraints (simple string, unique, immutable).
3344
network_format_registry.register_lazy(
3345
"Bazaar-NG branch, format 5\n",
3346
'bzrlib.repofmt.weaverepo',
3347
'RepositoryFormat5',
3348
)
3349
network_format_registry.register_lazy(
3350
"Bazaar-NG branch, format 6\n",
3351
'bzrlib.repofmt.weaverepo',
3352
'RepositoryFormat6',
3353
)
3354
3355
format_registry.register_extra_lazy(
3356
'bzrlib.repofmt.weaverepo',
3357
'RepositoryFormat4')
3358
format_registry.register_extra_lazy(
3359
'bzrlib.repofmt.weaverepo',
3360
'RepositoryFormat5')
3361
format_registry.register_extra_lazy(
3362
'bzrlib.repofmt.weaverepo',
3363
'RepositoryFormat6')
1508
@classmethod
1509
def find_format(klass, a_bzrdir):
1510
"""Return the format for the repository object in a_bzrdir.
1511
1512
This is used by bzr native formats that have a "format" file in
1513
the repository. Other methods may be used by different types of
1514
control directory.
1515
"""
1516
try:
1517
transport = a_bzrdir.get_repository_transport(None)
1518
format_string = transport.get_bytes("format")
1519
except errors.NoSuchFile:
1520
raise errors.NoRepositoryPresent(a_bzrdir)
1521
return klass._find_format(format_registry, 'repository', format_string)
1522
1523
def check_support_status(self, allow_unsupported, recommend_upgrade=True,
1524
basedir=None):
1525
RepositoryFormat.check_support_status(self,
1526
allow_unsupported=allow_unsupported, recommend_upgrade=recommend_upgrade,
1527
basedir=basedir)
1528
bzrdir.BzrFormat.check_support_status(self, allow_unsupported=allow_unsupported,
1529
recommend_upgrade=recommend_upgrade, basedir=basedir)
1530
3364
1531
3365
1532
# formats which have no format string are not discoverable or independently
3366
1533
# creatable on disk, so are not registered in format_registry. They're
3367
# all in bzrlib.repofmt.weaverepo now. When an instance of one of these is
3368
# needed, it's constructed directly by the BzrDir. Non-native formats where
1534
# all in bzrlib.repofmt.knitreponow. When an instance of one of these is
1535
# needed, it's constructed directly by the ControlDir. Non-native formats where
3369
1536
# the repository is not separately opened are similar.
3370
1537
3371
1538
format_registry.register_lazy(
3372
'Bazaar-NG Repository format 7',
3373
'bzrlib.repofmt.weaverepo',
3374
'RepositoryFormat7'
3375
)
3376
3377
format_registry.register_lazy(
3378
1539
'Bazaar-NG Knit Repository Format 1',
3379
1540
'bzrlib.repofmt.knitrepo',
3380
1541
'RepositoryFormatKnit1',
3397
1558
# NOTE: These are experimental in 0.92. Stable in 1.0 and above
3398
1559
format_registry.register_lazy(
3399
1560
'Bazaar pack repository format 1 (needs bzr 0.92)\n',
3400
'bzrlib.repofmt.pack_repo',
1561
'bzrlib.repofmt.knitpack_repo',
3401
1562
'RepositoryFormatKnitPack1',
3402
1563
)
3403
1564
format_registry.register_lazy(
3404
1565
'Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n',
3405
'bzrlib.repofmt.pack_repo',
1566
'bzrlib.repofmt.knitpack_repo',
3406
1567
'RepositoryFormatKnitPack3',
3407
1568
)
3408
1569
format_registry.register_lazy(
3409
1570
'Bazaar pack repository format 1 with rich root (needs bzr 1.0)\n',
3410
'bzrlib.repofmt.pack_repo',
1571
'bzrlib.repofmt.knitpack_repo',
3411
1572
'RepositoryFormatKnitPack4',
3412
1573
)
3413
1574
format_registry.register_lazy(
3414
1575
'Bazaar RepositoryFormatKnitPack5 (bzr 1.6)\n',
3415
'bzrlib.repofmt.pack_repo',
1576
'bzrlib.repofmt.knitpack_repo',
3416
1577
'RepositoryFormatKnitPack5',
3417
1578
)
3418
1579
format_registry.register_lazy(
3419
1580
'Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6.1)\n',
3420
'bzrlib.repofmt.pack_repo',
1581
'bzrlib.repofmt.knitpack_repo',
3421
1582
'RepositoryFormatKnitPack5RichRoot',
3422
1583
)
3423
1584
format_registry.register_lazy(
3424
1585
'Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6)\n',
3425
'bzrlib.repofmt.pack_repo',
1586
'bzrlib.repofmt.knitpack_repo',
3426
1587
'RepositoryFormatKnitPack5RichRootBroken',
3427
1588
)
3428
1589
format_registry.register_lazy(
3429
1590
'Bazaar RepositoryFormatKnitPack6 (bzr 1.9)\n',
3430
'bzrlib.repofmt.pack_repo',
1591
'bzrlib.repofmt.knitpack_repo',
3431
1592
'RepositoryFormatKnitPack6',
3432
1593
)
3433
1594
format_registry.register_lazy(
3434
1595
'Bazaar RepositoryFormatKnitPack6RichRoot (bzr 1.9)\n',
3435
'bzrlib.repofmt.pack_repo',
1596
'bzrlib.repofmt.knitpack_repo',
3436
1597
'RepositoryFormatKnitPack6RichRoot',
3437
1598
)
3438
1599
format_registry.register_lazy(
3446
1607
format_registry.register_lazy(
3447
1608
("Bazaar development format 2 with subtree support "
3448
1609
"(needs bzr.dev from before 1.8)\n"),
3449
'bzrlib.repofmt.pack_repo',
1610
'bzrlib.repofmt.knitpack_repo',
3450
1611
'RepositoryFormatPackDevelopment2Subtree',
3451
1612
)
3452
1613
format_registry.register_lazy(
3468
1629
InterRepository.get(other).method_name(parameters).
3469
1630
"""
3470
1631
3471
_walk_to_common_revisions_batch_size = 50
3472
1632
_optimisers = []
3473
1633
"""The available optimised InterRepository types."""
3474
1634
3489
1649
self.target.fetch(self.source, revision_id=revision_id)
3490
1650
3491
1651
@needs_write_lock
3492
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
3493
fetch_spec=None):
1652
def fetch(self, revision_id=None, find_ghosts=False):
3494
1653
"""Fetch the content required to construct revision_id.
3495
1654
3496
1655
The content is copied from self.source to self.target.
3497
1656
3498
1657
:param revision_id: if None all content is copied, if NULL_REVISION no
3499
1658
content is copied.
3500
:param pb: ignored.
3501
1659
:return: None.
3502
1660
"""
3503
ui.ui_factory.warn_experimental_format_fetch(self)
3504
from bzrlib.fetch import RepoFetcher
3505
# See <https://launchpad.net/bugs/456077> asking for a warning here
3506
if self.source._format.network_name() != self.target._format.network_name():
3507
ui.ui_factory.show_user_warning('cross_format_fetch',
3508
from_format=self.source._format,
3509
to_format=self.target._format)
3510
f = RepoFetcher(to_repository=self.target,
3511
from_repository=self.source,
3512
last_revision=revision_id,
3513
fetch_spec=fetch_spec,
3514
find_ghosts=find_ghosts)
3515
3516
def _walk_to_common_revisions(self, revision_ids, if_present_ids=None):
3517
"""Walk out from revision_ids in source to revisions target has.
3518
3519
:param revision_ids: The start point for the search.
3520
:return: A set of revision ids.
3521
"""
3522
target_graph = self.target.get_graph()
3523
revision_ids = frozenset(revision_ids)
3524
if if_present_ids:
3525
all_wanted_revs = revision_ids.union(if_present_ids)
3526
else:
3527
all_wanted_revs = revision_ids
3528
missing_revs = set()
3529
source_graph = self.source.get_graph()
3530
# ensure we don't pay silly lookup costs.
3531
searcher = source_graph._make_breadth_first_searcher(all_wanted_revs)
3532
null_set = frozenset([_mod_revision.NULL_REVISION])
3533
searcher_exhausted = False
3534
while True:
3535
next_revs = set()
3536
ghosts = set()
3537
# Iterate the searcher until we have enough next_revs
3538
while len(next_revs) < self._walk_to_common_revisions_batch_size:
3539
try:
3540
next_revs_part, ghosts_part = searcher.next_with_ghosts()
3541
next_revs.update(next_revs_part)
3542
ghosts.update(ghosts_part)
3543
except StopIteration:
3544
searcher_exhausted = True
3545
break
3546
# If there are ghosts in the source graph, and the caller asked for
3547
# them, make sure that they are present in the target.
3548
# We don't care about other ghosts as we can't fetch them and
3549
# haven't been asked to.
3550
ghosts_to_check = set(revision_ids.intersection(ghosts))
3551
revs_to_get = set(next_revs).union(ghosts_to_check)
3552
if revs_to_get:
3553
have_revs = set(target_graph.get_parent_map(revs_to_get))
3554
# we always have NULL_REVISION present.
3555
have_revs = have_revs.union(null_set)
3556
# Check if the target is missing any ghosts we need.
3557
ghosts_to_check.difference_update(have_revs)
3558
if ghosts_to_check:
3559
# One of the caller's revision_ids is a ghost in both the
3560
# source and the target.
3561
raise errors.NoSuchRevision(
3562
self.source, ghosts_to_check.pop())
3563
missing_revs.update(next_revs - have_revs)
3564
# Because we may have walked past the original stop point, make
3565
# sure everything is stopped
3566
stop_revs = searcher.find_seen_ancestors(have_revs)
3567
searcher.stop_searching_any(stop_revs)
3568
if searcher_exhausted:
3569
break
3570
return searcher.get_result()
1661
raise NotImplementedError(self.fetch)
3571
1662
3572
1663
@needs_read_lock
3573
1664
def search_missing_revision_ids(self,
3574
1665
revision_id=symbol_versioning.DEPRECATED_PARAMETER,
3575
find_ghosts=True, revision_ids=None, if_present_ids=None):
1666
find_ghosts=True, revision_ids=None, if_present_ids=None,
1667
limit=None):
3576
1668
"""Return the revision ids that source has that target does not.
3577
1669
3578
1670
:param revision_id: only return revision ids included by this
3586
1678
to fetch for tags, which may reference absent revisions.
3587
1679
:param find_ghosts: If True find missing revisions in deep history
3588
1680
rather than just finding the surface difference.
1681
:param limit: Maximum number of revisions to return, topologically
1682
ordered
3589
1683
:return: A bzrlib.graph.SearchResult.
3590
1684
"""
3591
if symbol_versioning.deprecated_passed(revision_id):
3592
symbol_versioning.warn(
3593
'search_missing_revision_ids(revision_id=...) was '
3594
'deprecated in 2.4. Use revision_ids=[...] instead.',
3595
DeprecationWarning, stacklevel=2)
3596
if revision_ids is not None:
3597
raise AssertionError(
3598
'revision_ids is mutually exclusive with revision_id')
3599
if revision_id is not None:
3600
revision_ids = [revision_id]
3601
del revision_id
3602
# stop searching at found target revisions.
3603
if not find_ghosts and (revision_ids is not None or if_present_ids is
3604
not None):
3605
return self._walk_to_common_revisions(revision_ids,
3606
if_present_ids=if_present_ids)
3607
# generic, possibly worst case, slow code path.
3608
target_ids = set(self.target.all_revision_ids())
3609
source_ids = self._present_source_revisions_for(
3610
revision_ids, if_present_ids)
3611
result_set = set(source_ids).difference(target_ids)
3612
return self.source.revision_ids_to_search_result(result_set)
3613
3614
def _present_source_revisions_for(self, revision_ids, if_present_ids=None):
3615
"""Returns set of all revisions in ancestry of revision_ids present in
3616
the source repo.
3617
3618
:param revision_ids: if None, all revisions in source are returned.
3619
:param if_present_ids: like revision_ids, but if any/all of these are
3620
absent no error is raised.
3621
"""
3622
if revision_ids is not None or if_present_ids is not None:
3623
# First, ensure all specified revisions exist. Callers expect
3624
# NoSuchRevision when they pass absent revision_ids here.
3625
if revision_ids is None:
3626
revision_ids = set()
3627
if if_present_ids is None:
3628
if_present_ids = set()
3629
revision_ids = set(revision_ids)
3630
if_present_ids = set(if_present_ids)
3631
all_wanted_ids = revision_ids.union(if_present_ids)
3632
graph = self.source.get_graph()
3633
present_revs = set(graph.get_parent_map(all_wanted_ids))
3634
missing = revision_ids.difference(present_revs)
3635
if missing:
3636
raise errors.NoSuchRevision(self.source, missing.pop())
3637
found_ids = all_wanted_ids.intersection(present_revs)
3638
source_ids = [rev_id for (rev_id, parents) in
3639
graph.iter_ancestry(found_ids)
3640
if rev_id != _mod_revision.NULL_REVISION
3641
and parents is not None]
3642
else:
3643
source_ids = self.source.all_revision_ids()
3644
return set(source_ids)
1685
raise NotImplementedError(self.search_missing_revision_ids)
3645
1686
3646
1687
@staticmethod
3647
1688
def _same_model(source, target):
3668
1709
"different serializers")
3669
1710
3670
1711
3671
class InterSameDataRepository(InterRepository):
3672
"""Code for converting between repositories that represent the same data.
3673
3674
Data format and model must match for this to work.
3675
"""
3676
3677
@classmethod
3678
def _get_repo_format_to_test(self):
3679
"""Repository format for testing with.
3680
3681
InterSameData can pull from subtree to subtree and from non-subtree to
3682
non-subtree, so we test this with the richest repository format.
3683
"""
3684
from bzrlib.repofmt import knitrepo
3685
return knitrepo.RepositoryFormatKnit3()
3686
3687
@staticmethod
3688
def is_compatible(source, target):
3689
return InterRepository._same_model(source, target)
3690
3691
3692
class InterDifferingSerializer(InterRepository):
3693
3694
@classmethod
3695
def _get_repo_format_to_test(self):
3696
return None
3697
3698
@staticmethod
3699
def is_compatible(source, target):
3700
"""Be compatible with Knit2 source and Knit3 target"""
3701
# This is redundant with format.check_conversion_target(), however that
3702
# raises an exception, and we just want to say "False" as in we won't
3703
# support converting between these formats.
3704
if 'IDS_never' in debug.debug_flags:
3705
return False
3706
if source.supports_rich_root() and not target.supports_rich_root():
3707
return False
3708
if (source._format.supports_tree_reference
3709
and not target._format.supports_tree_reference):
3710
return False
3711
if target._fallback_repositories and target._format.supports_chks:
3712
# IDS doesn't know how to copy CHKs for the parent inventories it
3713
# adds to stacked repos.
3714
return False
3715
if 'IDS_always' in debug.debug_flags:
3716
return True
3717
# Only use this code path for local source and target. IDS does far
3718
# too much IO (both bandwidth and roundtrips) over a network.
3719
if not source.bzrdir.transport.base.startswith('file:///'):
3720
return False
3721
if not target.bzrdir.transport.base.startswith('file:///'):
3722
return False
3723
return True
3724
3725
def _get_trees(self, revision_ids, cache):
3726
possible_trees = []
3727
for rev_id in revision_ids:
3728
if rev_id in cache:
3729
possible_trees.append((rev_id, cache[rev_id]))
3730
else:
3731
# Not cached, but inventory might be present anyway.
3732
try:
3733
tree = self.source.revision_tree(rev_id)
3734
except errors.NoSuchRevision:
3735
# Nope, parent is ghost.
3736
pass
3737
else:
3738
cache[rev_id] = tree
3739
possible_trees.append((rev_id, tree))
3740
return possible_trees
3741
3742
def _get_delta_for_revision(self, tree, parent_ids, possible_trees):
3743
"""Get the best delta and base for this revision.
3744
3745
:return: (basis_id, delta)
3746
"""
3747
deltas = []
3748
# Generate deltas against each tree, to find the shortest.
3749
texts_possibly_new_in_tree = set()
3750
for basis_id, basis_tree in possible_trees:
3751
delta = tree.inventory._make_delta(basis_tree.inventory)
3752
for old_path, new_path, file_id, new_entry in delta:
3753
if new_path is None:
3754
# This file_id isn't present in the new rev, so we don't
3755
# care about it.
3756
continue
3757
if not new_path:
3758
# Rich roots are handled elsewhere...
3759
continue
3760
kind = new_entry.kind
3761
if kind != 'directory' and kind != 'file':
3762
# No text record associated with this inventory entry.
3763
continue
3764
# This is a directory or file that has changed somehow.
3765
texts_possibly_new_in_tree.add((file_id, new_entry.revision))
3766
deltas.append((len(delta), basis_id, delta))
3767
deltas.sort()
3768
return deltas[0][1:]
3769
3770
def _fetch_parent_invs_for_stacking(self, parent_map, cache):
3771
"""Find all parent revisions that are absent, but for which the
3772
inventory is present, and copy those inventories.
3773
3774
This is necessary to preserve correctness when the source is stacked
3775
without fallbacks configured. (Note that in cases like upgrade the
3776
source may be not have _fallback_repositories even though it is
3777
stacked.)
3778
"""
3779
parent_revs = set()
3780
for parents in parent_map.values():
3781
parent_revs.update(parents)
3782
present_parents = self.source.get_parent_map(parent_revs)
3783
absent_parents = set(parent_revs).difference(present_parents)
3784
parent_invs_keys_for_stacking = self.source.inventories.get_parent_map(
3785
(rev_id,) for rev_id in absent_parents)
3786
parent_inv_ids = [key[-1] for key in parent_invs_keys_for_stacking]
3787
for parent_tree in self.source.revision_trees(parent_inv_ids):
3788
current_revision_id = parent_tree.get_revision_id()
3789
parents_parents_keys = parent_invs_keys_for_stacking[
3790
(current_revision_id,)]
3791
parents_parents = [key[-1] for key in parents_parents_keys]
3792
basis_id = _mod_revision.NULL_REVISION
3793
basis_tree = self.source.revision_tree(basis_id)
3794
delta = parent_tree.inventory._make_delta(basis_tree.inventory)
3795
self.target.add_inventory_by_delta(
3796
basis_id, delta, current_revision_id, parents_parents)
3797
cache[current_revision_id] = parent_tree
3798
3799
def _fetch_batch(self, revision_ids, basis_id, cache):
3800
"""Fetch across a few revisions.
3801
3802
:param revision_ids: The revisions to copy
3803
:param basis_id: The revision_id of a tree that must be in cache, used
3804
as a basis for delta when no other base is available
3805
:param cache: A cache of RevisionTrees that we can use.
3806
:return: The revision_id of the last converted tree. The RevisionTree
3807
for it will be in cache
3808
"""
3809
# Walk though all revisions; get inventory deltas, copy referenced
3810
# texts that delta references, insert the delta, revision and
3811
# signature.
3812
root_keys_to_create = set()
3813
text_keys = set()
3814
pending_deltas = []
3815
pending_revisions = []
3816
parent_map = self.source.get_parent_map(revision_ids)
3817
self._fetch_parent_invs_for_stacking(parent_map, cache)
3818
self.source._safe_to_return_from_cache = True
3819
for tree in self.source.revision_trees(revision_ids):
3820
# Find a inventory delta for this revision.
3821
# Find text entries that need to be copied, too.
3822
current_revision_id = tree.get_revision_id()
3823
parent_ids = parent_map.get(current_revision_id, ())
3824
parent_trees = self._get_trees(parent_ids, cache)
3825
possible_trees = list(parent_trees)
3826
if len(possible_trees) == 0:
3827
# There either aren't any parents, or the parents are ghosts,
3828
# so just use the last converted tree.
3829
possible_trees.append((basis_id, cache[basis_id]))
3830
basis_id, delta = self._get_delta_for_revision(tree, parent_ids,
3831
possible_trees)
3832
revision = self.source.get_revision(current_revision_id)
3833
pending_deltas.append((basis_id, delta,
3834
current_revision_id, revision.parent_ids))
3835
if self._converting_to_rich_root:
3836
self._revision_id_to_root_id[current_revision_id] = \
3837
tree.get_root_id()
3838
# Determine which texts are in present in this revision but not in
3839
# any of the available parents.
3840
texts_possibly_new_in_tree = set()
3841
for old_path, new_path, file_id, entry in delta:
3842
if new_path is None:
3843
# This file_id isn't present in the new rev
3844
continue
3845
if not new_path:
3846
# This is the root
3847
if not self.target.supports_rich_root():
3848
# The target doesn't support rich root, so we don't
3849
# copy
3850
continue
3851
if self._converting_to_rich_root:
3852
# This can't be copied normally, we have to insert
3853
# it specially
3854
root_keys_to_create.add((file_id, entry.revision))
3855
continue
3856
kind = entry.kind
3857
texts_possibly_new_in_tree.add((file_id, entry.revision))
3858
for basis_id, basis_tree in possible_trees:
3859
basis_inv = basis_tree.inventory
3860
for file_key in list(texts_possibly_new_in_tree):
3861
file_id, file_revision = file_key
3862
try:
3863
entry = basis_inv[file_id]
3864
except errors.NoSuchId:
3865
continue
3866
if entry.revision == file_revision:
3867
texts_possibly_new_in_tree.remove(file_key)
3868
text_keys.update(texts_possibly_new_in_tree)
3869
pending_revisions.append(revision)
3870
cache[current_revision_id] = tree
3871
basis_id = current_revision_id
3872
self.source._safe_to_return_from_cache = False
3873
# Copy file texts
3874
from_texts = self.source.texts
3875
to_texts = self.target.texts
3876
if root_keys_to_create:
3877
root_stream = _mod_fetch._new_root_data_stream(
3878
root_keys_to_create, self._revision_id_to_root_id, parent_map,
3879
self.source)
3880
to_texts.insert_record_stream(root_stream)
3881
to_texts.insert_record_stream(from_texts.get_record_stream(
3882
text_keys, self.target._format._fetch_order,
3883
not self.target._format._fetch_uses_deltas))
3884
# insert inventory deltas
3885
for delta in pending_deltas:
3886
self.target.add_inventory_by_delta(*delta)
3887
if self.target._fallback_repositories:
3888
# Make sure this stacked repository has all the parent inventories
3889
# for the new revisions that we are about to insert. We do this
3890
# before adding the revisions so that no revision is added until
3891
# all the inventories it may depend on are added.
3892
# Note that this is overzealous, as we may have fetched these in an
3893
# earlier batch.
3894
parent_ids = set()
3895
revision_ids = set()
3896
for revision in pending_revisions:
3897
revision_ids.add(revision.revision_id)
3898
parent_ids.update(revision.parent_ids)
3899
parent_ids.difference_update(revision_ids)
3900
parent_ids.discard(_mod_revision.NULL_REVISION)
3901
parent_map = self.source.get_parent_map(parent_ids)
3902
# we iterate over parent_map and not parent_ids because we don't
3903
# want to try copying any revision which is a ghost
3904
for parent_tree in self.source.revision_trees(parent_map):
3905
current_revision_id = parent_tree.get_revision_id()
3906
parents_parents = parent_map[current_revision_id]
3907
possible_trees = self._get_trees(parents_parents, cache)
3908
if len(possible_trees) == 0:
3909
# There either aren't any parents, or the parents are
3910
# ghosts, so just use the last converted tree.
3911
possible_trees.append((basis_id, cache[basis_id]))
3912
basis_id, delta = self._get_delta_for_revision(parent_tree,
3913
parents_parents, possible_trees)
3914
self.target.add_inventory_by_delta(
3915
basis_id, delta, current_revision_id, parents_parents)
3916
# insert signatures and revisions
3917
for revision in pending_revisions:
3918
try:
3919
signature = self.source.get_signature_text(
3920
revision.revision_id)
3921
self.target.add_signature_text(revision.revision_id,
3922
signature)
3923
except errors.NoSuchRevision:
3924
pass
3925
self.target.add_revision(revision.revision_id, revision)
3926
return basis_id
3927
3928
def _fetch_all_revisions(self, revision_ids, pb):
3929
"""Fetch everything for the list of revisions.
3930
3931
:param revision_ids: The list of revisions to fetch. Must be in
3932
topological order.
3933
:param pb: A ProgressTask
3934
:return: None
3935
"""
3936
basis_id, basis_tree = self._get_basis(revision_ids[0])
3937
batch_size = 100
3938
cache = lru_cache.LRUCache(100)
3939
cache[basis_id] = basis_tree
3940
del basis_tree # We don't want to hang on to it here
3941
hints = []
3942
a_graph = None
3943
3944
for offset in range(0, len(revision_ids), batch_size):
3945
self.target.start_write_group()
3946
try:
3947
pb.update('Transferring revisions', offset,
3948
len(revision_ids))
3949
batch = revision_ids[offset:offset+batch_size]
3950
basis_id = self._fetch_batch(batch, basis_id, cache)
3951
except:
3952
self.source._safe_to_return_from_cache = False
3953
self.target.abort_write_group()
3954
raise
3955
else:
3956
hint = self.target.commit_write_group()
3957
if hint:
3958
hints.extend(hint)
3959
if hints and self.target._format.pack_compresses:
3960
self.target.pack(hint=hints)
3961
pb.update('Transferring revisions', len(revision_ids),
3962
len(revision_ids))
3963
3964
@needs_write_lock
3965
def fetch(self, revision_id=None, pb=None, find_ghosts=False,
3966
fetch_spec=None):
3967
"""See InterRepository.fetch()."""
3968
if fetch_spec is not None:
3969
revision_ids = fetch_spec.get_keys()
3970
else:
3971
revision_ids = None
3972
ui.ui_factory.warn_experimental_format_fetch(self)
3973
if (not self.source.supports_rich_root()
3974
and self.target.supports_rich_root()):
3975
self._converting_to_rich_root = True
3976
self._revision_id_to_root_id = {}
3977
else:
3978
self._converting_to_rich_root = False
3979
# See <https://launchpad.net/bugs/456077> asking for a warning here
3980
if self.source._format.network_name() != self.target._format.network_name():
3981
ui.ui_factory.show_user_warning('cross_format_fetch',
3982
from_format=self.source._format,
3983
to_format=self.target._format)
3984
if revision_ids is None:
3985
if revision_id:
3986
search_revision_ids = [revision_id]
3987
else:
3988
search_revision_ids = None
3989
revision_ids = self.target.search_missing_revision_ids(self.source,
3990
revision_ids=search_revision_ids,
3991
find_ghosts=find_ghosts).get_keys()
3992
if not revision_ids:
3993
return 0, 0
3994
revision_ids = tsort.topo_sort(
3995
self.source.get_graph().get_parent_map(revision_ids))
3996
if not revision_ids:
3997
return 0, 0
3998
# Walk though all revisions; get inventory deltas, copy referenced
3999
# texts that delta references, insert the delta, revision and
4000
# signature.
4001
if pb is None:
4002
my_pb = ui.ui_factory.nested_progress_bar()
4003
pb = my_pb
4004
else:
4005
symbol_versioning.warn(
4006
symbol_versioning.deprecated_in((1, 14, 0))
4007
% "pb parameter to fetch()")
4008
my_pb = None
4009
try:
4010
self._fetch_all_revisions(revision_ids, pb)
4011
finally:
4012
if my_pb is not None:
4013
my_pb.finished()
4014
return len(revision_ids), 0
4015
4016
def _get_basis(self, first_revision_id):
4017
"""Get a revision and tree which exists in the target.
4018
4019
This assumes that first_revision_id is selected for transmission
4020
because all other ancestors are already present. If we can't find an
4021
ancestor we fall back to NULL_REVISION since we know that is safe.
4022
4023
:return: (basis_id, basis_tree)
4024
"""
4025
first_rev = self.source.get_revision(first_revision_id)
4026
try:
4027
basis_id = first_rev.parent_ids[0]
4028
# only valid as a basis if the target has it
4029
self.target.get_revision(basis_id)
4030
# Try to get a basis tree - if it's a ghost it will hit the
4031
# NoSuchRevision case.
4032
basis_tree = self.source.revision_tree(basis_id)
4033
except (IndexError, errors.NoSuchRevision):
4034
basis_id = _mod_revision.NULL_REVISION
4035
basis_tree = self.source.revision_tree(basis_id)
4036
return basis_id, basis_tree
4037
4038
4039
InterRepository.register_optimiser(InterDifferingSerializer)
4040
InterRepository.register_optimiser(InterSameDataRepository)
4041
4042
4043
1712
class CopyConverter(object):
4044
1713
"""A repository conversion tool which just performs a copy of the content.
4045
1714
4066
1735
# trigger an assertion if not such
4067
1736
repo._format.get_format_string()
4068
1737
self.repo_dir = repo.bzrdir
4069
pb.update('Moving repository to repository.backup')
1738
pb.update(gettext('Moving repository to repository.backup'))
4070
1739
self.repo_dir.transport.move('repository', 'repository.backup')
4071
1740
backup_transport = self.repo_dir.transport.clone('repository.backup')
4072
1741
repo._format.check_conversion_target(self.target_format)
4073
1742
self.source_repo = repo._format.open(self.repo_dir,
4074
1743
_found=True,
4075
1744
_override_transport=backup_transport)
4076
pb.update('Creating new repository')
1745
pb.update(gettext('Creating new repository'))
4077
1746
converted = self.target_format.initialize(self.repo_dir,
4078
1747
self.source_repo.is_shared())
4079
1748
converted.lock_write()
4080
1749
try:
4081
pb.update('Copying content')
1750
pb.update(gettext('Copying content'))
4082
1751
self.source_repo.copy_content_into(converted)
4083
1752
finally:
4084
1753
converted.unlock()
4085
pb.update('Deleting old repository content')
1754
pb.update(gettext('Deleting old repository content'))
4086
1755
self.repo_dir.transport.delete_tree('repository.backup')
4087
ui.ui_factory.note('repository converted')
1756
ui.ui_factory.note(gettext('repository converted'))
4088
1757
pb.finished()
4089
1758
4090
1759
4091
_unescape_map = {
4092
'apos':"'",
4093
'quot':'"',
4094
'amp':'&',
4095
'lt':'<',
4096
'gt':'>'
4097
}
4098
4099
4100
def _unescaper(match, _map=_unescape_map):
4101
code = match.group(1)
4102
try:
4103
return _map[code]
4104
except KeyError:
4105
if not code.startswith('#'):
4106
raise
4107
return unichr(int(code[1:])).encode('utf8')
4108
4109
4110
_unescape_re = None
4111
4112
4113
def _unescape_xml(data):
4114
"""Unescape predefined XML entities in a string of data."""
4115
global _unescape_re
4116
if _unescape_re is None:
4117
_unescape_re = re.compile('\&([^;]*);')
4118
return _unescape_re.sub(_unescaper, data)
4119
4120
4121
class _VersionedFileChecker(object):
4122
4123
def __init__(self, repository, text_key_references=None, ancestors=None):
4124
self.repository = repository
4125
self.text_index = self.repository._generate_text_key_index(
4126
text_key_references=text_key_references, ancestors=ancestors)
4127
4128
def calculate_file_version_parents(self, text_key):
4129
"""Calculate the correct parents for a file version according to
4130
the inventories.
4131
"""
4132
parent_keys = self.text_index[text_key]
4133
if parent_keys == [_mod_revision.NULL_REVISION]:
4134
return ()
4135
return tuple(parent_keys)
4136
4137
def check_file_version_parents(self, texts, progress_bar=None):
4138
"""Check the parents stored in a versioned file are correct.
4139
4140
It also detects file versions that are not referenced by their
4141
corresponding revision's inventory.
4142
4143
:returns: A tuple of (wrong_parents, dangling_file_versions).
4144
wrong_parents is a dict mapping {revision_id: (stored_parents,
4145
correct_parents)} for each revision_id where the stored parents
4146
are not correct. dangling_file_versions is a set of (file_id,
4147
revision_id) tuples for versions that are present in this versioned
4148
file, but not used by the corresponding inventory.
4149
"""
4150
local_progress = None
4151
if progress_bar is None:
4152
local_progress = ui.ui_factory.nested_progress_bar()
4153
progress_bar = local_progress
4154
try:
4155
return self._check_file_version_parents(texts, progress_bar)
4156
finally:
4157
if local_progress:
4158
local_progress.finished()
4159
4160
def _check_file_version_parents(self, texts, progress_bar):
4161
"""See check_file_version_parents."""
4162
wrong_parents = {}
4163
self.file_ids = set([file_id for file_id, _ in
4164
self.text_index.iterkeys()])
4165
# text keys is now grouped by file_id
4166
n_versions = len(self.text_index)
4167
progress_bar.update('loading text store', 0, n_versions)
4168
parent_map = self.repository.texts.get_parent_map(self.text_index)
4169
# On unlistable transports this could well be empty/error...
4170
text_keys = self.repository.texts.keys()
4171
unused_keys = frozenset(text_keys) - set(self.text_index)
4172
for num, key in enumerate(self.text_index.iterkeys()):
4173
progress_bar.update('checking text graph', num, n_versions)
4174
correct_parents = self.calculate_file_version_parents(key)
4175
try:
4176
knit_parents = parent_map[key]
4177
except errors.RevisionNotPresent:
4178
# Missing text!
4179
knit_parents = None
4180
if correct_parents != knit_parents:
4181
wrong_parents[key] = (knit_parents, correct_parents)
4182
return wrong_parents, unused_keys
4183
4184
4185
def _old_get_graph(repository, revision_id):
4186
"""DO NOT USE. That is all. I'm serious."""
4187
graph = repository.get_graph()
4188
revision_graph = dict(((key, value) for key, value in
4189
graph.iter_ancestry([revision_id]) if value is not None))
4190
return _strip_NULL_ghosts(revision_graph)
4191
4192
4193
1760
def _strip_NULL_ghosts(revision_graph):
4194
1761
"""Also don't use this. more compatibility code for unmigrated clients."""
4195
1762
# Filter ghosts, and null:
4201
1768
return revision_graph
4202
1769
4203
1770
4204
class StreamSink(object):
4205
"""An object that can insert a stream into a repository.
4206
4207
This interface handles the complexity of reserialising inventories and
4208
revisions from different formats, and allows unidirectional insertion into
4209
stacked repositories without looking for the missing basis parents
4210
beforehand.
4211
"""
4212
4213
def __init__(self, target_repo):
4214
self.target_repo = target_repo
4215
4216
def insert_stream(self, stream, src_format, resume_tokens):
4217
"""Insert a stream's content into the target repository.
4218
4219
:param src_format: a bzr repository format.
4220
4221
:return: a list of resume tokens and an iterable of keys additional
4222
items required before the insertion can be completed.
4223
"""
4224
self.target_repo.lock_write()
4225
try:
4226
if resume_tokens:
4227
self.target_repo.resume_write_group(resume_tokens)
4228
is_resume = True
4229
else:
4230
self.target_repo.start_write_group()
4231
is_resume = False
4232
try:
4233
# locked_insert_stream performs a commit|suspend.
4234
missing_keys = self.insert_stream_without_locking(stream,
4235
src_format, is_resume)
4236
if missing_keys:
4237
# suspend the write group and tell the caller what we is
4238
# missing. We know we can suspend or else we would not have
4239
# entered this code path. (All repositories that can handle
4240
# missing keys can handle suspending a write group).
4241
write_group_tokens = self.target_repo.suspend_write_group()
4242
return write_group_tokens, missing_keys
4243
hint = self.target_repo.commit_write_group()
4244
to_serializer = self.target_repo._format._serializer
4245
src_serializer = src_format._serializer
4246
if (to_serializer != src_serializer and
4247
self.target_repo._format.pack_compresses):
4248
self.target_repo.pack(hint=hint)
4249
return [], set()
4250
except:
4251
self.target_repo.abort_write_group(suppress_errors=True)
4252
raise
4253
finally:
4254
self.target_repo.unlock()
4255
4256
def insert_stream_without_locking(self, stream, src_format,
4257
is_resume=False):
4258
"""Insert a stream's content into the target repository.
4259
4260
This assumes that you already have a locked repository and an active
4261
write group.
4262
4263
:param src_format: a bzr repository format.
4264
:param is_resume: Passed down to get_missing_parent_inventories to
4265
indicate if we should be checking for missing texts at the same
4266
time.
4267
4268
:return: A set of keys that are missing.
4269
"""
4270
if not self.target_repo.is_write_locked():
4271
raise errors.ObjectNotLocked(self)
4272
if not self.target_repo.is_in_write_group():
4273
raise errors.BzrError('you must already be in a write group')
4274
to_serializer = self.target_repo._format._serializer
4275
src_serializer = src_format._serializer
4276
new_pack = None
4277
if to_serializer == src_serializer:
4278
# If serializers match and the target is a pack repository, set the
4279
# write cache size on the new pack. This avoids poor performance
4280
# on transports where append is unbuffered (such as
4281
# RemoteTransport). This is safe to do because nothing should read
4282
# back from the target repository while a stream with matching
4283
# serialization is being inserted.
4284
# The exception is that a delta record from the source that should
4285
# be a fulltext may need to be expanded by the target (see
4286
# test_fetch_revisions_with_deltas_into_pack); but we take care to
4287
# explicitly flush any buffered writes first in that rare case.
4288
try:
4289
new_pack = self.target_repo._pack_collection._new_pack
4290
except AttributeError:
4291
# Not a pack repository
4292
pass
4293
else:
4294
new_pack.set_write_cache_size(1024*1024)
4295
for substream_type, substream in stream:
4296
if 'stream' in debug.debug_flags:
4297
mutter('inserting substream: %s', substream_type)
4298
if substream_type == 'texts':
4299
self.target_repo.texts.insert_record_stream(substream)
4300
elif substream_type == 'inventories':
4301
if src_serializer == to_serializer:
4302
self.target_repo.inventories.insert_record_stream(
4303
substream)
4304
else:
4305
self._extract_and_insert_inventories(
4306
substream, src_serializer)
4307
elif substream_type == 'inventory-deltas':
4308
self._extract_and_insert_inventory_deltas(
4309
substream, src_serializer)
4310
elif substream_type == 'chk_bytes':
4311
# XXX: This doesn't support conversions, as it assumes the
4312
# conversion was done in the fetch code.
4313
self.target_repo.chk_bytes.insert_record_stream(substream)
4314
elif substream_type == 'revisions':
4315
# This may fallback to extract-and-insert more often than
4316
# required if the serializers are different only in terms of
4317
# the inventory.
4318
if src_serializer == to_serializer:
4319
self.target_repo.revisions.insert_record_stream(substream)
4320
else:
4321
self._extract_and_insert_revisions(substream,
4322
src_serializer)
4323
elif substream_type == 'signatures':
4324
self.target_repo.signatures.insert_record_stream(substream)
4325
else:
4326
raise AssertionError('kaboom! %s' % (substream_type,))
4327
# Done inserting data, and the missing_keys calculations will try to
4328
# read back from the inserted data, so flush the writes to the new pack
4329
# (if this is pack format).
4330
if new_pack is not None:
4331
new_pack._write_data('', flush=True)
4332
# Find all the new revisions (including ones from resume_tokens)
4333
missing_keys = self.target_repo.get_missing_parent_inventories(
4334
check_for_missing_texts=is_resume)
4335
try:
4336
for prefix, versioned_file in (
4337
('texts', self.target_repo.texts),
4338
('inventories', self.target_repo.inventories),
4339
('revisions', self.target_repo.revisions),
4340
('signatures', self.target_repo.signatures),
4341
('chk_bytes', self.target_repo.chk_bytes),
4342
):
4343
if versioned_file is None:
4344
continue
4345
# TODO: key is often going to be a StaticTuple object
4346
# I don't believe we can define a method by which
4347
# (prefix,) + StaticTuple will work, though we could
4348
# define a StaticTuple.sq_concat that would allow you to
4349
# pass in either a tuple or a StaticTuple as the second
4350
# object, so instead we could have:
4351
# StaticTuple(prefix) + key here...
4352
missing_keys.update((prefix,) + key for key in
4353
versioned_file.get_missing_compression_parent_keys())
4354
except NotImplementedError:
4355
# cannot even attempt suspending, and missing would have failed
4356
# during stream insertion.
4357
missing_keys = set()
4358
return missing_keys
4359
4360
def _extract_and_insert_inventory_deltas(self, substream, serializer):
4361
target_rich_root = self.target_repo._format.rich_root_data
4362
target_tree_refs = self.target_repo._format.supports_tree_reference
4363
for record in substream:
4364
# Insert the delta directly
4365
inventory_delta_bytes = record.get_bytes_as('fulltext')
4366
deserialiser = inventory_delta.InventoryDeltaDeserializer()
4367
try:
4368
parse_result = deserialiser.parse_text_bytes(
4369
inventory_delta_bytes)
4370
except inventory_delta.IncompatibleInventoryDelta, err:
4371
trace.mutter("Incompatible delta: %s", err.msg)
4372
raise errors.IncompatibleRevision(self.target_repo._format)
4373
basis_id, new_id, rich_root, tree_refs, inv_delta = parse_result
4374
revision_id = new_id
4375
parents = [key[0] for key in record.parents]
4376
self.target_repo.add_inventory_by_delta(
4377
basis_id, inv_delta, revision_id, parents)
4378
4379
def _extract_and_insert_inventories(self, substream, serializer,
4380
parse_delta=None):
4381
"""Generate a new inventory versionedfile in target, converting data.
4382
4383
The inventory is retrieved from the source, (deserializing it), and
4384
stored in the target (reserializing it in a different format).
4385
"""
4386
target_rich_root = self.target_repo._format.rich_root_data
4387
target_tree_refs = self.target_repo._format.supports_tree_reference
4388
for record in substream:
4389
# It's not a delta, so it must be a fulltext in the source
4390
# serializer's format.
4391
bytes = record.get_bytes_as('fulltext')
4392
revision_id = record.key[0]
4393
inv = serializer.read_inventory_from_string(bytes, revision_id)
4394
parents = [key[0] for key in record.parents]
4395
self.target_repo.add_inventory(revision_id, inv, parents)
4396
# No need to keep holding this full inv in memory when the rest of
4397
# the substream is likely to be all deltas.
4398
del inv
4399
4400
def _extract_and_insert_revisions(self, substream, serializer):
4401
for record in substream:
4402
bytes = record.get_bytes_as('fulltext')
4403
revision_id = record.key[0]
4404
rev = serializer.read_revision_from_string(bytes)
4405
if rev.revision_id != revision_id:
4406
raise AssertionError('wtf: %s != %s' % (rev, revision_id))
4407
self.target_repo.add_revision(revision_id, rev)
4408
4409
def finished(self):
4410
if self.target_repo._format._fetch_reconcile:
4411
self.target_repo.reconcile()
4412
4413
4414
class StreamSource(object):
4415
"""A source of a stream for fetching between repositories."""
4416
4417
def __init__(self, from_repository, to_format):
4418
"""Create a StreamSource streaming from from_repository."""
4419
self.from_repository = from_repository
4420
self.to_format = to_format
4421
self._record_counter = RecordCounter()
4422
4423
def delta_on_metadata(self):
4424
"""Return True if delta's are permitted on metadata streams.
4425
4426
That is on revisions and signatures.
4427
"""
4428
src_serializer = self.from_repository._format._serializer
4429
target_serializer = self.to_format._serializer
4430
return (self.to_format._fetch_uses_deltas and
4431
src_serializer == target_serializer)
4432
4433
def _fetch_revision_texts(self, revs):
4434
# fetch signatures first and then the revision texts
4435
# may need to be a InterRevisionStore call here.
4436
from_sf = self.from_repository.signatures
4437
# A missing signature is just skipped.
4438
keys = [(rev_id,) for rev_id in revs]
4439
signatures = versionedfile.filter_absent(from_sf.get_record_stream(
4440
keys,
4441
self.to_format._fetch_order,
4442
not self.to_format._fetch_uses_deltas))
4443
# If a revision has a delta, this is actually expanded inside the
4444
# insert_record_stream code now, which is an alternate fix for
4445
# bug #261339
4446
from_rf = self.from_repository.revisions
4447
revisions = from_rf.get_record_stream(
4448
keys,
4449
self.to_format._fetch_order,
4450
not self.delta_on_metadata())
4451
return [('signatures', signatures), ('revisions', revisions)]
4452
4453
def _generate_root_texts(self, revs):
4454
"""This will be called by get_stream between fetching weave texts and
4455
fetching the inventory weave.
4456
"""
4457
if self._rich_root_upgrade():
4458
return _mod_fetch.Inter1and2Helper(
4459
self.from_repository).generate_root_texts(revs)
4460
else:
4461
return []
4462
4463
def get_stream(self, search):
4464
phase = 'file'
4465
revs = search.get_keys()
4466
graph = self.from_repository.get_graph()
4467
revs = tsort.topo_sort(graph.get_parent_map(revs))
4468
data_to_fetch = self.from_repository.item_keys_introduced_by(revs)
4469
text_keys = []
4470
for knit_kind, file_id, revisions in data_to_fetch:
4471
if knit_kind != phase:
4472
phase = knit_kind
4473
# Make a new progress bar for this phase
4474
if knit_kind == "file":
4475
# Accumulate file texts
4476
text_keys.extend([(file_id, revision) for revision in
4477
revisions])
4478
elif knit_kind == "inventory":
4479
# Now copy the file texts.
4480
from_texts = self.from_repository.texts
4481
yield ('texts', from_texts.get_record_stream(
4482
text_keys, self.to_format._fetch_order,
4483
not self.to_format._fetch_uses_deltas))
4484
# Cause an error if a text occurs after we have done the
4485
# copy.
4486
text_keys = None
4487
# Before we process the inventory we generate the root
4488
# texts (if necessary) so that the inventories references
4489
# will be valid.
4490
for _ in self._generate_root_texts(revs):
4491
yield _
4492
# we fetch only the referenced inventories because we do not
4493
# know for unselected inventories whether all their required
4494
# texts are present in the other repository - it could be
4495
# corrupt.
4496
for info in self._get_inventory_stream(revs):
4497
yield info
4498
elif knit_kind == "signatures":
4499
# Nothing to do here; this will be taken care of when
4500
# _fetch_revision_texts happens.
4501
pass
4502
elif knit_kind == "revisions":
4503
for record in self._fetch_revision_texts(revs):
4504
yield record
4505
else:
4506
raise AssertionError("Unknown knit kind %r" % knit_kind)
4507
4508
def get_stream_for_missing_keys(self, missing_keys):
4509
# missing keys can only occur when we are byte copying and not
4510
# translating (because translation means we don't send
4511
# unreconstructable deltas ever).
4512
keys = {}
4513
keys['texts'] = set()
4514
keys['revisions'] = set()
4515
keys['inventories'] = set()
4516
keys['chk_bytes'] = set()
4517
keys['signatures'] = set()
4518
for key in missing_keys:
4519
keys[key[0]].add(key[1:])
4520
if len(keys['revisions']):
4521
# If we allowed copying revisions at this point, we could end up
4522
# copying a revision without copying its required texts: a
4523
# violation of the requirements for repository integrity.
4524
raise AssertionError(
4525
'cannot copy revisions to fill in missing deltas %s' % (
4526
keys['revisions'],))
4527
for substream_kind, keys in keys.iteritems():
4528
vf = getattr(self.from_repository, substream_kind)
4529
if vf is None and keys:
4530
raise AssertionError(
4531
"cannot fill in keys for a versioned file we don't"
4532
" have: %s needs %s" % (substream_kind, keys))
4533
if not keys:
4534
# No need to stream something we don't have
4535
continue
4536
if substream_kind == 'inventories':
4537
# Some missing keys are genuinely ghosts, filter those out.
4538
present = self.from_repository.inventories.get_parent_map(keys)
4539
revs = [key[0] for key in present]
4540
# Get the inventory stream more-or-less as we do for the
4541
# original stream; there's no reason to assume that records
4542
# direct from the source will be suitable for the sink. (Think
4543
# e.g. 2a -> 1.9-rich-root).
4544
for info in self._get_inventory_stream(revs, missing=True):
4545
yield info
4546
continue
4547
4548
# Ask for full texts always so that we don't need more round trips
4549
# after this stream.
4550
# Some of the missing keys are genuinely ghosts, so filter absent
4551
# records. The Sink is responsible for doing another check to
4552
# ensure that ghosts don't introduce missing data for future
4553
# fetches.
4554
stream = versionedfile.filter_absent(vf.get_record_stream(keys,
4555
self.to_format._fetch_order, True))
4556
yield substream_kind, stream
4557
4558
def inventory_fetch_order(self):
4559
if self._rich_root_upgrade():
4560
return 'topological'
4561
else:
4562
return self.to_format._fetch_order
4563
4564
def _rich_root_upgrade(self):
4565
return (not self.from_repository._format.rich_root_data and
4566
self.to_format.rich_root_data)
4567
4568
def _get_inventory_stream(self, revision_ids, missing=False):
4569
from_format = self.from_repository._format
4570
if (from_format.supports_chks and self.to_format.supports_chks and
4571
from_format.network_name() == self.to_format.network_name()):
4572
raise AssertionError(
4573
"this case should be handled by GroupCHKStreamSource")
4574
elif 'forceinvdeltas' in debug.debug_flags:
4575
return self._get_convertable_inventory_stream(revision_ids,
4576
delta_versus_null=missing)
4577
elif from_format.network_name() == self.to_format.network_name():
4578
# Same format.
4579
return self._get_simple_inventory_stream(revision_ids,
4580
missing=missing)
4581
elif (not from_format.supports_chks and not self.to_format.supports_chks
4582
and from_format._serializer == self.to_format._serializer):
4583
# Essentially the same format.
4584
return self._get_simple_inventory_stream(revision_ids,
4585
missing=missing)
4586
else:
4587
# Any time we switch serializations, we want to use an
4588
# inventory-delta based approach.
4589
return self._get_convertable_inventory_stream(revision_ids,
4590
delta_versus_null=missing)
4591
4592
def _get_simple_inventory_stream(self, revision_ids, missing=False):
4593
# NB: This currently reopens the inventory weave in source;
4594
# using a single stream interface instead would avoid this.
4595
from_weave = self.from_repository.inventories
4596
if missing:
4597
delta_closure = True
4598
else:
4599
delta_closure = not self.delta_on_metadata()
4600
yield ('inventories', from_weave.get_record_stream(
4601
[(rev_id,) for rev_id in revision_ids],
4602
self.inventory_fetch_order(), delta_closure))
4603
4604
def _get_convertable_inventory_stream(self, revision_ids,
4605
delta_versus_null=False):
4606
# The two formats are sufficiently different that there is no fast
4607
# path, so we need to send just inventorydeltas, which any
4608
# sufficiently modern client can insert into any repository.
4609
# The StreamSink code expects to be able to
4610
# convert on the target, so we need to put bytes-on-the-wire that can
4611
# be converted. That means inventory deltas (if the remote is <1.19,
4612
# RemoteStreamSink will fallback to VFS to insert the deltas).
4613
yield ('inventory-deltas',
4614
self._stream_invs_as_deltas(revision_ids,
4615
delta_versus_null=delta_versus_null))
4616
4617
def _stream_invs_as_deltas(self, revision_ids, delta_versus_null=False):
4618
"""Return a stream of inventory-deltas for the given rev ids.
4619
4620
:param revision_ids: The list of inventories to transmit
4621
:param delta_versus_null: Don't try to find a minimal delta for this
4622
entry, instead compute the delta versus the NULL_REVISION. This
4623
effectively streams a complete inventory. Used for stuff like
4624
filling in missing parents, etc.
4625
"""
4626
from_repo = self.from_repository
4627
revision_keys = [(rev_id,) for rev_id in revision_ids]
4628
parent_map = from_repo.inventories.get_parent_map(revision_keys)
4629
# XXX: possibly repos could implement a more efficient iter_inv_deltas
4630
# method...
4631
inventories = self.from_repository.iter_inventories(
4632
revision_ids, 'topological')
4633
format = from_repo._format
4634
invs_sent_so_far = set([_mod_revision.NULL_REVISION])
4635
inventory_cache = lru_cache.LRUCache(50)
4636
null_inventory = from_repo.revision_tree(
4637
_mod_revision.NULL_REVISION).inventory
4638
# XXX: ideally the rich-root/tree-refs flags would be per-revision, not
4639
# per-repo (e.g. streaming a non-rich-root revision out of a rich-root
4640
# repo back into a non-rich-root repo ought to be allowed)
4641
serializer = inventory_delta.InventoryDeltaSerializer(
4642
versioned_root=format.rich_root_data,
4643
tree_references=format.supports_tree_reference)
4644
for inv in inventories:
4645
key = (inv.revision_id,)
4646
parent_keys = parent_map.get(key, ())
4647
delta = None
4648
if not delta_versus_null and parent_keys:
4649
# The caller did not ask for complete inventories and we have
4650
# some parents that we can delta against. Make a delta against
4651
# each parent so that we can find the smallest.
4652
parent_ids = [parent_key[0] for parent_key in parent_keys]
4653
for parent_id in parent_ids:
4654
if parent_id not in invs_sent_so_far:
4655
# We don't know that the remote side has this basis, so
4656
# we can't use it.
4657
continue
4658
if parent_id == _mod_revision.NULL_REVISION:
4659
parent_inv = null_inventory
4660
else:
4661
parent_inv = inventory_cache.get(parent_id, None)
4662
if parent_inv is None:
4663
parent_inv = from_repo.get_inventory(parent_id)
4664
candidate_delta = inv._make_delta(parent_inv)
4665
if (delta is None or
4666
len(delta) > len(candidate_delta)):
4667
delta = candidate_delta
4668
basis_id = parent_id
4669
if delta is None:
4670
# Either none of the parents ended up being suitable, or we
4671
# were asked to delta against NULL
4672
basis_id = _mod_revision.NULL_REVISION
4673
delta = inv._make_delta(null_inventory)
4674
invs_sent_so_far.add(inv.revision_id)
4675
inventory_cache[inv.revision_id] = inv
4676
delta_serialized = ''.join(
4677
serializer.delta_to_lines(basis_id, key[-1], delta))
4678
yield versionedfile.FulltextContentFactory(
4679
key, parent_keys, None, delta_serialized)
4680
4681
4682
1771
def _iter_for_revno(repo, partial_history_cache, stop_index=None,
4683
1772
stop_revision=None):
4684
1773
"""Extend the partial history to include a given index
4694
1783
it is encountered, history extension will stop.
4695
1784
"""
4696
1785
start_revision = partial_history_cache[-1]
4697
iterator = repo.iter_reverse_revision_history(start_revision)
1786
graph = repo.get_graph()
1787
iterator = graph.iter_lefthand_ancestry(start_revision,
1788
(_mod_revision.NULL_REVISION,))
4698
1789
try:
4699
#skip the last revision in the list
1790
# skip the last revision in the list
4700
1791
iterator.next()
4701
1792
while True:
4702
1793
if (stop_index is not None and
4710
1801
# No more history
4711
1802
return
4712
1803
1804
1805
class _LazyListJoin(object):
1806
"""An iterable yielding the contents of many lists as one list.
1807
1808
Each iterator made from this will reflect the current contents of the lists
1809
at the time the iterator is made.
1810
1811
This is used by Repository's _make_parents_provider implementation so that
1812
it is safe to do::
1813
1814
pp = repo._make_parents_provider() # uses a list of fallback repos
1815
pp.add_fallback_repository(other_repo) # appends to that list
1816
result = pp.get_parent_map(...)
1817
# The result will include revs from other_repo
1818
"""
1819
1820
def __init__(self, *list_parts):
1821
self.list_parts = list_parts
1822
1823
def __iter__(self):
1824
full_list = []
1825
for list_part in self.list_parts:
1826
full_list.extend(list_part)
1827
return iter(full_list)
1828
1829
def __repr__(self):
1830
return "%s.%s(%s)" % (self.__module__, self.__class__.__name__,
1831
self.list_parts)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1