13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
17
"""Reconcilers are able to fix some potential data errors in a branch."""
19
from __future__ import absolute_import
20
__all__ = ['reconcile', 'Reconciler', 'RepoReconciler', 'KnitReconciler']
30
23
from bzrlib import (
33
revision as _mod_revision,
36
from bzrlib.trace import mutter
37
from bzrlib.tsort import topo_sort
38
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
39
from bzrlib.i18n import gettext
42
def reconcile(dir, canonicalize_chks=False):
29
from bzrlib import errors
31
from bzrlib.trace import mutter, note
32
from bzrlib.tsort import TopoSorter, topo_sort
35
def reconcile(dir, other=None):
43
36
"""Reconcile the data in dir.
45
38
Currently this is limited to a inventory 'reweave'.
49
42
Directly using Reconciler is recommended for library users that
50
43
desire fine grained control or analysis of the found issues.
52
:param canonicalize_chks: Make sure CHKs are in canonical form.
45
:param other: another bzrdir to reconcile against.
54
reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
47
reconciler = Reconciler(dir, other=other)
55
48
reconciler.reconcile()
58
51
class Reconciler(object):
59
52
"""Reconcilers are used to reconcile existing data."""
61
def __init__(self, dir, other=None, canonicalize_chks=False):
54
def __init__(self, dir, other=None):
62
55
"""Create a Reconciler."""
64
self.canonicalize_chks = canonicalize_chks
66
58
def reconcile(self):
67
59
"""Perform reconciliation.
69
61
After reconciliation the following attributes document found issues:
71
* `inconsistent_parents`: The number of revisions in the repository
72
whose ancestry was being reported incorrectly.
73
* `garbage_inventories`: The number of inventory objects without
74
revisions that were garbage collected.
75
* `fixed_branch_history`: None if there was no branch, False if the
76
branch history was correct, True if the branch history needed to be
62
inconsistent_parents: The number of revisions in the repository whose
63
ancestry was being reported incorrectly.
64
garbage_inventories: The number of inventory objects without revisions
65
that were garbage collected.
79
67
self.pb = ui.ui_factory.nested_progress_bar()
85
73
def _reconcile(self):
86
74
"""Helper function for performing reconciliation."""
87
self._reconcile_branch()
88
self._reconcile_repository()
90
def _reconcile_branch(self):
92
self.branch = self.bzrdir.open_branch()
93
except errors.NotBranchError:
94
# Nothing to check here
95
self.fixed_branch_history = None
97
ui.ui_factory.note(gettext('Reconciling branch %s') % self.branch.base)
98
branch_reconciler = self.branch.reconcile(thorough=True)
99
self.fixed_branch_history = branch_reconciler.fixed_history
101
def _reconcile_repository(self):
102
75
self.repo = self.bzrdir.find_repository()
103
ui.ui_factory.note(gettext('Reconciling repository %s') %
105
self.pb.update(gettext("Reconciling repository"), 0, 1)
106
if self.canonicalize_chks:
108
self.repo.reconcile_canonicalize_chks
109
except AttributeError:
110
raise errors.BzrError(
111
gettext("%s cannot canonicalize CHKs.") % (self.repo,))
112
repo_reconciler = self.repo.reconcile_canonicalize_chks()
114
repo_reconciler = self.repo.reconcile(thorough=True)
76
self.pb.note('Reconciling repository %s',
77
self.repo.bzrdir.root_transport.base)
78
repo_reconciler = self.repo.reconcile(thorough=True)
115
79
self.inconsistent_parents = repo_reconciler.inconsistent_parents
116
80
self.garbage_inventories = repo_reconciler.garbage_inventories
117
81
if repo_reconciler.aborted:
118
ui.ui_factory.note(gettext(
119
'Reconcile aborted: revision index has inconsistent parents.'))
120
ui.ui_factory.note(gettext(
121
'Run "bzr check" for more details.'))
123
ui.ui_factory.note(gettext('Reconciliation complete.'))
126
class BranchReconciler(object):
127
"""Reconciler that works on a branch."""
129
def __init__(self, a_branch, thorough=False):
130
self.fixed_history = None
131
self.thorough = thorough
132
self.branch = a_branch
135
operation = cleanup.OperationWithCleanups(self._reconcile)
136
self.add_cleanup = operation.add_cleanup
137
operation.run_simple()
139
def _reconcile(self):
140
self.branch.lock_write()
141
self.add_cleanup(self.branch.unlock)
142
self.pb = ui.ui_factory.nested_progress_bar()
143
self.add_cleanup(self.pb.finished)
144
self._reconcile_steps()
146
def _reconcile_steps(self):
147
self._reconcile_revision_history()
149
def _reconcile_revision_history(self):
150
last_revno, last_revision_id = self.branch.last_revision_info()
152
graph = self.branch.repository.get_graph()
154
for revid in graph.iter_lefthand_ancestry(
155
last_revision_id, (_mod_revision.NULL_REVISION,)):
156
real_history.append(revid)
157
except errors.RevisionNotPresent:
158
pass # Hit a ghost left hand parent
159
real_history.reverse()
160
if last_revno != len(real_history):
161
self.fixed_history = True
162
# Technically for Branch5 formats, it is more efficient to use
163
# set_revision_history, as this will regenerate it again.
164
# Not really worth a whole BranchReconciler class just for this,
166
ui.ui_factory.note(gettext('Fixing last revision info {0} '\
168
last_revno, len(real_history)))
169
self.branch.set_last_revision_info(len(real_history),
172
self.fixed_history = False
173
ui.ui_factory.note(gettext('revision_history ok.'))
83
'Reconcile aborted: revision index has inconsistent parents.')
85
'Run "bzr check" for more details.')
87
self.pb.note('Reconciliation complete.')
176
90
class RepoReconciler(object):
177
91
"""Reconciler that reconciles a repository.
179
93
The goal of repository reconciliation is to make any derived data
180
consistent with the core data committed by a user. This can involve
94
consistent with the core data committed by a user. This can involve
181
95
reindexing, or removing unreferenced data if that can interfere with
182
96
queries in a given repository.
247
160
# put a revision into the graph.
248
161
self._graph_revision(rev_id)
249
162
self._check_garbage_inventories()
250
# if there are no inconsistent_parents and
163
# if there are no inconsistent_parents and
251
164
# (no garbage inventories or we are not doing a thorough check)
252
if (not self.inconsistent_parents and
165
if (not self.inconsistent_parents and
253
166
(not self.garbage_inventories or not self.thorough)):
254
ui.ui_factory.note(gettext('Inventory ok.'))
167
self.pb.note('Inventory ok.')
256
self.pb.update(gettext('Backing up inventory'), 0, 0)
257
self.repo._backup_inventory()
258
ui.ui_factory.note(gettext('Backup inventory created.'))
259
new_inventories = self.repo._temp_inventories()
169
self.pb.update('Backing up inventory...', 0, 0)
170
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.repo.get_transaction())
171
self.pb.note('Backup Inventory created.')
172
# asking for '' should never return a non-empty weave
173
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
174
self.repo.get_transaction())
261
176
# we have topological order of revisions and non ghost parents ready.
262
177
self._setup_steps(len(self._rev_graph))
263
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
264
stream = self._change_inv_parents(
265
self.inventory.get_record_stream(revision_keys, 'unordered', True),
266
self._new_inv_parents,
268
new_inventories.insert_record_stream(stream)
269
# if this worked, the set of new_inventories.keys should equal
178
for rev_id in TopoSorter(self._rev_graph.items()).iter_topo_order():
179
parents = self._rev_graph[rev_id]
180
# double check this really is in topological order.
181
unavailable = [p for p in parents if p not in new_inventory_vf]
182
assert len(unavailable) == 0
183
# this entry has all the non ghost parents in the inventory
185
self._reweave_step('adding inventories')
186
if isinstance(new_inventory_vf, WeaveFile):
187
# It's really a WeaveFile, but we call straight into the
188
# Weave's add method to disable the auto-write-out behaviour.
189
# This is done to avoid a revision_count * time-to-write additional overhead on
191
new_inventory_vf._check_write_ok()
192
Weave._add_lines(new_inventory_vf, rev_id, parents,
193
self.inventory.get_lines(rev_id), None, None, None, False, True)
195
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
197
if isinstance(new_inventory_vf, WeaveFile):
198
new_inventory_vf._save()
199
# if this worked, the set of new_inventory_vf.names should equal
271
if not (set(new_inventories.keys()) ==
272
set([(revid,) for revid in self.pending])):
273
raise AssertionError()
274
self.pb.update(gettext('Writing weave'))
275
self.repo._activate_new_inventory()
201
assert set(new_inventory_vf.versions()) == self.pending
202
self.pb.update('Writing weave')
203
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.repo.get_transaction())
204
self.repo.control_weaves.delete('inventory.new', self.repo.get_transaction())
276
205
self.inventory = None
277
ui.ui_factory.note(gettext('Inventory regenerated.'))
279
def _new_inv_parents(self, revision_key):
280
"""Lookup ghost-filtered parents for revision_key."""
281
# Use the filtered ghostless parents list:
282
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
284
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
285
"""Adapt a record stream to reconcile the parents."""
286
for record in stream:
287
wanted_parents = get_parents(record.key)
288
if wanted_parents and wanted_parents[0] not in all_revision_keys:
289
# The check for the left most parent only handles knit
290
# compressors, but this code only applies to knit and weave
291
# repositories anyway.
292
bytes = record.get_bytes_as('fulltext')
293
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
295
adapted_record = AdapterFactory(record.key, wanted_parents, record)
297
self._reweave_step('adding inventories')
206
self.pb.note('Inventory regenerated.')
299
208
def _setup_steps(self, new_total):
300
209
"""Setup the markers we need to control the progress bar."""
307
216
# analyse revision id rev_id and put it in the stack.
308
217
self._reweave_step('loading revisions')
309
218
rev = self.repo.get_revision_reconcile(rev_id)
219
assert rev.revision_id == rev_id
311
221
for parent in rev.parent_ids:
312
222
if self._parent_is_available(parent):
313
223
parents.append(parent)
315
225
mutter('found ghost %s', parent)
316
self._rev_graph[rev_id] = parents
226
self._rev_graph[rev_id] = parents
227
if self._parents_are_inconsistent(rev_id, parents):
228
self.inconsistent_parents += 1
229
mutter('Inconsistent inventory parents: id {%s} '
230
'inventory claims %r, '
231
'available parents are %r, '
232
'unavailable parents are %r',
234
set(self.inventory.get_parents(rev_id)),
236
set(rev.parent_ids).difference(set(parents)))
238
def _parents_are_inconsistent(self, rev_id, parents):
239
"""Return True if the parents list of rev_id does not match the weave.
241
This detects inconsistencies based on the self.thorough value:
242
if thorough is on, the first parent value is checked as well as ghost
244
Otherwise only the ghost differences are evaluated.
246
weave_parents = self.inventory.get_parents(rev_id)
247
weave_missing_old_ghosts = set(weave_parents) != set(parents)
248
first_parent_is_wrong = (
249
len(weave_parents) and len(parents) and
250
parents[0] != weave_parents[0])
252
return weave_missing_old_ghosts or first_parent_is_wrong
254
return weave_missing_old_ghosts
318
256
def _check_garbage_inventories(self):
319
257
"""Check for garbage inventories which we cannot trust
368
304
def _load_indexes(self):
369
305
"""Load indexes for the reconciliation."""
370
306
self.transaction = self.repo.get_transaction()
371
self.pb.update(gettext('Reading indexes'), 0, 2)
372
self.inventory = self.repo.inventories
373
self.pb.update(gettext('Reading indexes'), 1, 2)
307
self.pb.update('Reading indexes.', 0, 2)
308
self.inventory = self.repo.get_inventory_weave()
309
self.pb.update('Reading indexes.', 1, 2)
374
310
self.repo._check_for_inconsistent_revision_parents()
375
self.revisions = self.repo.revisions
376
self.pb.update(gettext('Reading indexes'), 2, 2)
311
self.revisions = self.repo._revision_store.get_revision_file(self.transaction)
312
self.pb.update('Reading indexes.', 2, 2)
378
314
def _gc_inventory(self):
379
315
"""Remove inventories that are not referenced from the revision store."""
380
self.pb.update(gettext('Checking unused inventories'), 0, 1)
316
self.pb.update('Checking unused inventories.', 0, 1)
381
317
self._check_garbage_inventories()
382
self.pb.update(gettext('Checking unused inventories'), 1, 3)
318
self.pb.update('Checking unused inventories.', 1, 3)
383
319
if not self.garbage_inventories:
384
ui.ui_factory.note(gettext('Inventory ok.'))
320
self.pb.note('Inventory ok.')
386
self.pb.update(gettext('Backing up inventory'), 0, 0)
387
self.repo._backup_inventory()
388
ui.ui_factory.note(gettext('Backup Inventory created'))
322
self.pb.update('Backing up inventory...', 0, 0)
323
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.transaction)
324
self.pb.note('Backup Inventory created.')
389
325
# asking for '' should never return a non-empty weave
390
new_inventories = self.repo._temp_inventories()
326
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
391
329
# we have topological order of revisions and non ghost parents ready.
392
graph = self.revisions.get_parent_map(self.revisions.keys())
393
revision_keys = topo_sort(graph)
394
revision_ids = [key[-1] for key in revision_keys]
395
self._setup_steps(len(revision_keys))
396
stream = self._change_inv_parents(
397
self.inventory.get_record_stream(revision_keys, 'unordered', True),
400
new_inventories.insert_record_stream(stream)
330
self._setup_steps(len(self.revisions))
331
for rev_id in TopoSorter(self.revisions.get_graph().items()).iter_topo_order():
332
parents = self.revisions.get_parents(rev_id)
333
# double check this really is in topological order.
334
unavailable = [p for p in parents if p not in new_inventory_vf]
335
assert len(unavailable) == 0
336
# this entry has all the non ghost parents in the inventory
338
self._reweave_step('adding inventories')
339
# ugly but needed, weaves are just way tooooo slow else.
340
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
401
342
# if this worked, the set of new_inventory_vf.names should equal
402
# the revisionds list
403
if not(set(new_inventories.keys()) == set(revision_keys)):
404
raise AssertionError()
405
self.pb.update(gettext('Writing weave'))
406
self.repo._activate_new_inventory()
344
assert set(new_inventory_vf.versions()) == set(self.revisions.versions())
345
self.pb.update('Writing weave')
346
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.transaction)
347
self.repo.control_weaves.delete('inventory.new', self.transaction)
407
348
self.inventory = None
408
ui.ui_factory.note(gettext('Inventory regenerated.'))
349
self.pb.note('Inventory regenerated.')
351
def _check_garbage_inventories(self):
352
"""Check for garbage inventories which we cannot trust
354
We cant trust them because their pre-requisite file data may not
355
be present - all we know is that their revision was not installed.
357
inventories = set(self.inventory.versions())
358
revisions = set(self.revisions.versions())
359
garbage = inventories.difference(revisions)
360
self.garbage_inventories = len(garbage)
361
for revision_id in garbage:
362
mutter('Garbage inventory {%s} found.', revision_id)
410
364
def _fix_text_parents(self):
411
365
"""Fix bad versionedfile parent entries.
417
371
parent lists, and replaces the versionedfile with a corrected version.
419
373
transaction = self.repo.get_transaction()
420
versions = [key[-1] for key in self.revisions.keys()]
421
mutter('Prepopulating revision text cache with %d revisions',
423
vf_checker = self.repo._get_versioned_file_checker()
424
bad_parents, unused_versions = vf_checker.check_file_version_parents(
425
self.repo.texts, self.pb)
426
text_index = vf_checker.text_index
427
per_id_bad_parents = {}
428
for key in unused_versions:
429
# Ensure that every file with unused versions gets rewritten.
430
# NB: This is really not needed, reconcile != pack.
431
per_id_bad_parents[key[0]] = {}
432
# Generate per-knit/weave data.
433
for key, details in bad_parents.iteritems():
436
knit_parents = tuple([parent[-1] for parent in details[0]])
437
correct_parents = tuple([parent[-1] for parent in details[1]])
438
file_details = per_id_bad_parents.setdefault(file_id, {})
439
file_details[rev_id] = (knit_parents, correct_parents)
440
file_id_versions = {}
441
for text_key in text_index:
442
versions_list = file_id_versions.setdefault(text_key[0], [])
443
versions_list.append(text_key[1])
444
# Do the reconcile of individual weaves.
445
for num, file_id in enumerate(per_id_bad_parents):
446
self.pb.update(gettext('Fixing text parents'), num,
447
len(per_id_bad_parents))
448
versions_with_bad_parents = per_id_bad_parents[file_id]
449
id_unused_versions = set(key[-1] for key in unused_versions
450
if key[0] == file_id)
451
if file_id in file_id_versions:
452
file_versions = file_id_versions[file_id]
454
# This id was present in the disk store but is not referenced
455
# by any revision at all.
457
self._fix_text_parent(file_id, versions_with_bad_parents,
458
id_unused_versions, file_versions)
374
revision_versions = repository._RevisionTextVersionCache(self.repo)
375
versions = self.revisions.versions()
376
revision_versions.prepopulate_revs(versions)
377
for num, file_id in enumerate(self.repo.weave_store):
378
self.pb.update('Fixing text parents', num,
379
len(self.repo.weave_store))
380
vf = self.repo.weave_store.get_weave(file_id, transaction)
381
vf_checker = self.repo.get_versioned_file_checker(
382
versions, revision_versions)
383
versions_with_bad_parents = vf_checker.check_file_version_parents(
385
if len(versions_with_bad_parents) == 0:
387
self._fix_text_parent(file_id, vf, versions_with_bad_parents)
460
def _fix_text_parent(self, file_id, versions_with_bad_parents,
461
unused_versions, all_versions):
389
def _fix_text_parent(self, file_id, vf, versions_with_bad_parents):
462
390
"""Fix bad versionedfile entries in a single versioned file."""
463
mutter('fixing text parent: %r (%d versions)', file_id,
464
len(versions_with_bad_parents))
465
mutter('(%d are unused)', len(unused_versions))
466
new_file_id = 'temp:%s' % file_id
391
new_vf = self.repo.weave_store.get_empty('temp:%s' % file_id,
469
for version in all_versions:
470
if version in unused_versions:
472
elif version in versions_with_bad_parents:
394
for version in vf.versions():
395
if version in versions_with_bad_parents:
473
396
parents = versions_with_bad_parents[version][1]
475
pmap = self.repo.texts.get_parent_map([(file_id, version)])
476
parents = [key[-1] for key in pmap[(file_id, version)]]
477
new_parents[(new_file_id, version)] = [
478
(new_file_id, parent) for parent in parents]
479
needed_keys.add((file_id, version))
480
def fix_parents(stream):
481
for record in stream:
482
bytes = record.get_bytes_as('fulltext')
483
new_key = (new_file_id, record.key[-1])
484
parents = new_parents[new_key]
485
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
486
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
487
self.repo._remove_file_id(new_file_id)
488
self.repo.texts.insert_record_stream(fix_parents(stream))
489
self.repo._remove_file_id(file_id)
491
self.repo._move_file_id(new_file_id, file_id)
494
class PackReconciler(RepoReconciler):
495
"""Reconciler that reconciles a pack based repository.
497
Garbage inventories do not affect ancestry queries, and removal is
498
considerably more expensive as there is no separate versioned file for
499
them, so they are not cleaned. In short it is currently a no-op.
501
In future this may be a good place to hook in annotation cache checking,
502
index recreation etc.
505
# XXX: The index corruption that _fix_text_parents performs is needed for
506
# packs, but not yet implemented. The basic approach is to:
507
# - lock the names list
508
# - perform a customised pack() that regenerates data as needed
509
# - unlock the names list
510
# https://bugs.launchpad.net/bzr/+bug/154173
512
def __init__(self, repo, other=None, thorough=False,
513
canonicalize_chks=False):
514
super(PackReconciler, self).__init__(repo, other=other,
516
self.canonicalize_chks = canonicalize_chks
518
def _reconcile_steps(self):
519
"""Perform the steps to reconcile this repository."""
520
if not self.thorough:
522
collection = self.repo._pack_collection
523
collection.ensure_loaded()
524
collection.lock_names()
525
self.add_cleanup(collection._unlock_names)
526
packs = collection.all_packs()
527
all_revisions = self.repo.all_revision_ids()
528
total_inventories = len(list(
529
collection.inventory_index.combined_index.iter_all_entries()))
530
if len(all_revisions):
531
if self.canonicalize_chks:
532
reconcile_meth = self.repo._canonicalize_chks_pack
534
reconcile_meth = self.repo._reconcile_pack
535
new_pack = reconcile_meth(collection, packs, ".reconcile",
536
all_revisions, self.pb)
537
if new_pack is not None:
538
self._discard_and_save(packs)
540
# only make a new pack when there is data to copy.
541
self._discard_and_save(packs)
542
self.garbage_inventories = total_inventories - len(list(
543
collection.inventory_index.combined_index.iter_all_entries()))
545
def _discard_and_save(self, packs):
546
"""Discard some packs from the repository.
548
This removes them from the memory index, saves the in-memory index
549
which makes the newly reconciled pack visible and hides the packs to be
550
discarded, and finally renames the packs being discarded into the
551
obsolete packs directory.
553
:param packs: The packs to discard.
556
self.repo._pack_collection._remove_pack_from_memory(pack)
557
self.repo._pack_collection._save_pack_names()
558
self.repo._pack_collection._obsolete_packs(packs)
398
parents = vf.get_parents(version)
399
new_parents[version] = parents
400
for version in topo_sort(new_parents.items()):
401
new_vf.add_lines(version, new_parents[version],
402
vf.get_lines(version))
403
self.repo.weave_store.copy(new_vf, file_id, self.transaction)
404
self.repo.weave_store.delete('temp:%s' % file_id, self.transaction)
added
removed
bzrlib/reconcile.py
