1
# Copyright (C) 2006-2010 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
19
from __future__ import absolute_import
33
revision as _mod_revision,
36
from bzrlib.trace import mutter
37
from bzrlib.tsort import topo_sort
38
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
39
from bzrlib.i18n import gettext
42
def reconcile(dir, canonicalize_chks=False):
43
"""Reconcile the data in dir.
45
Currently this is limited to a inventory 'reweave'.
47
This is a convenience method, for using a Reconciler object.
49
Directly using Reconciler is recommended for library users that
50
desire fine grained control or analysis of the found issues.
52
:param canonicalize_chks: Make sure CHKs are in canonical form.
54
reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
55
reconciler.reconcile()
58
class Reconciler(object):
59
"""Reconcilers are used to reconcile existing data."""
61
def __init__(self, dir, other=None, canonicalize_chks=False):
62
"""Create a Reconciler."""
64
self.canonicalize_chks = canonicalize_chks
67
"""Perform reconciliation.
69
After reconciliation the following attributes document found issues:
71
* `inconsistent_parents`: The number of revisions in the repository
72
whose ancestry was being reported incorrectly.
73
* `garbage_inventories`: The number of inventory objects without
74
revisions that were garbage collected.
75
* `fixed_branch_history`: None if there was no branch, False if the
76
branch history was correct, True if the branch history needed to be
79
self.pb = ui.ui_factory.nested_progress_bar()
86
"""Helper function for performing reconciliation."""
87
self._reconcile_branch()
88
self._reconcile_repository()
90
def _reconcile_branch(self):
92
self.branch = self.bzrdir.open_branch()
93
except errors.NotBranchError:
94
# Nothing to check here
95
self.fixed_branch_history = None
97
ui.ui_factory.note(gettext('Reconciling branch %s') % self.branch.base)
98
branch_reconciler = self.branch.reconcile(thorough=True)
99
self.fixed_branch_history = branch_reconciler.fixed_history
101
def _reconcile_repository(self):
102
self.repo = self.bzrdir.find_repository()
103
ui.ui_factory.note(gettext('Reconciling repository %s') %
105
self.pb.update(gettext("Reconciling repository"), 0, 1)
106
if self.canonicalize_chks:
108
self.repo.reconcile_canonicalize_chks
109
except AttributeError:
110
raise errors.BzrError(
111
gettext("%s cannot canonicalize CHKs.") % (self.repo,))
112
repo_reconciler = self.repo.reconcile_canonicalize_chks()
114
repo_reconciler = self.repo.reconcile(thorough=True)
115
self.inconsistent_parents = repo_reconciler.inconsistent_parents
116
self.garbage_inventories = repo_reconciler.garbage_inventories
117
if repo_reconciler.aborted:
118
ui.ui_factory.note(gettext(
119
'Reconcile aborted: revision index has inconsistent parents.'))
120
ui.ui_factory.note(gettext(
121
'Run "bzr check" for more details.'))
123
ui.ui_factory.note(gettext('Reconciliation complete.'))
126
class BranchReconciler(object):
127
"""Reconciler that works on a branch."""
129
def __init__(self, a_branch, thorough=False):
130
self.fixed_history = None
131
self.thorough = thorough
132
self.branch = a_branch
135
operation = cleanup.OperationWithCleanups(self._reconcile)
136
self.add_cleanup = operation.add_cleanup
137
operation.run_simple()
139
def _reconcile(self):
140
self.branch.lock_write()
141
self.add_cleanup(self.branch.unlock)
142
self.pb = ui.ui_factory.nested_progress_bar()
143
self.add_cleanup(self.pb.finished)
144
self._reconcile_steps()
146
def _reconcile_steps(self):
147
self._reconcile_revision_history()
149
def _reconcile_revision_history(self):
150
last_revno, last_revision_id = self.branch.last_revision_info()
152
graph = self.branch.repository.get_graph()
154
for revid in graph.iter_lefthand_ancestry(
155
last_revision_id, (_mod_revision.NULL_REVISION,)):
156
real_history.append(revid)
157
except errors.RevisionNotPresent:
158
pass # Hit a ghost left hand parent
159
real_history.reverse()
160
if last_revno != len(real_history):
161
self.fixed_history = True
162
# Technically for Branch5 formats, it is more efficient to use
163
# set_revision_history, as this will regenerate it again.
164
# Not really worth a whole BranchReconciler class just for this,
166
ui.ui_factory.note(gettext('Fixing last revision info {0} '\
168
last_revno, len(real_history)))
169
self.branch.set_last_revision_info(len(real_history),
172
self.fixed_history = False
173
ui.ui_factory.note(gettext('revision_history ok.'))
176
class RepoReconciler(object):
177
"""Reconciler that reconciles a repository.
179
The goal of repository reconciliation is to make any derived data
180
consistent with the core data committed by a user. This can involve
181
reindexing, or removing unreferenced data if that can interfere with
182
queries in a given repository.
184
Currently this consists of an inventory reweave with revision cross-checks.
187
def __init__(self, repo, other=None, thorough=False):
188
"""Construct a RepoReconciler.
190
:param thorough: perform a thorough check which may take longer but
191
will correct non-data loss issues such as incorrect
194
self.garbage_inventories = 0
195
self.inconsistent_parents = 0
198
self.thorough = thorough
201
"""Perform reconciliation.
203
After reconciliation the following attributes document found issues:
205
* `inconsistent_parents`: The number of revisions in the repository
206
whose ancestry was being reported incorrectly.
207
* `garbage_inventories`: The number of inventory objects without
208
revisions that were garbage collected.
210
operation = cleanup.OperationWithCleanups(self._reconcile)
211
self.add_cleanup = operation.add_cleanup
212
operation.run_simple()
214
def _reconcile(self):
215
self.repo.lock_write()
216
self.add_cleanup(self.repo.unlock)
217
self.pb = ui.ui_factory.nested_progress_bar()
218
self.add_cleanup(self.pb.finished)
219
self._reconcile_steps()
221
def _reconcile_steps(self):
222
"""Perform the steps to reconcile this repository."""
223
self._reweave_inventory()
225
def _reweave_inventory(self):
226
"""Regenerate the inventory weave for the repository from scratch.
228
This is a smart function: it will only do the reweave if doing it
229
will correct data issues. The self.thorough flag controls whether
230
only data-loss causing issues (!self.thorough) or all issues
231
(self.thorough) are treated as requiring the reweave.
233
transaction = self.repo.get_transaction()
234
self.pb.update(gettext('Reading inventory data'))
235
self.inventory = self.repo.inventories
236
self.revisions = self.repo.revisions
237
# the total set of revisions to process
238
self.pending = set([key[-1] for key in self.revisions.keys()])
240
# mapping from revision_id to parents
242
# errors that we detect
243
self.inconsistent_parents = 0
244
# we need the revision id of each revision and its available parents list
245
self._setup_steps(len(self.pending))
246
for rev_id in self.pending:
247
# put a revision into the graph.
248
self._graph_revision(rev_id)
249
self._check_garbage_inventories()
250
# if there are no inconsistent_parents and
251
# (no garbage inventories or we are not doing a thorough check)
252
if (not self.inconsistent_parents and
253
(not self.garbage_inventories or not self.thorough)):
254
ui.ui_factory.note(gettext('Inventory ok.'))
256
self.pb.update(gettext('Backing up inventory'), 0, 0)
257
self.repo._backup_inventory()
258
ui.ui_factory.note(gettext('Backup inventory created.'))
259
new_inventories = self.repo._temp_inventories()
261
# we have topological order of revisions and non ghost parents ready.
262
self._setup_steps(len(self._rev_graph))
263
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
264
stream = self._change_inv_parents(
265
self.inventory.get_record_stream(revision_keys, 'unordered', True),
266
self._new_inv_parents,
268
new_inventories.insert_record_stream(stream)
269
# if this worked, the set of new_inventories.keys should equal
271
if not (set(new_inventories.keys()) ==
272
set([(revid,) for revid in self.pending])):
273
raise AssertionError()
274
self.pb.update(gettext('Writing weave'))
275
self.repo._activate_new_inventory()
276
self.inventory = None
277
ui.ui_factory.note(gettext('Inventory regenerated.'))
279
def _new_inv_parents(self, revision_key):
280
"""Lookup ghost-filtered parents for revision_key."""
281
# Use the filtered ghostless parents list:
282
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
284
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
285
"""Adapt a record stream to reconcile the parents."""
286
for record in stream:
287
wanted_parents = get_parents(record.key)
288
if wanted_parents and wanted_parents[0] not in all_revision_keys:
289
# The check for the left most parent only handles knit
290
# compressors, but this code only applies to knit and weave
291
# repositories anyway.
292
bytes = record.get_bytes_as('fulltext')
293
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
295
adapted_record = AdapterFactory(record.key, wanted_parents, record)
297
self._reweave_step('adding inventories')
299
def _setup_steps(self, new_total):
300
"""Setup the markers we need to control the progress bar."""
301
self.total = new_total
304
def _graph_revision(self, rev_id):
305
"""Load a revision into the revision graph."""
306
# pick a random revision
307
# analyse revision id rev_id and put it in the stack.
308
self._reweave_step('loading revisions')
309
rev = self.repo.get_revision_reconcile(rev_id)
311
for parent in rev.parent_ids:
312
if self._parent_is_available(parent):
313
parents.append(parent)
315
mutter('found ghost %s', parent)
316
self._rev_graph[rev_id] = parents
318
def _check_garbage_inventories(self):
319
"""Check for garbage inventories which we cannot trust
321
We cant trust them because their pre-requisite file data may not
322
be present - all we know is that their revision was not installed.
324
if not self.thorough:
326
inventories = set(self.inventory.keys())
327
revisions = set(self.revisions.keys())
328
garbage = inventories.difference(revisions)
329
self.garbage_inventories = len(garbage)
330
for revision_key in garbage:
331
mutter('Garbage inventory {%s} found.', revision_key[-1])
333
def _parent_is_available(self, parent):
334
"""True if parent is a fully available revision
336
A fully available revision has a inventory and a revision object in the
339
if parent in self._rev_graph:
341
inv_present = (1 == len(self.inventory.get_parent_map([(parent,)])))
342
return (inv_present and self.repo.has_revision(parent))
344
def _reweave_step(self, message):
345
"""Mark a single step of regeneration complete."""
346
self.pb.update(message, self.count, self.total)
350
class KnitReconciler(RepoReconciler):
351
"""Reconciler that reconciles a knit format repository.
353
This will detect garbage inventories and remove them in thorough mode.
356
def _reconcile_steps(self):
357
"""Perform the steps to reconcile this repository."""
361
except errors.BzrCheckError:
364
# knits never suffer this
366
self._fix_text_parents()
368
def _load_indexes(self):
369
"""Load indexes for the reconciliation."""
370
self.transaction = self.repo.get_transaction()
371
self.pb.update(gettext('Reading indexes'), 0, 2)
372
self.inventory = self.repo.inventories
373
self.pb.update(gettext('Reading indexes'), 1, 2)
374
self.repo._check_for_inconsistent_revision_parents()
375
self.revisions = self.repo.revisions
376
self.pb.update(gettext('Reading indexes'), 2, 2)
378
def _gc_inventory(self):
379
"""Remove inventories that are not referenced from the revision store."""
380
self.pb.update(gettext('Checking unused inventories'), 0, 1)
381
self._check_garbage_inventories()
382
self.pb.update(gettext('Checking unused inventories'), 1, 3)
383
if not self.garbage_inventories:
384
ui.ui_factory.note(gettext('Inventory ok.'))
386
self.pb.update(gettext('Backing up inventory'), 0, 0)
387
self.repo._backup_inventory()
388
ui.ui_factory.note(gettext('Backup Inventory created'))
389
# asking for '' should never return a non-empty weave
390
new_inventories = self.repo._temp_inventories()
391
# we have topological order of revisions and non ghost parents ready.
392
graph = self.revisions.get_parent_map(self.revisions.keys())
393
revision_keys = topo_sort(graph)
394
revision_ids = [key[-1] for key in revision_keys]
395
self._setup_steps(len(revision_keys))
396
stream = self._change_inv_parents(
397
self.inventory.get_record_stream(revision_keys, 'unordered', True),
400
new_inventories.insert_record_stream(stream)
401
# if this worked, the set of new_inventory_vf.names should equal
402
# the revisionds list
403
if not(set(new_inventories.keys()) == set(revision_keys)):
404
raise AssertionError()
405
self.pb.update(gettext('Writing weave'))
406
self.repo._activate_new_inventory()
407
self.inventory = None
408
ui.ui_factory.note(gettext('Inventory regenerated.'))
410
def _fix_text_parents(self):
411
"""Fix bad versionedfile parent entries.
413
It is possible for the parents entry in a versionedfile entry to be
414
inconsistent with the values in the revision and inventory.
416
This method finds entries with such inconsistencies, corrects their
417
parent lists, and replaces the versionedfile with a corrected version.
419
transaction = self.repo.get_transaction()
420
versions = [key[-1] for key in self.revisions.keys()]
421
mutter('Prepopulating revision text cache with %d revisions',
423
vf_checker = self.repo._get_versioned_file_checker()
424
bad_parents, unused_versions = vf_checker.check_file_version_parents(
425
self.repo.texts, self.pb)
426
text_index = vf_checker.text_index
427
per_id_bad_parents = {}
428
for key in unused_versions:
429
# Ensure that every file with unused versions gets rewritten.
430
# NB: This is really not needed, reconcile != pack.
431
per_id_bad_parents[key[0]] = {}
432
# Generate per-knit/weave data.
433
for key, details in bad_parents.iteritems():
436
knit_parents = tuple([parent[-1] for parent in details[0]])
437
correct_parents = tuple([parent[-1] for parent in details[1]])
438
file_details = per_id_bad_parents.setdefault(file_id, {})
439
file_details[rev_id] = (knit_parents, correct_parents)
440
file_id_versions = {}
441
for text_key in text_index:
442
versions_list = file_id_versions.setdefault(text_key[0], [])
443
versions_list.append(text_key[1])
444
# Do the reconcile of individual weaves.
445
for num, file_id in enumerate(per_id_bad_parents):
446
self.pb.update(gettext('Fixing text parents'), num,
447
len(per_id_bad_parents))
448
versions_with_bad_parents = per_id_bad_parents[file_id]
449
id_unused_versions = set(key[-1] for key in unused_versions
450
if key[0] == file_id)
451
if file_id in file_id_versions:
452
file_versions = file_id_versions[file_id]
454
# This id was present in the disk store but is not referenced
455
# by any revision at all.
457
self._fix_text_parent(file_id, versions_with_bad_parents,
458
id_unused_versions, file_versions)
460
def _fix_text_parent(self, file_id, versions_with_bad_parents,
461
unused_versions, all_versions):
462
"""Fix bad versionedfile entries in a single versioned file."""
463
mutter('fixing text parent: %r (%d versions)', file_id,
464
len(versions_with_bad_parents))
465
mutter('(%d are unused)', len(unused_versions))
466
new_file_id = 'temp:%s' % file_id
469
for version in all_versions:
470
if version in unused_versions:
472
elif version in versions_with_bad_parents:
473
parents = versions_with_bad_parents[version][1]
475
pmap = self.repo.texts.get_parent_map([(file_id, version)])
476
parents = [key[-1] for key in pmap[(file_id, version)]]
477
new_parents[(new_file_id, version)] = [
478
(new_file_id, parent) for parent in parents]
479
needed_keys.add((file_id, version))
480
def fix_parents(stream):
481
for record in stream:
482
bytes = record.get_bytes_as('fulltext')
483
new_key = (new_file_id, record.key[-1])
484
parents = new_parents[new_key]
485
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
486
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
487
self.repo._remove_file_id(new_file_id)
488
self.repo.texts.insert_record_stream(fix_parents(stream))
489
self.repo._remove_file_id(file_id)
491
self.repo._move_file_id(new_file_id, file_id)
494
class PackReconciler(RepoReconciler):
495
"""Reconciler that reconciles a pack based repository.
497
Garbage inventories do not affect ancestry queries, and removal is
498
considerably more expensive as there is no separate versioned file for
499
them, so they are not cleaned. In short it is currently a no-op.
501
In future this may be a good place to hook in annotation cache checking,
502
index recreation etc.
505
# XXX: The index corruption that _fix_text_parents performs is needed for
506
# packs, but not yet implemented. The basic approach is to:
507
# - lock the names list
508
# - perform a customised pack() that regenerates data as needed
509
# - unlock the names list
510
# https://bugs.launchpad.net/bzr/+bug/154173
512
def __init__(self, repo, other=None, thorough=False,
513
canonicalize_chks=False):
514
super(PackReconciler, self).__init__(repo, other=other,
516
self.canonicalize_chks = canonicalize_chks
518
def _reconcile_steps(self):
519
"""Perform the steps to reconcile this repository."""
520
if not self.thorough:
522
collection = self.repo._pack_collection
523
collection.ensure_loaded()
524
collection.lock_names()
525
self.add_cleanup(collection._unlock_names)
526
packs = collection.all_packs()
527
all_revisions = self.repo.all_revision_ids()
528
total_inventories = len(list(
529
collection.inventory_index.combined_index.iter_all_entries()))
530
if len(all_revisions):
531
if self.canonicalize_chks:
532
reconcile_meth = self.repo._canonicalize_chks_pack
534
reconcile_meth = self.repo._reconcile_pack
535
new_pack = reconcile_meth(collection, packs, ".reconcile",
536
all_revisions, self.pb)
537
if new_pack is not None:
538
self._discard_and_save(packs)
540
# only make a new pack when there is data to copy.
541
self._discard_and_save(packs)
542
self.garbage_inventories = total_inventories - len(list(
543
collection.inventory_index.combined_index.iter_all_entries()))
545
def _discard_and_save(self, packs):
546
"""Discard some packs from the repository.
548
This removes them from the memory index, saves the in-memory index
549
which makes the newly reconciled pack visible and hides the packs to be
550
discarded, and finally renames the packs being discarded into the
551
obsolete packs directory.
553
:param packs: The packs to discard.
556
self.repo._pack_collection._remove_pack_from_memory(pack)
557
self.repo._pack_collection._save_pack_names()
558
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
