1
# Copyright (C) 2006-2010 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
32
revision as _mod_revision,
35
from bzrlib.trace import mutter
36
from bzrlib.tsort import topo_sort
37
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
38
from bzrlib.i18n import gettext
41
def reconcile(dir, canonicalize_chks=False):
42
"""Reconcile the data in dir.
44
Currently this is limited to a inventory 'reweave'.
46
This is a convenience method, for using a Reconciler object.
48
Directly using Reconciler is recommended for library users that
49
desire fine grained control or analysis of the found issues.
51
:param canonicalize_chks: Make sure CHKs are in canonical form.
53
reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
54
reconciler.reconcile()
57
class Reconciler(object):
58
"""Reconcilers are used to reconcile existing data."""
60
def __init__(self, dir, other=None, canonicalize_chks=False):
61
"""Create a Reconciler."""
63
self.canonicalize_chks = canonicalize_chks
66
"""Perform reconciliation.
68
After reconciliation the following attributes document found issues:
70
* `inconsistent_parents`: The number of revisions in the repository
71
whose ancestry was being reported incorrectly.
72
* `garbage_inventories`: The number of inventory objects without
73
revisions that were garbage collected.
74
* `fixed_branch_history`: None if there was no branch, False if the
75
branch history was correct, True if the branch history needed to be
78
self.pb = ui.ui_factory.nested_progress_bar()
85
"""Helper function for performing reconciliation."""
86
self._reconcile_branch()
87
self._reconcile_repository()
89
def _reconcile_branch(self):
91
self.branch = self.bzrdir.open_branch()
92
except errors.NotBranchError:
93
# Nothing to check here
94
self.fixed_branch_history = None
96
ui.ui_factory.note(gettext('Reconciling branch %s') % self.branch.base)
97
branch_reconciler = self.branch.reconcile(thorough=True)
98
self.fixed_branch_history = branch_reconciler.fixed_history
100
def _reconcile_repository(self):
101
self.repo = self.bzrdir.find_repository()
102
ui.ui_factory.note(gettext('Reconciling repository %s') %
104
self.pb.update(gettext("Reconciling repository"), 0, 1)
105
if self.canonicalize_chks:
107
self.repo.reconcile_canonicalize_chks
108
except AttributeError:
109
raise errors.BzrError(
110
gettext("%s cannot canonicalize CHKs.") % (self.repo,))
111
repo_reconciler = self.repo.reconcile_canonicalize_chks()
113
repo_reconciler = self.repo.reconcile(thorough=True)
114
self.inconsistent_parents = repo_reconciler.inconsistent_parents
115
self.garbage_inventories = repo_reconciler.garbage_inventories
116
if repo_reconciler.aborted:
117
ui.ui_factory.note(gettext(
118
'Reconcile aborted: revision index has inconsistent parents.'))
119
ui.ui_factory.note(gettext(
120
'Run "bzr check" for more details.'))
122
ui.ui_factory.note(gettext('Reconciliation complete.'))
125
class BranchReconciler(object):
126
"""Reconciler that works on a branch."""
128
def __init__(self, a_branch, thorough=False):
129
self.fixed_history = None
130
self.thorough = thorough
131
self.branch = a_branch
134
operation = cleanup.OperationWithCleanups(self._reconcile)
135
self.add_cleanup = operation.add_cleanup
136
operation.run_simple()
138
def _reconcile(self):
139
self.branch.lock_write()
140
self.add_cleanup(self.branch.unlock)
141
self.pb = ui.ui_factory.nested_progress_bar()
142
self.add_cleanup(self.pb.finished)
143
self._reconcile_steps()
145
def _reconcile_steps(self):
146
self._reconcile_revision_history()
148
def _reconcile_revision_history(self):
149
last_revno, last_revision_id = self.branch.last_revision_info()
151
graph = self.branch.repository.get_graph()
153
for revid in graph.iter_lefthand_ancestry(
154
last_revision_id, (_mod_revision.NULL_REVISION,)):
155
real_history.append(revid)
156
except errors.RevisionNotPresent:
157
pass # Hit a ghost left hand parent
158
real_history.reverse()
159
if last_revno != len(real_history):
160
self.fixed_history = True
161
# Technically for Branch5 formats, it is more efficient to use
162
# set_revision_history, as this will regenerate it again.
163
# Not really worth a whole BranchReconciler class just for this,
165
ui.ui_factory.note(gettext('Fixing last revision info {0} '\
167
last_revno, len(real_history)))
168
self.branch.set_last_revision_info(len(real_history),
171
self.fixed_history = False
172
ui.ui_factory.note(gettext('revision_history ok.'))
175
class RepoReconciler(object):
176
"""Reconciler that reconciles a repository.
178
The goal of repository reconciliation is to make any derived data
179
consistent with the core data committed by a user. This can involve
180
reindexing, or removing unreferenced data if that can interfere with
181
queries in a given repository.
183
Currently this consists of an inventory reweave with revision cross-checks.
186
def __init__(self, repo, other=None, thorough=False):
187
"""Construct a RepoReconciler.
189
:param thorough: perform a thorough check which may take longer but
190
will correct non-data loss issues such as incorrect
193
self.garbage_inventories = 0
194
self.inconsistent_parents = 0
197
self.thorough = thorough
200
"""Perform reconciliation.
202
After reconciliation the following attributes document found issues:
204
* `inconsistent_parents`: The number of revisions in the repository
205
whose ancestry was being reported incorrectly.
206
* `garbage_inventories`: The number of inventory objects without
207
revisions that were garbage collected.
209
operation = cleanup.OperationWithCleanups(self._reconcile)
210
self.add_cleanup = operation.add_cleanup
211
operation.run_simple()
213
def _reconcile(self):
214
self.repo.lock_write()
215
self.add_cleanup(self.repo.unlock)
216
self.pb = ui.ui_factory.nested_progress_bar()
217
self.add_cleanup(self.pb.finished)
218
self._reconcile_steps()
220
def _reconcile_steps(self):
221
"""Perform the steps to reconcile this repository."""
222
self._reweave_inventory()
224
def _reweave_inventory(self):
225
"""Regenerate the inventory weave for the repository from scratch.
227
This is a smart function: it will only do the reweave if doing it
228
will correct data issues. The self.thorough flag controls whether
229
only data-loss causing issues (!self.thorough) or all issues
230
(self.thorough) are treated as requiring the reweave.
232
transaction = self.repo.get_transaction()
233
self.pb.update(gettext('Reading inventory data'))
234
self.inventory = self.repo.inventories
235
self.revisions = self.repo.revisions
236
# the total set of revisions to process
237
self.pending = set([key[-1] for key in self.revisions.keys()])
239
# mapping from revision_id to parents
241
# errors that we detect
242
self.inconsistent_parents = 0
243
# we need the revision id of each revision and its available parents list
244
self._setup_steps(len(self.pending))
245
for rev_id in self.pending:
246
# put a revision into the graph.
247
self._graph_revision(rev_id)
248
self._check_garbage_inventories()
249
# if there are no inconsistent_parents and
250
# (no garbage inventories or we are not doing a thorough check)
251
if (not self.inconsistent_parents and
252
(not self.garbage_inventories or not self.thorough)):
253
ui.ui_factory.note(gettext('Inventory ok.'))
255
self.pb.update(gettext('Backing up inventory'), 0, 0)
256
self.repo._backup_inventory()
257
ui.ui_factory.note(gettext('Backup inventory created.'))
258
new_inventories = self.repo._temp_inventories()
260
# we have topological order of revisions and non ghost parents ready.
261
self._setup_steps(len(self._rev_graph))
262
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
263
stream = self._change_inv_parents(
264
self.inventory.get_record_stream(revision_keys, 'unordered', True),
265
self._new_inv_parents,
267
new_inventories.insert_record_stream(stream)
268
# if this worked, the set of new_inventories.keys should equal
270
if not (set(new_inventories.keys()) ==
271
set([(revid,) for revid in self.pending])):
272
raise AssertionError()
273
self.pb.update(gettext('Writing weave'))
274
self.repo._activate_new_inventory()
275
self.inventory = None
276
ui.ui_factory.note(gettext('Inventory regenerated.'))
278
def _new_inv_parents(self, revision_key):
279
"""Lookup ghost-filtered parents for revision_key."""
280
# Use the filtered ghostless parents list:
281
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
283
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
284
"""Adapt a record stream to reconcile the parents."""
285
for record in stream:
286
wanted_parents = get_parents(record.key)
287
if wanted_parents and wanted_parents[0] not in all_revision_keys:
288
# The check for the left most parent only handles knit
289
# compressors, but this code only applies to knit and weave
290
# repositories anyway.
291
bytes = record.get_bytes_as('fulltext')
292
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
294
adapted_record = AdapterFactory(record.key, wanted_parents, record)
296
self._reweave_step('adding inventories')
298
def _setup_steps(self, new_total):
299
"""Setup the markers we need to control the progress bar."""
300
self.total = new_total
303
def _graph_revision(self, rev_id):
304
"""Load a revision into the revision graph."""
305
# pick a random revision
306
# analyse revision id rev_id and put it in the stack.
307
self._reweave_step('loading revisions')
308
rev = self.repo.get_revision_reconcile(rev_id)
310
for parent in rev.parent_ids:
311
if self._parent_is_available(parent):
312
parents.append(parent)
314
mutter('found ghost %s', parent)
315
self._rev_graph[rev_id] = parents
317
def _check_garbage_inventories(self):
318
"""Check for garbage inventories which we cannot trust
320
We cant trust them because their pre-requisite file data may not
321
be present - all we know is that their revision was not installed.
323
if not self.thorough:
325
inventories = set(self.inventory.keys())
326
revisions = set(self.revisions.keys())
327
garbage = inventories.difference(revisions)
328
self.garbage_inventories = len(garbage)
329
for revision_key in garbage:
330
mutter('Garbage inventory {%s} found.', revision_key[-1])
332
def _parent_is_available(self, parent):
333
"""True if parent is a fully available revision
335
A fully available revision has a inventory and a revision object in the
338
if parent in self._rev_graph:
340
inv_present = (1 == len(self.inventory.get_parent_map([(parent,)])))
341
return (inv_present and self.repo.has_revision(parent))
343
def _reweave_step(self, message):
344
"""Mark a single step of regeneration complete."""
345
self.pb.update(message, self.count, self.total)
349
class KnitReconciler(RepoReconciler):
350
"""Reconciler that reconciles a knit format repository.
352
This will detect garbage inventories and remove them in thorough mode.
355
def _reconcile_steps(self):
356
"""Perform the steps to reconcile this repository."""
360
except errors.BzrCheckError:
363
# knits never suffer this
365
self._fix_text_parents()
367
def _load_indexes(self):
368
"""Load indexes for the reconciliation."""
369
self.transaction = self.repo.get_transaction()
370
self.pb.update(gettext('Reading indexes'), 0, 2)
371
self.inventory = self.repo.inventories
372
self.pb.update(gettext('Reading indexes'), 1, 2)
373
self.repo._check_for_inconsistent_revision_parents()
374
self.revisions = self.repo.revisions
375
self.pb.update(gettext('Reading indexes'), 2, 2)
377
def _gc_inventory(self):
378
"""Remove inventories that are not referenced from the revision store."""
379
self.pb.update(gettext('Checking unused inventories'), 0, 1)
380
self._check_garbage_inventories()
381
self.pb.update(gettext('Checking unused inventories'), 1, 3)
382
if not self.garbage_inventories:
383
ui.ui_factory.note(gettext('Inventory ok.'))
385
self.pb.update(gettext('Backing up inventory'), 0, 0)
386
self.repo._backup_inventory()
387
ui.ui_factory.note(gettext('Backup Inventory created'))
388
# asking for '' should never return a non-empty weave
389
new_inventories = self.repo._temp_inventories()
390
# we have topological order of revisions and non ghost parents ready.
391
graph = self.revisions.get_parent_map(self.revisions.keys())
392
revision_keys = topo_sort(graph)
393
revision_ids = [key[-1] for key in revision_keys]
394
self._setup_steps(len(revision_keys))
395
stream = self._change_inv_parents(
396
self.inventory.get_record_stream(revision_keys, 'unordered', True),
399
new_inventories.insert_record_stream(stream)
400
# if this worked, the set of new_inventory_vf.names should equal
401
# the revisionds list
402
if not(set(new_inventories.keys()) == set(revision_keys)):
403
raise AssertionError()
404
self.pb.update(gettext('Writing weave'))
405
self.repo._activate_new_inventory()
406
self.inventory = None
407
ui.ui_factory.note(gettext('Inventory regenerated.'))
409
def _fix_text_parents(self):
410
"""Fix bad versionedfile parent entries.
412
It is possible for the parents entry in a versionedfile entry to be
413
inconsistent with the values in the revision and inventory.
415
This method finds entries with such inconsistencies, corrects their
416
parent lists, and replaces the versionedfile with a corrected version.
418
transaction = self.repo.get_transaction()
419
versions = [key[-1] for key in self.revisions.keys()]
420
mutter('Prepopulating revision text cache with %d revisions',
422
vf_checker = self.repo._get_versioned_file_checker()
423
bad_parents, unused_versions = vf_checker.check_file_version_parents(
424
self.repo.texts, self.pb)
425
text_index = vf_checker.text_index
426
per_id_bad_parents = {}
427
for key in unused_versions:
428
# Ensure that every file with unused versions gets rewritten.
429
# NB: This is really not needed, reconcile != pack.
430
per_id_bad_parents[key[0]] = {}
431
# Generate per-knit/weave data.
432
for key, details in bad_parents.iteritems():
435
knit_parents = tuple([parent[-1] for parent in details[0]])
436
correct_parents = tuple([parent[-1] for parent in details[1]])
437
file_details = per_id_bad_parents.setdefault(file_id, {})
438
file_details[rev_id] = (knit_parents, correct_parents)
439
file_id_versions = {}
440
for text_key in text_index:
441
versions_list = file_id_versions.setdefault(text_key[0], [])
442
versions_list.append(text_key[1])
443
# Do the reconcile of individual weaves.
444
for num, file_id in enumerate(per_id_bad_parents):
445
self.pb.update(gettext('Fixing text parents'), num,
446
len(per_id_bad_parents))
447
versions_with_bad_parents = per_id_bad_parents[file_id]
448
id_unused_versions = set(key[-1] for key in unused_versions
449
if key[0] == file_id)
450
if file_id in file_id_versions:
451
file_versions = file_id_versions[file_id]
453
# This id was present in the disk store but is not referenced
454
# by any revision at all.
456
self._fix_text_parent(file_id, versions_with_bad_parents,
457
id_unused_versions, file_versions)
459
def _fix_text_parent(self, file_id, versions_with_bad_parents,
460
unused_versions, all_versions):
461
"""Fix bad versionedfile entries in a single versioned file."""
462
mutter('fixing text parent: %r (%d versions)', file_id,
463
len(versions_with_bad_parents))
464
mutter('(%d are unused)', len(unused_versions))
465
new_file_id = 'temp:%s' % file_id
468
for version in all_versions:
469
if version in unused_versions:
471
elif version in versions_with_bad_parents:
472
parents = versions_with_bad_parents[version][1]
474
pmap = self.repo.texts.get_parent_map([(file_id, version)])
475
parents = [key[-1] for key in pmap[(file_id, version)]]
476
new_parents[(new_file_id, version)] = [
477
(new_file_id, parent) for parent in parents]
478
needed_keys.add((file_id, version))
479
def fix_parents(stream):
480
for record in stream:
481
bytes = record.get_bytes_as('fulltext')
482
new_key = (new_file_id, record.key[-1])
483
parents = new_parents[new_key]
484
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
485
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
486
self.repo._remove_file_id(new_file_id)
487
self.repo.texts.insert_record_stream(fix_parents(stream))
488
self.repo._remove_file_id(file_id)
490
self.repo._move_file_id(new_file_id, file_id)
493
class PackReconciler(RepoReconciler):
494
"""Reconciler that reconciles a pack based repository.
496
Garbage inventories do not affect ancestry queries, and removal is
497
considerably more expensive as there is no separate versioned file for
498
them, so they are not cleaned. In short it is currently a no-op.
500
In future this may be a good place to hook in annotation cache checking,
501
index recreation etc.
504
# XXX: The index corruption that _fix_text_parents performs is needed for
505
# packs, but not yet implemented. The basic approach is to:
506
# - lock the names list
507
# - perform a customised pack() that regenerates data as needed
508
# - unlock the names list
509
# https://bugs.launchpad.net/bzr/+bug/154173
511
def __init__(self, repo, other=None, thorough=False,
512
canonicalize_chks=False):
513
super(PackReconciler, self).__init__(repo, other=other,
515
self.canonicalize_chks = canonicalize_chks
517
def _reconcile_steps(self):
518
"""Perform the steps to reconcile this repository."""
519
if not self.thorough:
521
collection = self.repo._pack_collection
522
collection.ensure_loaded()
523
collection.lock_names()
524
self.add_cleanup(collection._unlock_names)
525
packs = collection.all_packs()
526
all_revisions = self.repo.all_revision_ids()
527
total_inventories = len(list(
528
collection.inventory_index.combined_index.iter_all_entries()))
529
if len(all_revisions):
530
if self.canonicalize_chks:
531
reconcile_meth = self.repo._canonicalize_chks_pack
533
reconcile_meth = self.repo._reconcile_pack
534
new_pack = reconcile_meth(collection, packs, ".reconcile",
535
all_revisions, self.pb)
536
if new_pack is not None:
537
self._discard_and_save(packs)
539
# only make a new pack when there is data to copy.
540
self._discard_and_save(packs)
541
self.garbage_inventories = total_inventories - len(list(
542
collection.inventory_index.combined_index.iter_all_entries()))
544
def _discard_and_save(self, packs):
545
"""Discard some packs from the repository.
547
This removes them from the memory index, saves the in-memory index
548
which makes the newly reconciled pack visible and hides the packs to be
549
discarded, and finally renames the packs being discarded into the
550
obsolete packs directory.
552
:param packs: The packs to discard.
555
self.repo._pack_collection._remove_pack_from_memory(pack)
556
self.repo._pack_collection._save_pack_names()
557
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
