1
# Copyright (C) 2006-2010 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
32
revision as _mod_revision,
35
from bzrlib.trace import mutter
36
from bzrlib.tsort import topo_sort
37
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
40
def reconcile(dir, canonicalize_chks=False):
41
"""Reconcile the data in dir.
43
Currently this is limited to a inventory 'reweave'.
45
This is a convenience method, for using a Reconciler object.
47
Directly using Reconciler is recommended for library users that
48
desire fine grained control or analysis of the found issues.
50
:param canonicalize_chks: Make sure CHKs are in canonical form.
52
reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
53
reconciler.reconcile()
56
class Reconciler(object):
57
"""Reconcilers are used to reconcile existing data."""
59
def __init__(self, dir, other=None, canonicalize_chks=False):
60
"""Create a Reconciler."""
62
self.canonicalize_chks = canonicalize_chks
65
"""Perform reconciliation.
67
After reconciliation the following attributes document found issues:
69
* `inconsistent_parents`: The number of revisions in the repository
70
whose ancestry was being reported incorrectly.
71
* `garbage_inventories`: The number of inventory objects without
72
revisions that were garbage collected.
73
* `fixed_branch_history`: None if there was no branch, False if the
74
branch history was correct, True if the branch history needed to be
77
self.pb = ui.ui_factory.nested_progress_bar()
84
"""Helper function for performing reconciliation."""
85
self._reconcile_branch()
86
self._reconcile_repository()
88
def _reconcile_branch(self):
90
self.branch = self.bzrdir.open_branch()
91
except errors.NotBranchError:
92
# Nothing to check here
93
self.fixed_branch_history = None
95
ui.ui_factory.note('Reconciling branch %s' % self.branch.base)
96
branch_reconciler = self.branch.reconcile(thorough=True)
97
self.fixed_branch_history = branch_reconciler.fixed_history
99
def _reconcile_repository(self):
100
self.repo = self.bzrdir.find_repository()
101
ui.ui_factory.note('Reconciling repository %s' %
103
self.pb.update("Reconciling repository", 0, 1)
104
if self.canonicalize_chks:
106
self.repo.reconcile_canonicalize_chks
107
except AttributeError:
108
raise errors.BzrError(
109
"%s cannot canonicalize CHKs." % (self.repo,))
110
repo_reconciler = self.repo.reconcile_canonicalize_chks()
112
repo_reconciler = self.repo.reconcile(thorough=True)
113
self.inconsistent_parents = repo_reconciler.inconsistent_parents
114
self.garbage_inventories = repo_reconciler.garbage_inventories
115
if repo_reconciler.aborted:
117
'Reconcile aborted: revision index has inconsistent parents.')
119
'Run "bzr check" for more details.')
121
ui.ui_factory.note('Reconciliation complete.')
124
class BranchReconciler(object):
125
"""Reconciler that works on a branch."""
127
def __init__(self, a_branch, thorough=False):
128
self.fixed_history = None
129
self.thorough = thorough
130
self.branch = a_branch
133
operation = cleanup.OperationWithCleanups(self._reconcile)
134
self.add_cleanup = operation.add_cleanup
135
operation.run_simple()
137
def _reconcile(self):
138
self.branch.lock_write()
139
self.add_cleanup(self.branch.unlock)
140
self.pb = ui.ui_factory.nested_progress_bar()
141
self.add_cleanup(self.pb.finished)
142
self._reconcile_steps()
144
def _reconcile_steps(self):
145
self._reconcile_revision_history()
147
def _reconcile_revision_history(self):
148
last_revno, last_revision_id = self.branch.last_revision_info()
150
graph = self.branch.repository.get_graph()
152
for revid in graph.iter_lefthand_ancestry(
153
last_revision_id, (_mod_revision.NULL_REVISION,)):
154
real_history.append(revid)
155
except errors.RevisionNotPresent:
156
pass # Hit a ghost left hand parent
157
real_history.reverse()
158
if last_revno != len(real_history):
159
self.fixed_history = True
160
# Technically for Branch5 formats, it is more efficient to use
161
# set_revision_history, as this will regenerate it again.
162
# Not really worth a whole BranchReconciler class just for this,
164
ui.ui_factory.note('Fixing last revision info %s => %s' % (
165
last_revno, len(real_history)))
166
self.branch.set_last_revision_info(len(real_history),
169
self.fixed_history = False
170
ui.ui_factory.note('revision_history ok.')
173
class RepoReconciler(object):
174
"""Reconciler that reconciles a repository.
176
The goal of repository reconciliation is to make any derived data
177
consistent with the core data committed by a user. This can involve
178
reindexing, or removing unreferenced data if that can interfere with
179
queries in a given repository.
181
Currently this consists of an inventory reweave with revision cross-checks.
184
def __init__(self, repo, other=None, thorough=False):
185
"""Construct a RepoReconciler.
187
:param thorough: perform a thorough check which may take longer but
188
will correct non-data loss issues such as incorrect
191
self.garbage_inventories = 0
192
self.inconsistent_parents = 0
195
self.thorough = thorough
198
"""Perform reconciliation.
200
After reconciliation the following attributes document found issues:
202
* `inconsistent_parents`: The number of revisions in the repository
203
whose ancestry was being reported incorrectly.
204
* `garbage_inventories`: The number of inventory objects without
205
revisions that were garbage collected.
207
operation = cleanup.OperationWithCleanups(self._reconcile)
208
self.add_cleanup = operation.add_cleanup
209
operation.run_simple()
211
def _reconcile(self):
212
self.repo.lock_write()
213
self.add_cleanup(self.repo.unlock)
214
self.pb = ui.ui_factory.nested_progress_bar()
215
self.add_cleanup(self.pb.finished)
216
self._reconcile_steps()
218
def _reconcile_steps(self):
219
"""Perform the steps to reconcile this repository."""
220
self._reweave_inventory()
222
def _reweave_inventory(self):
223
"""Regenerate the inventory weave for the repository from scratch.
225
This is a smart function: it will only do the reweave if doing it
226
will correct data issues. The self.thorough flag controls whether
227
only data-loss causing issues (!self.thorough) or all issues
228
(self.thorough) are treated as requiring the reweave.
230
transaction = self.repo.get_transaction()
231
self.pb.update('Reading inventory data')
232
self.inventory = self.repo.inventories
233
self.revisions = self.repo.revisions
234
# the total set of revisions to process
235
self.pending = set([key[-1] for key in self.revisions.keys()])
237
# mapping from revision_id to parents
239
# errors that we detect
240
self.inconsistent_parents = 0
241
# we need the revision id of each revision and its available parents list
242
self._setup_steps(len(self.pending))
243
for rev_id in self.pending:
244
# put a revision into the graph.
245
self._graph_revision(rev_id)
246
self._check_garbage_inventories()
247
# if there are no inconsistent_parents and
248
# (no garbage inventories or we are not doing a thorough check)
249
if (not self.inconsistent_parents and
250
(not self.garbage_inventories or not self.thorough)):
251
ui.ui_factory.note('Inventory ok.')
253
self.pb.update('Backing up inventory', 0, 0)
254
self.repo._backup_inventory()
255
ui.ui_factory.note('Backup inventory created.')
256
new_inventories = self.repo._temp_inventories()
258
# we have topological order of revisions and non ghost parents ready.
259
self._setup_steps(len(self._rev_graph))
260
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
261
stream = self._change_inv_parents(
262
self.inventory.get_record_stream(revision_keys, 'unordered', True),
263
self._new_inv_parents,
265
new_inventories.insert_record_stream(stream)
266
# if this worked, the set of new_inventories.keys should equal
268
if not (set(new_inventories.keys()) ==
269
set([(revid,) for revid in self.pending])):
270
raise AssertionError()
271
self.pb.update('Writing weave')
272
self.repo._activate_new_inventory()
273
self.inventory = None
274
ui.ui_factory.note('Inventory regenerated.')
276
def _new_inv_parents(self, revision_key):
277
"""Lookup ghost-filtered parents for revision_key."""
278
# Use the filtered ghostless parents list:
279
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
281
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
282
"""Adapt a record stream to reconcile the parents."""
283
for record in stream:
284
wanted_parents = get_parents(record.key)
285
if wanted_parents and wanted_parents[0] not in all_revision_keys:
286
# The check for the left most parent only handles knit
287
# compressors, but this code only applies to knit and weave
288
# repositories anyway.
289
bytes = record.get_bytes_as('fulltext')
290
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
292
adapted_record = AdapterFactory(record.key, wanted_parents, record)
294
self._reweave_step('adding inventories')
296
def _setup_steps(self, new_total):
297
"""Setup the markers we need to control the progress bar."""
298
self.total = new_total
301
def _graph_revision(self, rev_id):
302
"""Load a revision into the revision graph."""
303
# pick a random revision
304
# analyse revision id rev_id and put it in the stack.
305
self._reweave_step('loading revisions')
306
rev = self.repo.get_revision_reconcile(rev_id)
308
for parent in rev.parent_ids:
309
if self._parent_is_available(parent):
310
parents.append(parent)
312
mutter('found ghost %s', parent)
313
self._rev_graph[rev_id] = parents
315
def _check_garbage_inventories(self):
316
"""Check for garbage inventories which we cannot trust
318
We cant trust them because their pre-requisite file data may not
319
be present - all we know is that their revision was not installed.
321
if not self.thorough:
323
inventories = set(self.inventory.keys())
324
revisions = set(self.revisions.keys())
325
garbage = inventories.difference(revisions)
326
self.garbage_inventories = len(garbage)
327
for revision_key in garbage:
328
mutter('Garbage inventory {%s} found.', revision_key[-1])
330
def _parent_is_available(self, parent):
331
"""True if parent is a fully available revision
333
A fully available revision has a inventory and a revision object in the
336
if parent in self._rev_graph:
338
inv_present = (1 == len(self.inventory.get_parent_map([(parent,)])))
339
return (inv_present and self.repo.has_revision(parent))
341
def _reweave_step(self, message):
342
"""Mark a single step of regeneration complete."""
343
self.pb.update(message, self.count, self.total)
347
class KnitReconciler(RepoReconciler):
348
"""Reconciler that reconciles a knit format repository.
350
This will detect garbage inventories and remove them in thorough mode.
353
def _reconcile_steps(self):
354
"""Perform the steps to reconcile this repository."""
358
except errors.BzrCheckError:
361
# knits never suffer this
363
self._fix_text_parents()
365
def _load_indexes(self):
366
"""Load indexes for the reconciliation."""
367
self.transaction = self.repo.get_transaction()
368
self.pb.update('Reading indexes', 0, 2)
369
self.inventory = self.repo.inventories
370
self.pb.update('Reading indexes', 1, 2)
371
self.repo._check_for_inconsistent_revision_parents()
372
self.revisions = self.repo.revisions
373
self.pb.update('Reading indexes', 2, 2)
375
def _gc_inventory(self):
376
"""Remove inventories that are not referenced from the revision store."""
377
self.pb.update('Checking unused inventories', 0, 1)
378
self._check_garbage_inventories()
379
self.pb.update('Checking unused inventories', 1, 3)
380
if not self.garbage_inventories:
381
ui.ui_factory.note('Inventory ok.')
383
self.pb.update('Backing up inventory', 0, 0)
384
self.repo._backup_inventory()
385
ui.ui_factory.note('Backup Inventory created')
386
# asking for '' should never return a non-empty weave
387
new_inventories = self.repo._temp_inventories()
388
# we have topological order of revisions and non ghost parents ready.
389
graph = self.revisions.get_parent_map(self.revisions.keys())
390
revision_keys = topo_sort(graph)
391
revision_ids = [key[-1] for key in revision_keys]
392
self._setup_steps(len(revision_keys))
393
stream = self._change_inv_parents(
394
self.inventory.get_record_stream(revision_keys, 'unordered', True),
397
new_inventories.insert_record_stream(stream)
398
# if this worked, the set of new_inventory_vf.names should equal
399
# the revisionds list
400
if not(set(new_inventories.keys()) == set(revision_keys)):
401
raise AssertionError()
402
self.pb.update('Writing weave')
403
self.repo._activate_new_inventory()
404
self.inventory = None
405
ui.ui_factory.note('Inventory regenerated.')
407
def _fix_text_parents(self):
408
"""Fix bad versionedfile parent entries.
410
It is possible for the parents entry in a versionedfile entry to be
411
inconsistent with the values in the revision and inventory.
413
This method finds entries with such inconsistencies, corrects their
414
parent lists, and replaces the versionedfile with a corrected version.
416
transaction = self.repo.get_transaction()
417
versions = [key[-1] for key in self.revisions.keys()]
418
mutter('Prepopulating revision text cache with %d revisions',
420
vf_checker = self.repo._get_versioned_file_checker()
421
bad_parents, unused_versions = vf_checker.check_file_version_parents(
422
self.repo.texts, self.pb)
423
text_index = vf_checker.text_index
424
per_id_bad_parents = {}
425
for key in unused_versions:
426
# Ensure that every file with unused versions gets rewritten.
427
# NB: This is really not needed, reconcile != pack.
428
per_id_bad_parents[key[0]] = {}
429
# Generate per-knit/weave data.
430
for key, details in bad_parents.iteritems():
433
knit_parents = tuple([parent[-1] for parent in details[0]])
434
correct_parents = tuple([parent[-1] for parent in details[1]])
435
file_details = per_id_bad_parents.setdefault(file_id, {})
436
file_details[rev_id] = (knit_parents, correct_parents)
437
file_id_versions = {}
438
for text_key in text_index:
439
versions_list = file_id_versions.setdefault(text_key[0], [])
440
versions_list.append(text_key[1])
441
# Do the reconcile of individual weaves.
442
for num, file_id in enumerate(per_id_bad_parents):
443
self.pb.update('Fixing text parents', num,
444
len(per_id_bad_parents))
445
versions_with_bad_parents = per_id_bad_parents[file_id]
446
id_unused_versions = set(key[-1] for key in unused_versions
447
if key[0] == file_id)
448
if file_id in file_id_versions:
449
file_versions = file_id_versions[file_id]
451
# This id was present in the disk store but is not referenced
452
# by any revision at all.
454
self._fix_text_parent(file_id, versions_with_bad_parents,
455
id_unused_versions, file_versions)
457
def _fix_text_parent(self, file_id, versions_with_bad_parents,
458
unused_versions, all_versions):
459
"""Fix bad versionedfile entries in a single versioned file."""
460
mutter('fixing text parent: %r (%d versions)', file_id,
461
len(versions_with_bad_parents))
462
mutter('(%d are unused)', len(unused_versions))
463
new_file_id = 'temp:%s' % file_id
466
for version in all_versions:
467
if version in unused_versions:
469
elif version in versions_with_bad_parents:
470
parents = versions_with_bad_parents[version][1]
472
pmap = self.repo.texts.get_parent_map([(file_id, version)])
473
parents = [key[-1] for key in pmap[(file_id, version)]]
474
new_parents[(new_file_id, version)] = [
475
(new_file_id, parent) for parent in parents]
476
needed_keys.add((file_id, version))
477
def fix_parents(stream):
478
for record in stream:
479
bytes = record.get_bytes_as('fulltext')
480
new_key = (new_file_id, record.key[-1])
481
parents = new_parents[new_key]
482
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
483
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
484
self.repo._remove_file_id(new_file_id)
485
self.repo.texts.insert_record_stream(fix_parents(stream))
486
self.repo._remove_file_id(file_id)
488
self.repo._move_file_id(new_file_id, file_id)
491
class PackReconciler(RepoReconciler):
492
"""Reconciler that reconciles a pack based repository.
494
Garbage inventories do not affect ancestry queries, and removal is
495
considerably more expensive as there is no separate versioned file for
496
them, so they are not cleaned. In short it is currently a no-op.
498
In future this may be a good place to hook in annotation cache checking,
499
index recreation etc.
502
# XXX: The index corruption that _fix_text_parents performs is needed for
503
# packs, but not yet implemented. The basic approach is to:
504
# - lock the names list
505
# - perform a customised pack() that regenerates data as needed
506
# - unlock the names list
507
# https://bugs.launchpad.net/bzr/+bug/154173
509
def __init__(self, repo, other=None, thorough=False,
510
canonicalize_chks=False):
511
super(PackReconciler, self).__init__(repo, other=other,
513
self.canonicalize_chks = canonicalize_chks
515
def _reconcile_steps(self):
516
"""Perform the steps to reconcile this repository."""
517
if not self.thorough:
519
collection = self.repo._pack_collection
520
collection.ensure_loaded()
521
collection.lock_names()
522
self.add_cleanup(collection._unlock_names)
523
packs = collection.all_packs()
524
all_revisions = self.repo.all_revision_ids()
525
total_inventories = len(list(
526
collection.inventory_index.combined_index.iter_all_entries()))
527
if len(all_revisions):
528
if self.canonicalize_chks:
529
reconcile_meth = self.repo._canonicalize_chks_pack
531
reconcile_meth = self.repo._reconcile_pack
532
new_pack = reconcile_meth(collection, packs, ".reconcile",
533
all_revisions, self.pb)
534
if new_pack is not None:
535
self._discard_and_save(packs)
537
# only make a new pack when there is data to copy.
538
self._discard_and_save(packs)
539
self.garbage_inventories = total_inventories - len(list(
540
collection.inventory_index.combined_index.iter_all_entries()))
542
def _discard_and_save(self, packs):
543
"""Discard some packs from the repository.
545
This removes them from the memory index, saves the in-memory index
546
which makes the newly reconciled pack visible and hides the packs to be
547
discarded, and finally renames the packs being discarded into the
548
obsolete packs directory.
550
:param packs: The packs to discard.
553
self.repo._pack_collection._remove_pack_from_memory(pack)
554
self.repo._pack_collection._save_pack_names()
555
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
