1
# Copyright (C) 2006-2010 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
34
from bzrlib.trace import mutter
35
from bzrlib.tsort import topo_sort
36
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
39
def reconcile(dir, canonicalize_chks=False):
40
"""Reconcile the data in dir.
42
Currently this is limited to a inventory 'reweave'.
44
This is a convenience method, for using a Reconciler object.
46
Directly using Reconciler is recommended for library users that
47
desire fine grained control or analysis of the found issues.
49
:param canonicalize_chks: Make sure CHKs are in canonical form.
51
reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
52
reconciler.reconcile()
55
class Reconciler(object):
56
"""Reconcilers are used to reconcile existing data."""
58
def __init__(self, dir, other=None, canonicalize_chks=False):
59
"""Create a Reconciler."""
61
self.canonicalize_chks = canonicalize_chks
64
"""Perform reconciliation.
66
After reconciliation the following attributes document found issues:
67
inconsistent_parents: The number of revisions in the repository whose
68
ancestry was being reported incorrectly.
69
garbage_inventories: The number of inventory objects without revisions
70
that were garbage collected.
71
fixed_branch_history: None if there was no branch, False if the branch
72
history was correct, True if the branch history
73
needed to be re-normalized.
75
self.pb = ui.ui_factory.nested_progress_bar()
82
"""Helper function for performing reconciliation."""
83
self._reconcile_branch()
84
self._reconcile_repository()
86
def _reconcile_branch(self):
88
self.branch = self.bzrdir.open_branch()
89
except errors.NotBranchError:
90
# Nothing to check here
91
self.fixed_branch_history = None
93
ui.ui_factory.note('Reconciling branch %s' % self.branch.base)
94
branch_reconciler = self.branch.reconcile(thorough=True)
95
self.fixed_branch_history = branch_reconciler.fixed_history
97
def _reconcile_repository(self):
98
self.repo = self.bzrdir.find_repository()
99
ui.ui_factory.note('Reconciling repository %s' %
101
self.pb.update("Reconciling repository", 0, 1)
102
if self.canonicalize_chks:
104
self.repo.reconcile_canonicalize_chks
105
except AttributeError:
106
raise errors.BzrError(
107
"%s cannot canonicalize CHKs." % (self.repo,))
108
repo_reconciler = self.repo.reconcile_canonicalize_chks()
110
repo_reconciler = self.repo.reconcile(thorough=True)
111
self.inconsistent_parents = repo_reconciler.inconsistent_parents
112
self.garbage_inventories = repo_reconciler.garbage_inventories
113
if repo_reconciler.aborted:
115
'Reconcile aborted: revision index has inconsistent parents.')
117
'Run "bzr check" for more details.')
119
ui.ui_factory.note('Reconciliation complete.')
122
class BranchReconciler(object):
123
"""Reconciler that works on a branch."""
125
def __init__(self, a_branch, thorough=False):
126
self.fixed_history = None
127
self.thorough = thorough
128
self.branch = a_branch
131
operation = cleanup.OperationWithCleanups(self._reconcile)
132
self.add_cleanup = operation.add_cleanup
133
operation.run_simple()
135
def _reconcile(self):
136
self.branch.lock_write()
137
self.add_cleanup(self.branch.unlock)
138
self.pb = ui.ui_factory.nested_progress_bar()
139
self.add_cleanup(self.pb.finished)
140
self._reconcile_steps()
142
def _reconcile_steps(self):
143
self._reconcile_revision_history()
145
def _reconcile_revision_history(self):
146
repo = self.branch.repository
147
last_revno, last_revision_id = self.branch.last_revision_info()
150
for revid in repo.iter_reverse_revision_history(
152
real_history.append(revid)
153
except errors.RevisionNotPresent:
154
pass # Hit a ghost left hand parent
155
real_history.reverse()
156
if last_revno != len(real_history):
157
self.fixed_history = True
158
# Technically for Branch5 formats, it is more efficient to use
159
# set_revision_history, as this will regenerate it again.
160
# Not really worth a whole BranchReconciler class just for this,
162
ui.ui_factory.note('Fixing last revision info %s => %s' % (
163
last_revno, len(real_history)))
164
self.branch.set_last_revision_info(len(real_history),
167
self.fixed_history = False
168
ui.ui_factory.note('revision_history ok.')
171
class RepoReconciler(object):
172
"""Reconciler that reconciles a repository.
174
The goal of repository reconciliation is to make any derived data
175
consistent with the core data committed by a user. This can involve
176
reindexing, or removing unreferenced data if that can interfere with
177
queries in a given repository.
179
Currently this consists of an inventory reweave with revision cross-checks.
182
def __init__(self, repo, other=None, thorough=False):
183
"""Construct a RepoReconciler.
185
:param thorough: perform a thorough check which may take longer but
186
will correct non-data loss issues such as incorrect
189
self.garbage_inventories = 0
190
self.inconsistent_parents = 0
193
self.thorough = thorough
196
"""Perform reconciliation.
198
After reconciliation the following attributes document found issues:
199
inconsistent_parents: The number of revisions in the repository whose
200
ancestry was being reported incorrectly.
201
garbage_inventories: The number of inventory objects without revisions
202
that were garbage collected.
204
operation = cleanup.OperationWithCleanups(self._reconcile)
205
self.add_cleanup = operation.add_cleanup
206
operation.run_simple()
208
def _reconcile(self):
209
self.repo.lock_write()
210
self.add_cleanup(self.repo.unlock)
211
self.pb = ui.ui_factory.nested_progress_bar()
212
self.add_cleanup(self.pb.finished)
213
self._reconcile_steps()
215
def _reconcile_steps(self):
216
"""Perform the steps to reconcile this repository."""
217
self._reweave_inventory()
219
def _reweave_inventory(self):
220
"""Regenerate the inventory weave for the repository from scratch.
222
This is a smart function: it will only do the reweave if doing it
223
will correct data issues. The self.thorough flag controls whether
224
only data-loss causing issues (!self.thorough) or all issues
225
(self.thorough) are treated as requiring the reweave.
227
# local because needing to know about WeaveFile is a wart we want to hide
228
from bzrlib.weave import WeaveFile, Weave
229
transaction = self.repo.get_transaction()
230
self.pb.update('Reading inventory data')
231
self.inventory = self.repo.inventories
232
self.revisions = self.repo.revisions
233
# the total set of revisions to process
234
self.pending = set([key[-1] for key in self.revisions.keys()])
236
# mapping from revision_id to parents
238
# errors that we detect
239
self.inconsistent_parents = 0
240
# we need the revision id of each revision and its available parents list
241
self._setup_steps(len(self.pending))
242
for rev_id in self.pending:
243
# put a revision into the graph.
244
self._graph_revision(rev_id)
245
self._check_garbage_inventories()
246
# if there are no inconsistent_parents and
247
# (no garbage inventories or we are not doing a thorough check)
248
if (not self.inconsistent_parents and
249
(not self.garbage_inventories or not self.thorough)):
250
ui.ui_factory.note('Inventory ok.')
252
self.pb.update('Backing up inventory', 0, 0)
253
self.repo._backup_inventory()
254
ui.ui_factory.note('Backup inventory created.')
255
new_inventories = self.repo._temp_inventories()
257
# we have topological order of revisions and non ghost parents ready.
258
self._setup_steps(len(self._rev_graph))
259
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
260
stream = self._change_inv_parents(
261
self.inventory.get_record_stream(revision_keys, 'unordered', True),
262
self._new_inv_parents,
264
new_inventories.insert_record_stream(stream)
265
# if this worked, the set of new_inventories.keys should equal
267
if not (set(new_inventories.keys()) ==
268
set([(revid,) for revid in self.pending])):
269
raise AssertionError()
270
self.pb.update('Writing weave')
271
self.repo._activate_new_inventory()
272
self.inventory = None
273
ui.ui_factory.note('Inventory regenerated.')
275
def _new_inv_parents(self, revision_key):
276
"""Lookup ghost-filtered parents for revision_key."""
277
# Use the filtered ghostless parents list:
278
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
280
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
281
"""Adapt a record stream to reconcile the parents."""
282
for record in stream:
283
wanted_parents = get_parents(record.key)
284
if wanted_parents and wanted_parents[0] not in all_revision_keys:
285
# The check for the left most parent only handles knit
286
# compressors, but this code only applies to knit and weave
287
# repositories anyway.
288
bytes = record.get_bytes_as('fulltext')
289
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
291
adapted_record = AdapterFactory(record.key, wanted_parents, record)
293
self._reweave_step('adding inventories')
295
def _setup_steps(self, new_total):
296
"""Setup the markers we need to control the progress bar."""
297
self.total = new_total
300
def _graph_revision(self, rev_id):
301
"""Load a revision into the revision graph."""
302
# pick a random revision
303
# analyse revision id rev_id and put it in the stack.
304
self._reweave_step('loading revisions')
305
rev = self.repo.get_revision_reconcile(rev_id)
307
for parent in rev.parent_ids:
308
if self._parent_is_available(parent):
309
parents.append(parent)
311
mutter('found ghost %s', parent)
312
self._rev_graph[rev_id] = parents
314
def _check_garbage_inventories(self):
315
"""Check for garbage inventories which we cannot trust
317
We cant trust them because their pre-requisite file data may not
318
be present - all we know is that their revision was not installed.
320
if not self.thorough:
322
inventories = set(self.inventory.keys())
323
revisions = set(self.revisions.keys())
324
garbage = inventories.difference(revisions)
325
self.garbage_inventories = len(garbage)
326
for revision_key in garbage:
327
mutter('Garbage inventory {%s} found.', revision_key[-1])
329
def _parent_is_available(self, parent):
330
"""True if parent is a fully available revision
332
A fully available revision has a inventory and a revision object in the
335
if parent in self._rev_graph:
337
inv_present = (1 == len(self.inventory.get_parent_map([(parent,)])))
338
return (inv_present and self.repo.has_revision(parent))
340
def _reweave_step(self, message):
341
"""Mark a single step of regeneration complete."""
342
self.pb.update(message, self.count, self.total)
346
class KnitReconciler(RepoReconciler):
347
"""Reconciler that reconciles a knit format repository.
349
This will detect garbage inventories and remove them in thorough mode.
352
def _reconcile_steps(self):
353
"""Perform the steps to reconcile this repository."""
357
except errors.BzrCheckError:
360
# knits never suffer this
362
self._fix_text_parents()
364
def _load_indexes(self):
365
"""Load indexes for the reconciliation."""
366
self.transaction = self.repo.get_transaction()
367
self.pb.update('Reading indexes', 0, 2)
368
self.inventory = self.repo.inventories
369
self.pb.update('Reading indexes', 1, 2)
370
self.repo._check_for_inconsistent_revision_parents()
371
self.revisions = self.repo.revisions
372
self.pb.update('Reading indexes', 2, 2)
374
def _gc_inventory(self):
375
"""Remove inventories that are not referenced from the revision store."""
376
self.pb.update('Checking unused inventories', 0, 1)
377
self._check_garbage_inventories()
378
self.pb.update('Checking unused inventories', 1, 3)
379
if not self.garbage_inventories:
380
ui.ui_factory.note('Inventory ok.')
382
self.pb.update('Backing up inventory', 0, 0)
383
self.repo._backup_inventory()
384
ui.ui_factory.note('Backup Inventory created')
385
# asking for '' should never return a non-empty weave
386
new_inventories = self.repo._temp_inventories()
387
# we have topological order of revisions and non ghost parents ready.
388
graph = self.revisions.get_parent_map(self.revisions.keys())
389
revision_keys = topo_sort(graph)
390
revision_ids = [key[-1] for key in revision_keys]
391
self._setup_steps(len(revision_keys))
392
stream = self._change_inv_parents(
393
self.inventory.get_record_stream(revision_keys, 'unordered', True),
396
new_inventories.insert_record_stream(stream)
397
# if this worked, the set of new_inventory_vf.names should equal
398
# the revisionds list
399
if not(set(new_inventories.keys()) == set(revision_keys)):
400
raise AssertionError()
401
self.pb.update('Writing weave')
402
self.repo._activate_new_inventory()
403
self.inventory = None
404
ui.ui_factory.note('Inventory regenerated.')
406
def _fix_text_parents(self):
407
"""Fix bad versionedfile parent entries.
409
It is possible for the parents entry in a versionedfile entry to be
410
inconsistent with the values in the revision and inventory.
412
This method finds entries with such inconsistencies, corrects their
413
parent lists, and replaces the versionedfile with a corrected version.
415
transaction = self.repo.get_transaction()
416
versions = [key[-1] for key in self.revisions.keys()]
417
mutter('Prepopulating revision text cache with %d revisions',
419
vf_checker = self.repo._get_versioned_file_checker()
420
bad_parents, unused_versions = vf_checker.check_file_version_parents(
421
self.repo.texts, self.pb)
422
text_index = vf_checker.text_index
423
per_id_bad_parents = {}
424
for key in unused_versions:
425
# Ensure that every file with unused versions gets rewritten.
426
# NB: This is really not needed, reconcile != pack.
427
per_id_bad_parents[key[0]] = {}
428
# Generate per-knit/weave data.
429
for key, details in bad_parents.iteritems():
432
knit_parents = tuple([parent[-1] for parent in details[0]])
433
correct_parents = tuple([parent[-1] for parent in details[1]])
434
file_details = per_id_bad_parents.setdefault(file_id, {})
435
file_details[rev_id] = (knit_parents, correct_parents)
436
file_id_versions = {}
437
for text_key in text_index:
438
versions_list = file_id_versions.setdefault(text_key[0], [])
439
versions_list.append(text_key[1])
440
# Do the reconcile of individual weaves.
441
for num, file_id in enumerate(per_id_bad_parents):
442
self.pb.update('Fixing text parents', num,
443
len(per_id_bad_parents))
444
versions_with_bad_parents = per_id_bad_parents[file_id]
445
id_unused_versions = set(key[-1] for key in unused_versions
446
if key[0] == file_id)
447
if file_id in file_id_versions:
448
file_versions = file_id_versions[file_id]
450
# This id was present in the disk store but is not referenced
451
# by any revision at all.
453
self._fix_text_parent(file_id, versions_with_bad_parents,
454
id_unused_versions, file_versions)
456
def _fix_text_parent(self, file_id, versions_with_bad_parents,
457
unused_versions, all_versions):
458
"""Fix bad versionedfile entries in a single versioned file."""
459
mutter('fixing text parent: %r (%d versions)', file_id,
460
len(versions_with_bad_parents))
461
mutter('(%d are unused)', len(unused_versions))
462
new_file_id = 'temp:%s' % file_id
465
for version in all_versions:
466
if version in unused_versions:
468
elif version in versions_with_bad_parents:
469
parents = versions_with_bad_parents[version][1]
471
pmap = self.repo.texts.get_parent_map([(file_id, version)])
472
parents = [key[-1] for key in pmap[(file_id, version)]]
473
new_parents[(new_file_id, version)] = [
474
(new_file_id, parent) for parent in parents]
475
needed_keys.add((file_id, version))
476
def fix_parents(stream):
477
for record in stream:
478
bytes = record.get_bytes_as('fulltext')
479
new_key = (new_file_id, record.key[-1])
480
parents = new_parents[new_key]
481
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
482
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
483
self.repo._remove_file_id(new_file_id)
484
self.repo.texts.insert_record_stream(fix_parents(stream))
485
self.repo._remove_file_id(file_id)
487
self.repo._move_file_id(new_file_id, file_id)
490
class PackReconciler(RepoReconciler):
491
"""Reconciler that reconciles a pack based repository.
493
Garbage inventories do not affect ancestry queries, and removal is
494
considerably more expensive as there is no separate versioned file for
495
them, so they are not cleaned. In short it is currently a no-op.
497
In future this may be a good place to hook in annotation cache checking,
498
index recreation etc.
501
# XXX: The index corruption that _fix_text_parents performs is needed for
502
# packs, but not yet implemented. The basic approach is to:
503
# - lock the names list
504
# - perform a customised pack() that regenerates data as needed
505
# - unlock the names list
506
# https://bugs.launchpad.net/bzr/+bug/154173
508
def __init__(self, repo, other=None, thorough=False,
509
canonicalize_chks=False):
510
super(PackReconciler, self).__init__(repo, other=other,
512
self.canonicalize_chks = canonicalize_chks
514
def _reconcile_steps(self):
515
"""Perform the steps to reconcile this repository."""
516
if not self.thorough:
518
collection = self.repo._pack_collection
519
collection.ensure_loaded()
520
collection.lock_names()
521
self.add_cleanup(collection._unlock_names)
522
packs = collection.all_packs()
523
all_revisions = self.repo.all_revision_ids()
524
total_inventories = len(list(
525
collection.inventory_index.combined_index.iter_all_entries()))
526
if len(all_revisions):
527
if self.canonicalize_chks:
528
reconcile_meth = self.repo._canonicalize_chks_pack
530
reconcile_meth = self.repo._reconcile_pack
531
new_pack = reconcile_meth(collection, packs, ".reconcile",
532
all_revisions, self.pb)
533
if new_pack is not None:
534
self._discard_and_save(packs)
536
# only make a new pack when there is data to copy.
537
self._discard_and_save(packs)
538
self.garbage_inventories = total_inventories - len(list(
539
collection.inventory_index.combined_index.iter_all_entries()))
541
def _discard_and_save(self, packs):
542
"""Discard some packs from the repository.
544
This removes them from the memory index, saves the in-memory index
545
which makes the newly reconciled pack visible and hides the packs to be
546
discarded, and finally renames the packs being discarded into the
547
obsolete packs directory.
549
:param packs: The packs to discard.
552
self.repo._pack_collection._remove_pack_from_memory(pack)
553
self.repo._pack_collection._save_pack_names()
554
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
