1
# Copyright (C) 2006-2010 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
35
from bzrlib.trace import mutter
36
from bzrlib.tsort import topo_sort
37
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
40
def reconcile(dir, other=None):
41
"""Reconcile the data in dir.
43
Currently this is limited to a inventory 'reweave'.
45
This is a convenience method, for using a Reconciler object.
47
Directly using Reconciler is recommended for library users that
48
desire fine grained control or analysis of the found issues.
50
:param other: another bzrdir to reconcile against.
52
reconciler = Reconciler(dir, other=other)
53
reconciler.reconcile()
56
class Reconciler(object):
57
"""Reconcilers are used to reconcile existing data."""
59
def __init__(self, dir, other=None):
60
"""Create a Reconciler."""
64
"""Perform reconciliation.
66
After reconciliation the following attributes document found issues:
67
inconsistent_parents: The number of revisions in the repository whose
68
ancestry was being reported incorrectly.
69
garbage_inventories: The number of inventory objects without revisions
70
that were garbage collected.
71
fixed_branch_history: None if there was no branch, False if the branch
72
history was correct, True if the branch history
73
needed to be re-normalized.
75
self.pb = ui.ui_factory.nested_progress_bar()
82
"""Helper function for performing reconciliation."""
83
self._reconcile_branch()
84
self._reconcile_repository()
86
def _reconcile_branch(self):
88
self.branch = self.bzrdir.open_branch()
89
except errors.NotBranchError:
90
# Nothing to check here
91
self.fixed_branch_history = None
93
ui.ui_factory.note('Reconciling branch %s' % self.branch.base)
94
branch_reconciler = self.branch.reconcile(thorough=True)
95
self.fixed_branch_history = branch_reconciler.fixed_history
97
def _reconcile_repository(self):
98
self.repo = self.bzrdir.find_repository()
99
ui.ui_factory.note('Reconciling repository %s' %
100
self.repo.bzrdir.root_transport.base)
101
self.pb.update("Reconciling repository", 0, 1)
102
repo_reconciler = self.repo.reconcile(thorough=True)
103
self.inconsistent_parents = repo_reconciler.inconsistent_parents
104
self.garbage_inventories = repo_reconciler.garbage_inventories
105
if repo_reconciler.aborted:
107
'Reconcile aborted: revision index has inconsistent parents.')
109
'Run "bzr check" for more details.')
111
ui.ui_factory.note('Reconciliation complete.')
114
class BranchReconciler(object):
115
"""Reconciler that works on a branch."""
117
def __init__(self, a_branch, thorough=False):
118
self.fixed_history = None
119
self.thorough = thorough
120
self.branch = a_branch
123
operation = cleanup.OperationWithCleanups(self._reconcile)
124
self.add_cleanup = operation.add_cleanup
125
operation.run_simple()
127
def _reconcile(self):
128
self.branch.lock_write()
129
self.add_cleanup(self.branch.unlock)
130
self.pb = ui.ui_factory.nested_progress_bar()
131
self.add_cleanup(self.pb.finished)
132
self._reconcile_steps()
134
def _reconcile_steps(self):
135
self._reconcile_revision_history()
137
def _reconcile_revision_history(self):
138
repo = self.branch.repository
139
last_revno, last_revision_id = self.branch.last_revision_info()
142
for revid in repo.iter_reverse_revision_history(
144
real_history.append(revid)
145
except errors.RevisionNotPresent:
146
pass # Hit a ghost left hand parent
147
real_history.reverse()
148
if last_revno != len(real_history):
149
self.fixed_history = True
150
# Technically for Branch5 formats, it is more efficient to use
151
# set_revision_history, as this will regenerate it again.
152
# Not really worth a whole BranchReconciler class just for this,
154
ui.ui_factory.note('Fixing last revision info %s => %s' % (
155
last_revno, len(real_history)))
156
self.branch.set_last_revision_info(len(real_history),
159
self.fixed_history = False
160
ui.ui_factory.note('revision_history ok.')
163
class RepoReconciler(object):
164
"""Reconciler that reconciles a repository.
166
The goal of repository reconciliation is to make any derived data
167
consistent with the core data committed by a user. This can involve
168
reindexing, or removing unreferenced data if that can interfere with
169
queries in a given repository.
171
Currently this consists of an inventory reweave with revision cross-checks.
174
def __init__(self, repo, other=None, thorough=False):
175
"""Construct a RepoReconciler.
177
:param thorough: perform a thorough check which may take longer but
178
will correct non-data loss issues such as incorrect
181
self.garbage_inventories = 0
182
self.inconsistent_parents = 0
185
self.thorough = thorough
188
"""Perform reconciliation.
190
After reconciliation the following attributes document found issues:
191
inconsistent_parents: The number of revisions in the repository whose
192
ancestry was being reported incorrectly.
193
garbage_inventories: The number of inventory objects without revisions
194
that were garbage collected.
196
operation = cleanup.OperationWithCleanups(self._reconcile)
197
self.add_cleanup = operation.add_cleanup
198
operation.run_simple()
200
def _reconcile(self):
201
self.repo.lock_write()
202
self.add_cleanup(self.repo.unlock)
203
self.pb = ui.ui_factory.nested_progress_bar()
204
self.add_cleanup(self.pb.finished)
205
self._reconcile_steps()
207
def _reconcile_steps(self):
208
"""Perform the steps to reconcile this repository."""
209
self._reweave_inventory()
211
def _reweave_inventory(self):
212
"""Regenerate the inventory weave for the repository from scratch.
214
This is a smart function: it will only do the reweave if doing it
215
will correct data issues. The self.thorough flag controls whether
216
only data-loss causing issues (!self.thorough) or all issues
217
(self.thorough) are treated as requiring the reweave.
219
# local because needing to know about WeaveFile is a wart we want to hide
220
from bzrlib.weave import WeaveFile, Weave
221
transaction = self.repo.get_transaction()
222
self.pb.update('Reading inventory data')
223
self.inventory = self.repo.inventories
224
self.revisions = self.repo.revisions
225
# the total set of revisions to process
226
self.pending = set([key[-1] for key in self.revisions.keys()])
228
# mapping from revision_id to parents
230
# errors that we detect
231
self.inconsistent_parents = 0
232
# we need the revision id of each revision and its available parents list
233
self._setup_steps(len(self.pending))
234
for rev_id in self.pending:
235
# put a revision into the graph.
236
self._graph_revision(rev_id)
237
self._check_garbage_inventories()
238
# if there are no inconsistent_parents and
239
# (no garbage inventories or we are not doing a thorough check)
240
if (not self.inconsistent_parents and
241
(not self.garbage_inventories or not self.thorough)):
242
ui.ui_factory.note('Inventory ok.')
244
self.pb.update('Backing up inventory', 0, 0)
245
self.repo._backup_inventory()
246
ui.ui_factory.note('Backup inventory created.')
247
new_inventories = self.repo._temp_inventories()
249
# we have topological order of revisions and non ghost parents ready.
250
self._setup_steps(len(self._rev_graph))
251
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
252
stream = self._change_inv_parents(
253
self.inventory.get_record_stream(revision_keys, 'unordered', True),
254
self._new_inv_parents,
256
new_inventories.insert_record_stream(stream)
257
# if this worked, the set of new_inventories.keys should equal
259
if not (set(new_inventories.keys()) ==
260
set([(revid,) for revid in self.pending])):
261
raise AssertionError()
262
self.pb.update('Writing weave')
263
self.repo._activate_new_inventory()
264
self.inventory = None
265
ui.ui_factory.note('Inventory regenerated.')
267
def _new_inv_parents(self, revision_key):
268
"""Lookup ghost-filtered parents for revision_key."""
269
# Use the filtered ghostless parents list:
270
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
272
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
273
"""Adapt a record stream to reconcile the parents."""
274
for record in stream:
275
wanted_parents = get_parents(record.key)
276
if wanted_parents and wanted_parents[0] not in all_revision_keys:
277
# The check for the left most parent only handles knit
278
# compressors, but this code only applies to knit and weave
279
# repositories anyway.
280
bytes = record.get_bytes_as('fulltext')
281
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
283
adapted_record = AdapterFactory(record.key, wanted_parents, record)
285
self._reweave_step('adding inventories')
287
def _setup_steps(self, new_total):
288
"""Setup the markers we need to control the progress bar."""
289
self.total = new_total
292
def _graph_revision(self, rev_id):
293
"""Load a revision into the revision graph."""
294
# pick a random revision
295
# analyse revision id rev_id and put it in the stack.
296
self._reweave_step('loading revisions')
297
rev = self.repo.get_revision_reconcile(rev_id)
299
for parent in rev.parent_ids:
300
if self._parent_is_available(parent):
301
parents.append(parent)
303
mutter('found ghost %s', parent)
304
self._rev_graph[rev_id] = parents
306
def _check_garbage_inventories(self):
307
"""Check for garbage inventories which we cannot trust
309
We cant trust them because their pre-requisite file data may not
310
be present - all we know is that their revision was not installed.
312
if not self.thorough:
314
inventories = set(self.inventory.keys())
315
revisions = set(self.revisions.keys())
316
garbage = inventories.difference(revisions)
317
self.garbage_inventories = len(garbage)
318
for revision_key in garbage:
319
mutter('Garbage inventory {%s} found.', revision_key[-1])
321
def _parent_is_available(self, parent):
322
"""True if parent is a fully available revision
324
A fully available revision has a inventory and a revision object in the
327
if parent in self._rev_graph:
329
inv_present = (1 == len(self.inventory.get_parent_map([(parent,)])))
330
return (inv_present and self.repo.has_revision(parent))
332
def _reweave_step(self, message):
333
"""Mark a single step of regeneration complete."""
334
self.pb.update(message, self.count, self.total)
338
class KnitReconciler(RepoReconciler):
339
"""Reconciler that reconciles a knit format repository.
341
This will detect garbage inventories and remove them in thorough mode.
344
def _reconcile_steps(self):
345
"""Perform the steps to reconcile this repository."""
349
except errors.BzrCheckError:
352
# knits never suffer this
354
self._fix_text_parents()
356
def _load_indexes(self):
357
"""Load indexes for the reconciliation."""
358
self.transaction = self.repo.get_transaction()
359
self.pb.update('Reading indexes', 0, 2)
360
self.inventory = self.repo.inventories
361
self.pb.update('Reading indexes', 1, 2)
362
self.repo._check_for_inconsistent_revision_parents()
363
self.revisions = self.repo.revisions
364
self.pb.update('Reading indexes', 2, 2)
366
def _gc_inventory(self):
367
"""Remove inventories that are not referenced from the revision store."""
368
self.pb.update('Checking unused inventories', 0, 1)
369
self._check_garbage_inventories()
370
self.pb.update('Checking unused inventories', 1, 3)
371
if not self.garbage_inventories:
372
ui.ui_factory.note('Inventory ok.')
374
self.pb.update('Backing up inventory', 0, 0)
375
self.repo._backup_inventory()
376
ui.ui_factory.note('Backup Inventory created')
377
# asking for '' should never return a non-empty weave
378
new_inventories = self.repo._temp_inventories()
379
# we have topological order of revisions and non ghost parents ready.
380
graph = self.revisions.get_parent_map(self.revisions.keys())
381
revision_keys = topo_sort(graph)
382
revision_ids = [key[-1] for key in revision_keys]
383
self._setup_steps(len(revision_keys))
384
stream = self._change_inv_parents(
385
self.inventory.get_record_stream(revision_keys, 'unordered', True),
388
new_inventories.insert_record_stream(stream)
389
# if this worked, the set of new_inventory_vf.names should equal
390
# the revisionds list
391
if not(set(new_inventories.keys()) == set(revision_keys)):
392
raise AssertionError()
393
self.pb.update('Writing weave')
394
self.repo._activate_new_inventory()
395
self.inventory = None
396
ui.ui_factory.note('Inventory regenerated.')
398
def _fix_text_parents(self):
399
"""Fix bad versionedfile parent entries.
401
It is possible for the parents entry in a versionedfile entry to be
402
inconsistent with the values in the revision and inventory.
404
This method finds entries with such inconsistencies, corrects their
405
parent lists, and replaces the versionedfile with a corrected version.
407
transaction = self.repo.get_transaction()
408
versions = [key[-1] for key in self.revisions.keys()]
409
mutter('Prepopulating revision text cache with %d revisions',
411
vf_checker = self.repo._get_versioned_file_checker()
412
bad_parents, unused_versions = vf_checker.check_file_version_parents(
413
self.repo.texts, self.pb)
414
text_index = vf_checker.text_index
415
per_id_bad_parents = {}
416
for key in unused_versions:
417
# Ensure that every file with unused versions gets rewritten.
418
# NB: This is really not needed, reconcile != pack.
419
per_id_bad_parents[key[0]] = {}
420
# Generate per-knit/weave data.
421
for key, details in bad_parents.iteritems():
424
knit_parents = tuple([parent[-1] for parent in details[0]])
425
correct_parents = tuple([parent[-1] for parent in details[1]])
426
file_details = per_id_bad_parents.setdefault(file_id, {})
427
file_details[rev_id] = (knit_parents, correct_parents)
428
file_id_versions = {}
429
for text_key in text_index:
430
versions_list = file_id_versions.setdefault(text_key[0], [])
431
versions_list.append(text_key[1])
432
# Do the reconcile of individual weaves.
433
for num, file_id in enumerate(per_id_bad_parents):
434
self.pb.update('Fixing text parents', num,
435
len(per_id_bad_parents))
436
versions_with_bad_parents = per_id_bad_parents[file_id]
437
id_unused_versions = set(key[-1] for key in unused_versions
438
if key[0] == file_id)
439
if file_id in file_id_versions:
440
file_versions = file_id_versions[file_id]
442
# This id was present in the disk store but is not referenced
443
# by any revision at all.
445
self._fix_text_parent(file_id, versions_with_bad_parents,
446
id_unused_versions, file_versions)
448
def _fix_text_parent(self, file_id, versions_with_bad_parents,
449
unused_versions, all_versions):
450
"""Fix bad versionedfile entries in a single versioned file."""
451
mutter('fixing text parent: %r (%d versions)', file_id,
452
len(versions_with_bad_parents))
453
mutter('(%d are unused)', len(unused_versions))
454
new_file_id = 'temp:%s' % file_id
457
for version in all_versions:
458
if version in unused_versions:
460
elif version in versions_with_bad_parents:
461
parents = versions_with_bad_parents[version][1]
463
pmap = self.repo.texts.get_parent_map([(file_id, version)])
464
parents = [key[-1] for key in pmap[(file_id, version)]]
465
new_parents[(new_file_id, version)] = [
466
(new_file_id, parent) for parent in parents]
467
needed_keys.add((file_id, version))
468
def fix_parents(stream):
469
for record in stream:
470
bytes = record.get_bytes_as('fulltext')
471
new_key = (new_file_id, record.key[-1])
472
parents = new_parents[new_key]
473
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
474
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
475
self.repo._remove_file_id(new_file_id)
476
self.repo.texts.insert_record_stream(fix_parents(stream))
477
self.repo._remove_file_id(file_id)
479
self.repo._move_file_id(new_file_id, file_id)
482
class PackReconciler(RepoReconciler):
483
"""Reconciler that reconciles a pack based repository.
485
Garbage inventories do not affect ancestry queries, and removal is
486
considerably more expensive as there is no separate versioned file for
487
them, so they are not cleaned. In short it is currently a no-op.
489
In future this may be a good place to hook in annotation cache checking,
490
index recreation etc.
493
# XXX: The index corruption that _fix_text_parents performs is needed for
494
# packs, but not yet implemented. The basic approach is to:
495
# - lock the names list
496
# - perform a customised pack() that regenerates data as needed
497
# - unlock the names list
498
# https://bugs.edge.launchpad.net/bzr/+bug/154173
500
def _reconcile_steps(self):
501
"""Perform the steps to reconcile this repository."""
502
if not self.thorough:
504
collection = self.repo._pack_collection
505
collection.ensure_loaded()
506
collection.lock_names()
507
self.add_cleanup(collection._unlock_names)
508
packs = collection.all_packs()
509
all_revisions = self.repo.all_revision_ids()
510
total_inventories = len(list(
511
collection.inventory_index.combined_index.iter_all_entries()))
512
if len(all_revisions):
513
new_pack = self.repo._reconcile_pack(collection, packs,
514
".reconcile", all_revisions, self.pb)
515
if new_pack is not None:
516
self._discard_and_save(packs)
518
# only make a new pack when there is data to copy.
519
self._discard_and_save(packs)
520
self.garbage_inventories = total_inventories - len(list(
521
collection.inventory_index.combined_index.iter_all_entries()))
523
def _discard_and_save(self, packs):
524
"""Discard some packs from the repository.
526
This removes them from the memory index, saves the in-memory index
527
which makes the newly reconciled pack visible and hides the packs to be
528
discarded, and finally renames the packs being discarded into the
529
obsolete packs directory.
531
:param packs: The packs to discard.
534
self.repo._pack_collection._remove_pack_from_memory(pack)
535
self.repo._pack_collection._save_pack_names()
536
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
