1
# Copyright (C) 2005, 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
35
from bzrlib.trace import mutter, note
36
from bzrlib.tsort import topo_sort
37
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
40
def reconcile(dir, other=None):
41
"""Reconcile the data in dir.
43
Currently this is limited to a inventory 'reweave'.
45
This is a convenience method, for using a Reconciler object.
47
Directly using Reconciler is recommended for library users that
48
desire fine grained control or analysis of the found issues.
50
:param other: another bzrdir to reconcile against.
52
reconciler = Reconciler(dir, other=other)
53
reconciler.reconcile()
56
class Reconciler(object):
57
"""Reconcilers are used to reconcile existing data."""
59
def __init__(self, dir, other=None):
60
"""Create a Reconciler."""
64
"""Perform reconciliation.
66
After reconciliation the following attributes document found issues:
67
inconsistent_parents: The number of revisions in the repository whose
68
ancestry was being reported incorrectly.
69
garbage_inventories: The number of inventory objects without revisions
70
that were garbage collected.
71
fixed_branch_history: None if there was no branch, False if the branch
72
history was correct, True if the branch history
73
needed to be re-normalized.
75
self.pb = ui.ui_factory.nested_progress_bar()
82
"""Helper function for performing reconciliation."""
83
self._reconcile_branch()
84
self._reconcile_repository()
86
def _reconcile_branch(self):
88
self.branch = self.bzrdir.open_branch()
89
except errors.NotBranchError:
90
# Nothing to check here
91
self.fixed_branch_history = None
93
ui.ui_factory.note('Reconciling branch %s' % self.branch.base)
94
branch_reconciler = self.branch.reconcile(thorough=True)
95
self.fixed_branch_history = branch_reconciler.fixed_history
97
def _reconcile_repository(self):
98
self.repo = self.bzrdir.find_repository()
99
ui.ui_factory.note('Reconciling repository %s' %
100
self.repo.bzrdir.root_transport.base)
101
self.pb.update("Reconciling repository", 0, 1)
102
repo_reconciler = self.repo.reconcile(thorough=True)
103
self.inconsistent_parents = repo_reconciler.inconsistent_parents
104
self.garbage_inventories = repo_reconciler.garbage_inventories
105
if repo_reconciler.aborted:
107
'Reconcile aborted: revision index has inconsistent parents.')
109
'Run "bzr check" for more details.')
111
ui.ui_factory.note('Reconciliation complete.')
114
class BranchReconciler(object):
115
"""Reconciler that works on a branch."""
117
def __init__(self, a_branch, thorough=False):
118
self.fixed_history = None
119
self.thorough = thorough
120
self.branch = a_branch
123
self.branch.lock_write()
125
self.pb = ui.ui_factory.nested_progress_bar()
127
self._reconcile_steps()
133
def _reconcile_steps(self):
134
self._reconcile_revision_history()
136
def _reconcile_revision_history(self):
137
repo = self.branch.repository
138
last_revno, last_revision_id = self.branch.last_revision_info()
141
for revid in repo.iter_reverse_revision_history(
143
real_history.append(revid)
144
except errors.RevisionNotPresent:
145
pass # Hit a ghost left hand parent
146
real_history.reverse()
147
if last_revno != len(real_history):
148
self.fixed_history = True
149
# Technically for Branch5 formats, it is more efficient to use
150
# set_revision_history, as this will regenerate it again.
151
# Not really worth a whole BranchReconciler class just for this,
153
ui.ui_factory.note('Fixing last revision info %s => %s' % (
154
last_revno, len(real_history)))
155
self.branch.set_last_revision_info(len(real_history),
158
self.fixed_history = False
159
ui.ui_factory.note('revision_history ok.')
162
class RepoReconciler(object):
163
"""Reconciler that reconciles a repository.
165
The goal of repository reconciliation is to make any derived data
166
consistent with the core data committed by a user. This can involve
167
reindexing, or removing unreferenced data if that can interfere with
168
queries in a given repository.
170
Currently this consists of an inventory reweave with revision cross-checks.
173
def __init__(self, repo, other=None, thorough=False):
174
"""Construct a RepoReconciler.
176
:param thorough: perform a thorough check which may take longer but
177
will correct non-data loss issues such as incorrect
180
self.garbage_inventories = 0
181
self.inconsistent_parents = 0
184
self.thorough = thorough
187
"""Perform reconciliation.
189
After reconciliation the following attributes document found issues:
190
inconsistent_parents: The number of revisions in the repository whose
191
ancestry was being reported incorrectly.
192
garbage_inventories: The number of inventory objects without revisions
193
that were garbage collected.
195
self.repo.lock_write()
197
self.pb = ui.ui_factory.nested_progress_bar()
199
self._reconcile_steps()
205
def _reconcile_steps(self):
206
"""Perform the steps to reconcile this repository."""
207
self._reweave_inventory()
209
def _reweave_inventory(self):
210
"""Regenerate the inventory weave for the repository from scratch.
212
This is a smart function: it will only do the reweave if doing it
213
will correct data issues. The self.thorough flag controls whether
214
only data-loss causing issues (!self.thorough) or all issues
215
(self.thorough) are treated as requiring the reweave.
217
# local because needing to know about WeaveFile is a wart we want to hide
218
from bzrlib.weave import WeaveFile, Weave
219
transaction = self.repo.get_transaction()
220
self.pb.update('Reading inventory data')
221
self.inventory = self.repo.inventories
222
self.revisions = self.repo.revisions
223
# the total set of revisions to process
224
self.pending = set([key[-1] for key in self.revisions.keys()])
226
# mapping from revision_id to parents
228
# errors that we detect
229
self.inconsistent_parents = 0
230
# we need the revision id of each revision and its available parents list
231
self._setup_steps(len(self.pending))
232
for rev_id in self.pending:
233
# put a revision into the graph.
234
self._graph_revision(rev_id)
235
self._check_garbage_inventories()
236
# if there are no inconsistent_parents and
237
# (no garbage inventories or we are not doing a thorough check)
238
if (not self.inconsistent_parents and
239
(not self.garbage_inventories or not self.thorough)):
240
ui.ui_factory.note('Inventory ok.')
242
self.pb.update('Backing up inventory', 0, 0)
243
self.repo._backup_inventory()
244
ui.ui_factory.note('Backup inventory created.')
245
new_inventories = self.repo._temp_inventories()
247
# we have topological order of revisions and non ghost parents ready.
248
self._setup_steps(len(self._rev_graph))
249
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
250
stream = self._change_inv_parents(
251
self.inventory.get_record_stream(revision_keys, 'unordered', True),
252
self._new_inv_parents,
254
new_inventories.insert_record_stream(stream)
255
# if this worked, the set of new_inventories.keys should equal
257
if not (set(new_inventories.keys()) ==
258
set([(revid,) for revid in self.pending])):
259
raise AssertionError()
260
self.pb.update('Writing weave')
261
self.repo._activate_new_inventory()
262
self.inventory = None
263
ui.ui_factory.note('Inventory regenerated.')
265
def _new_inv_parents(self, revision_key):
266
"""Lookup ghost-filtered parents for revision_key."""
267
# Use the filtered ghostless parents list:
268
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
270
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
271
"""Adapt a record stream to reconcile the parents."""
272
for record in stream:
273
wanted_parents = get_parents(record.key)
274
if wanted_parents and wanted_parents[0] not in all_revision_keys:
275
# The check for the left most parent only handles knit
276
# compressors, but this code only applies to knit and weave
277
# repositories anyway.
278
bytes = record.get_bytes_as('fulltext')
279
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
281
adapted_record = AdapterFactory(record.key, wanted_parents, record)
283
self._reweave_step('adding inventories')
285
def _setup_steps(self, new_total):
286
"""Setup the markers we need to control the progress bar."""
287
self.total = new_total
290
def _graph_revision(self, rev_id):
291
"""Load a revision into the revision graph."""
292
# pick a random revision
293
# analyse revision id rev_id and put it in the stack.
294
self._reweave_step('loading revisions')
295
rev = self.repo.get_revision_reconcile(rev_id)
297
for parent in rev.parent_ids:
298
if self._parent_is_available(parent):
299
parents.append(parent)
301
mutter('found ghost %s', parent)
302
self._rev_graph[rev_id] = parents
304
def _check_garbage_inventories(self):
305
"""Check for garbage inventories which we cannot trust
307
We cant trust them because their pre-requisite file data may not
308
be present - all we know is that their revision was not installed.
310
if not self.thorough:
312
inventories = set(self.inventory.keys())
313
revisions = set(self.revisions.keys())
314
garbage = inventories.difference(revisions)
315
self.garbage_inventories = len(garbage)
316
for revision_key in garbage:
317
mutter('Garbage inventory {%s} found.', revision_key[-1])
319
def _parent_is_available(self, parent):
320
"""True if parent is a fully available revision
322
A fully available revision has a inventory and a revision object in the
325
if parent in self._rev_graph:
327
inv_present = (1 == len(self.inventory.get_parent_map([(parent,)])))
328
return (inv_present and self.repo.has_revision(parent))
330
def _reweave_step(self, message):
331
"""Mark a single step of regeneration complete."""
332
self.pb.update(message, self.count, self.total)
336
class KnitReconciler(RepoReconciler):
337
"""Reconciler that reconciles a knit format repository.
339
This will detect garbage inventories and remove them in thorough mode.
342
def _reconcile_steps(self):
343
"""Perform the steps to reconcile this repository."""
347
except errors.BzrCheckError:
350
# knits never suffer this
352
self._fix_text_parents()
354
def _load_indexes(self):
355
"""Load indexes for the reconciliation."""
356
self.transaction = self.repo.get_transaction()
357
self.pb.update('Reading indexes', 0, 2)
358
self.inventory = self.repo.inventories
359
self.pb.update('Reading indexes', 1, 2)
360
self.repo._check_for_inconsistent_revision_parents()
361
self.revisions = self.repo.revisions
362
self.pb.update('Reading indexes', 2, 2)
364
def _gc_inventory(self):
365
"""Remove inventories that are not referenced from the revision store."""
366
self.pb.update('Checking unused inventories', 0, 1)
367
self._check_garbage_inventories()
368
self.pb.update('Checking unused inventories', 1, 3)
369
if not self.garbage_inventories:
370
ui.ui_factory.note('Inventory ok.')
372
self.pb.update('Backing up inventory', 0, 0)
373
self.repo._backup_inventory()
374
ui.ui_factory.note('Backup Inventory created')
375
# asking for '' should never return a non-empty weave
376
new_inventories = self.repo._temp_inventories()
377
# we have topological order of revisions and non ghost parents ready.
378
graph = self.revisions.get_parent_map(self.revisions.keys())
379
revision_keys = topo_sort(graph)
380
revision_ids = [key[-1] for key in revision_keys]
381
self._setup_steps(len(revision_keys))
382
stream = self._change_inv_parents(
383
self.inventory.get_record_stream(revision_keys, 'unordered', True),
386
new_inventories.insert_record_stream(stream)
387
# if this worked, the set of new_inventory_vf.names should equal
388
# the revisionds list
389
if not(set(new_inventories.keys()) == set(revision_keys)):
390
raise AssertionError()
391
self.pb.update('Writing weave')
392
self.repo._activate_new_inventory()
393
self.inventory = None
394
ui.ui_factory.note('Inventory regenerated.')
396
def _fix_text_parents(self):
397
"""Fix bad versionedfile parent entries.
399
It is possible for the parents entry in a versionedfile entry to be
400
inconsistent with the values in the revision and inventory.
402
This method finds entries with such inconsistencies, corrects their
403
parent lists, and replaces the versionedfile with a corrected version.
405
transaction = self.repo.get_transaction()
406
versions = [key[-1] for key in self.revisions.keys()]
407
mutter('Prepopulating revision text cache with %d revisions',
409
vf_checker = self.repo._get_versioned_file_checker()
410
bad_parents, unused_versions = vf_checker.check_file_version_parents(
411
self.repo.texts, self.pb)
412
text_index = vf_checker.text_index
413
per_id_bad_parents = {}
414
for key in unused_versions:
415
# Ensure that every file with unused versions gets rewritten.
416
# NB: This is really not needed, reconcile != pack.
417
per_id_bad_parents[key[0]] = {}
418
# Generate per-knit/weave data.
419
for key, details in bad_parents.iteritems():
422
knit_parents = tuple([parent[-1] for parent in details[0]])
423
correct_parents = tuple([parent[-1] for parent in details[1]])
424
file_details = per_id_bad_parents.setdefault(file_id, {})
425
file_details[rev_id] = (knit_parents, correct_parents)
426
file_id_versions = {}
427
for text_key in text_index:
428
versions_list = file_id_versions.setdefault(text_key[0], [])
429
versions_list.append(text_key[1])
430
# Do the reconcile of individual weaves.
431
for num, file_id in enumerate(per_id_bad_parents):
432
self.pb.update('Fixing text parents', num,
433
len(per_id_bad_parents))
434
versions_with_bad_parents = per_id_bad_parents[file_id]
435
id_unused_versions = set(key[-1] for key in unused_versions
436
if key[0] == file_id)
437
if file_id in file_id_versions:
438
file_versions = file_id_versions[file_id]
440
# This id was present in the disk store but is not referenced
441
# by any revision at all.
443
self._fix_text_parent(file_id, versions_with_bad_parents,
444
id_unused_versions, file_versions)
446
def _fix_text_parent(self, file_id, versions_with_bad_parents,
447
unused_versions, all_versions):
448
"""Fix bad versionedfile entries in a single versioned file."""
449
mutter('fixing text parent: %r (%d versions)', file_id,
450
len(versions_with_bad_parents))
451
mutter('(%d are unused)', len(unused_versions))
452
new_file_id = 'temp:%s' % file_id
455
for version in all_versions:
456
if version in unused_versions:
458
elif version in versions_with_bad_parents:
459
parents = versions_with_bad_parents[version][1]
461
pmap = self.repo.texts.get_parent_map([(file_id, version)])
462
parents = [key[-1] for key in pmap[(file_id, version)]]
463
new_parents[(new_file_id, version)] = [
464
(new_file_id, parent) for parent in parents]
465
needed_keys.add((file_id, version))
466
def fix_parents(stream):
467
for record in stream:
468
bytes = record.get_bytes_as('fulltext')
469
new_key = (new_file_id, record.key[-1])
470
parents = new_parents[new_key]
471
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
472
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
473
self.repo._remove_file_id(new_file_id)
474
self.repo.texts.insert_record_stream(fix_parents(stream))
475
self.repo._remove_file_id(file_id)
477
self.repo._move_file_id(new_file_id, file_id)
480
class PackReconciler(RepoReconciler):
481
"""Reconciler that reconciles a pack based repository.
483
Garbage inventories do not affect ancestry queries, and removal is
484
considerably more expensive as there is no separate versioned file for
485
them, so they are not cleaned. In short it is currently a no-op.
487
In future this may be a good place to hook in annotation cache checking,
488
index recreation etc.
491
# XXX: The index corruption that _fix_text_parents performs is needed for
492
# packs, but not yet implemented. The basic approach is to:
493
# - lock the names list
494
# - perform a customised pack() that regenerates data as needed
495
# - unlock the names list
496
# https://bugs.edge.launchpad.net/bzr/+bug/154173
498
def _reconcile_steps(self):
499
"""Perform the steps to reconcile this repository."""
500
if not self.thorough:
502
collection = self.repo._pack_collection
503
collection.ensure_loaded()
504
collection.lock_names()
506
packs = collection.all_packs()
507
all_revisions = self.repo.all_revision_ids()
508
total_inventories = len(list(
509
collection.inventory_index.combined_index.iter_all_entries()))
510
if len(all_revisions):
511
new_pack = self.repo._reconcile_pack(collection, packs,
512
".reconcile", all_revisions, self.pb)
513
if new_pack is not None:
514
self._discard_and_save(packs)
516
# only make a new pack when there is data to copy.
517
self._discard_and_save(packs)
518
self.garbage_inventories = total_inventories - len(list(
519
collection.inventory_index.combined_index.iter_all_entries()))
521
collection._unlock_names()
523
def _discard_and_save(self, packs):
524
"""Discard some packs from the repository.
526
This removes them from the memory index, saves the in-memory index
527
which makes the newly reconciled pack visible and hides the packs to be
528
discarded, and finally renames the packs being discarded into the
529
obsolete packs directory.
531
:param packs: The packs to discard.
534
self.repo._pack_collection._remove_pack_from_memory(pack)
535
self.repo._pack_collection._save_pack_names()
536
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
