1
# Copyright (C) 2005, 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
35
from bzrlib.trace import mutter, note
36
from bzrlib.tsort import topo_sort
37
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
40
def reconcile(dir, other=None):
41
"""Reconcile the data in dir.
43
Currently this is limited to a inventory 'reweave'.
45
This is a convenience method, for using a Reconciler object.
47
Directly using Reconciler is recommended for library users that
48
desire fine grained control or analysis of the found issues.
50
:param other: another bzrdir to reconcile against.
52
reconciler = Reconciler(dir, other=other)
53
reconciler.reconcile()
56
class Reconciler(object):
57
"""Reconcilers are used to reconcile existing data."""
59
def __init__(self, dir, other=None):
60
"""Create a Reconciler."""
64
"""Perform reconciliation.
66
After reconciliation the following attributes document found issues:
67
inconsistent_parents: The number of revisions in the repository whose
68
ancestry was being reported incorrectly.
69
garbage_inventories: The number of inventory objects without revisions
70
that were garbage collected.
71
fixed_branch_history: None if there was no branch, False if the branch
72
history was correct, True if the branch history
73
needed to be re-normalized.
75
self.pb = ui.ui_factory.nested_progress_bar()
82
"""Helper function for performing reconciliation."""
83
self._reconcile_branch()
84
self._reconcile_repository()
86
def _reconcile_branch(self):
88
self.branch = self.bzrdir.open_branch()
89
except errors.NotBranchError:
90
# Nothing to check here
91
self.fixed_branch_history = None
93
self.pb.note('Reconciling branch %s',
95
branch_reconciler = self.branch.reconcile(thorough=True)
96
self.fixed_branch_history = branch_reconciler.fixed_history
98
def _reconcile_repository(self):
99
self.repo = self.bzrdir.find_repository()
100
self.pb.note('Reconciling repository %s',
101
self.repo.bzrdir.root_transport.base)
102
self.pb.update("Reconciling repository", 0, 1)
103
repo_reconciler = self.repo.reconcile(thorough=True)
104
self.inconsistent_parents = repo_reconciler.inconsistent_parents
105
self.garbage_inventories = repo_reconciler.garbage_inventories
106
if repo_reconciler.aborted:
108
'Reconcile aborted: revision index has inconsistent parents.')
110
'Run "bzr check" for more details.')
112
self.pb.note('Reconciliation complete.')
115
class BranchReconciler(object):
116
"""Reconciler that works on a branch."""
118
def __init__(self, a_branch, thorough=False):
119
self.fixed_history = None
120
self.thorough = thorough
121
self.branch = a_branch
124
self.branch.lock_write()
126
self.pb = ui.ui_factory.nested_progress_bar()
128
self._reconcile_steps()
134
def _reconcile_steps(self):
135
self._reconcile_revision_history()
137
def _reconcile_revision_history(self):
138
repo = self.branch.repository
139
last_revno, last_revision_id = self.branch.last_revision_info()
142
for revid in repo.iter_reverse_revision_history(
144
real_history.append(revid)
145
except errors.RevisionNotPresent:
146
pass # Hit a ghost left hand parent
147
real_history.reverse()
148
if last_revno != len(real_history):
149
self.fixed_history = True
150
# Technically for Branch5 formats, it is more efficient to use
151
# set_revision_history, as this will regenerate it again.
152
# Not really worth a whole BranchReconciler class just for this,
154
self.pb.note('Fixing last revision info %s => %s',
155
last_revno, len(real_history))
156
self.branch.set_last_revision_info(len(real_history),
159
self.fixed_history = False
160
self.pb.note('revision_history ok.')
163
class RepoReconciler(object):
164
"""Reconciler that reconciles a repository.
166
The goal of repository reconciliation is to make any derived data
167
consistent with the core data committed by a user. This can involve
168
reindexing, or removing unreferenced data if that can interfere with
169
queries in a given repository.
171
Currently this consists of an inventory reweave with revision cross-checks.
174
def __init__(self, repo, other=None, thorough=False):
175
"""Construct a RepoReconciler.
177
:param thorough: perform a thorough check which may take longer but
178
will correct non-data loss issues such as incorrect
181
self.garbage_inventories = 0
182
self.inconsistent_parents = 0
185
self.thorough = thorough
188
"""Perform reconciliation.
190
After reconciliation the following attributes document found issues:
191
inconsistent_parents: The number of revisions in the repository whose
192
ancestry was being reported incorrectly.
193
garbage_inventories: The number of inventory objects without revisions
194
that were garbage collected.
196
self.repo.lock_write()
198
self.pb = ui.ui_factory.nested_progress_bar()
200
self._reconcile_steps()
206
def _reconcile_steps(self):
207
"""Perform the steps to reconcile this repository."""
208
self._reweave_inventory()
210
def _reweave_inventory(self):
211
"""Regenerate the inventory weave for the repository from scratch.
213
This is a smart function: it will only do the reweave if doing it
214
will correct data issues. The self.thorough flag controls whether
215
only data-loss causing issues (!self.thorough) or all issues
216
(self.thorough) are treated as requiring the reweave.
218
# local because needing to know about WeaveFile is a wart we want to hide
219
from bzrlib.weave import WeaveFile, Weave
220
transaction = self.repo.get_transaction()
221
self.pb.update('Reading inventory data')
222
self.inventory = self.repo.inventories
223
self.revisions = self.repo.revisions
224
# the total set of revisions to process
225
self.pending = set([key[-1] for key in self.revisions.keys()])
227
# mapping from revision_id to parents
229
# errors that we detect
230
self.inconsistent_parents = 0
231
# we need the revision id of each revision and its available parents list
232
self._setup_steps(len(self.pending))
233
for rev_id in self.pending:
234
# put a revision into the graph.
235
self._graph_revision(rev_id)
236
self._check_garbage_inventories()
237
# if there are no inconsistent_parents and
238
# (no garbage inventories or we are not doing a thorough check)
239
if (not self.inconsistent_parents and
240
(not self.garbage_inventories or not self.thorough)):
241
self.pb.note('Inventory ok.')
243
self.pb.update('Backing up inventory', 0, 0)
244
self.repo._backup_inventory()
245
self.pb.note('Backup inventory created.')
246
new_inventories = self.repo._temp_inventories()
248
# we have topological order of revisions and non ghost parents ready.
249
self._setup_steps(len(self._rev_graph))
250
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
251
stream = self._change_inv_parents(
252
self.inventory.get_record_stream(revision_keys, 'unordered', True),
253
self._new_inv_parents,
255
new_inventories.insert_record_stream(stream)
256
# if this worked, the set of new_inventories.keys should equal
258
if not (set(new_inventories.keys()) ==
259
set([(revid,) for revid in self.pending])):
260
raise AssertionError()
261
self.pb.update('Writing weave')
262
self.repo._activate_new_inventory()
263
self.inventory = None
264
self.pb.note('Inventory regenerated.')
266
def _new_inv_parents(self, revision_key):
267
"""Lookup ghost-filtered parents for revision_key."""
268
# Use the filtered ghostless parents list:
269
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
271
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
272
"""Adapt a record stream to reconcile the parents."""
273
for record in stream:
274
wanted_parents = get_parents(record.key)
275
if wanted_parents and wanted_parents[0] not in all_revision_keys:
276
# The check for the left most parent only handles knit
277
# compressors, but this code only applies to knit and weave
278
# repositories anyway.
279
bytes = record.get_bytes_as('fulltext')
280
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
282
adapted_record = AdapterFactory(record.key, wanted_parents, record)
284
self._reweave_step('adding inventories')
286
def _setup_steps(self, new_total):
287
"""Setup the markers we need to control the progress bar."""
288
self.total = new_total
291
def _graph_revision(self, rev_id):
292
"""Load a revision into the revision graph."""
293
# pick a random revision
294
# analyse revision id rev_id and put it in the stack.
295
self._reweave_step('loading revisions')
296
rev = self.repo.get_revision_reconcile(rev_id)
298
for parent in rev.parent_ids:
299
if self._parent_is_available(parent):
300
parents.append(parent)
302
mutter('found ghost %s', parent)
303
self._rev_graph[rev_id] = parents
305
def _check_garbage_inventories(self):
306
"""Check for garbage inventories which we cannot trust
308
We cant trust them because their pre-requisite file data may not
309
be present - all we know is that their revision was not installed.
311
if not self.thorough:
313
inventories = set(self.inventory.keys())
314
revisions = set(self.revisions.keys())
315
garbage = inventories.difference(revisions)
316
self.garbage_inventories = len(garbage)
317
for revision_key in garbage:
318
mutter('Garbage inventory {%s} found.', revision_key[-1])
320
def _parent_is_available(self, parent):
321
"""True if parent is a fully available revision
323
A fully available revision has a inventory and a revision object in the
326
if parent in self._rev_graph:
328
inv_present = (1 == len(self.inventory.get_parent_map([(parent,)])))
329
return (inv_present and self.repo.has_revision(parent))
331
def _reweave_step(self, message):
332
"""Mark a single step of regeneration complete."""
333
self.pb.update(message, self.count, self.total)
337
class KnitReconciler(RepoReconciler):
338
"""Reconciler that reconciles a knit format repository.
340
This will detect garbage inventories and remove them in thorough mode.
343
def _reconcile_steps(self):
344
"""Perform the steps to reconcile this repository."""
348
except errors.BzrCheckError:
351
# knits never suffer this
353
self._fix_text_parents()
355
def _load_indexes(self):
356
"""Load indexes for the reconciliation."""
357
self.transaction = self.repo.get_transaction()
358
self.pb.update('Reading indexes', 0, 2)
359
self.inventory = self.repo.inventories
360
self.pb.update('Reading indexes', 1, 2)
361
self.repo._check_for_inconsistent_revision_parents()
362
self.revisions = self.repo.revisions
363
self.pb.update('Reading indexes', 2, 2)
365
def _gc_inventory(self):
366
"""Remove inventories that are not referenced from the revision store."""
367
self.pb.update('Checking unused inventories', 0, 1)
368
self._check_garbage_inventories()
369
self.pb.update('Checking unused inventories', 1, 3)
370
if not self.garbage_inventories:
371
self.pb.note('Inventory ok.')
373
self.pb.update('Backing up inventory', 0, 0)
374
self.repo._backup_inventory()
375
self.pb.note('Backup Inventory created')
376
# asking for '' should never return a non-empty weave
377
new_inventories = self.repo._temp_inventories()
378
# we have topological order of revisions and non ghost parents ready.
379
graph = self.revisions.get_parent_map(self.revisions.keys())
380
revision_keys = topo_sort(graph)
381
revision_ids = [key[-1] for key in revision_keys]
382
self._setup_steps(len(revision_keys))
383
stream = self._change_inv_parents(
384
self.inventory.get_record_stream(revision_keys, 'unordered', True),
387
new_inventories.insert_record_stream(stream)
388
# if this worked, the set of new_inventory_vf.names should equal
389
# the revisionds list
390
if not(set(new_inventories.keys()) == set(revision_keys)):
391
raise AssertionError()
392
self.pb.update('Writing weave')
393
self.repo._activate_new_inventory()
394
self.inventory = None
395
self.pb.note('Inventory regenerated.')
397
def _fix_text_parents(self):
398
"""Fix bad versionedfile parent entries.
400
It is possible for the parents entry in a versionedfile entry to be
401
inconsistent with the values in the revision and inventory.
403
This method finds entries with such inconsistencies, corrects their
404
parent lists, and replaces the versionedfile with a corrected version.
406
transaction = self.repo.get_transaction()
407
versions = [key[-1] for key in self.revisions.keys()]
408
mutter('Prepopulating revision text cache with %d revisions',
410
vf_checker = self.repo._get_versioned_file_checker()
411
bad_parents, unused_versions = vf_checker.check_file_version_parents(
412
self.repo.texts, self.pb)
413
text_index = vf_checker.text_index
414
per_id_bad_parents = {}
415
for key in unused_versions:
416
# Ensure that every file with unused versions gets rewritten.
417
# NB: This is really not needed, reconcile != pack.
418
per_id_bad_parents[key[0]] = {}
419
# Generate per-knit/weave data.
420
for key, details in bad_parents.iteritems():
423
knit_parents = tuple([parent[-1] for parent in details[0]])
424
correct_parents = tuple([parent[-1] for parent in details[1]])
425
file_details = per_id_bad_parents.setdefault(file_id, {})
426
file_details[rev_id] = (knit_parents, correct_parents)
427
file_id_versions = {}
428
for text_key in text_index:
429
versions_list = file_id_versions.setdefault(text_key[0], [])
430
versions_list.append(text_key[1])
431
# Do the reconcile of individual weaves.
432
for num, file_id in enumerate(per_id_bad_parents):
433
self.pb.update('Fixing text parents', num,
434
len(per_id_bad_parents))
435
versions_with_bad_parents = per_id_bad_parents[file_id]
436
id_unused_versions = set(key[-1] for key in unused_versions
437
if key[0] == file_id)
438
if file_id in file_id_versions:
439
file_versions = file_id_versions[file_id]
441
# This id was present in the disk store but is not referenced
442
# by any revision at all.
444
self._fix_text_parent(file_id, versions_with_bad_parents,
445
id_unused_versions, file_versions)
447
def _fix_text_parent(self, file_id, versions_with_bad_parents,
448
unused_versions, all_versions):
449
"""Fix bad versionedfile entries in a single versioned file."""
450
mutter('fixing text parent: %r (%d versions)', file_id,
451
len(versions_with_bad_parents))
452
mutter('(%d are unused)', len(unused_versions))
453
new_file_id = 'temp:%s' % file_id
456
for version in all_versions:
457
if version in unused_versions:
459
elif version in versions_with_bad_parents:
460
parents = versions_with_bad_parents[version][1]
462
pmap = self.repo.texts.get_parent_map([(file_id, version)])
463
parents = [key[-1] for key in pmap[(file_id, version)]]
464
new_parents[(new_file_id, version)] = [
465
(new_file_id, parent) for parent in parents]
466
needed_keys.add((file_id, version))
467
def fix_parents(stream):
468
for record in stream:
469
bytes = record.get_bytes_as('fulltext')
470
new_key = (new_file_id, record.key[-1])
471
parents = new_parents[new_key]
472
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
473
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
474
self.repo._remove_file_id(new_file_id)
475
self.repo.texts.insert_record_stream(fix_parents(stream))
476
self.repo._remove_file_id(file_id)
478
self.repo._move_file_id(new_file_id, file_id)
481
class PackReconciler(RepoReconciler):
482
"""Reconciler that reconciles a pack based repository.
484
Garbage inventories do not affect ancestry queries, and removal is
485
considerably more expensive as there is no separate versioned file for
486
them, so they are not cleaned. In short it is currently a no-op.
488
In future this may be a good place to hook in annotation cache checking,
489
index recreation etc.
492
# XXX: The index corruption that _fix_text_parents performs is needed for
493
# packs, but not yet implemented. The basic approach is to:
494
# - lock the names list
495
# - perform a customised pack() that regenerates data as needed
496
# - unlock the names list
497
# https://bugs.edge.launchpad.net/bzr/+bug/154173
499
def _reconcile_steps(self):
500
"""Perform the steps to reconcile this repository."""
501
if not self.thorough:
503
collection = self.repo._pack_collection
504
collection.ensure_loaded()
505
collection.lock_names()
507
packs = collection.all_packs()
508
all_revisions = self.repo.all_revision_ids()
509
total_inventories = len(list(
510
collection.inventory_index.combined_index.iter_all_entries()))
511
if len(all_revisions):
512
new_pack = self.repo._reconcile_pack(collection, packs,
513
".reconcile", all_revisions, self.pb)
514
if new_pack is not None:
515
self._discard_and_save(packs)
517
# only make a new pack when there is data to copy.
518
self._discard_and_save(packs)
519
self.garbage_inventories = total_inventories - len(list(
520
collection.inventory_index.combined_index.iter_all_entries()))
522
collection._unlock_names()
524
def _discard_and_save(self, packs):
525
"""Discard some packs from the repository.
527
This removes them from the memory index, saves the in-memory index
528
which makes the newly reconciled pack visible and hides the packs to be
529
discarded, and finally renames the packs being discarded into the
530
obsolete packs directory.
532
:param packs: The packs to discard.
535
self.repo._pack_collection._remove_pack_from_memory(pack)
536
self.repo._pack_collection._save_pack_names()
537
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
