1
# Copyright (C) 2005, 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
35
from bzrlib.trace import mutter, note
36
from bzrlib.tsort import TopoSorter
39
def reconcile(dir, other=None):
40
"""Reconcile the data in dir.
42
Currently this is limited to a inventory 'reweave'.
44
This is a convenience method, for using a Reconciler object.
46
Directly using Reconciler is recommended for library users that
47
desire fine grained control or analysis of the found issues.
49
:param other: another bzrdir to reconcile against.
51
reconciler = Reconciler(dir, other=other)
52
reconciler.reconcile()
55
class Reconciler(object):
56
"""Reconcilers are used to reconcile existing data."""
58
def __init__(self, dir, other=None):
59
"""Create a Reconciler."""
63
"""Perform reconciliation.
65
After reconciliation the following attributes document found issues:
66
inconsistent_parents: The number of revisions in the repository whose
67
ancestry was being reported incorrectly.
68
garbage_inventories: The number of inventory objects without revisions
69
that were garbage collected.
71
self.pb = ui.ui_factory.nested_progress_bar()
78
"""Helper function for performing reconciliation."""
79
self.repo = self.bzrdir.find_repository()
80
self.pb.note('Reconciling repository %s',
81
self.repo.bzrdir.root_transport.base)
82
self.pb.update("Reconciling repository", 0, 1)
83
repo_reconciler = self.repo.reconcile(thorough=True)
84
self.inconsistent_parents = repo_reconciler.inconsistent_parents
85
self.garbage_inventories = repo_reconciler.garbage_inventories
86
if repo_reconciler.aborted:
88
'Reconcile aborted: revision index has inconsistent parents.')
90
'Run "bzr check" for more details.')
92
self.pb.note('Reconciliation complete.')
95
class RepoReconciler(object):
96
"""Reconciler that reconciles a repository.
98
The goal of repository reconciliation is to make any derived data
99
consistent with the core data committed by a user. This can involve
100
reindexing, or removing unreferenced data if that can interfere with
101
queries in a given repository.
103
Currently this consists of an inventory reweave with revision cross-checks.
106
def __init__(self, repo, other=None, thorough=False):
107
"""Construct a RepoReconciler.
109
:param thorough: perform a thorough check which may take longer but
110
will correct non-data loss issues such as incorrect
113
self.garbage_inventories = 0
114
self.inconsistent_parents = 0
117
self.thorough = thorough
120
"""Perform reconciliation.
122
After reconciliation the following attributes document found issues:
123
inconsistent_parents: The number of revisions in the repository whose
124
ancestry was being reported incorrectly.
125
garbage_inventories: The number of inventory objects without revisions
126
that were garbage collected.
128
self.repo.lock_write()
130
self.pb = ui.ui_factory.nested_progress_bar()
132
self._reconcile_steps()
138
def _reconcile_steps(self):
139
"""Perform the steps to reconcile this repository."""
140
self._reweave_inventory()
142
def _reweave_inventory(self):
143
"""Regenerate the inventory weave for the repository from scratch.
145
This is a smart function: it will only do the reweave if doing it
146
will correct data issues. The self.thorough flag controls whether
147
only data-loss causing issues (!self.thorough) or all issues
148
(self.thorough) are treated as requiring the reweave.
150
# local because needing to know about WeaveFile is a wart we want to hide
151
from bzrlib.weave import WeaveFile, Weave
152
transaction = self.repo.get_transaction()
153
self.pb.update('Reading inventory data.')
154
self.inventory = self.repo.get_inventory_weave()
155
# the total set of revisions to process
156
self.pending = set([rev_id for rev_id in self.repo._revision_store.all_revision_ids(transaction)])
158
# mapping from revision_id to parents
160
# errors that we detect
161
self.inconsistent_parents = 0
162
# we need the revision id of each revision and its available parents list
163
self._setup_steps(len(self.pending))
164
for rev_id in self.pending:
165
# put a revision into the graph.
166
self._graph_revision(rev_id)
167
self._check_garbage_inventories()
168
# if there are no inconsistent_parents and
169
# (no garbage inventories or we are not doing a thorough check)
170
if (not self.inconsistent_parents and
171
(not self.garbage_inventories or not self.thorough)):
172
self.pb.note('Inventory ok.')
174
self.pb.update('Backing up inventory...', 0, 0)
175
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.repo.get_transaction())
176
self.pb.note('Backup Inventory created.')
177
# asking for '' should never return a non-empty weave
178
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
179
self.repo.get_transaction())
181
# we have topological order of revisions and non ghost parents ready.
182
self._setup_steps(len(self._rev_graph))
183
for rev_id in TopoSorter(self._rev_graph.items()).iter_topo_order():
184
parents = self._rev_graph[rev_id]
185
# double check this really is in topological order.
186
unavailable = [p for p in parents if p not in new_inventory_vf]
187
assert len(unavailable) == 0
188
# this entry has all the non ghost parents in the inventory
190
self._reweave_step('adding inventories')
191
if isinstance(new_inventory_vf, WeaveFile):
192
# It's really a WeaveFile, but we call straight into the
193
# Weave's add method to disable the auto-write-out behaviour.
194
# This is done to avoid a revision_count * time-to-write additional overhead on
196
new_inventory_vf._check_write_ok()
197
Weave._add_lines(new_inventory_vf, rev_id, parents,
198
self.inventory.get_lines(rev_id), None, None, None, False, True)
200
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
202
if isinstance(new_inventory_vf, WeaveFile):
203
new_inventory_vf._save()
204
# if this worked, the set of new_inventory_vf.names should equal
206
assert set(new_inventory_vf.versions()) == self.pending
207
self.pb.update('Writing weave')
208
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.repo.get_transaction())
209
self.repo.control_weaves.delete('inventory.new', self.repo.get_transaction())
210
self.inventory = None
211
self.pb.note('Inventory regenerated.')
213
def _setup_steps(self, new_total):
214
"""Setup the markers we need to control the progress bar."""
215
self.total = new_total
218
def _graph_revision(self, rev_id):
219
"""Load a revision into the revision graph."""
220
# pick a random revision
221
# analyse revision id rev_id and put it in the stack.
222
self._reweave_step('loading revisions')
223
rev = self.repo.get_revision_reconcile(rev_id)
224
assert rev.revision_id == rev_id
226
for parent in rev.parent_ids:
227
if self._parent_is_available(parent):
228
parents.append(parent)
230
mutter('found ghost %s', parent)
231
self._rev_graph[rev_id] = parents
232
if self._parents_are_inconsistent(rev_id, parents):
233
self.inconsistent_parents += 1
234
mutter('Inconsistent inventory parents: id {%s} '
235
'inventory claims %r, '
236
'available parents are %r, '
237
'unavailable parents are %r',
239
set(self.inventory.get_parents(rev_id)),
241
set(rev.parent_ids).difference(set(parents)))
243
def _parents_are_inconsistent(self, rev_id, parents):
244
"""Return True if the parents list of rev_id does not match the weave.
246
This detects inconsistencies based on the self.thorough value:
247
if thorough is on, the first parent value is checked as well as ghost
249
Otherwise only the ghost differences are evaluated.
251
weave_parents = self.inventory.get_parents(rev_id)
252
weave_missing_old_ghosts = set(weave_parents) != set(parents)
253
first_parent_is_wrong = (
254
len(weave_parents) and len(parents) and
255
parents[0] != weave_parents[0])
257
return weave_missing_old_ghosts or first_parent_is_wrong
259
return weave_missing_old_ghosts
261
def _check_garbage_inventories(self):
262
"""Check for garbage inventories which we cannot trust
264
We cant trust them because their pre-requisite file data may not
265
be present - all we know is that their revision was not installed.
267
if not self.thorough:
269
inventories = set(self.inventory.versions())
270
revisions = set(self._rev_graph.keys())
271
garbage = inventories.difference(revisions)
272
self.garbage_inventories = len(garbage)
273
for revision_id in garbage:
274
mutter('Garbage inventory {%s} found.', revision_id)
276
def _parent_is_available(self, parent):
277
"""True if parent is a fully available revision
279
A fully available revision has a inventory and a revision object in the
282
return (parent in self._rev_graph or
283
(parent in self.inventory and self.repo.has_revision(parent)))
285
def _reweave_step(self, message):
286
"""Mark a single step of regeneration complete."""
287
self.pb.update(message, self.count, self.total)
291
class KnitReconciler(RepoReconciler):
292
"""Reconciler that reconciles a knit format repository.
294
This will detect garbage inventories and remove them in thorough mode.
297
def _reconcile_steps(self):
298
"""Perform the steps to reconcile this repository."""
302
except errors.BzrCheckError:
305
# knits never suffer this
307
self._fix_text_parents()
309
def _load_indexes(self):
310
"""Load indexes for the reconciliation."""
311
self.transaction = self.repo.get_transaction()
312
self.pb.update('Reading indexes.', 0, 2)
313
self.inventory = self.repo.get_inventory_weave()
314
self.pb.update('Reading indexes.', 1, 2)
315
self.repo._check_for_inconsistent_revision_parents()
316
self.revisions = self.repo._revision_store.get_revision_file(self.transaction)
317
self.pb.update('Reading indexes.', 2, 2)
319
def _gc_inventory(self):
320
"""Remove inventories that are not referenced from the revision store."""
321
self.pb.update('Checking unused inventories.', 0, 1)
322
self._check_garbage_inventories()
323
self.pb.update('Checking unused inventories.', 1, 3)
324
if not self.garbage_inventories:
325
self.pb.note('Inventory ok.')
327
self.pb.update('Backing up inventory...', 0, 0)
328
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.transaction)
329
self.pb.note('Backup Inventory created.')
330
# asking for '' should never return a non-empty weave
331
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
334
# we have topological order of revisions and non ghost parents ready.
335
self._setup_steps(len(self.revisions))
336
for rev_id in TopoSorter(self.revisions.get_graph().items()).iter_topo_order():
337
parents = self.revisions.get_parents(rev_id)
338
# double check this really is in topological order.
339
unavailable = [p for p in parents if p not in new_inventory_vf]
340
assert len(unavailable) == 0
341
# this entry has all the non ghost parents in the inventory
343
self._reweave_step('adding inventories')
344
# ugly but needed, weaves are just way tooooo slow else.
345
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
347
# if this worked, the set of new_inventory_vf.names should equal
349
assert set(new_inventory_vf.versions()) == set(self.revisions.versions())
350
self.pb.update('Writing weave')
351
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.transaction)
352
self.repo.control_weaves.delete('inventory.new', self.transaction)
353
self.inventory = None
354
self.pb.note('Inventory regenerated.')
356
def _check_garbage_inventories(self):
357
"""Check for garbage inventories which we cannot trust
359
We cant trust them because their pre-requisite file data may not
360
be present - all we know is that their revision was not installed.
362
inventories = set(self.inventory.versions())
363
revisions = set(self.revisions.versions())
364
garbage = inventories.difference(revisions)
365
self.garbage_inventories = len(garbage)
366
for revision_id in garbage:
367
mutter('Garbage inventory {%s} found.', revision_id)
369
def _fix_text_parents(self):
370
"""Fix bad versionedfile parent entries.
372
It is possible for the parents entry in a versionedfile entry to be
373
inconsistent with the values in the revision and inventory.
375
This method finds entries with such inconsistencies, corrects their
376
parent lists, and replaces the versionedfile with a corrected version.
378
transaction = self.repo.get_transaction()
379
revision_versions = repository._RevisionTextVersionCache(self.repo)
380
versions = self.revisions.versions()
381
mutter('Prepopulating revision text cache with %d revisions',
383
revision_versions.prepopulate_revs(versions)
384
used_file_versions = revision_versions.used_file_versions()
385
for num, file_id in enumerate(self.repo.weave_store):
386
self.pb.update('Fixing text parents', num,
387
len(self.repo.weave_store))
388
vf = self.repo.weave_store.get_weave(file_id, transaction)
389
vf_checker = self.repo.get_versioned_file_checker(
390
vf.versions(), revision_versions)
391
versions_with_bad_parents, dangling_file_versions = \
392
vf_checker.check_file_version_parents(vf, file_id)
393
if (len(versions_with_bad_parents) == 0 and
394
len(dangling_file_versions) == 0):
396
full_text_versions = set()
397
unused_versions = set()
398
for dangling_version in dangling_file_versions:
399
version = dangling_version[1]
400
if dangling_version in used_file_versions:
401
# This version *is* used by some revision, even though it
402
# isn't used by its own revision! We make sure any
403
# revision referencing it is stored as a fulltext
404
# This avoids bug 155730: it means that clients looking at
405
# inventories to determine the versions to fetch will not
406
# miss a required version. (So clients can assume that if
407
# they have a complete revision graph, and fetch all file
408
# versions named by those revisions inventories, then they
409
# will not have any missing parents for 'delta' knit
411
# XXX: A better, but more difficult and slower fix would be
412
# to rewrite the inventories referencing this version.
413
full_text_versions.add(version)
415
# This version is totally unreferenced. It should be
417
unused_versions.add(version)
418
self._fix_text_parent(file_id, vf, versions_with_bad_parents,
419
full_text_versions, unused_versions)
421
def _fix_text_parent(self, file_id, vf, versions_with_bad_parents,
422
full_text_versions, unused_versions):
423
"""Fix bad versionedfile entries in a single versioned file."""
424
mutter('fixing text parent: %r (%d versions)', file_id,
425
len(versions_with_bad_parents))
426
mutter('(%d need to be full texts, %d are unused)',
427
len(full_text_versions), len(unused_versions))
428
new_vf = self.repo.weave_store.get_empty('temp:%s' % file_id,
431
for version in vf.versions():
432
if version in versions_with_bad_parents:
433
parents = versions_with_bad_parents[version][1]
435
parents = vf.get_parents(version)
436
new_parents[version] = parents
437
for version in TopoSorter(new_parents.items()).iter_topo_order():
438
if version in unused_versions:
440
lines = vf.get_lines(version)
441
parents = new_parents[version]
442
if parents and (parents[0] in full_text_versions):
443
# Force this record to be a fulltext, not a delta.
444
new_vf._add(version, lines, parents, False,
445
None, None, None, False)
447
new_vf.add_lines(version, parents, lines)
448
self.repo.weave_store.copy(new_vf, file_id, self.transaction)
449
self.repo.weave_store.delete('temp:%s' % file_id, self.transaction)
452
class PackReconciler(RepoReconciler):
453
"""Reconciler that reconciles a pack based repository.
455
Garbage inventories do not affect ancestry queries, and removal is
456
considerably more expensive as there is no separate versioned file for
457
them, so they are not cleaned. In short it is currently a no-op.
459
In future this may be a good place to hook in annotation cache checking,
460
index recreation etc.
463
# XXX: The index corruption that _fix_text_parents performs is needed for
464
# packs, but not yet implemented. The basic approach is to:
465
# - lock the names list
466
# - perform a customised pack() that regenerates data as needed
467
# - unlock the names list
468
# https://bugs.edge.launchpad.net/bzr/+bug/154173
470
def _reconcile_steps(self):
471
"""Perform the steps to reconcile this repository."""
472
if not self.thorough:
474
collection = self.repo._pack_collection
475
collection.ensure_loaded()
476
collection.lock_names()
478
packs = collection.all_packs()
479
all_revisions = self.repo.all_revision_ids()
480
total_inventories = len(list(
481
collection.inventory_index.combined_index.iter_all_entries()))
482
if len(all_revisions):
483
self._packer = repofmt.pack_repo.ReconcilePacker(
484
collection, packs, ".reconcile", all_revisions)
485
new_pack = self._packer.pack(pb=self.pb)
486
if new_pack is not None:
487
self._discard_and_save(packs)
489
# only make a new pack when there is data to copy.
490
self._discard_and_save(packs)
491
self.garbage_inventories = total_inventories - len(list(
492
collection.inventory_index.combined_index.iter_all_entries()))
494
collection._unlock_names()
496
def _discard_and_save(self, packs):
497
"""Discard some packs from the repository.
499
This removes them from the memory index, saves the in-memory index
500
which makes the newly reconciled pack visible and hides the packs to be
501
discarded, and finally renames the packs being discarded into the
502
obsolete packs directory.
504
:param packs: The packs to discard.
507
self.repo._pack_collection._remove_pack_from_memory(pack)
508
self.repo._pack_collection._save_pack_names()
509
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
