1
# Copyright (C) 2005, 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Reconcilers are able to fix some potential data errors in a branch."""
29
from bzrlib.trace import mutter
30
from bzrlib.tsort import TopoSorter
33
def reconcile(dir, other=None):
34
"""Reconcile the data in dir.
36
Currently this is limited to a inventory 'reweave'.
38
This is a convenience method, for using a Reconciler object.
40
Directly using Reconciler is recommended for library users that
41
desire fine grained control or analysis of the found issues.
43
:param other: another bzrdir to reconcile against.
45
reconciler = Reconciler(dir, other=other)
46
reconciler.reconcile()
49
class Reconciler(object):
50
"""Reconcilers are used to reconcile existing data."""
52
def __init__(self, dir, other=None):
53
"""Create a Reconciler."""
57
"""Perform reconciliation.
59
After reconciliation the following attributes document found issues:
60
inconsistent_parents: The number of revisions in the repository whose
61
ancestry was being reported incorrectly.
62
garbage_inventories: The number of inventory objects without revisions
63
that were garbage collected.
65
self.pb = ui.ui_factory.nested_progress_bar()
72
"""Helper function for performing reconciliation."""
73
self.repo = self.bzrdir.find_repository()
74
self.pb.note('Reconciling repository %s',
75
self.repo.bzrdir.root_transport.base)
76
repo_reconciler = self.repo.reconcile(thorough=True)
77
self.inconsistent_parents = repo_reconciler.inconsistent_parents
78
self.garbage_inventories = repo_reconciler.garbage_inventories
79
self.pb.note('Reconciliation complete.')
82
class RepoReconciler(object):
83
"""Reconciler that reconciles a repository.
85
The goal of repository reconciliation is to make any derived data
86
consistent with the core data committed by a user. This can involve
87
reindexing, or removing unreferenced data if that can interfere with
88
queries in a given repository.
90
Currently this consists of an inventory reweave with revision cross-checks.
93
def __init__(self, repo, other=None, thorough=False):
94
"""Construct a RepoReconciler.
96
:param thorough: perform a thorough check which may take longer but
97
will correct non-data loss issues such as incorrect
100
self.garbage_inventories = 0
101
self.inconsistent_parents = 0
103
self.thorough = thorough
106
"""Perform reconciliation.
108
After reconciliation the following attributes document found issues:
109
inconsistent_parents: The number of revisions in the repository whose
110
ancestry was being reported incorrectly.
111
garbage_inventories: The number of inventory objects without revisions
112
that were garbage collected.
114
self.repo.lock_write()
116
self.pb = ui.ui_factory.nested_progress_bar()
118
self._reconcile_steps()
124
def _reconcile_steps(self):
125
"""Perform the steps to reconcile this repository."""
126
self._reweave_inventory()
128
def _reweave_inventory(self):
129
"""Regenerate the inventory weave for the repository from scratch.
131
This is a smart function: it will only do the reweave if doing it
132
will correct data issues. The self.thorough flag controls whether
133
only data-loss causing issues (!self.thorough) or all issues
134
(self.thorough) are treated as requiring the reweave.
136
# local because needing to know about WeaveFile is a wart we want to hide
137
from bzrlib.weave import WeaveFile, Weave
138
transaction = self.repo.get_transaction()
139
self.pb.update('Reading inventory data.')
140
self.inventory = self.repo.get_inventory_weave()
141
# the total set of revisions to process
142
self.pending = set([rev_id for rev_id in self.repo._revision_store.all_revision_ids(transaction)])
144
# mapping from revision_id to parents
146
# errors that we detect
147
self.inconsistent_parents = 0
148
# we need the revision id of each revision and its available parents list
149
self._setup_steps(len(self.pending))
150
for rev_id in self.pending:
151
# put a revision into the graph.
152
self._graph_revision(rev_id)
153
self._check_garbage_inventories()
154
# if there are no inconsistent_parents and
155
# (no garbage inventories or we are not doing a thorough check)
156
if (not self.inconsistent_parents and
157
(not self.garbage_inventories or not self.thorough)):
158
self.pb.note('Inventory ok.')
160
self.pb.update('Backing up inventory...', 0, 0)
161
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.repo.get_transaction())
162
self.pb.note('Backup Inventory created.')
163
# asking for '' should never return a non-empty weave
164
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
165
self.repo.get_transaction())
167
# we have topological order of revisions and non ghost parents ready.
168
self._setup_steps(len(self._rev_graph))
169
for rev_id in TopoSorter(self._rev_graph.items()).iter_topo_order():
170
parents = self._rev_graph[rev_id]
171
# double check this really is in topological order.
172
unavailable = [p for p in parents if p not in new_inventory_vf]
173
assert len(unavailable) == 0
174
# this entry has all the non ghost parents in the inventory
176
self._reweave_step('adding inventories')
177
if isinstance(new_inventory_vf, WeaveFile):
178
# It's really a WeaveFile, but we call straight into the
179
# Weave's add method to disable the auto-write-out behaviour.
180
# This is done to avoid a revision_count * time-to-write additional overhead on
182
new_inventory_vf._check_write_ok()
183
Weave._add_lines(new_inventory_vf, rev_id, parents,
184
self.inventory.get_lines(rev_id), None, None, None, False, True)
186
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
188
if isinstance(new_inventory_vf, WeaveFile):
189
new_inventory_vf._save()
190
# if this worked, the set of new_inventory_vf.names should equal
192
assert set(new_inventory_vf.versions()) == self.pending
193
self.pb.update('Writing weave')
194
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.repo.get_transaction())
195
self.repo.control_weaves.delete('inventory.new', self.repo.get_transaction())
196
self.inventory = None
197
self.pb.note('Inventory regenerated.')
199
def _setup_steps(self, new_total):
200
"""Setup the markers we need to control the progress bar."""
201
self.total = new_total
204
def _graph_revision(self, rev_id):
205
"""Load a revision into the revision graph."""
206
# pick a random revision
207
# analyse revision id rev_id and put it in the stack.
208
self._reweave_step('loading revisions')
209
rev = self.repo.get_revision_reconcile(rev_id)
210
assert rev.revision_id == rev_id
212
for parent in rev.parent_ids:
213
if self._parent_is_available(parent):
214
parents.append(parent)
216
mutter('found ghost %s', parent)
217
self._rev_graph[rev_id] = parents
218
if self._parents_are_inconsistent(rev_id, parents):
219
self.inconsistent_parents += 1
220
mutter('Inconsistent inventory parents: id {%s} '
221
'inventory claims %r, '
222
'available parents are %r, '
223
'unavailable parents are %r',
225
set(self.inventory.get_parents(rev_id)),
227
set(rev.parent_ids).difference(set(parents)))
229
def _parents_are_inconsistent(self, rev_id, parents):
230
"""Return True if the parents list of rev_id does not match the weave.
232
This detects inconsistencies based on the self.thorough value:
233
if thorough is on, the first parent value is checked as well as ghost
235
Otherwise only the ghost differences are evaluated.
237
weave_parents = self.inventory.get_parents(rev_id)
238
weave_missing_old_ghosts = set(weave_parents) != set(parents)
239
first_parent_is_wrong = (
240
len(weave_parents) and len(parents) and
241
parents[0] != weave_parents[0])
243
return weave_missing_old_ghosts or first_parent_is_wrong
245
return weave_missing_old_ghosts
247
def _check_garbage_inventories(self):
248
"""Check for garbage inventories which we cannot trust
250
We cant trust them because their pre-requisite file data may not
251
be present - all we know is that their revision was not installed.
253
if not self.thorough:
255
inventories = set(self.inventory.versions())
256
revisions = set(self._rev_graph.keys())
257
garbage = inventories.difference(revisions)
258
self.garbage_inventories = len(garbage)
259
for revision_id in garbage:
260
mutter('Garbage inventory {%s} found.', revision_id)
262
def _parent_is_available(self, parent):
263
"""True if parent is a fully available revision
265
A fully available revision has a inventory and a revision object in the
268
return (parent in self._rev_graph or
269
(parent in self.inventory and self.repo.has_revision(parent)))
271
def _reweave_step(self, message):
272
"""Mark a single step of regeneration complete."""
273
self.pb.update(message, self.count, self.total)
277
class KnitReconciler(RepoReconciler):
278
"""Reconciler that reconciles a knit format repository.
280
This will detect garbage inventories and remove them in thorough mode.
283
def _reconcile_steps(self):
284
"""Perform the steps to reconcile this repository."""
287
# knits never suffer this
290
def _load_indexes(self):
291
"""Load indexes for the reconciliation."""
292
self.transaction = self.repo.get_transaction()
293
self.pb.update('Reading indexes.', 0, 2)
294
self.inventory = self.repo.get_inventory_weave()
295
self.pb.update('Reading indexes.', 1, 2)
296
self.revisions = self.repo._revision_store.get_revision_file(self.transaction)
297
self.pb.update('Reading indexes.', 2, 2)
299
def _gc_inventory(self):
300
"""Remove inventories that are not referenced from the revision store."""
301
self.pb.update('Checking unused inventories.', 0, 1)
302
self._check_garbage_inventories()
303
self.pb.update('Checking unused inventories.', 1, 3)
304
if not self.garbage_inventories:
305
self.pb.note('Inventory ok.')
307
self.pb.update('Backing up inventory...', 0, 0)
308
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.transaction)
309
self.pb.note('Backup Inventory created.')
310
# asking for '' should never return a non-empty weave
311
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
314
# we have topological order of revisions and non ghost parents ready.
315
self._setup_steps(len(self.revisions))
316
for rev_id in TopoSorter(self.revisions.get_graph().items()).iter_topo_order():
317
parents = self.revisions.get_parents(rev_id)
318
# double check this really is in topological order.
319
unavailable = [p for p in parents if p not in new_inventory_vf]
320
assert len(unavailable) == 0
321
# this entry has all the non ghost parents in the inventory
323
self._reweave_step('adding inventories')
324
# ugly but needed, weaves are just way tooooo slow else.
325
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
327
# if this worked, the set of new_inventory_vf.names should equal
329
assert set(new_inventory_vf.versions()) == set(self.revisions.versions())
330
self.pb.update('Writing weave')
331
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.transaction)
332
self.repo.control_weaves.delete('inventory.new', self.transaction)
333
self.inventory = None
334
self.pb.note('Inventory regenerated.')
336
def _check_garbage_inventories(self):
337
"""Check for garbage inventories which we cannot trust
339
We cant trust them because their pre-requisite file data may not
340
be present - all we know is that their revision was not installed.
342
inventories = set(self.inventory.versions())
343
revisions = set(self.revisions.versions())
344
garbage = inventories.difference(revisions)
345
self.garbage_inventories = len(garbage)
346
for revision_id in garbage:
347
mutter('Garbage inventory {%s} found.', revision_id)
added
removed
bzrlib/reconcile.py
