13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
"""Reconcilers are able to fix some potential data errors in a branch."""
29
29
from bzrlib import (
32
revision as _mod_revision,
35
from bzrlib.trace import mutter, note
36
from bzrlib.tsort import TopoSorter
35
from bzrlib.trace import mutter
36
from bzrlib.tsort import topo_sort
37
37
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
40
def reconcile(dir, other=None):
38
from bzrlib.i18n import gettext
41
def reconcile(dir, canonicalize_chks=False):
41
42
"""Reconcile the data in dir.
43
44
Currently this is limited to a inventory 'reweave'.
47
48
Directly using Reconciler is recommended for library users that
48
49
desire fine grained control or analysis of the found issues.
50
:param other: another bzrdir to reconcile against.
51
:param canonicalize_chks: Make sure CHKs are in canonical form.
52
reconciler = Reconciler(dir, other=other)
53
reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
53
54
reconciler.reconcile()
56
57
class Reconciler(object):
57
58
"""Reconcilers are used to reconcile existing data."""
59
def __init__(self, dir, other=None):
60
def __init__(self, dir, other=None, canonicalize_chks=False):
60
61
"""Create a Reconciler."""
63
self.canonicalize_chks = canonicalize_chks
63
65
def reconcile(self):
64
66
"""Perform reconciliation.
66
68
After reconciliation the following attributes document found issues:
67
inconsistent_parents: The number of revisions in the repository whose
68
ancestry was being reported incorrectly.
69
garbage_inventories: The number of inventory objects without revisions
70
that were garbage collected.
71
fixed_branch_history: None if there was no branch, False if the branch
72
history was correct, True if the branch history
73
needed to be re-normalized.
70
* `inconsistent_parents`: The number of revisions in the repository
71
whose ancestry was being reported incorrectly.
72
* `garbage_inventories`: The number of inventory objects without
73
revisions that were garbage collected.
74
* `fixed_branch_history`: None if there was no branch, False if the
75
branch history was correct, True if the branch history needed to be
75
78
self.pb = ui.ui_factory.nested_progress_bar()
90
93
# Nothing to check here
91
94
self.fixed_branch_history = None
93
self.pb.note('Reconciling branch %s',
96
ui.ui_factory.note(gettext('Reconciling branch %s') % self.branch.base)
95
97
branch_reconciler = self.branch.reconcile(thorough=True)
96
98
self.fixed_branch_history = branch_reconciler.fixed_history
98
100
def _reconcile_repository(self):
99
101
self.repo = self.bzrdir.find_repository()
100
self.pb.note('Reconciling repository %s',
101
self.repo.bzrdir.root_transport.base)
102
self.pb.update("Reconciling repository", 0, 1)
103
repo_reconciler = self.repo.reconcile(thorough=True)
102
ui.ui_factory.note(gettext('Reconciling repository %s') %
104
self.pb.update(gettext("Reconciling repository"), 0, 1)
105
if self.canonicalize_chks:
107
self.repo.reconcile_canonicalize_chks
108
except AttributeError:
109
raise errors.BzrError(
110
gettext("%s cannot canonicalize CHKs.") % (self.repo,))
111
repo_reconciler = self.repo.reconcile_canonicalize_chks()
113
repo_reconciler = self.repo.reconcile(thorough=True)
104
114
self.inconsistent_parents = repo_reconciler.inconsistent_parents
105
115
self.garbage_inventories = repo_reconciler.garbage_inventories
106
116
if repo_reconciler.aborted:
108
'Reconcile aborted: revision index has inconsistent parents.')
110
'Run "bzr check" for more details.')
117
ui.ui_factory.note(gettext(
118
'Reconcile aborted: revision index has inconsistent parents.'))
119
ui.ui_factory.note(gettext(
120
'Run "bzr check" for more details.'))
112
self.pb.note('Reconciliation complete.')
122
ui.ui_factory.note(gettext('Reconciliation complete.'))
115
125
class BranchReconciler(object):
121
131
self.branch = a_branch
123
133
def reconcile(self):
134
operation = cleanup.OperationWithCleanups(self._reconcile)
135
self.add_cleanup = operation.add_cleanup
136
operation.run_simple()
138
def _reconcile(self):
124
139
self.branch.lock_write()
126
self.pb = ui.ui_factory.nested_progress_bar()
128
self._reconcile_steps()
140
self.add_cleanup(self.branch.unlock)
141
self.pb = ui.ui_factory.nested_progress_bar()
142
self.add_cleanup(self.pb.finished)
143
self._reconcile_steps()
134
145
def _reconcile_steps(self):
135
146
self._reconcile_revision_history()
137
148
def _reconcile_revision_history(self):
138
repo = self.branch.repository
139
149
last_revno, last_revision_id = self.branch.last_revision_info()
140
real_history = list(repo.iter_reverse_revision_history(
151
graph = self.branch.repository.get_graph()
153
for revid in graph.iter_lefthand_ancestry(
154
last_revision_id, (_mod_revision.NULL_REVISION,)):
155
real_history.append(revid)
156
except errors.RevisionNotPresent:
157
pass # Hit a ghost left hand parent
142
158
real_history.reverse()
143
159
if last_revno != len(real_history):
144
160
self.fixed_history = True
146
162
# set_revision_history, as this will regenerate it again.
147
163
# Not really worth a whole BranchReconciler class just for this,
149
self.pb.note('Fixing last revision info %s => %s',
150
last_revno, len(real_history))
165
ui.ui_factory.note(gettext('Fixing last revision info {0} '\
167
last_revno, len(real_history)))
151
168
self.branch.set_last_revision_info(len(real_history),
152
169
last_revision_id)
154
171
self.fixed_history = False
155
self.pb.note('revision_history ok.')
172
ui.ui_factory.note(gettext('revision_history ok.'))
158
175
class RepoReconciler(object):
159
176
"""Reconciler that reconciles a repository.
161
178
The goal of repository reconciliation is to make any derived data
162
consistent with the core data committed by a user. This can involve
179
consistent with the core data committed by a user. This can involve
163
180
reindexing, or removing unreferenced data if that can interfere with
164
181
queries in a given repository.
182
199
def reconcile(self):
183
200
"""Perform reconciliation.
185
202
After reconciliation the following attributes document found issues:
186
inconsistent_parents: The number of revisions in the repository whose
187
ancestry was being reported incorrectly.
188
garbage_inventories: The number of inventory objects without revisions
189
that were garbage collected.
204
* `inconsistent_parents`: The number of revisions in the repository
205
whose ancestry was being reported incorrectly.
206
* `garbage_inventories`: The number of inventory objects without
207
revisions that were garbage collected.
209
operation = cleanup.OperationWithCleanups(self._reconcile)
210
self.add_cleanup = operation.add_cleanup
211
operation.run_simple()
213
def _reconcile(self):
191
214
self.repo.lock_write()
193
self.pb = ui.ui_factory.nested_progress_bar()
195
self._reconcile_steps()
215
self.add_cleanup(self.repo.unlock)
216
self.pb = ui.ui_factory.nested_progress_bar()
217
self.add_cleanup(self.pb.finished)
218
self._reconcile_steps()
201
220
def _reconcile_steps(self):
202
221
"""Perform the steps to reconcile this repository."""
205
224
def _reweave_inventory(self):
206
225
"""Regenerate the inventory weave for the repository from scratch.
208
This is a smart function: it will only do the reweave if doing it
227
This is a smart function: it will only do the reweave if doing it
209
228
will correct data issues. The self.thorough flag controls whether
210
229
only data-loss causing issues (!self.thorough) or all issues
211
230
(self.thorough) are treated as requiring the reweave.
213
# local because needing to know about WeaveFile is a wart we want to hide
214
from bzrlib.weave import WeaveFile, Weave
215
232
transaction = self.repo.get_transaction()
216
self.pb.update('Reading inventory data.')
233
self.pb.update(gettext('Reading inventory data'))
217
234
self.inventory = self.repo.inventories
218
235
self.revisions = self.repo.revisions
219
236
# the total set of revisions to process
229
246
# put a revision into the graph.
230
247
self._graph_revision(rev_id)
231
248
self._check_garbage_inventories()
232
# if there are no inconsistent_parents and
249
# if there are no inconsistent_parents and
233
250
# (no garbage inventories or we are not doing a thorough check)
234
if (not self.inconsistent_parents and
251
if (not self.inconsistent_parents and
235
252
(not self.garbage_inventories or not self.thorough)):
236
self.pb.note('Inventory ok.')
253
ui.ui_factory.note(gettext('Inventory ok.'))
238
self.pb.update('Backing up inventory...', 0, 0)
255
self.pb.update(gettext('Backing up inventory'), 0, 0)
239
256
self.repo._backup_inventory()
240
self.pb.note('Backup Inventory created.')
257
ui.ui_factory.note(gettext('Backup inventory created.'))
241
258
new_inventories = self.repo._temp_inventories()
243
260
# we have topological order of revisions and non ghost parents ready.
244
261
self._setup_steps(len(self._rev_graph))
245
revision_keys = [(rev_id,) for rev_id in
246
TopoSorter(self._rev_graph.items()).iter_topo_order()]
262
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
247
263
stream = self._change_inv_parents(
248
264
self.inventory.get_record_stream(revision_keys, 'unordered', True),
249
265
self._new_inv_parents,
254
270
if not (set(new_inventories.keys()) ==
255
271
set([(revid,) for revid in self.pending])):
256
272
raise AssertionError()
257
self.pb.update('Writing weave')
273
self.pb.update(gettext('Writing weave'))
258
274
self.repo._activate_new_inventory()
259
275
self.inventory = None
260
self.pb.note('Inventory regenerated.')
276
ui.ui_factory.note(gettext('Inventory regenerated.'))
262
278
def _new_inv_parents(self, revision_key):
263
279
"""Lookup ghost-filtered parents for revision_key."""
351
367
def _load_indexes(self):
352
368
"""Load indexes for the reconciliation."""
353
369
self.transaction = self.repo.get_transaction()
354
self.pb.update('Reading indexes.', 0, 2)
370
self.pb.update(gettext('Reading indexes'), 0, 2)
355
371
self.inventory = self.repo.inventories
356
self.pb.update('Reading indexes.', 1, 2)
372
self.pb.update(gettext('Reading indexes'), 1, 2)
357
373
self.repo._check_for_inconsistent_revision_parents()
358
374
self.revisions = self.repo.revisions
359
self.pb.update('Reading indexes.', 2, 2)
375
self.pb.update(gettext('Reading indexes'), 2, 2)
361
377
def _gc_inventory(self):
362
378
"""Remove inventories that are not referenced from the revision store."""
363
self.pb.update('Checking unused inventories.', 0, 1)
379
self.pb.update(gettext('Checking unused inventories'), 0, 1)
364
380
self._check_garbage_inventories()
365
self.pb.update('Checking unused inventories.', 1, 3)
381
self.pb.update(gettext('Checking unused inventories'), 1, 3)
366
382
if not self.garbage_inventories:
367
self.pb.note('Inventory ok.')
383
ui.ui_factory.note(gettext('Inventory ok.'))
369
self.pb.update('Backing up inventory...', 0, 0)
385
self.pb.update(gettext('Backing up inventory'), 0, 0)
370
386
self.repo._backup_inventory()
371
self.pb.note('Backup Inventory created.')
387
ui.ui_factory.note(gettext('Backup Inventory created'))
372
388
# asking for '' should never return a non-empty weave
373
389
new_inventories = self.repo._temp_inventories()
374
390
# we have topological order of revisions and non ghost parents ready.
375
391
graph = self.revisions.get_parent_map(self.revisions.keys())
376
revision_keys = list(TopoSorter(graph).iter_topo_order())
392
revision_keys = topo_sort(graph)
377
393
revision_ids = [key[-1] for key in revision_keys]
378
394
self._setup_steps(len(revision_keys))
379
395
stream = self._change_inv_parents(
385
401
# the revisionds list
386
402
if not(set(new_inventories.keys()) == set(revision_keys)):
387
403
raise AssertionError()
388
self.pb.update('Writing weave')
404
self.pb.update(gettext('Writing weave'))
389
405
self.repo._activate_new_inventory()
390
406
self.inventory = None
391
self.pb.note('Inventory regenerated.')
407
ui.ui_factory.note(gettext('Inventory regenerated.'))
393
409
def _fix_text_parents(self):
394
410
"""Fix bad versionedfile parent entries.
426
442
versions_list.append(text_key[1])
427
443
# Do the reconcile of individual weaves.
428
444
for num, file_id in enumerate(per_id_bad_parents):
429
self.pb.update('Fixing text parents', num,
445
self.pb.update(gettext('Fixing text parents'), num,
430
446
len(per_id_bad_parents))
431
447
versions_with_bad_parents = per_id_bad_parents[file_id]
432
448
id_unused_versions = set(key[-1] for key in unused_versions
490
506
# - lock the names list
491
507
# - perform a customised pack() that regenerates data as needed
492
508
# - unlock the names list
493
# https://bugs.edge.launchpad.net/bzr/+bug/154173
509
# https://bugs.launchpad.net/bzr/+bug/154173
511
def __init__(self, repo, other=None, thorough=False,
512
canonicalize_chks=False):
513
super(PackReconciler, self).__init__(repo, other=other,
515
self.canonicalize_chks = canonicalize_chks
495
517
def _reconcile_steps(self):
496
518
"""Perform the steps to reconcile this repository."""
499
521
collection = self.repo._pack_collection
500
522
collection.ensure_loaded()
501
523
collection.lock_names()
503
packs = collection.all_packs()
504
all_revisions = self.repo.all_revision_ids()
505
total_inventories = len(list(
506
collection.inventory_index.combined_index.iter_all_entries()))
507
if len(all_revisions):
508
self._packer = repofmt.pack_repo.ReconcilePacker(
509
collection, packs, ".reconcile", all_revisions)
510
new_pack = self._packer.pack(pb=self.pb)
511
if new_pack is not None:
512
self._discard_and_save(packs)
524
self.add_cleanup(collection._unlock_names)
525
packs = collection.all_packs()
526
all_revisions = self.repo.all_revision_ids()
527
total_inventories = len(list(
528
collection.inventory_index.combined_index.iter_all_entries()))
529
if len(all_revisions):
530
if self.canonicalize_chks:
531
reconcile_meth = self.repo._canonicalize_chks_pack
514
# only make a new pack when there is data to copy.
533
reconcile_meth = self.repo._reconcile_pack
534
new_pack = reconcile_meth(collection, packs, ".reconcile",
535
all_revisions, self.pb)
536
if new_pack is not None:
515
537
self._discard_and_save(packs)
516
self.garbage_inventories = total_inventories - len(list(
517
collection.inventory_index.combined_index.iter_all_entries()))
519
collection._unlock_names()
539
# only make a new pack when there is data to copy.
540
self._discard_and_save(packs)
541
self.garbage_inventories = total_inventories - len(list(
542
collection.inventory_index.combined_index.iter_all_entries()))
521
544
def _discard_and_save(self, packs):
522
545
"""Discard some packs from the repository.
added
removed
bzrlib/reconcile.py
