13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
17
"""Reconcilers are able to fix some potential data errors in a branch."""
29
29
from bzrlib import (
32
revision as _mod_revision,
35
from bzrlib.trace import mutter
36
from bzrlib.tsort import topo_sort
35
from bzrlib.trace import mutter, note
36
from bzrlib.tsort import TopoSorter
37
37
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
40
def reconcile(dir, canonicalize_chks=False):
40
def reconcile(dir, other=None):
41
41
"""Reconcile the data in dir.
43
43
Currently this is limited to a inventory 'reweave'.
47
47
Directly using Reconciler is recommended for library users that
48
48
desire fine grained control or analysis of the found issues.
50
:param canonicalize_chks: Make sure CHKs are in canonical form.
50
:param other: another bzrdir to reconcile against.
52
reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
52
reconciler = Reconciler(dir, other=other)
53
53
reconciler.reconcile()
56
56
class Reconciler(object):
57
57
"""Reconcilers are used to reconcile existing data."""
59
def __init__(self, dir, other=None, canonicalize_chks=False):
59
def __init__(self, dir, other=None):
60
60
"""Create a Reconciler."""
62
self.canonicalize_chks = canonicalize_chks
64
63
def reconcile(self):
65
64
"""Perform reconciliation.
67
66
After reconciliation the following attributes document found issues:
69
* `inconsistent_parents`: The number of revisions in the repository
70
whose ancestry was being reported incorrectly.
71
* `garbage_inventories`: The number of inventory objects without
72
revisions that were garbage collected.
73
* `fixed_branch_history`: None if there was no branch, False if the
74
branch history was correct, True if the branch history needed to be
67
inconsistent_parents: The number of revisions in the repository whose
68
ancestry was being reported incorrectly.
69
garbage_inventories: The number of inventory objects without revisions
70
that were garbage collected.
71
fixed_branch_history: None if there was no branch, False if the branch
72
history was correct, True if the branch history
73
needed to be re-normalized.
77
75
self.pb = ui.ui_factory.nested_progress_bar()
92
90
# Nothing to check here
93
91
self.fixed_branch_history = None
95
ui.ui_factory.note('Reconciling branch %s' % self.branch.base)
93
self.pb.note('Reconciling branch %s',
96
95
branch_reconciler = self.branch.reconcile(thorough=True)
97
96
self.fixed_branch_history = branch_reconciler.fixed_history
99
98
def _reconcile_repository(self):
100
99
self.repo = self.bzrdir.find_repository()
101
ui.ui_factory.note('Reconciling repository %s' %
100
self.pb.note('Reconciling repository %s',
101
self.repo.bzrdir.root_transport.base)
103
102
self.pb.update("Reconciling repository", 0, 1)
104
if self.canonicalize_chks:
106
self.repo.reconcile_canonicalize_chks
107
except AttributeError:
108
raise errors.BzrError(
109
"%s cannot canonicalize CHKs." % (self.repo,))
110
repo_reconciler = self.repo.reconcile_canonicalize_chks()
112
repo_reconciler = self.repo.reconcile(thorough=True)
103
repo_reconciler = self.repo.reconcile(thorough=True)
113
104
self.inconsistent_parents = repo_reconciler.inconsistent_parents
114
105
self.garbage_inventories = repo_reconciler.garbage_inventories
115
106
if repo_reconciler.aborted:
117
108
'Reconcile aborted: revision index has inconsistent parents.')
119
110
'Run "bzr check" for more details.')
121
ui.ui_factory.note('Reconciliation complete.')
112
self.pb.note('Reconciliation complete.')
124
115
class BranchReconciler(object):
130
121
self.branch = a_branch
132
123
def reconcile(self):
133
operation = cleanup.OperationWithCleanups(self._reconcile)
134
self.add_cleanup = operation.add_cleanup
135
operation.run_simple()
137
def _reconcile(self):
138
124
self.branch.lock_write()
139
self.add_cleanup(self.branch.unlock)
140
self.pb = ui.ui_factory.nested_progress_bar()
141
self.add_cleanup(self.pb.finished)
142
self._reconcile_steps()
126
self.pb = ui.ui_factory.nested_progress_bar()
128
self._reconcile_steps()
144
134
def _reconcile_steps(self):
145
135
self._reconcile_revision_history()
147
137
def _reconcile_revision_history(self):
138
repo = self.branch.repository
148
139
last_revno, last_revision_id = self.branch.last_revision_info()
150
graph = self.branch.repository.get_graph()
152
for revid in graph.iter_lefthand_ancestry(
153
last_revision_id, (_mod_revision.NULL_REVISION,)):
154
real_history.append(revid)
155
except errors.RevisionNotPresent:
156
pass # Hit a ghost left hand parent
140
real_history = list(repo.iter_reverse_revision_history(
157
142
real_history.reverse()
158
143
if last_revno != len(real_history):
159
144
self.fixed_history = True
161
146
# set_revision_history, as this will regenerate it again.
162
147
# Not really worth a whole BranchReconciler class just for this,
164
ui.ui_factory.note('Fixing last revision info %s => %s' % (
165
last_revno, len(real_history)))
149
self.pb.note('Fixing last revision info %s => %s',
150
last_revno, len(real_history))
166
151
self.branch.set_last_revision_info(len(real_history),
167
152
last_revision_id)
169
154
self.fixed_history = False
170
ui.ui_factory.note('revision_history ok.')
155
self.pb.note('revision_history ok.')
173
158
class RepoReconciler(object):
174
159
"""Reconciler that reconciles a repository.
176
161
The goal of repository reconciliation is to make any derived data
177
consistent with the core data committed by a user. This can involve
162
consistent with the core data committed by a user. This can involve
178
163
reindexing, or removing unreferenced data if that can interfere with
179
164
queries in a given repository.
197
182
def reconcile(self):
198
183
"""Perform reconciliation.
200
185
After reconciliation the following attributes document found issues:
202
* `inconsistent_parents`: The number of revisions in the repository
203
whose ancestry was being reported incorrectly.
204
* `garbage_inventories`: The number of inventory objects without
205
revisions that were garbage collected.
186
inconsistent_parents: The number of revisions in the repository whose
187
ancestry was being reported incorrectly.
188
garbage_inventories: The number of inventory objects without revisions
189
that were garbage collected.
207
operation = cleanup.OperationWithCleanups(self._reconcile)
208
self.add_cleanup = operation.add_cleanup
209
operation.run_simple()
211
def _reconcile(self):
212
191
self.repo.lock_write()
213
self.add_cleanup(self.repo.unlock)
214
self.pb = ui.ui_factory.nested_progress_bar()
215
self.add_cleanup(self.pb.finished)
216
self._reconcile_steps()
193
self.pb = ui.ui_factory.nested_progress_bar()
195
self._reconcile_steps()
218
201
def _reconcile_steps(self):
219
202
"""Perform the steps to reconcile this repository."""
222
205
def _reweave_inventory(self):
223
206
"""Regenerate the inventory weave for the repository from scratch.
225
This is a smart function: it will only do the reweave if doing it
208
This is a smart function: it will only do the reweave if doing it
226
209
will correct data issues. The self.thorough flag controls whether
227
210
only data-loss causing issues (!self.thorough) or all issues
228
211
(self.thorough) are treated as requiring the reweave.
213
# local because needing to know about WeaveFile is a wart we want to hide
214
from bzrlib.weave import WeaveFile, Weave
230
215
transaction = self.repo.get_transaction()
231
self.pb.update('Reading inventory data')
216
self.pb.update('Reading inventory data.')
232
217
self.inventory = self.repo.inventories
233
218
self.revisions = self.repo.revisions
234
219
# the total set of revisions to process
244
229
# put a revision into the graph.
245
230
self._graph_revision(rev_id)
246
231
self._check_garbage_inventories()
247
# if there are no inconsistent_parents and
232
# if there are no inconsistent_parents and
248
233
# (no garbage inventories or we are not doing a thorough check)
249
if (not self.inconsistent_parents and
234
if (not self.inconsistent_parents and
250
235
(not self.garbage_inventories or not self.thorough)):
251
ui.ui_factory.note('Inventory ok.')
236
self.pb.note('Inventory ok.')
253
self.pb.update('Backing up inventory', 0, 0)
238
self.pb.update('Backing up inventory...', 0, 0)
254
239
self.repo._backup_inventory()
255
ui.ui_factory.note('Backup inventory created.')
240
self.pb.note('Backup Inventory created.')
256
241
new_inventories = self.repo._temp_inventories()
258
243
# we have topological order of revisions and non ghost parents ready.
259
244
self._setup_steps(len(self._rev_graph))
260
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
245
revision_keys = [(rev_id,) for rev_id in
246
TopoSorter(self._rev_graph.items()).iter_topo_order()]
261
247
stream = self._change_inv_parents(
262
248
self.inventory.get_record_stream(revision_keys, 'unordered', True),
263
249
self._new_inv_parents,
271
257
self.pb.update('Writing weave')
272
258
self.repo._activate_new_inventory()
273
259
self.inventory = None
274
ui.ui_factory.note('Inventory regenerated.')
260
self.pb.note('Inventory regenerated.')
276
262
def _new_inv_parents(self, revision_key):
277
263
"""Lookup ghost-filtered parents for revision_key."""
365
351
def _load_indexes(self):
366
352
"""Load indexes for the reconciliation."""
367
353
self.transaction = self.repo.get_transaction()
368
self.pb.update('Reading indexes', 0, 2)
354
self.pb.update('Reading indexes.', 0, 2)
369
355
self.inventory = self.repo.inventories
370
self.pb.update('Reading indexes', 1, 2)
356
self.pb.update('Reading indexes.', 1, 2)
371
357
self.repo._check_for_inconsistent_revision_parents()
372
358
self.revisions = self.repo.revisions
373
self.pb.update('Reading indexes', 2, 2)
359
self.pb.update('Reading indexes.', 2, 2)
375
361
def _gc_inventory(self):
376
362
"""Remove inventories that are not referenced from the revision store."""
377
self.pb.update('Checking unused inventories', 0, 1)
363
self.pb.update('Checking unused inventories.', 0, 1)
378
364
self._check_garbage_inventories()
379
self.pb.update('Checking unused inventories', 1, 3)
365
self.pb.update('Checking unused inventories.', 1, 3)
380
366
if not self.garbage_inventories:
381
ui.ui_factory.note('Inventory ok.')
367
self.pb.note('Inventory ok.')
383
self.pb.update('Backing up inventory', 0, 0)
369
self.pb.update('Backing up inventory...', 0, 0)
384
370
self.repo._backup_inventory()
385
ui.ui_factory.note('Backup Inventory created')
371
self.pb.note('Backup Inventory created.')
386
372
# asking for '' should never return a non-empty weave
387
373
new_inventories = self.repo._temp_inventories()
388
374
# we have topological order of revisions and non ghost parents ready.
389
375
graph = self.revisions.get_parent_map(self.revisions.keys())
390
revision_keys = topo_sort(graph)
376
revision_keys = list(TopoSorter(graph).iter_topo_order())
391
377
revision_ids = [key[-1] for key in revision_keys]
392
378
self._setup_steps(len(revision_keys))
393
379
stream = self._change_inv_parents(
402
388
self.pb.update('Writing weave')
403
389
self.repo._activate_new_inventory()
404
390
self.inventory = None
405
ui.ui_factory.note('Inventory regenerated.')
391
self.pb.note('Inventory regenerated.')
407
393
def _fix_text_parents(self):
408
394
"""Fix bad versionedfile parent entries.
504
490
# - lock the names list
505
491
# - perform a customised pack() that regenerates data as needed
506
492
# - unlock the names list
507
# https://bugs.launchpad.net/bzr/+bug/154173
509
def __init__(self, repo, other=None, thorough=False,
510
canonicalize_chks=False):
511
super(PackReconciler, self).__init__(repo, other=other,
513
self.canonicalize_chks = canonicalize_chks
493
# https://bugs.edge.launchpad.net/bzr/+bug/154173
515
495
def _reconcile_steps(self):
516
496
"""Perform the steps to reconcile this repository."""
519
499
collection = self.repo._pack_collection
520
500
collection.ensure_loaded()
521
501
collection.lock_names()
522
self.add_cleanup(collection._unlock_names)
523
packs = collection.all_packs()
524
all_revisions = self.repo.all_revision_ids()
525
total_inventories = len(list(
526
collection.inventory_index.combined_index.iter_all_entries()))
527
if len(all_revisions):
528
if self.canonicalize_chks:
529
reconcile_meth = self.repo._canonicalize_chks_pack
503
packs = collection.all_packs()
504
all_revisions = self.repo.all_revision_ids()
505
total_inventories = len(list(
506
collection.inventory_index.combined_index.iter_all_entries()))
507
if len(all_revisions):
508
self._packer = repofmt.pack_repo.ReconcilePacker(
509
collection, packs, ".reconcile", all_revisions)
510
new_pack = self._packer.pack(pb=self.pb)
511
if new_pack is not None:
512
self._discard_and_save(packs)
531
reconcile_meth = self.repo._reconcile_pack
532
new_pack = reconcile_meth(collection, packs, ".reconcile",
533
all_revisions, self.pb)
534
if new_pack is not None:
514
# only make a new pack when there is data to copy.
535
515
self._discard_and_save(packs)
537
# only make a new pack when there is data to copy.
538
self._discard_and_save(packs)
539
self.garbage_inventories = total_inventories - len(list(
540
collection.inventory_index.combined_index.iter_all_entries()))
516
self.garbage_inventories = total_inventories - len(list(
517
collection.inventory_index.combined_index.iter_all_entries()))
519
collection._unlock_names()
542
521
def _discard_and_save(self, packs):
543
522
"""Discard some packs from the repository.
added
removed
bzrlib/reconcile.py
