46
46
Directly using Reconciler is recommended for library users that
47
47
desire fine grained control or analysis of the found issues.
49
:param other: another bzrdir to reconcile against.
49
:param canonicalize_chks: Make sure CHKs are in canonical form.
51
reconciler = Reconciler(dir, other=other)
51
reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
52
52
reconciler.reconcile()
55
55
class Reconciler(object):
56
56
"""Reconcilers are used to reconcile existing data."""
58
def __init__(self, dir, other=None):
58
def __init__(self, dir, other=None, canonicalize_chks=False):
59
59
"""Create a Reconciler."""
61
self.canonicalize_chks = canonicalize_chks
62
63
def reconcile(self):
63
64
"""Perform reconciliation.
65
66
After reconciliation the following attributes document found issues:
66
67
inconsistent_parents: The number of revisions in the repository whose
67
68
ancestry was being reported incorrectly.
68
69
garbage_inventories: The number of inventory objects without revisions
69
70
that were garbage collected.
71
fixed_branch_history: None if there was no branch, False if the branch
72
history was correct, True if the branch history
73
needed to be re-normalized.
71
75
self.pb = ui.ui_factory.nested_progress_bar()
77
81
def _reconcile(self):
78
82
"""Helper function for performing reconciliation."""
83
self._reconcile_branch()
84
self._reconcile_repository()
86
def _reconcile_branch(self):
88
self.branch = self.bzrdir.open_branch()
89
except errors.NotBranchError:
90
# Nothing to check here
91
self.fixed_branch_history = None
93
ui.ui_factory.note('Reconciling branch %s' % self.branch.base)
94
branch_reconciler = self.branch.reconcile(thorough=True)
95
self.fixed_branch_history = branch_reconciler.fixed_history
97
def _reconcile_repository(self):
79
98
self.repo = self.bzrdir.find_repository()
80
self.pb.note('Reconciling repository %s',
81
self.repo.bzrdir.root_transport.base)
99
ui.ui_factory.note('Reconciling repository %s' %
82
101
self.pb.update("Reconciling repository", 0, 1)
83
repo_reconciler = self.repo.reconcile(thorough=True)
102
if self.canonicalize_chks:
104
self.repo.reconcile_canonicalize_chks
105
except AttributeError:
106
raise errors.BzrError(
107
"%s cannot canonicalize CHKs." % (self.repo,))
108
repo_reconciler = self.repo.reconcile_canonicalize_chks()
110
repo_reconciler = self.repo.reconcile(thorough=True)
84
111
self.inconsistent_parents = repo_reconciler.inconsistent_parents
85
112
self.garbage_inventories = repo_reconciler.garbage_inventories
86
113
if repo_reconciler.aborted:
88
115
'Reconcile aborted: revision index has inconsistent parents.')
90
117
'Run "bzr check" for more details.')
92
self.pb.note('Reconciliation complete.')
119
ui.ui_factory.note('Reconciliation complete.')
122
class BranchReconciler(object):
123
"""Reconciler that works on a branch."""
125
def __init__(self, a_branch, thorough=False):
126
self.fixed_history = None
127
self.thorough = thorough
128
self.branch = a_branch
131
operation = cleanup.OperationWithCleanups(self._reconcile)
132
self.add_cleanup = operation.add_cleanup
133
operation.run_simple()
135
def _reconcile(self):
136
self.branch.lock_write()
137
self.add_cleanup(self.branch.unlock)
138
self.pb = ui.ui_factory.nested_progress_bar()
139
self.add_cleanup(self.pb.finished)
140
self._reconcile_steps()
142
def _reconcile_steps(self):
143
self._reconcile_revision_history()
145
def _reconcile_revision_history(self):
146
repo = self.branch.repository
147
last_revno, last_revision_id = self.branch.last_revision_info()
150
for revid in repo.iter_reverse_revision_history(
152
real_history.append(revid)
153
except errors.RevisionNotPresent:
154
pass # Hit a ghost left hand parent
155
real_history.reverse()
156
if last_revno != len(real_history):
157
self.fixed_history = True
158
# Technically for Branch5 formats, it is more efficient to use
159
# set_revision_history, as this will regenerate it again.
160
# Not really worth a whole BranchReconciler class just for this,
162
ui.ui_factory.note('Fixing last revision info %s => %s' % (
163
last_revno, len(real_history)))
164
self.branch.set_last_revision_info(len(real_history),
167
self.fixed_history = False
168
ui.ui_factory.note('revision_history ok.')
95
171
class RepoReconciler(object):
96
172
"""Reconciler that reconciles a repository.
98
174
The goal of repository reconciliation is to make any derived data
99
consistent with the core data committed by a user. This can involve
175
consistent with the core data committed by a user. This can involve
100
176
reindexing, or removing unreferenced data if that can interfere with
101
177
queries in a given repository.
165
241
# put a revision into the graph.
166
242
self._graph_revision(rev_id)
167
243
self._check_garbage_inventories()
168
# if there are no inconsistent_parents and
244
# if there are no inconsistent_parents and
169
245
# (no garbage inventories or we are not doing a thorough check)
170
if (not self.inconsistent_parents and
246
if (not self.inconsistent_parents and
171
247
(not self.garbage_inventories or not self.thorough)):
172
self.pb.note('Inventory ok.')
248
ui.ui_factory.note('Inventory ok.')
174
self.pb.update('Backing up inventory...', 0, 0)
175
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.repo.get_transaction())
176
self.pb.note('Backup Inventory created.')
177
# asking for '' should never return a non-empty weave
178
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
179
self.repo.get_transaction())
250
self.pb.update('Backing up inventory', 0, 0)
251
self.repo._backup_inventory()
252
ui.ui_factory.note('Backup inventory created.')
253
new_inventories = self.repo._temp_inventories()
181
255
# we have topological order of revisions and non ghost parents ready.
182
256
self._setup_steps(len(self._rev_graph))
183
for rev_id in TopoSorter(self._rev_graph.items()).iter_topo_order():
184
parents = self._rev_graph[rev_id]
185
# double check this really is in topological order.
186
unavailable = [p for p in parents if p not in new_inventory_vf]
187
assert len(unavailable) == 0
188
# this entry has all the non ghost parents in the inventory
190
self._reweave_step('adding inventories')
191
if isinstance(new_inventory_vf, WeaveFile):
192
# It's really a WeaveFile, but we call straight into the
193
# Weave's add method to disable the auto-write-out behaviour.
194
# This is done to avoid a revision_count * time-to-write additional overhead on
196
new_inventory_vf._check_write_ok()
197
Weave._add_lines(new_inventory_vf, rev_id, parents,
198
self.inventory.get_lines(rev_id), None, None, None, False, True)
200
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
202
if isinstance(new_inventory_vf, WeaveFile):
203
new_inventory_vf._save()
204
# if this worked, the set of new_inventory_vf.names should equal
257
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
258
stream = self._change_inv_parents(
259
self.inventory.get_record_stream(revision_keys, 'unordered', True),
260
self._new_inv_parents,
262
new_inventories.insert_record_stream(stream)
263
# if this worked, the set of new_inventories.keys should equal
206
assert set(new_inventory_vf.versions()) == self.pending
265
if not (set(new_inventories.keys()) ==
266
set([(revid,) for revid in self.pending])):
267
raise AssertionError()
207
268
self.pb.update('Writing weave')
208
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.repo.get_transaction())
209
self.repo.control_weaves.delete('inventory.new', self.repo.get_transaction())
269
self.repo._activate_new_inventory()
210
270
self.inventory = None
211
self.pb.note('Inventory regenerated.')
271
ui.ui_factory.note('Inventory regenerated.')
273
def _new_inv_parents(self, revision_key):
274
"""Lookup ghost-filtered parents for revision_key."""
275
# Use the filtered ghostless parents list:
276
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
278
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
279
"""Adapt a record stream to reconcile the parents."""
280
for record in stream:
281
wanted_parents = get_parents(record.key)
282
if wanted_parents and wanted_parents[0] not in all_revision_keys:
283
# The check for the left most parent only handles knit
284
# compressors, but this code only applies to knit and weave
285
# repositories anyway.
286
bytes = record.get_bytes_as('fulltext')
287
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
289
adapted_record = AdapterFactory(record.key, wanted_parents, record)
291
self._reweave_step('adding inventories')
213
293
def _setup_steps(self, new_total):
214
294
"""Setup the markers we need to control the progress bar."""
221
301
# analyse revision id rev_id and put it in the stack.
222
302
self._reweave_step('loading revisions')
223
303
rev = self.repo.get_revision_reconcile(rev_id)
224
assert rev.revision_id == rev_id
226
305
for parent in rev.parent_ids:
227
306
if self._parent_is_available(parent):
228
307
parents.append(parent)
230
309
mutter('found ghost %s', parent)
231
self._rev_graph[rev_id] = parents
232
if self._parents_are_inconsistent(rev_id, parents):
233
self.inconsistent_parents += 1
234
mutter('Inconsistent inventory parents: id {%s} '
235
'inventory claims %r, '
236
'available parents are %r, '
237
'unavailable parents are %r',
239
set(self.inventory.get_parents(rev_id)),
241
set(rev.parent_ids).difference(set(parents)))
243
def _parents_are_inconsistent(self, rev_id, parents):
244
"""Return True if the parents list of rev_id does not match the weave.
246
This detects inconsistencies based on the self.thorough value:
247
if thorough is on, the first parent value is checked as well as ghost
249
Otherwise only the ghost differences are evaluated.
251
weave_parents = self.inventory.get_parents(rev_id)
252
weave_missing_old_ghosts = set(weave_parents) != set(parents)
253
first_parent_is_wrong = (
254
len(weave_parents) and len(parents) and
255
parents[0] != weave_parents[0])
257
return weave_missing_old_ghosts or first_parent_is_wrong
259
return weave_missing_old_ghosts
310
self._rev_graph[rev_id] = parents
261
312
def _check_garbage_inventories(self):
262
313
"""Check for garbage inventories which we cannot trust
309
362
def _load_indexes(self):
310
363
"""Load indexes for the reconciliation."""
311
364
self.transaction = self.repo.get_transaction()
312
self.pb.update('Reading indexes.', 0, 2)
313
self.inventory = self.repo.get_inventory_weave()
314
self.pb.update('Reading indexes.', 1, 2)
365
self.pb.update('Reading indexes', 0, 2)
366
self.inventory = self.repo.inventories
367
self.pb.update('Reading indexes', 1, 2)
315
368
self.repo._check_for_inconsistent_revision_parents()
316
self.revisions = self.repo._revision_store.get_revision_file(self.transaction)
317
self.pb.update('Reading indexes.', 2, 2)
369
self.revisions = self.repo.revisions
370
self.pb.update('Reading indexes', 2, 2)
319
372
def _gc_inventory(self):
320
373
"""Remove inventories that are not referenced from the revision store."""
321
self.pb.update('Checking unused inventories.', 0, 1)
374
self.pb.update('Checking unused inventories', 0, 1)
322
375
self._check_garbage_inventories()
323
self.pb.update('Checking unused inventories.', 1, 3)
376
self.pb.update('Checking unused inventories', 1, 3)
324
377
if not self.garbage_inventories:
325
self.pb.note('Inventory ok.')
378
ui.ui_factory.note('Inventory ok.')
327
self.pb.update('Backing up inventory...', 0, 0)
328
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.transaction)
329
self.pb.note('Backup Inventory created.')
380
self.pb.update('Backing up inventory', 0, 0)
381
self.repo._backup_inventory()
382
ui.ui_factory.note('Backup Inventory created')
330
383
# asking for '' should never return a non-empty weave
331
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
384
new_inventories = self.repo._temp_inventories()
334
385
# we have topological order of revisions and non ghost parents ready.
335
self._setup_steps(len(self.revisions))
336
for rev_id in TopoSorter(self.revisions.get_graph().items()).iter_topo_order():
337
parents = self.revisions.get_parents(rev_id)
338
# double check this really is in topological order.
339
unavailable = [p for p in parents if p not in new_inventory_vf]
340
assert len(unavailable) == 0
341
# this entry has all the non ghost parents in the inventory
343
self._reweave_step('adding inventories')
344
# ugly but needed, weaves are just way tooooo slow else.
345
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
386
graph = self.revisions.get_parent_map(self.revisions.keys())
387
revision_keys = topo_sort(graph)
388
revision_ids = [key[-1] for key in revision_keys]
389
self._setup_steps(len(revision_keys))
390
stream = self._change_inv_parents(
391
self.inventory.get_record_stream(revision_keys, 'unordered', True),
394
new_inventories.insert_record_stream(stream)
347
395
# if this worked, the set of new_inventory_vf.names should equal
349
assert set(new_inventory_vf.versions()) == set(self.revisions.versions())
396
# the revisionds list
397
if not(set(new_inventories.keys()) == set(revision_keys)):
398
raise AssertionError()
350
399
self.pb.update('Writing weave')
351
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.transaction)
352
self.repo.control_weaves.delete('inventory.new', self.transaction)
400
self.repo._activate_new_inventory()
353
401
self.inventory = None
354
self.pb.note('Inventory regenerated.')
356
def _check_garbage_inventories(self):
357
"""Check for garbage inventories which we cannot trust
359
We cant trust them because their pre-requisite file data may not
360
be present - all we know is that their revision was not installed.
362
inventories = set(self.inventory.versions())
363
revisions = set(self.revisions.versions())
364
garbage = inventories.difference(revisions)
365
self.garbage_inventories = len(garbage)
366
for revision_id in garbage:
367
mutter('Garbage inventory {%s} found.', revision_id)
402
ui.ui_factory.note('Inventory regenerated.')
369
404
def _fix_text_parents(self):
370
405
"""Fix bad versionedfile parent entries.
376
411
parent lists, and replaces the versionedfile with a corrected version.
378
413
transaction = self.repo.get_transaction()
379
versions = self.revisions.versions()
414
versions = [key[-1] for key in self.revisions.keys()]
380
415
mutter('Prepopulating revision text cache with %d revisions',
382
vf_checker = self.repo.get_versioned_file_checker()
383
# List all weaves before altering, to avoid race conditions when we
384
# delete unused weaves.
385
weaves = list(enumerate(self.repo.weave_store))
386
for num, file_id in weaves:
417
vf_checker = self.repo._get_versioned_file_checker()
418
bad_parents, unused_versions = vf_checker.check_file_version_parents(
419
self.repo.texts, self.pb)
420
text_index = vf_checker.text_index
421
per_id_bad_parents = {}
422
for key in unused_versions:
423
# Ensure that every file with unused versions gets rewritten.
424
# NB: This is really not needed, reconcile != pack.
425
per_id_bad_parents[key[0]] = {}
426
# Generate per-knit/weave data.
427
for key, details in bad_parents.iteritems():
430
knit_parents = tuple([parent[-1] for parent in details[0]])
431
correct_parents = tuple([parent[-1] for parent in details[1]])
432
file_details = per_id_bad_parents.setdefault(file_id, {})
433
file_details[rev_id] = (knit_parents, correct_parents)
434
file_id_versions = {}
435
for text_key in text_index:
436
versions_list = file_id_versions.setdefault(text_key[0], [])
437
versions_list.append(text_key[1])
438
# Do the reconcile of individual weaves.
439
for num, file_id in enumerate(per_id_bad_parents):
387
440
self.pb.update('Fixing text parents', num,
388
len(self.repo.weave_store))
389
vf = self.repo.weave_store.get_weave(file_id, transaction)
390
versions_with_bad_parents, unused_versions = \
391
vf_checker.check_file_version_parents(vf, file_id,
393
if (len(versions_with_bad_parents) == 0 and
394
len(unused_versions) == 0):
396
full_text_versions = set()
397
self._fix_text_parent(file_id, vf, versions_with_bad_parents,
398
full_text_versions, unused_versions)
441
len(per_id_bad_parents))
442
versions_with_bad_parents = per_id_bad_parents[file_id]
443
id_unused_versions = set(key[-1] for key in unused_versions
444
if key[0] == file_id)
445
if file_id in file_id_versions:
446
file_versions = file_id_versions[file_id]
448
# This id was present in the disk store but is not referenced
449
# by any revision at all.
451
self._fix_text_parent(file_id, versions_with_bad_parents,
452
id_unused_versions, file_versions)
400
def _fix_text_parent(self, file_id, vf, versions_with_bad_parents,
401
full_text_versions, unused_versions):
454
def _fix_text_parent(self, file_id, versions_with_bad_parents,
455
unused_versions, all_versions):
402
456
"""Fix bad versionedfile entries in a single versioned file."""
403
457
mutter('fixing text parent: %r (%d versions)', file_id,
404
458
len(versions_with_bad_parents))
405
mutter('(%d need to be full texts, %d are unused)',
406
len(full_text_versions), len(unused_versions))
407
new_vf = self.repo.weave_store.get_empty('temp:%s' % file_id,
459
mutter('(%d are unused)', len(unused_versions))
460
new_file_id = 'temp:%s' % file_id
410
for version in vf.versions():
463
for version in all_versions:
411
464
if version in unused_versions:
413
466
elif version in versions_with_bad_parents:
414
467
parents = versions_with_bad_parents[version][1]
416
parents = vf.get_parents(version)
417
new_parents[version] = parents
418
if not len(new_parents):
419
# No used versions, remove the VF.
420
self.repo.weave_store.delete(file_id, self.transaction)
422
for version in TopoSorter(new_parents.items()).iter_topo_order():
423
lines = vf.get_lines(version)
424
parents = new_parents[version]
425
if parents and (parents[0] in full_text_versions):
426
# Force this record to be a fulltext, not a delta.
427
new_vf._add(version, lines, parents, False,
428
None, None, None, False)
430
new_vf.add_lines(version, parents, lines)
431
self.repo.weave_store.copy(new_vf, file_id, self.transaction)
432
self.repo.weave_store.delete('temp:%s' % file_id, self.transaction)
469
pmap = self.repo.texts.get_parent_map([(file_id, version)])
470
parents = [key[-1] for key in pmap[(file_id, version)]]
471
new_parents[(new_file_id, version)] = [
472
(new_file_id, parent) for parent in parents]
473
needed_keys.add((file_id, version))
474
def fix_parents(stream):
475
for record in stream:
476
bytes = record.get_bytes_as('fulltext')
477
new_key = (new_file_id, record.key[-1])
478
parents = new_parents[new_key]
479
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
480
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
481
self.repo._remove_file_id(new_file_id)
482
self.repo.texts.insert_record_stream(fix_parents(stream))
483
self.repo._remove_file_id(file_id)
485
self.repo._move_file_id(new_file_id, file_id)
435
488
class PackReconciler(RepoReconciler):
457
516
collection = self.repo._pack_collection
458
517
collection.ensure_loaded()
459
518
collection.lock_names()
461
packs = collection.all_packs()
462
all_revisions = self.repo.all_revision_ids()
463
total_inventories = len(list(
464
collection.inventory_index.combined_index.iter_all_entries()))
465
if len(all_revisions):
466
self._packer = repofmt.pack_repo.ReconcilePacker(
467
collection, packs, ".reconcile", all_revisions)
468
new_pack = self._packer.pack(pb=self.pb)
469
if new_pack is not None:
470
self._discard_and_save(packs)
519
self.add_cleanup(collection._unlock_names)
520
packs = collection.all_packs()
521
all_revisions = self.repo.all_revision_ids()
522
total_inventories = len(list(
523
collection.inventory_index.combined_index.iter_all_entries()))
524
if len(all_revisions):
525
if self.canonicalize_chks:
526
reconcile_meth = self.repo._canonicalize_chks_pack
472
# only make a new pack when there is data to copy.
528
reconcile_meth = self.repo._reconcile_pack
529
new_pack = reconcile_meth(collection, packs, ".reconcile",
530
all_revisions, self.pb)
531
if new_pack is not None:
473
532
self._discard_and_save(packs)
474
self.garbage_inventories = total_inventories - len(list(
475
collection.inventory_index.combined_index.iter_all_entries()))
477
collection._unlock_names()
534
# only make a new pack when there is data to copy.
535
self._discard_and_save(packs)
536
self.garbage_inventories = total_inventories - len(list(
537
collection.inventory_index.combined_index.iter_all_entries()))
479
539
def _discard_and_save(self, packs):
480
540
"""Discard some packs from the repository.
added
removed
bzrlib/reconcile.py
