81
76
def _reconcile(self):
82
77
"""Helper function for performing reconciliation."""
83
self._reconcile_branch()
84
self._reconcile_repository()
86
def _reconcile_branch(self):
88
self.branch = self.bzrdir.open_branch()
89
except errors.NotBranchError:
90
# Nothing to check here
91
self.fixed_branch_history = None
93
ui.ui_factory.note('Reconciling branch %s' % self.branch.base)
94
branch_reconciler = self.branch.reconcile(thorough=True)
95
self.fixed_branch_history = branch_reconciler.fixed_history
97
def _reconcile_repository(self):
98
78
self.repo = self.bzrdir.find_repository()
99
ui.ui_factory.note('Reconciling repository %s' %
100
self.repo.bzrdir.root_transport.base)
101
self.pb.update("Reconciling repository", 0, 1)
79
self.pb.note('Reconciling repository %s',
80
self.repo.bzrdir.root_transport.base)
102
81
repo_reconciler = self.repo.reconcile(thorough=True)
103
82
self.inconsistent_parents = repo_reconciler.inconsistent_parents
104
83
self.garbage_inventories = repo_reconciler.garbage_inventories
105
84
if repo_reconciler.aborted:
107
86
'Reconcile aborted: revision index has inconsistent parents.')
109
88
'Run "bzr check" for more details.')
111
ui.ui_factory.note('Reconciliation complete.')
114
class BranchReconciler(object):
115
"""Reconciler that works on a branch."""
117
def __init__(self, a_branch, thorough=False):
118
self.fixed_history = None
119
self.thorough = thorough
120
self.branch = a_branch
123
operation = cleanup.OperationWithCleanups(self._reconcile)
124
self.add_cleanup = operation.add_cleanup
125
operation.run_simple()
127
def _reconcile(self):
128
self.branch.lock_write()
129
self.add_cleanup(self.branch.unlock)
130
self.pb = ui.ui_factory.nested_progress_bar()
131
self.add_cleanup(self.pb.finished)
132
self._reconcile_steps()
134
def _reconcile_steps(self):
135
self._reconcile_revision_history()
137
def _reconcile_revision_history(self):
138
repo = self.branch.repository
139
last_revno, last_revision_id = self.branch.last_revision_info()
142
for revid in repo.iter_reverse_revision_history(
144
real_history.append(revid)
145
except errors.RevisionNotPresent:
146
pass # Hit a ghost left hand parent
147
real_history.reverse()
148
if last_revno != len(real_history):
149
self.fixed_history = True
150
# Technically for Branch5 formats, it is more efficient to use
151
# set_revision_history, as this will regenerate it again.
152
# Not really worth a whole BranchReconciler class just for this,
154
ui.ui_factory.note('Fixing last revision info %s => %s' % (
155
last_revno, len(real_history)))
156
self.branch.set_last_revision_info(len(real_history),
159
self.fixed_history = False
160
ui.ui_factory.note('revision_history ok.')
90
self.pb.note('Reconciliation complete.')
163
93
class RepoReconciler(object):
164
94
"""Reconciler that reconciles a repository.
166
96
The goal of repository reconciliation is to make any derived data
167
consistent with the core data committed by a user. This can involve
97
consistent with the core data committed by a user. This can involve
168
98
reindexing, or removing unreferenced data if that can interfere with
169
99
queries in a given repository.
235
163
# put a revision into the graph.
236
164
self._graph_revision(rev_id)
237
165
self._check_garbage_inventories()
238
# if there are no inconsistent_parents and
166
# if there are no inconsistent_parents and
239
167
# (no garbage inventories or we are not doing a thorough check)
240
if (not self.inconsistent_parents and
168
if (not self.inconsistent_parents and
241
169
(not self.garbage_inventories or not self.thorough)):
242
ui.ui_factory.note('Inventory ok.')
170
self.pb.note('Inventory ok.')
244
self.pb.update('Backing up inventory', 0, 0)
245
self.repo._backup_inventory()
246
ui.ui_factory.note('Backup inventory created.')
247
new_inventories = self.repo._temp_inventories()
172
self.pb.update('Backing up inventory...', 0, 0)
173
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.repo.get_transaction())
174
self.pb.note('Backup Inventory created.')
175
# asking for '' should never return a non-empty weave
176
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
177
self.repo.get_transaction())
249
179
# we have topological order of revisions and non ghost parents ready.
250
180
self._setup_steps(len(self._rev_graph))
251
revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
252
stream = self._change_inv_parents(
253
self.inventory.get_record_stream(revision_keys, 'unordered', True),
254
self._new_inv_parents,
256
new_inventories.insert_record_stream(stream)
257
# if this worked, the set of new_inventories.keys should equal
181
for rev_id in TopoSorter(self._rev_graph.items()).iter_topo_order():
182
parents = self._rev_graph[rev_id]
183
# double check this really is in topological order.
184
unavailable = [p for p in parents if p not in new_inventory_vf]
185
assert len(unavailable) == 0
186
# this entry has all the non ghost parents in the inventory
188
self._reweave_step('adding inventories')
189
if isinstance(new_inventory_vf, WeaveFile):
190
# It's really a WeaveFile, but we call straight into the
191
# Weave's add method to disable the auto-write-out behaviour.
192
# This is done to avoid a revision_count * time-to-write additional overhead on
194
new_inventory_vf._check_write_ok()
195
Weave._add_lines(new_inventory_vf, rev_id, parents,
196
self.inventory.get_lines(rev_id), None, None, None, False, True)
198
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
200
if isinstance(new_inventory_vf, WeaveFile):
201
new_inventory_vf._save()
202
# if this worked, the set of new_inventory_vf.names should equal
259
if not (set(new_inventories.keys()) ==
260
set([(revid,) for revid in self.pending])):
261
raise AssertionError()
204
assert set(new_inventory_vf.versions()) == self.pending
262
205
self.pb.update('Writing weave')
263
self.repo._activate_new_inventory()
206
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.repo.get_transaction())
207
self.repo.control_weaves.delete('inventory.new', self.repo.get_transaction())
264
208
self.inventory = None
265
ui.ui_factory.note('Inventory regenerated.')
267
def _new_inv_parents(self, revision_key):
268
"""Lookup ghost-filtered parents for revision_key."""
269
# Use the filtered ghostless parents list:
270
return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
272
def _change_inv_parents(self, stream, get_parents, all_revision_keys):
273
"""Adapt a record stream to reconcile the parents."""
274
for record in stream:
275
wanted_parents = get_parents(record.key)
276
if wanted_parents and wanted_parents[0] not in all_revision_keys:
277
# The check for the left most parent only handles knit
278
# compressors, but this code only applies to knit and weave
279
# repositories anyway.
280
bytes = record.get_bytes_as('fulltext')
281
yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
283
adapted_record = AdapterFactory(record.key, wanted_parents, record)
285
self._reweave_step('adding inventories')
209
self.pb.note('Inventory regenerated.')
287
211
def _setup_steps(self, new_total):
288
212
"""Setup the markers we need to control the progress bar."""
295
219
# analyse revision id rev_id and put it in the stack.
296
220
self._reweave_step('loading revisions')
297
221
rev = self.repo.get_revision_reconcile(rev_id)
222
assert rev.revision_id == rev_id
299
224
for parent in rev.parent_ids:
300
225
if self._parent_is_available(parent):
301
226
parents.append(parent)
303
228
mutter('found ghost %s', parent)
304
self._rev_graph[rev_id] = parents
229
self._rev_graph[rev_id] = parents
230
if self._parents_are_inconsistent(rev_id, parents):
231
self.inconsistent_parents += 1
232
mutter('Inconsistent inventory parents: id {%s} '
233
'inventory claims %r, '
234
'available parents are %r, '
235
'unavailable parents are %r',
237
set(self.inventory.get_parents(rev_id)),
239
set(rev.parent_ids).difference(set(parents)))
241
def _parents_are_inconsistent(self, rev_id, parents):
242
"""Return True if the parents list of rev_id does not match the weave.
244
This detects inconsistencies based on the self.thorough value:
245
if thorough is on, the first parent value is checked as well as ghost
247
Otherwise only the ghost differences are evaluated.
249
weave_parents = self.inventory.get_parents(rev_id)
250
weave_missing_old_ghosts = set(weave_parents) != set(parents)
251
first_parent_is_wrong = (
252
len(weave_parents) and len(parents) and
253
parents[0] != weave_parents[0])
255
return weave_missing_old_ghosts or first_parent_is_wrong
257
return weave_missing_old_ghosts
306
259
def _check_garbage_inventories(self):
307
260
"""Check for garbage inventories which we cannot trust
356
307
def _load_indexes(self):
357
308
"""Load indexes for the reconciliation."""
358
309
self.transaction = self.repo.get_transaction()
359
self.pb.update('Reading indexes', 0, 2)
360
self.inventory = self.repo.inventories
361
self.pb.update('Reading indexes', 1, 2)
310
self.pb.update('Reading indexes.', 0, 2)
311
self.inventory = self.repo.get_inventory_weave()
312
self.pb.update('Reading indexes.', 1, 2)
362
313
self.repo._check_for_inconsistent_revision_parents()
363
self.revisions = self.repo.revisions
364
self.pb.update('Reading indexes', 2, 2)
314
self.revisions = self.repo._revision_store.get_revision_file(self.transaction)
315
self.pb.update('Reading indexes.', 2, 2)
366
317
def _gc_inventory(self):
367
318
"""Remove inventories that are not referenced from the revision store."""
368
self.pb.update('Checking unused inventories', 0, 1)
319
self.pb.update('Checking unused inventories.', 0, 1)
369
320
self._check_garbage_inventories()
370
self.pb.update('Checking unused inventories', 1, 3)
321
self.pb.update('Checking unused inventories.', 1, 3)
371
322
if not self.garbage_inventories:
372
ui.ui_factory.note('Inventory ok.')
323
self.pb.note('Inventory ok.')
374
self.pb.update('Backing up inventory', 0, 0)
375
self.repo._backup_inventory()
376
ui.ui_factory.note('Backup Inventory created')
325
self.pb.update('Backing up inventory...', 0, 0)
326
self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.transaction)
327
self.pb.note('Backup Inventory created.')
377
328
# asking for '' should never return a non-empty weave
378
new_inventories = self.repo._temp_inventories()
329
new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
379
332
# we have topological order of revisions and non ghost parents ready.
380
graph = self.revisions.get_parent_map(self.revisions.keys())
381
revision_keys = topo_sort(graph)
382
revision_ids = [key[-1] for key in revision_keys]
383
self._setup_steps(len(revision_keys))
384
stream = self._change_inv_parents(
385
self.inventory.get_record_stream(revision_keys, 'unordered', True),
388
new_inventories.insert_record_stream(stream)
333
self._setup_steps(len(self.revisions))
334
for rev_id in TopoSorter(self.revisions.get_graph().items()).iter_topo_order():
335
parents = self.revisions.get_parents(rev_id)
336
# double check this really is in topological order.
337
unavailable = [p for p in parents if p not in new_inventory_vf]
338
assert len(unavailable) == 0
339
# this entry has all the non ghost parents in the inventory
341
self._reweave_step('adding inventories')
342
# ugly but needed, weaves are just way tooooo slow else.
343
new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
389
345
# if this worked, the set of new_inventory_vf.names should equal
390
# the revisionds list
391
if not(set(new_inventories.keys()) == set(revision_keys)):
392
raise AssertionError()
347
assert set(new_inventory_vf.versions()) == set(self.revisions.versions())
393
348
self.pb.update('Writing weave')
394
self.repo._activate_new_inventory()
349
self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.transaction)
350
self.repo.control_weaves.delete('inventory.new', self.transaction)
395
351
self.inventory = None
396
ui.ui_factory.note('Inventory regenerated.')
352
self.pb.note('Inventory regenerated.')
354
def _check_garbage_inventories(self):
355
"""Check for garbage inventories which we cannot trust
357
We cant trust them because their pre-requisite file data may not
358
be present - all we know is that their revision was not installed.
360
inventories = set(self.inventory.versions())
361
revisions = set(self.revisions.versions())
362
garbage = inventories.difference(revisions)
363
self.garbage_inventories = len(garbage)
364
for revision_id in garbage:
365
mutter('Garbage inventory {%s} found.', revision_id)
398
367
def _fix_text_parents(self):
399
368
"""Fix bad versionedfile parent entries.
405
374
parent lists, and replaces the versionedfile with a corrected version.
407
376
transaction = self.repo.get_transaction()
408
versions = [key[-1] for key in self.revisions.keys()]
377
revision_versions = repository._RevisionTextVersionCache(self.repo)
378
versions = self.revisions.versions()
409
379
mutter('Prepopulating revision text cache with %d revisions',
411
vf_checker = self.repo._get_versioned_file_checker()
412
bad_parents, unused_versions = vf_checker.check_file_version_parents(
413
self.repo.texts, self.pb)
414
text_index = vf_checker.text_index
415
per_id_bad_parents = {}
416
for key in unused_versions:
417
# Ensure that every file with unused versions gets rewritten.
418
# NB: This is really not needed, reconcile != pack.
419
per_id_bad_parents[key[0]] = {}
420
# Generate per-knit/weave data.
421
for key, details in bad_parents.iteritems():
424
knit_parents = tuple([parent[-1] for parent in details[0]])
425
correct_parents = tuple([parent[-1] for parent in details[1]])
426
file_details = per_id_bad_parents.setdefault(file_id, {})
427
file_details[rev_id] = (knit_parents, correct_parents)
428
file_id_versions = {}
429
for text_key in text_index:
430
versions_list = file_id_versions.setdefault(text_key[0], [])
431
versions_list.append(text_key[1])
432
# Do the reconcile of individual weaves.
433
for num, file_id in enumerate(per_id_bad_parents):
381
revision_versions.prepopulate_revs(versions)
382
used_file_versions = revision_versions.used_file_versions()
383
for num, file_id in enumerate(self.repo.weave_store):
434
384
self.pb.update('Fixing text parents', num,
435
len(per_id_bad_parents))
436
versions_with_bad_parents = per_id_bad_parents[file_id]
437
id_unused_versions = set(key[-1] for key in unused_versions
438
if key[0] == file_id)
439
if file_id in file_id_versions:
440
file_versions = file_id_versions[file_id]
442
# This id was present in the disk store but is not referenced
443
# by any revision at all.
445
self._fix_text_parent(file_id, versions_with_bad_parents,
446
id_unused_versions, file_versions)
385
len(self.repo.weave_store))
386
vf = self.repo.weave_store.get_weave(file_id, transaction)
387
vf_checker = self.repo.get_versioned_file_checker(
388
vf.versions(), revision_versions)
389
versions_with_bad_parents, dangling_file_versions = \
390
vf_checker.check_file_version_parents(vf, file_id)
391
if (len(versions_with_bad_parents) == 0 and
392
len(dangling_file_versions) == 0):
394
full_text_versions = set()
395
unused_versions = set()
396
for dangling_version in dangling_file_versions:
397
version = dangling_version[1]
398
if dangling_version in used_file_versions:
399
# This version *is* used by some revision, even though it
400
# isn't used by its own revision! We make sure any
401
# revision referencing it is stored as a fulltext
402
# This avoids bug 155730: it means that clients looking at
403
# inventories to determine the versions to fetch will not
404
# miss a required version. (So clients can assume that if
405
# they have a complete revision graph, and fetch all file
406
# versions named by those revisions inventories, then they
407
# will not have any missing parents for 'delta' knit
409
# XXX: A better, but more difficult and slower fix would be
410
# to rewrite the inventories referencing this version.
411
full_text_versions.add(version)
413
# This version is totally unreferenced. It should be
415
unused_versions.add(version)
416
self._fix_text_parent(file_id, vf, versions_with_bad_parents,
417
full_text_versions, unused_versions)
448
def _fix_text_parent(self, file_id, versions_with_bad_parents,
449
unused_versions, all_versions):
419
def _fix_text_parent(self, file_id, vf, versions_with_bad_parents,
420
full_text_versions, unused_versions):
450
421
"""Fix bad versionedfile entries in a single versioned file."""
451
422
mutter('fixing text parent: %r (%d versions)', file_id,
452
423
len(versions_with_bad_parents))
453
mutter('(%d are unused)', len(unused_versions))
454
new_file_id = 'temp:%s' % file_id
424
mutter('(%d need to be full texts, %d are unused)',
425
len(full_text_versions), len(unused_versions))
426
new_vf = self.repo.weave_store.get_empty('temp:%s' % file_id,
457
for version in all_versions:
429
for version in vf.versions():
430
if version in versions_with_bad_parents:
431
parents = versions_with_bad_parents[version][1]
433
parents = vf.get_parents(version)
434
new_parents[version] = parents
435
for version in TopoSorter(new_parents.items()).iter_topo_order():
458
436
if version in unused_versions:
460
elif version in versions_with_bad_parents:
461
parents = versions_with_bad_parents[version][1]
438
lines = vf.get_lines(version)
439
parents = new_parents[version]
440
if parents and (parents[0] in full_text_versions):
441
# Force this record to be a fulltext, not a delta.
442
new_vf._add(version, lines, parents, False,
443
None, None, None, False)
463
pmap = self.repo.texts.get_parent_map([(file_id, version)])
464
parents = [key[-1] for key in pmap[(file_id, version)]]
465
new_parents[(new_file_id, version)] = [
466
(new_file_id, parent) for parent in parents]
467
needed_keys.add((file_id, version))
468
def fix_parents(stream):
469
for record in stream:
470
bytes = record.get_bytes_as('fulltext')
471
new_key = (new_file_id, record.key[-1])
472
parents = new_parents[new_key]
473
yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
474
stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
475
self.repo._remove_file_id(new_file_id)
476
self.repo.texts.insert_record_stream(fix_parents(stream))
477
self.repo._remove_file_id(file_id)
479
self.repo._move_file_id(new_file_id, file_id)
445
new_vf.add_lines(version, parents, lines)
446
self.repo.weave_store.copy(new_vf, file_id, self.transaction)
447
self.repo.weave_store.delete('temp:%s' % file_id, self.transaction)
482
450
class PackReconciler(RepoReconciler):
500
468
def _reconcile_steps(self):
501
469
"""Perform the steps to reconcile this repository."""
502
if not self.thorough:
504
collection = self.repo._pack_collection
505
collection.ensure_loaded()
506
collection.lock_names()
507
self.add_cleanup(collection._unlock_names)
508
packs = collection.all_packs()
509
all_revisions = self.repo.all_revision_ids()
510
total_inventories = len(list(
511
collection.inventory_index.combined_index.iter_all_entries()))
512
if len(all_revisions):
513
new_pack = self.repo._reconcile_pack(collection, packs,
514
".reconcile", all_revisions, self.pb)
515
if new_pack is not None:
516
self._discard_and_save(packs)
518
# only make a new pack when there is data to copy.
519
self._discard_and_save(packs)
520
self.garbage_inventories = total_inventories - len(list(
521
collection.inventory_index.combined_index.iter_all_entries()))
523
def _discard_and_save(self, packs):
524
"""Discard some packs from the repository.
526
This removes them from the memory index, saves the in-memory index
527
which makes the newly reconciled pack visible and hides the packs to be
528
discarded, and finally renames the packs being discarded into the
529
obsolete packs directory.
531
:param packs: The packs to discard.
534
self.repo._pack_collection._remove_pack_from_memory(pack)
535
self.repo._pack_collection._save_pack_names()
536
self.repo._pack_collection._obsolete_packs(packs)
added
removed
bzrlib/reconcile.py
