32
32
# raising them. If there's more than one exception it'd be good to see them
35
"""Checking of bzr objects.
37
check_refs is a concept used for optimising check. Objects that depend on other
38
objects (e.g. tree on repository) can list the objects they would be requesting
39
so that when the dependent object is checked, matches can be pulled out and
40
evaluated in-line rather than re-reading the same data many times.
41
check_refs are tuples (kind, value). Currently defined kinds are:
43
* 'trees', where value is a revid and the looked up objects are revision trees.
44
* 'lefthand-distance', where value is a revid and the looked up objects are the
45
distance along the lefthand path to NULL for that revid.
46
* 'revision-existence', where value is a revid, and the result is True or False
47
indicating that the revision was found/not found.
54
from bzrlib.branch import Branch
55
from bzrlib.controldir import ControlDir
56
from bzrlib.revision import NULL_REVISION
57
from bzrlib.trace import note
58
from bzrlib.workingtree import WorkingTree
59
from bzrlib.i18n import gettext
35
from bzrlib import errors
36
from bzrlib import repository as _mod_repository
37
from bzrlib import revision
38
from bzrlib.errors import BzrCheckError
40
from bzrlib.trace import log_error, note
61
42
class Check(object):
62
43
"""Check a repository"""
64
def __init__(self, repository, check_repo=True):
65
self.repository = repository
67
def report_results(self, verbose):
68
raise NotImplementedError(self.report_results)
71
class VersionedFileCheck(Check):
72
"""Check a versioned file repository"""
74
45
# The Check object interacts with InventoryEntry.check, etc.
76
def __init__(self, repository, check_repo=True):
47
def __init__(self, repository):
77
48
self.repository = repository
49
self.checked_text_cnt = 0
78
50
self.checked_rev_cnt = 0
52
self.repeated_text_cnt = 0
80
53
self.missing_parent_links = {}
81
54
self.missing_inventory_sha_cnt = 0
82
55
self.missing_revision_cnt = 0
83
self.checked_weaves = set()
56
# maps (file-id, version) -> sha1; used by InventoryFile._check
57
self.checked_texts = {}
58
self.checked_weaves = {}
84
59
self.unreferenced_versions = set()
85
60
self.inconsistent_parents = []
86
self.rich_roots = repository.supports_rich_root()
87
self.text_key_references = {}
88
self.check_repo = check_repo
89
self.other_results = []
90
# Plain text lines to include in the report
91
self._report_items = []
92
# Keys we are looking for; may be large and need spilling to disk.
93
# key->(type(revision/inventory/text/signature/map), sha1, first-referer)
94
self.pending_keys = {}
95
# Ancestors map for all of revisions being checked; while large helper
96
# functions we call would create it anyway, so better to have once and
100
def check(self, callback_refs=None, check_repo=True):
101
if callback_refs is None:
103
63
self.repository.lock_read()
104
self.progress = ui.ui_factory.nested_progress_bar()
64
self.progress = bzrlib.ui.ui_factory.nested_progress_bar()
106
self.progress.update(gettext('check'), 0, 4)
108
self.progress.update(gettext('checking revisions'), 0)
109
self.check_revisions()
110
self.progress.update(gettext('checking commit contents'), 1)
111
self.repository._check_inventories(self)
112
self.progress.update(gettext('checking file graphs'), 2)
113
# check_weaves is done after the revision scan so that
114
# revision index is known to be valid.
116
self.progress.update(gettext('checking branches and trees'), 3)
118
repo = self.repository
119
# calculate all refs, and callback the objects requesting them.
121
wanting_items = set()
122
# Current crude version calculates everything and calls
123
# everything at once. Doing a queue and popping as things are
124
# satisfied would be cheaper on memory [but few people have
125
# huge numbers of working trees today. TODO: fix before
129
for ref, wantlist in callback_refs.iteritems():
130
wanting_items.update(wantlist)
133
refs[ref] = repo.revision_tree(value)
134
elif kind == 'lefthand-distance':
136
elif kind == 'revision-existence':
137
existences.add(value)
139
raise AssertionError(
140
'unknown ref kind for ref %s' % ref)
141
node_distances = repo.get_graph().find_lefthand_distances(distances)
142
for key, distance in node_distances.iteritems():
143
refs[('lefthand-distance', key)] = distance
144
if key in existences and distance > 0:
145
refs[('revision-existence', key)] = True
146
existences.remove(key)
147
parent_map = repo.get_graph().get_parent_map(existences)
148
for key in parent_map:
149
refs[('revision-existence', key)] = True
150
existences.remove(key)
151
for key in existences:
152
refs[('revision-existence', key)] = False
153
for item in wanting_items:
154
if isinstance(item, WorkingTree):
156
if isinstance(item, Branch):
157
self.other_results.append(item.check(refs))
66
self.progress.update('retrieving inventory', 0, 2)
67
# do not put in init, as it should be done with progess,
68
# and inside the lock.
69
self.inventory_weave = self.repository.get_inventory_weave()
70
self.progress.update('checking revision graph', 1)
71
self.check_revision_graph()
74
while revno < len(self.planned_revisions):
75
rev_id = self.planned_revisions[revno]
76
self.progress.update('checking revision', revno,
77
len(self.planned_revisions))
79
self.check_one_rev(rev_id)
80
# check_weaves is done after the revision scan so that
81
# revision index is known to be valid.
159
84
self.progress.finished()
160
85
self.repository.unlock()
162
def _check_revisions(self, revisions_iterator):
163
"""Check revision objects by decorating a generator.
165
:param revisions_iterator: An iterator of(revid, Revision-or-None).
166
:return: A generator of the contents of revisions_iterator.
168
self.planned_revisions = set()
169
for revid, revision in revisions_iterator:
170
yield revid, revision
171
self._check_one_rev(revid, revision)
172
# Flatten the revisions we found to guarantee consistent later
174
self.planned_revisions = list(self.planned_revisions)
175
# TODO: extract digital signatures as items to callback on too.
177
def check_revisions(self):
178
"""Scan revisions, checking data directly available as we go."""
179
revision_iterator = self.repository._iter_revisions(None)
180
revision_iterator = self._check_revisions(revision_iterator)
181
# We read the all revisions here:
182
# - doing this allows later code to depend on the revision index.
183
# - we can fill out existence flags at this point
184
# - we can read the revision inventory sha at this point
185
# - we can check properties and serialisers etc.
186
if not self.repository._format.revision_graph_can_have_wrong_parents:
187
# The check against the index isn't needed.
87
def check_revision_graph(self):
88
if not self.repository.revision_graph_can_have_wrong_parents():
89
# This check is not necessary.
188
90
self.revs_with_bad_parents_in_index = None
189
for thing in revision_iterator:
192
bad_revisions = self.repository._find_inconsistent_revision_parents(
194
self.revs_with_bad_parents_in_index = list(bad_revisions)
92
bad_revisions = self.repository._find_inconsistent_revision_parents()
93
self.revs_with_bad_parents_in_index = list(bad_revisions)
95
def plan_revisions(self):
96
repository = self.repository
97
self.planned_revisions = repository.all_revision_ids()
99
inventoried = set(self.inventory_weave.versions())
100
awol = set(self.planned_revisions) - inventoried
102
raise BzrCheckError('Stored revisions missing from inventory'
103
'{%s}' % ','.join([f for f in awol]))
196
105
def report_results(self, verbose):
198
self._report_repo_results(verbose)
199
for result in self.other_results:
200
result.report_results(verbose)
202
def _report_repo_results(self, verbose):
203
note(gettext('checked repository {0} format {1}').format(
204
self.repository.user_url,
205
self.repository._format))
206
note(gettext('%6d revisions'), self.checked_rev_cnt)
207
note(gettext('%6d file-ids'), len(self.checked_weaves))
209
note(gettext('%6d unreferenced text versions'),
210
len(self.unreferenced_versions))
211
if verbose and len(self.unreferenced_versions):
212
for file_id, revision_id in self.unreferenced_versions:
213
note(gettext('unreferenced version: {{{0}}} in {1}').format(revision_id,
106
note('checked repository %s format %s',
107
self.repository.bzrdir.root_transport,
108
self.repository._format)
109
note('%6d revisions', self.checked_rev_cnt)
110
note('%6d file-ids', len(self.checked_weaves))
111
note('%6d unique file texts', self.checked_text_cnt)
112
note('%6d repeated file texts', self.repeated_text_cnt)
113
note('%6d unreferenced text versions',
114
len(self.unreferenced_versions))
215
115
if self.missing_inventory_sha_cnt:
216
note(gettext('%6d revisions are missing inventory_sha1'),
116
note('%6d revisions are missing inventory_sha1',
217
117
self.missing_inventory_sha_cnt)
218
118
if self.missing_revision_cnt:
219
note(gettext('%6d revisions are mentioned but not present'),
119
note('%6d revisions are mentioned but not present',
220
120
self.missing_revision_cnt)
221
121
if len(self.ghosts):
222
note(gettext('%6d ghost revisions'), len(self.ghosts))
122
note('%6d ghost revisions', len(self.ghosts))
224
124
for ghost in self.ghosts:
225
125
note(' %s', ghost)
226
126
if len(self.missing_parent_links):
227
note(gettext('%6d revisions missing parents in ancestry'),
127
note('%6d revisions missing parents in ancestry',
228
128
len(self.missing_parent_links))
230
130
for link, linkers in self.missing_parent_links.items():
231
note(gettext(' %s should be in the ancestry for:'), link)
131
note(' %s should be in the ancestry for:', link)
232
132
for linker in linkers:
233
133
note(' * %s', linker)
135
for file_id, revision_id in self.unreferenced_versions:
136
log_error('unreferenced version: {%s} in %s', revision_id,
234
138
if len(self.inconsistent_parents):
235
note(gettext('%6d inconsistent parents'), len(self.inconsistent_parents))
139
note('%6d inconsistent parents', len(self.inconsistent_parents))
237
141
for info in self.inconsistent_parents:
238
142
revision_id, file_id, found_parents, correct_parents = info
239
note(gettext(' * {0} version {1} has parents {2!r} '
240
'but should have {3!r}').format(
241
file_id, revision_id, found_parents,
143
note(' * %s version %s has parents %r '
145
% (file_id, revision_id, found_parents,
242
146
correct_parents))
243
147
if self.revs_with_bad_parents_in_index:
245
'%6d revisions have incorrect parents in the revision index'),
148
note('%6d revisions have incorrect parents in the revision index',
246
149
len(self.revs_with_bad_parents_in_index))
248
151
for item in self.revs_with_bad_parents_in_index:
249
152
revision_id, index_parents, actual_parents = item
251
' {0} has wrong parents in index: '
252
'{1!r} should be {2!r}').format(
253
revision_id, index_parents, actual_parents))
254
for item in self._report_items:
257
def _check_one_rev(self, rev_id, rev):
258
"""Cross-check one revision.
260
:param rev_id: A revision id to check.
261
:param rev: A revision or None to indicate a missing revision.
154
' %s has wrong parents in index: '
156
revision_id, index_parents, actual_parents)
158
def check_one_rev(self, rev_id):
159
"""Check one revision.
161
rev_id - the one to check
163
rev = self.repository.get_revision(rev_id)
263
165
if rev.revision_id != rev_id:
264
self._report_items.append(gettext(
265
'Mismatched internal revid {{{0}}} and index revid {{{1}}}').format(
266
rev.revision_id, rev_id))
267
rev_id = rev.revision_id
268
# Check this revision tree etc, and count as seen when we encounter a
270
self.planned_revisions.add(rev_id)
272
self.ghosts.discard(rev_id)
273
# Count all parents as ghosts if we haven't seen them yet.
166
raise BzrCheckError('wrong internal revision id in revision {%s}'
274
169
for parent in rev.parent_ids:
275
170
if not parent in self.planned_revisions:
276
self.ghosts.add(parent)
278
self.ancestors[rev_id] = tuple(rev.parent_ids) or (NULL_REVISION,)
279
self.add_pending_item(rev_id, ('inventories', rev_id), 'inventory',
171
missing_links = self.missing_parent_links.get(parent, [])
172
missing_links.append(rev_id)
173
self.missing_parent_links[parent] = missing_links
174
# list based so somewhat slow,
175
# TODO have a planned_revisions list and set.
176
if self.repository.has_revision(parent):
177
missing_ancestry = self.repository.get_ancestry(parent)
178
for missing in missing_ancestry:
179
if (missing is not None
180
and missing not in self.planned_revisions):
181
self.planned_revisions.append(missing)
183
self.ghosts.append(rev_id)
185
if rev.inventory_sha1:
186
inv_sha1 = self.repository.get_inventory_sha1(rev_id)
187
if inv_sha1 != rev.inventory_sha1:
188
raise BzrCheckError('Inventory sha1 hash doesn\'t match'
189
' value in revision {%s}' % rev_id)
190
self._check_revision_tree(rev_id)
281
191
self.checked_rev_cnt += 1
283
def add_pending_item(self, referer, key, kind, sha1):
284
"""Add a reference to a sha1 to be cross checked against a key.
286
:param referer: The referer that expects key to have sha1.
287
:param key: A storage key e.g. ('texts', 'foo@bar-20040504-1234')
288
:param kind: revision/inventory/text/map/signature
289
:param sha1: A hex sha1 or None if no sha1 is known.
291
existing = self.pending_keys.get(key)
293
if sha1 != existing[1]:
294
self._report_items.append(gettext('Multiple expected sha1s for {0}. {{{1}}}'
295
' expects {{{2}}}, {{{3}}} expects {{{4}}}').format(
296
key, referer, sha1, existing[1], existing[0]))
298
self.pending_keys[key] = (kind, sha1, referer)
300
193
def check_weaves(self):
301
194
"""Check all the weaves we can get our hands on.
304
storebar = ui.ui_factory.nested_progress_bar()
306
self._check_weaves(storebar)
310
def _check_weaves(self, storebar):
311
storebar.update('text-index', 0, 2)
312
if self.repository._format.fast_deltas:
313
# We haven't considered every fileid instance so far.
314
weave_checker = self.repository._get_versioned_file_checker(
315
ancestors=self.ancestors)
317
weave_checker = self.repository._get_versioned_file_checker(
318
text_key_references=self.text_key_references,
319
ancestors=self.ancestors)
320
storebar.update('file-graph', 1)
321
result = weave_checker.check_file_version_parents(
322
self.repository.texts)
323
self.checked_weaves = weave_checker.file_ids
324
bad_parents, unused_versions = result
325
bad_parents = bad_parents.items()
326
for text_key, (stored_parents, correct_parents) in bad_parents:
327
# XXX not ready for id join/split operations.
328
weave_id = text_key[0]
329
revision_id = text_key[-1]
330
weave_parents = tuple([parent[-1] for parent in stored_parents])
331
correct_parents = tuple([parent[-1] for parent in correct_parents])
332
self.inconsistent_parents.append(
333
(revision_id, weave_id, weave_parents, correct_parents))
334
self.unreferenced_versions.update(unused_versions)
336
def _add_entry_to_text_key_references(self, inv, entry):
337
if not self.rich_roots and entry.name == '':
339
key = (entry.file_id, entry.revision)
340
self.text_key_references.setdefault(key, False)
341
if entry.revision == inv.revision_id:
342
self.text_key_references[key] = True
345
def scan_branch(branch, needed_refs, to_unlock):
346
"""Scan a branch for refs.
348
:param branch: The branch to schedule for checking.
349
:param needed_refs: Refs we are accumulating.
350
:param to_unlock: The unlock list accumulating.
198
if self.repository.weave_store.listable():
199
weave_ids = list(self.repository.weave_store)
200
n_weaves = len(weave_ids) + 1
201
self.progress.update('checking versionedfile', 0, n_weaves)
202
self.inventory_weave.check(progress_bar=self.progress)
203
files_in_revisions = {}
204
revisions_of_files = {}
205
weave_checker = self.repository._get_versioned_file_checker()
206
for i, weave_id in enumerate(weave_ids):
207
self.progress.update('checking versionedfile', i, n_weaves)
208
w = self.repository.weave_store.get_weave(weave_id,
209
self.repository.get_transaction())
210
# No progress here, because it looks ugly.
212
result = weave_checker.check_file_version_parents(w, weave_id)
213
bad_parents, unused_versions = result
214
bad_parents = bad_parents.items()
215
for revision_id, (weave_parents, correct_parents) in bad_parents:
216
self.inconsistent_parents.append(
217
(revision_id, weave_id, weave_parents, correct_parents))
218
for revision_id in unused_versions:
219
self.unreferenced_versions.add((weave_id, revision_id))
220
self.checked_weaves[weave_id] = True
222
def _check_revision_tree(self, rev_id):
223
tree = self.repository.revision_tree(rev_id)
227
if file_id in seen_ids:
228
raise BzrCheckError('duplicated file_id {%s} '
229
'in inventory for revision {%s}'
231
seen_ids[file_id] = True
234
ie.check(self, rev_id, inv, tree)
236
for path, ie in inv.iter_entries():
237
if path in seen_names:
238
raise BzrCheckError('duplicated path %s '
239
'in inventory for revision {%s}'
241
seen_names[path] = True
244
def check(branch, verbose):
245
"""Run consistency checks on a branch.
247
Results are reported through logging.
249
:raise BzrCheckError: if there's a consistency error.
352
note(gettext("Checking branch at '%s'.") % (branch.base,))
353
251
branch.lock_read()
354
to_unlock.append(branch)
355
branch_refs = branch._get_check_refs()
356
for ref in branch_refs:
357
reflist = needed_refs.setdefault(ref, [])
358
reflist.append(branch)
361
def scan_tree(base_tree, tree, needed_refs, to_unlock):
362
"""Scan a tree for refs.
364
:param base_tree: The original tree check opened, used to detect duplicate
366
:param tree: The tree to schedule for checking.
367
:param needed_refs: Refs we are accumulating.
368
:param to_unlock: The unlock list accumulating.
370
if base_tree is not None and tree.basedir == base_tree.basedir:
372
note(gettext("Checking working tree at '%s'.") % (tree.basedir,))
374
to_unlock.append(tree)
375
tree_refs = tree._get_check_refs()
376
for ref in tree_refs:
377
reflist = needed_refs.setdefault(ref, [])
381
def check_dwim(path, verbose, do_branch=False, do_repo=False, do_tree=False):
382
"""Check multiple objects.
384
If errors occur they are accumulated and reported as far as possible, and
385
an exception raised at the end of the process.
388
base_tree, branch, repo, relpath = \
389
ControlDir.open_containing_tree_branch_or_repository(path)
390
except errors.NotBranchError:
391
base_tree = branch = repo = None
396
if base_tree is not None:
397
# If the tree is a lightweight checkout we won't see it in
398
# repo.find_branches - add now.
400
scan_tree(None, base_tree, needed_refs, to_unlock)
401
branch = base_tree.branch
402
if branch is not None:
405
# The branch is in a shared repository
406
repo = branch.repository
409
to_unlock.append(repo)
410
branches = repo.find_branches(using=True)
412
if do_branch or do_tree:
413
for branch in branches:
416
tree = branch.bzrdir.open_workingtree()
418
except (errors.NotLocalUrl, errors.NoWorkingTree):
421
scan_tree(base_tree, tree, needed_refs, to_unlock)
423
scan_branch(branch, needed_refs, to_unlock)
424
if do_branch and not branches:
425
note(gettext("No branch found at specified location."))
426
if do_tree and base_tree is None and not saw_tree:
427
note(gettext("No working tree found at specified location."))
428
if do_repo or do_branch or do_tree:
430
note(gettext("Checking repository at '%s'.")
432
result = repo.check(None, callback_refs=needed_refs,
434
result.report_results(verbose)
437
note(gettext("No working tree found at specified location."))
439
note(gettext("No branch found at specified location."))
441
note(gettext("No repository found at specified location."))
253
branch_result = branch.check()
254
repo_result = branch.repository.check([branch.last_revision()])
443
for thing in to_unlock:
257
branch_result.report_results(verbose)
258
repo_result.report_results(verbose)
added
removed
bzrlib/check.py
