22
23
# TODO: Check for extra files in the control directory.
24
# TODO: Check revision, inventory and entry objects have all
25
# TODO: Check revision, inventory and entry objects have all
27
# TODO: Get every revision in the revision-store even if they're not
28
# referenced by history and make sure they're all valid.
30
# TODO: Perhaps have a way to record errors other than by raising exceptions;
31
# would perhaps be enough to accumulate exception objects in a list without
32
# raising them. If there's more than one exception it'd be good to see them
35
"""Checking of bzr objects.
37
check_refs is a concept used for optimising check. Objects that depend on other
38
objects (e.g. tree on repository) can list the objects they would be requesting
39
so that when the dependent object is checked, matches can be pulled out and
40
evaluated in-line rather than re-reading the same data many times.
41
check_refs are tuples (kind, value). Currently defined kinds are:
42
* 'trees', where value is a revid and the looked up objects are revision trees.
43
* 'lefthand-distance', where value is a revid and the looked up objects are the
44
distance along the lefthand path to NULL for that revid.
45
* 'revision-existence', where value is a revid, and the result is True or False
46
indicating that the revision was found/not found.
49
from bzrlib import errors
50
from bzrlib.branch import Branch
51
from bzrlib.bzrdir import BzrDir
52
from bzrlib.revision import NULL_REVISION
53
from bzrlib.symbol_versioning import deprecated_function, deprecated_in
54
from bzrlib.trace import note
56
from bzrlib.workingtree import WorkingTree
30
from bzrlib.trace import note, warning
31
from bzrlib.osutils import rename, sha_string, fingerprint_file, sha_strings
32
from bzrlib.trace import mutter
33
from bzrlib.errors import BzrCheckError, NoSuchRevision
34
from bzrlib.inventory import ROOT_ID
35
from bzrlib.branch import gen_root_id
58
38
class Check(object):
59
"""Check a repository"""
61
# The Check object interacts with InventoryEntry.check, etc.
63
def __init__(self, repository, check_repo=True):
64
self.repository = repository
40
def __init__(self, branch):
44
branch.weave_store.enable_cache = True
45
branch.control_weaves.enable_cache = True
49
branch.weave_store.enable_cache = False
50
branch.control_weaves.enable_cache = False
57
self.checked_text_cnt = 0
65
58
self.checked_rev_cnt = 0
67
self.missing_parent_links = {}
59
self.repeated_text_cnt = 0
68
60
self.missing_inventory_sha_cnt = 0
69
61
self.missing_revision_cnt = 0
70
self.checked_weaves = set()
71
self.unreferenced_versions = set()
72
self.inconsistent_parents = []
73
self.rich_roots = repository.supports_rich_root()
74
self.text_key_references = {}
75
self.check_repo = check_repo
76
self.other_results = []
77
# Plain text lines to include in the report
78
self._report_items = []
79
# Keys we are looking for; may be large and need spilling to disk.
80
# key->(type(revision/inventory/text/signature/map), sha1, first-referer)
81
self.pending_keys = {}
82
# Ancestors map for all of revisions being checked; while large helper
83
# functions we call would create it anyway, so better to have once and
87
def check(self, callback_refs=None, check_repo=True):
88
if callback_refs is None:
90
self.repository.lock_read()
91
self.progress = bzrlib.ui.ui_factory.nested_progress_bar()
93
self.progress.update('check', 0, 4)
95
self.progress.update('checking revisions', 0)
96
self.check_revisions()
97
self.progress.update('checking commit contents', 1)
98
self.repository._check_inventories(self)
99
self.progress.update('checking file graphs', 2)
100
# check_weaves is done after the revision scan so that
101
# revision index is known to be valid.
103
self.progress.update('checking branches and trees', 3)
105
repo = self.repository
106
# calculate all refs, and callback the objects requesting them.
108
wanting_items = set()
109
# Current crude version calculates everything and calls
110
# everything at once. Doing a queue and popping as things are
111
# satisfied would be cheaper on memory [but few people have
112
# huge numbers of working trees today. TODO: fix before
116
for ref, wantlist in callback_refs.iteritems():
117
wanting_items.update(wantlist)
120
refs[ref] = repo.revision_tree(value)
121
elif kind == 'lefthand-distance':
123
elif kind == 'revision-existence':
124
existences.add(value)
126
raise AssertionError(
127
'unknown ref kind for ref %s' % ref)
128
node_distances = repo.get_graph().find_lefthand_distances(distances)
129
for key, distance in node_distances.iteritems():
130
refs[('lefthand-distance', key)] = distance
131
if key in existences and distance > 0:
132
refs[('revision-existence', key)] = True
133
existences.remove(key)
134
parent_map = repo.get_graph().get_parent_map(existences)
135
for key in parent_map:
136
refs[('revision-existence', key)] = True
137
existences.remove(key)
138
for key in existences:
139
refs[('revision-existence', key)] = False
140
for item in wanting_items:
141
if isinstance(item, WorkingTree):
143
if isinstance(item, Branch):
144
self.other_results.append(item.check(refs))
146
self.progress.finished()
147
self.repository.unlock()
149
def _check_revisions(self, revisions_iterator):
150
"""Check revision objects by decorating a generator.
152
:param revisions_iterator: An iterator of(revid, Revision-or-None).
153
:return: A generator of the contents of revisions_iterator.
155
self.planned_revisions = set()
156
for revid, revision in revisions_iterator:
157
yield revid, revision
158
self._check_one_rev(revid, revision)
159
# Flatten the revisions we found to guarantee consistent later
161
self.planned_revisions = list(self.planned_revisions)
162
# TODO: extract digital signatures as items to callback on too.
164
def check_revisions(self):
165
"""Scan revisions, checking data directly available as we go."""
166
revision_iterator = self.repository._iter_revisions(None)
167
revision_iterator = self._check_revisions(revision_iterator)
168
# We read the all revisions here:
169
# - doing this allows later code to depend on the revision index.
170
# - we can fill out existence flags at this point
171
# - we can read the revision inventory sha at this point
172
# - we can check properties and serialisers etc.
173
if not self.repository.revision_graph_can_have_wrong_parents():
174
# The check against the index isn't needed.
175
self.revs_with_bad_parents_in_index = None
176
for thing in revision_iterator:
179
bad_revisions = self.repository._find_inconsistent_revision_parents(
181
self.revs_with_bad_parents_in_index = list(bad_revisions)
183
def report_results(self, verbose):
185
self._report_repo_results(verbose)
186
for result in self.other_results:
187
result.report_results(verbose)
189
def _report_repo_results(self, verbose):
190
note('checked repository %s format %s',
191
self.repository.user_url,
192
self.repository._format)
62
# maps (file-id, version) -> sha1
63
self.checked_texts = {}
65
history = branch.revision_history()
67
revcount = len(history)
70
self.progress = bzrlib.ui.ui_factory.progress_bar()
71
for rev_id in history:
72
self.progress.update('checking revision', revno, revcount)
74
self.check_one_rev(rev_id, last_rev_id)
80
def report_results(self):
81
note('checked branch %s format %d',
83
self.branch._branch_format)
193
85
note('%6d revisions', self.checked_rev_cnt)
194
note('%6d file-ids', len(self.checked_weaves))
196
note('%6d unreferenced text versions',
197
len(self.unreferenced_versions))
198
if verbose and len(self.unreferenced_versions):
199
for file_id, revision_id in self.unreferenced_versions:
200
note('unreferenced version: {%s} in %s', revision_id,
86
note('%6d unique file texts', self.checked_text_cnt)
87
note('%6d repeated file texts', self.repeated_text_cnt)
202
88
if self.missing_inventory_sha_cnt:
203
note('%6d revisions are missing inventory_sha1',
89
note('%d revisions are missing inventory_sha1',
204
90
self.missing_inventory_sha_cnt)
205
91
if self.missing_revision_cnt:
206
note('%6d revisions are mentioned but not present',
92
note('%d revisions are mentioned but not present',
207
93
self.missing_revision_cnt)
209
note('%6d ghost revisions', len(self.ghosts))
211
for ghost in self.ghosts:
213
if len(self.missing_parent_links):
214
note('%6d revisions missing parents in ancestry',
215
len(self.missing_parent_links))
217
for link, linkers in self.missing_parent_links.items():
218
note(' %s should be in the ancestry for:', link)
219
for linker in linkers:
220
note(' * %s', linker)
221
if len(self.inconsistent_parents):
222
note('%6d inconsistent parents', len(self.inconsistent_parents))
224
for info in self.inconsistent_parents:
225
revision_id, file_id, found_parents, correct_parents = info
226
note(' * %s version %s has parents %r '
228
% (file_id, revision_id, found_parents,
230
if self.revs_with_bad_parents_in_index:
231
note('%6d revisions have incorrect parents in the revision index',
232
len(self.revs_with_bad_parents_in_index))
234
for item in self.revs_with_bad_parents_in_index:
235
revision_id, index_parents, actual_parents = item
237
' %s has wrong parents in index: '
239
revision_id, index_parents, actual_parents)
240
for item in self._report_items:
243
def _check_one_rev(self, rev_id, rev):
244
"""Cross-check one revision.
246
:param rev_id: A revision id to check.
247
:param rev: A revision or None to indicate a missing revision.
96
def check_one_rev(self, rev_id, last_rev_id):
97
"""Check one revision.
99
rev_id - the one to check
101
last_rev_id - the previous one on the mainline, if any.
104
# mutter(' revision {%s}' % rev_id)
106
rev = branch.get_revision(rev_id)
249
107
if rev.revision_id != rev_id:
250
self._report_items.append(
251
'Mismatched internal revid {%s} and index revid {%s}' % (
252
rev.revision_id, rev_id))
253
rev_id = rev.revision_id
254
# Check this revision tree etc, and count as seen when we encounter a
256
self.planned_revisions.add(rev_id)
258
self.ghosts.discard(rev_id)
259
# Count all parents as ghosts if we haven't seen them yet.
260
for parent in rev.parent_ids:
261
if not parent in self.planned_revisions:
262
self.ghosts.add(parent)
108
raise BzrCheckError('wrong internal revision id in revision {%s}'
111
# check the previous history entry is a parent of this entry
113
if last_rev_id is None:
114
raise BzrCheckError("revision {%s} has %d parents, but is the "
115
"start of the branch"
116
% (rev_id, len(rev.parent_ids)))
117
for parent_id in rev.parent_ids:
118
if parent_id == last_rev_id:
121
raise BzrCheckError("previous revision {%s} not listed among "
123
% (last_rev_id, rev_id))
125
raise BzrCheckError("revision {%s} has no parents listed "
126
"but preceded by {%s}"
127
% (rev_id, last_rev_id))
129
if rev.inventory_sha1:
130
inv_sha1 = branch.get_inventory_sha1(rev_id)
131
if inv_sha1 != rev.inventory_sha1:
132
raise BzrCheckError('Inventory sha1 hash doesn\'t match'
133
' value in revision {%s}' % rev_id)
135
missing_inventory_sha_cnt += 1
136
mutter("no inventory_sha1 on revision {%s}" % rev_id)
137
self._check_revision_tree(rev_id)
138
self.checked_rev_cnt += 1
140
def _check_revision_tree(self, rev_id):
141
tree = self.branch.revision_tree(rev_id)
145
if file_id in seen_ids:
146
raise BzrCheckError('duplicated file_id {%s} '
147
'in inventory for revision {%s}'
149
seen_ids[file_id] = True
151
self._check_one_entry(rev_id, inv, tree, file_id)
153
for path, ie in inv.iter_entries():
154
if path in seen_names:
155
raise BzrCheckError('duplicated path %s '
156
'in inventory for revision {%s}'
158
seen_names[path] = True
264
self.ancestors[rev_id] = tuple(rev.parent_ids) or (NULL_REVISION,)
265
self.add_pending_item(rev_id, ('inventories', rev_id), 'inventory',
267
self.checked_rev_cnt += 1
269
def add_pending_item(self, referer, key, kind, sha1):
270
"""Add a reference to a sha1 to be cross checked against a key.
272
:param referer: The referer that expects key to have sha1.
273
:param key: A storage key e.g. ('texts', 'foo@bar-20040504-1234')
274
:param kind: revision/inventory/text/map/signature
275
:param sha1: A hex sha1 or None if no sha1 is known.
277
existing = self.pending_keys.get(key)
279
if sha1 != existing[1]:
280
self._report_items.append('Multiple expected sha1s for %s. {%s}'
281
' expects {%s}, {%s} expects {%s}', (
282
key, referer, sha1, existing[1], existing[0]))
284
self.pending_keys[key] = (kind, sha1, referer)
286
def check_weaves(self):
287
"""Check all the weaves we can get our hands on.
290
storebar = bzrlib.ui.ui_factory.nested_progress_bar()
292
self._check_weaves(storebar)
296
def _check_weaves(self, storebar):
297
storebar.update('text-index', 0, 2)
298
if self.repository._format.fast_deltas:
299
# We haven't considered every fileid instance so far.
300
weave_checker = self.repository._get_versioned_file_checker(
301
ancestors=self.ancestors)
303
weave_checker = self.repository._get_versioned_file_checker(
304
text_key_references=self.text_key_references,
305
ancestors=self.ancestors)
306
storebar.update('file-graph', 1)
307
result = weave_checker.check_file_version_parents(
308
self.repository.texts)
309
self.checked_weaves = weave_checker.file_ids
310
bad_parents, unused_versions = result
311
bad_parents = bad_parents.items()
312
for text_key, (stored_parents, correct_parents) in bad_parents:
313
# XXX not ready for id join/split operations.
314
weave_id = text_key[0]
315
revision_id = text_key[-1]
316
weave_parents = tuple([parent[-1] for parent in stored_parents])
317
correct_parents = tuple([parent[-1] for parent in correct_parents])
318
self.inconsistent_parents.append(
319
(revision_id, weave_id, weave_parents, correct_parents))
320
self.unreferenced_versions.update(unused_versions)
322
def _add_entry_to_text_key_references(self, inv, entry):
323
if not self.rich_roots and entry.name == '':
325
key = (entry.file_id, entry.revision)
326
self.text_key_references.setdefault(key, False)
327
if entry.revision == inv.revision_id:
328
self.text_key_references[key] = True
331
@deprecated_function(deprecated_in((1,6,0)))
332
def check(branch, verbose):
333
"""Run consistency checks on a branch.
335
Results are reported through logging.
337
Deprecated in 1.6. Please use check_dwim instead.
339
:raise BzrCheckError: if there's a consistency error.
341
check_branch(branch, verbose)
344
@deprecated_function(deprecated_in((1,16,0)))
345
def check_branch(branch, verbose):
346
"""Run consistency checks on a branch.
348
Results are reported through logging.
350
:raise BzrCheckError: if there's a consistency error.
355
for ref in branch._get_check_refs():
356
needed_refs.setdefault(ref, []).append(branch)
357
result = branch.repository.check([branch.last_revision()], needed_refs)
358
branch_result = result.other_results[0]
361
branch_result.report_results(verbose)
364
def scan_branch(branch, needed_refs, to_unlock):
365
"""Scan a branch for refs.
367
:param branch: The branch to schedule for checking.
368
:param needed_refs: Refs we are accumulating.
369
:param to_unlock: The unlock list accumulating.
371
note("Checking branch at '%s'." % (branch.base,))
373
to_unlock.append(branch)
374
branch_refs = branch._get_check_refs()
375
for ref in branch_refs:
376
reflist = needed_refs.setdefault(ref, [])
377
reflist.append(branch)
380
def scan_tree(base_tree, tree, needed_refs, to_unlock):
381
"""Scan a tree for refs.
383
:param base_tree: The original tree check opened, used to detect duplicate
385
:param tree: The tree to schedule for checking.
386
:param needed_refs: Refs we are accumulating.
387
:param to_unlock: The unlock list accumulating.
389
if base_tree is not None and tree.basedir == base_tree.basedir:
391
note("Checking working tree at '%s'." % (tree.basedir,))
393
to_unlock.append(tree)
394
tree_refs = tree._get_check_refs()
395
for ref in tree_refs:
396
reflist = needed_refs.setdefault(ref, [])
400
def check_dwim(path, verbose, do_branch=False, do_repo=False, do_tree=False):
401
"""Check multiple objects.
403
If errors occur they are accumulated and reported as far as possible, and
404
an exception raised at the end of the process.
407
base_tree, branch, repo, relpath = \
408
BzrDir.open_containing_tree_branch_or_repository(path)
409
except errors.NotBranchError:
410
base_tree = branch = repo = None
415
if base_tree is not None:
416
# If the tree is a lightweight checkout we won't see it in
417
# repo.find_branches - add now.
419
scan_tree(None, base_tree, needed_refs, to_unlock)
420
branch = base_tree.branch
421
if branch is not None:
424
# The branch is in a shared repository
425
repo = branch.repository
428
to_unlock.append(repo)
429
branches = repo.find_branches(using=True)
431
if do_branch or do_tree:
432
for branch in branches:
435
tree = branch.bzrdir.open_workingtree()
437
except (errors.NotLocalUrl, errors.NoWorkingTree):
440
scan_tree(base_tree, tree, needed_refs, to_unlock)
442
scan_branch(branch, needed_refs, to_unlock)
443
if do_branch and not branches:
444
note("No branch found at specified location.")
445
if do_tree and base_tree is None and not saw_tree:
446
note("No working tree found at specified location.")
447
if do_repo or do_branch or do_tree:
449
note("Checking repository at '%s'."
451
result = repo.check(None, callback_refs=needed_refs,
453
result.report_results(verbose)
456
note("No working tree found at specified location.")
458
note("No branch found at specified location.")
460
note("No repository found at specified location.")
462
for thing in to_unlock:
161
def _check_one_entry(self, rev_id, inv, tree, file_id):
163
if ie.parent_id != None:
164
if not inv.has_id(ie.parent_id):
165
raise BzrCheckError('missing parent {%s} in inventory for revision {%s}'
166
% (ie.parent_id, rev_id))
167
if ie.kind == 'file':
168
text_version = ie.text_version
169
t = (file_id, text_version)
170
if t in self.checked_texts:
171
prev_sha = self.checked_texts[t]
172
if prev_sha != ie.text_sha1:
173
raise BzrCheckError('mismatched sha1 on {%s} in {%s}' %
176
self.repeated_text_cnt += 1
178
mutter('check version {%s} of {%s}', rev_id, file_id)
179
file_lines = tree.get_file_lines(file_id)
180
self.checked_text_cnt += 1
181
if ie.text_size != sum(map(len, file_lines)):
182
raise BzrCheckError('text {%s} wrong size' % ie.text_id)
183
if ie.text_sha1 != sha_strings(file_lines):
184
raise BzrCheckError('text {%s} wrong sha1' % ie.text_id)
185
self.checked_texts[t] = ie.text_sha1
186
elif ie.kind == 'directory':
187
if ie.text_sha1 != None or ie.text_size != None or ie.text_id != None:
188
raise BzrCheckError('directory {%s} has text in revision {%s}'
190
elif ie.kind == 'root_directory':
193
raise BzrCheckError('unknown entry kind %r in revision {%s}' %
198
"""Run consistency checks on a branch."""
added
removed
bzrlib/check.py
