32
32
# raising them. If there's more than one exception it'd be good to see them
35
"""Checking of bzr objects.
37
check_refs is a concept used for optimising check. Objects that depend on other
38
objects (e.g. tree on repository) can list the objects they would be requesting
39
so that when the dependent object is checked, matches can be pulled out and
40
evaluated in-line rather than re-reading the same data many times.
41
check_refs are tuples (kind, value). Currently defined kinds are:
43
* 'trees', where value is a revid and the looked up objects are revision trees.
44
* 'lefthand-distance', where value is a revid and the looked up objects are the
45
distance along the lefthand path to NULL for that revid.
46
* 'revision-existence', where value is a revid, and the result is True or False
47
indicating that the revision was found/not found.
54
from bzrlib.branch import Branch
55
from bzrlib.bzrdir import BzrDir
56
from bzrlib.revision import NULL_REVISION
35
from bzrlib.errors import BzrCheckError
57
37
from bzrlib.trace import note
58
from bzrlib.workingtree import WorkingTree
61
39
class Check(object):
62
40
"""Check a repository"""
64
def __init__(self, repository, check_repo=True):
65
self.repository = repository
67
def report_results(self, verbose):
68
raise NotImplementedError(self.report_results)
71
class VersionedFileCheck(Check):
72
"""Check a versioned file repository"""
74
42
# The Check object interacts with InventoryEntry.check, etc.
76
def __init__(self, repository, check_repo=True):
44
def __init__(self, repository):
77
45
self.repository = repository
46
self.checked_text_cnt = 0
78
47
self.checked_rev_cnt = 0
49
self.repeated_text_cnt = 0
80
50
self.missing_parent_links = {}
81
51
self.missing_inventory_sha_cnt = 0
82
52
self.missing_revision_cnt = 0
83
self.checked_weaves = set()
84
self.unreferenced_versions = set()
85
self.inconsistent_parents = []
86
self.rich_roots = repository.supports_rich_root()
87
self.text_key_references = {}
88
self.check_repo = check_repo
89
self.other_results = []
90
# Plain text lines to include in the report
91
self._report_items = []
92
# Keys we are looking for; may be large and need spilling to disk.
93
# key->(type(revision/inventory/text/signature/map), sha1, first-referer)
94
self.pending_keys = {}
95
# Ancestors map for all of revisions being checked; while large helper
96
# functions we call would create it anyway, so better to have once and
53
# maps (file-id, version) -> sha1; used by InventoryFile._check
54
self.checked_texts = {}
55
self.checked_weaves = {}
100
def check(self, callback_refs=None, check_repo=True):
101
if callback_refs is None:
103
58
self.repository.lock_read()
104
self.progress = ui.ui_factory.nested_progress_bar()
59
self.progress = bzrlib.ui.ui_factory.nested_progress_bar()
106
self.progress.update('check', 0, 4)
108
self.progress.update('checking revisions', 0)
109
self.check_revisions()
110
self.progress.update('checking commit contents', 1)
111
self.repository._check_inventories(self)
112
self.progress.update('checking file graphs', 2)
113
# check_weaves is done after the revision scan so that
114
# revision index is known to be valid.
116
self.progress.update('checking branches and trees', 3)
118
repo = self.repository
119
# calculate all refs, and callback the objects requesting them.
121
wanting_items = set()
122
# Current crude version calculates everything and calls
123
# everything at once. Doing a queue and popping as things are
124
# satisfied would be cheaper on memory [but few people have
125
# huge numbers of working trees today. TODO: fix before
129
for ref, wantlist in callback_refs.iteritems():
130
wanting_items.update(wantlist)
133
refs[ref] = repo.revision_tree(value)
134
elif kind == 'lefthand-distance':
136
elif kind == 'revision-existence':
137
existences.add(value)
139
raise AssertionError(
140
'unknown ref kind for ref %s' % ref)
141
node_distances = repo.get_graph().find_lefthand_distances(distances)
142
for key, distance in node_distances.iteritems():
143
refs[('lefthand-distance', key)] = distance
144
if key in existences and distance > 0:
145
refs[('revision-existence', key)] = True
146
existences.remove(key)
147
parent_map = repo.get_graph().get_parent_map(existences)
148
for key in parent_map:
149
refs[('revision-existence', key)] = True
150
existences.remove(key)
151
for key in existences:
152
refs[('revision-existence', key)] = False
153
for item in wanting_items:
154
if isinstance(item, WorkingTree):
156
if isinstance(item, Branch):
157
self.other_results.append(item.check(refs))
61
self.progress.update('retrieving inventory', 0, 0)
62
# do not put in init, as it should be done with progess,
63
# and inside the lock.
64
self.inventory_weave = self.repository.get_inventory_weave()
68
while revno < len(self.planned_revisions):
69
rev_id = self.planned_revisions[revno]
70
self.progress.update('checking revision', revno,
71
len(self.planned_revisions))
73
self.check_one_rev(rev_id)
159
75
self.progress.finished()
160
76
self.repository.unlock()
162
def _check_revisions(self, revisions_iterator):
163
"""Check revision objects by decorating a generator.
165
:param revisions_iterator: An iterator of(revid, Revision-or-None).
166
:return: A generator of the contents of revisions_iterator.
168
self.planned_revisions = set()
169
for revid, revision in revisions_iterator:
170
yield revid, revision
171
self._check_one_rev(revid, revision)
172
# Flatten the revisions we found to guarantee consistent later
78
def plan_revisions(self):
79
repository = self.repository
80
self.planned_revisions = set(repository.all_revision_ids())
82
inventoried = set(self.inventory_weave.versions())
83
awol = self.planned_revisions - inventoried
85
raise BzrCheckError('Stored revisions missing from inventory'
86
'{%s}' % ','.join([f for f in awol]))
174
87
self.planned_revisions = list(self.planned_revisions)
175
# TODO: extract digital signatures as items to callback on too.
177
def check_revisions(self):
178
"""Scan revisions, checking data directly available as we go."""
179
revision_iterator = self.repository._iter_revisions(None)
180
revision_iterator = self._check_revisions(revision_iterator)
181
# We read the all revisions here:
182
# - doing this allows later code to depend on the revision index.
183
# - we can fill out existence flags at this point
184
# - we can read the revision inventory sha at this point
185
# - we can check properties and serialisers etc.
186
if not self.repository._format.revision_graph_can_have_wrong_parents:
187
# The check against the index isn't needed.
188
self.revs_with_bad_parents_in_index = None
189
for thing in revision_iterator:
192
bad_revisions = self.repository._find_inconsistent_revision_parents(
194
self.revs_with_bad_parents_in_index = list(bad_revisions)
196
89
def report_results(self, verbose):
198
self._report_repo_results(verbose)
199
for result in self.other_results:
200
result.report_results(verbose)
202
def _report_repo_results(self, verbose):
203
90
note('checked repository %s format %s',
204
self.repository.user_url,
205
self.repository._format)
91
self.repository.bzrdir.root_transport,
92
self.repository._format)
206
93
note('%6d revisions', self.checked_rev_cnt)
207
note('%6d file-ids', len(self.checked_weaves))
209
note('%6d unreferenced text versions',
210
len(self.unreferenced_versions))
211
if verbose and len(self.unreferenced_versions):
212
for file_id, revision_id in self.unreferenced_versions:
213
note('unreferenced version: {%s} in %s', revision_id,
94
note('%6d unique file texts', self.checked_text_cnt)
95
note('%6d repeated file texts', self.repeated_text_cnt)
96
note('%6d weaves', len(self.checked_weaves))
215
97
if self.missing_inventory_sha_cnt:
216
98
note('%6d revisions are missing inventory_sha1',
217
99
self.missing_inventory_sha_cnt)
231
113
note(' %s should be in the ancestry for:', link)
232
114
for linker in linkers:
233
115
note(' * %s', linker)
234
if len(self.inconsistent_parents):
235
note('%6d inconsistent parents', len(self.inconsistent_parents))
237
for info in self.inconsistent_parents:
238
revision_id, file_id, found_parents, correct_parents = info
239
note(' * %s version %s has parents %r '
241
% (file_id, revision_id, found_parents,
243
if self.revs_with_bad_parents_in_index:
244
note('%6d revisions have incorrect parents in the revision index',
245
len(self.revs_with_bad_parents_in_index))
247
for item in self.revs_with_bad_parents_in_index:
248
revision_id, index_parents, actual_parents = item
250
' %s has wrong parents in index: '
252
revision_id, index_parents, actual_parents)
253
for item in self._report_items:
256
def _check_one_rev(self, rev_id, rev):
257
"""Cross-check one revision.
259
:param rev_id: A revision id to check.
260
:param rev: A revision or None to indicate a missing revision.
117
def check_one_rev(self, rev_id):
118
"""Check one revision.
120
rev_id - the one to check
122
rev = self.repository.get_revision(rev_id)
262
124
if rev.revision_id != rev_id:
263
self._report_items.append(
264
'Mismatched internal revid {%s} and index revid {%s}' % (
265
rev.revision_id, rev_id))
266
rev_id = rev.revision_id
267
# Check this revision tree etc, and count as seen when we encounter a
269
self.planned_revisions.add(rev_id)
271
self.ghosts.discard(rev_id)
272
# Count all parents as ghosts if we haven't seen them yet.
125
raise BzrCheckError('wrong internal revision id in revision {%s}'
273
128
for parent in rev.parent_ids:
274
129
if not parent in self.planned_revisions:
275
self.ghosts.add(parent)
277
self.ancestors[rev_id] = tuple(rev.parent_ids) or (NULL_REVISION,)
278
self.add_pending_item(rev_id, ('inventories', rev_id), 'inventory',
130
missing_links = self.missing_parent_links.get(parent, [])
131
missing_links.append(rev_id)
132
self.missing_parent_links[parent] = missing_links
133
# list based so somewhat slow,
134
# TODO have a planned_revisions list and set.
135
if self.repository.has_revision(parent):
136
missing_ancestry = self.repository.get_ancestry(parent)
137
for missing in missing_ancestry:
138
if (missing is not None
139
and missing not in self.planned_revisions):
140
self.planned_revisions.append(missing)
142
self.ghosts.append(rev_id)
144
if rev.inventory_sha1:
145
inv_sha1 = self.repository.get_inventory_sha1(rev_id)
146
if inv_sha1 != rev.inventory_sha1:
147
raise BzrCheckError('Inventory sha1 hash doesn\'t match'
148
' value in revision {%s}' % rev_id)
149
self._check_revision_tree(rev_id)
280
150
self.checked_rev_cnt += 1
282
def add_pending_item(self, referer, key, kind, sha1):
283
"""Add a reference to a sha1 to be cross checked against a key.
285
:param referer: The referer that expects key to have sha1.
286
:param key: A storage key e.g. ('texts', 'foo@bar-20040504-1234')
287
:param kind: revision/inventory/text/map/signature
288
:param sha1: A hex sha1 or None if no sha1 is known.
290
existing = self.pending_keys.get(key)
292
if sha1 != existing[1]:
293
self._report_items.append('Multiple expected sha1s for %s. {%s}'
294
' expects {%s}, {%s} expects {%s}', (
295
key, referer, sha1, existing[1], existing[0]))
297
self.pending_keys[key] = (kind, sha1, referer)
299
152
def check_weaves(self):
300
153
"""Check all the weaves we can get our hands on.
303
storebar = ui.ui_factory.nested_progress_bar()
305
self._check_weaves(storebar)
309
def _check_weaves(self, storebar):
310
storebar.update('text-index', 0, 2)
311
if self.repository._format.fast_deltas:
312
# We haven't considered every fileid instance so far.
313
weave_checker = self.repository._get_versioned_file_checker(
314
ancestors=self.ancestors)
316
weave_checker = self.repository._get_versioned_file_checker(
317
text_key_references=self.text_key_references,
318
ancestors=self.ancestors)
319
storebar.update('file-graph', 1)
320
result = weave_checker.check_file_version_parents(
321
self.repository.texts)
322
self.checked_weaves = weave_checker.file_ids
323
bad_parents, unused_versions = result
324
bad_parents = bad_parents.items()
325
for text_key, (stored_parents, correct_parents) in bad_parents:
326
# XXX not ready for id join/split operations.
327
weave_id = text_key[0]
328
revision_id = text_key[-1]
329
weave_parents = tuple([parent[-1] for parent in stored_parents])
330
correct_parents = tuple([parent[-1] for parent in correct_parents])
331
self.inconsistent_parents.append(
332
(revision_id, weave_id, weave_parents, correct_parents))
333
self.unreferenced_versions.update(unused_versions)
335
def _add_entry_to_text_key_references(self, inv, entry):
336
if not self.rich_roots and entry.name == '':
338
key = (entry.file_id, entry.revision)
339
self.text_key_references.setdefault(key, False)
340
if entry.revision == inv.revision_id:
341
self.text_key_references[key] = True
344
def scan_branch(branch, needed_refs, to_unlock):
345
"""Scan a branch for refs.
347
:param branch: The branch to schedule for checking.
348
:param needed_refs: Refs we are accumulating.
349
:param to_unlock: The unlock list accumulating.
157
if self.repository.weave_store.listable():
158
weave_ids = list(self.repository.weave_store)
159
n_weaves = len(weave_ids)
160
self.progress.update('checking weave', 0, n_weaves)
161
self.inventory_weave.check(progress_bar=self.progress)
162
for i, weave_id in enumerate(weave_ids):
163
self.progress.update('checking weave', i, n_weaves)
164
w = self.repository.weave_store.get_weave(weave_id,
165
self.repository.get_transaction())
166
# No progress here, because it looks ugly.
168
self.checked_weaves[weave_id] = True
170
def _check_revision_tree(self, rev_id):
171
tree = self.repository.revision_tree(rev_id)
175
if file_id in seen_ids:
176
raise BzrCheckError('duplicated file_id {%s} '
177
'in inventory for revision {%s}'
179
seen_ids[file_id] = True
182
ie.check(self, rev_id, inv, tree)
184
for path, ie in inv.iter_entries():
185
if path in seen_names:
186
raise BzrCheckError('duplicated path %s '
187
'in inventory for revision {%s}'
189
seen_names[path] = True
192
def check(branch, verbose):
193
"""Run consistency checks on a branch.
195
Results are reported through logging.
197
:raise BzrCheckError: if there's a consistency error.
351
note("Checking branch at '%s'." % (branch.base,))
352
199
branch.lock_read()
353
to_unlock.append(branch)
354
branch_refs = branch._get_check_refs()
355
for ref in branch_refs:
356
reflist = needed_refs.setdefault(ref, [])
357
reflist.append(branch)
360
def scan_tree(base_tree, tree, needed_refs, to_unlock):
361
"""Scan a tree for refs.
363
:param base_tree: The original tree check opened, used to detect duplicate
365
:param tree: The tree to schedule for checking.
366
:param needed_refs: Refs we are accumulating.
367
:param to_unlock: The unlock list accumulating.
369
if base_tree is not None and tree.basedir == base_tree.basedir:
371
note("Checking working tree at '%s'." % (tree.basedir,))
373
to_unlock.append(tree)
374
tree_refs = tree._get_check_refs()
375
for ref in tree_refs:
376
reflist = needed_refs.setdefault(ref, [])
380
def check_dwim(path, verbose, do_branch=False, do_repo=False, do_tree=False):
381
"""Check multiple objects.
383
If errors occur they are accumulated and reported as far as possible, and
384
an exception raised at the end of the process.
387
base_tree, branch, repo, relpath = \
388
BzrDir.open_containing_tree_branch_or_repository(path)
389
except errors.NotBranchError:
390
base_tree = branch = repo = None
395
if base_tree is not None:
396
# If the tree is a lightweight checkout we won't see it in
397
# repo.find_branches - add now.
399
scan_tree(None, base_tree, needed_refs, to_unlock)
400
branch = base_tree.branch
401
if branch is not None:
404
# The branch is in a shared repository
405
repo = branch.repository
408
to_unlock.append(repo)
409
branches = repo.find_branches(using=True)
411
if do_branch or do_tree:
412
for branch in branches:
415
tree = branch.bzrdir.open_workingtree()
417
except (errors.NotLocalUrl, errors.NoWorkingTree):
420
scan_tree(base_tree, tree, needed_refs, to_unlock)
422
scan_branch(branch, needed_refs, to_unlock)
423
if do_branch and not branches:
424
note("No branch found at specified location.")
425
if do_tree and base_tree is None and not saw_tree:
426
note("No working tree found at specified location.")
427
if do_repo or do_branch or do_tree:
429
note("Checking repository at '%s'."
431
result = repo.check(None, callback_refs=needed_refs,
433
result.report_results(verbose)
436
note("No working tree found at specified location.")
438
note("No branch found at specified location.")
440
note("No repository found at specified location.")
201
branch_result = branch.check()
202
repo_result = branch.repository.check([branch.last_revision()])
442
for thing in to_unlock:
205
branch_result.report_results(verbose)
206
repo_result.report_results(verbose)
added
removed
bzrlib/check.py
