32
32
# raising them. If there's more than one exception it'd be good to see them
35
"""Checking of bzr objects.
37
check_refs is a concept used for optimising check. Objects that depend on other
38
objects (e.g. tree on repository) can list the objects they would be requesting
39
so that when the dependent object is checked, matches can be pulled out and
40
evaluated in-line rather than re-reading the same data many times.
41
check_refs are tuples (kind, value). Currently defined kinds are:
42
* 'trees', where value is a revid and the looked up objects are revision trees.
43
* 'lefthand-distance', where value is a revid and the looked up objects are the
44
distance along the lefthand path to NULL for that revid.
45
* 'revision-existence', where value is a revid, and the result is True or False
46
indicating that the revision was found/not found.
35
from bzrlib import errors, osutils
36
from bzrlib import repository as _mod_repository
37
from bzrlib import revision
53
38
from bzrlib.branch import Branch
54
39
from bzrlib.bzrdir import BzrDir
55
from bzrlib.revision import NULL_REVISION
40
from bzrlib.errors import BzrCheckError
41
from bzrlib.repository import Repository
56
42
from bzrlib.symbol_versioning import deprecated_function, deprecated_in
57
from bzrlib.trace import note
43
from bzrlib.trace import log_error, note
58
45
from bzrlib.workingtree import WorkingTree
60
47
class Check(object):
63
50
# The Check object interacts with InventoryEntry.check, etc.
65
def __init__(self, repository, check_repo=True):
52
def __init__(self, repository):
66
53
self.repository = repository
54
self.checked_text_cnt = 0
67
55
self.checked_rev_cnt = 0
57
self.repeated_text_cnt = 0
69
58
self.missing_parent_links = {}
70
59
self.missing_inventory_sha_cnt = 0
71
60
self.missing_revision_cnt = 0
61
# maps (file-id, version) -> sha1; used by InventoryFile._check
62
self.checked_texts = {}
72
63
self.checked_weaves = set()
73
64
self.unreferenced_versions = set()
74
65
self.inconsistent_parents = []
75
66
self.rich_roots = repository.supports_rich_root()
76
67
self.text_key_references = {}
77
self.check_repo = check_repo
78
self.other_results = []
79
# Plain text lines to include in the report
80
self._report_items = []
81
# Keys we are looking for; may be large and need spilling to disk.
82
# key->(type(revision/inventory/text/signature/map), sha1, first-referer)
83
self.pending_keys = {}
84
# Ancestors map for all of revisions being checked; while large helper
85
# functions we call would create it anyway, so better to have once and
89
def check(self, callback_refs=None, check_repo=True):
90
if callback_refs is None:
92
70
self.repository.lock_read()
93
self.progress = ui.ui_factory.nested_progress_bar()
71
self.progress = bzrlib.ui.ui_factory.nested_progress_bar()
95
self.progress.update('check', 0, 4)
97
self.progress.update('checking revisions', 0)
98
self.check_revisions()
99
self.progress.update('checking commit contents', 1)
100
self.repository._check_inventories(self)
101
self.progress.update('checking file graphs', 2)
102
# check_weaves is done after the revision scan so that
103
# revision index is known to be valid.
105
self.progress.update('checking branches and trees', 3)
107
repo = self.repository
108
# calculate all refs, and callback the objects requesting them.
110
wanting_items = set()
111
# Current crude version calculates everything and calls
112
# everything at once. Doing a queue and popping as things are
113
# satisfied would be cheaper on memory [but few people have
114
# huge numbers of working trees today. TODO: fix before
118
for ref, wantlist in callback_refs.iteritems():
119
wanting_items.update(wantlist)
122
refs[ref] = repo.revision_tree(value)
123
elif kind == 'lefthand-distance':
125
elif kind == 'revision-existence':
126
existences.add(value)
128
raise AssertionError(
129
'unknown ref kind for ref %s' % ref)
130
node_distances = repo.get_graph().find_lefthand_distances(distances)
131
for key, distance in node_distances.iteritems():
132
refs[('lefthand-distance', key)] = distance
133
if key in existences and distance > 0:
134
refs[('revision-existence', key)] = True
135
existences.remove(key)
136
parent_map = repo.get_graph().get_parent_map(existences)
137
for key in parent_map:
138
refs[('revision-existence', key)] = True
139
existences.remove(key)
140
for key in existences:
141
refs[('revision-existence', key)] = False
142
for item in wanting_items:
143
if isinstance(item, WorkingTree):
145
if isinstance(item, Branch):
146
self.other_results.append(item.check(refs))
73
self.progress.update('retrieving inventory', 0, 2)
74
# do not put in init, as it should be done with progess,
75
# and inside the lock.
76
self.inventory_weave = self.repository.inventories
77
self.progress.update('checking revision graph', 1)
78
self.check_revision_graph()
81
while revno < len(self.planned_revisions):
82
rev_id = self.planned_revisions[revno]
83
self.progress.update('checking revision', revno,
84
len(self.planned_revisions))
86
self.check_one_rev(rev_id)
87
# check_weaves is done after the revision scan so that
88
# revision index is known to be valid.
148
91
self.progress.finished()
149
92
self.repository.unlock()
151
def _check_revisions(self, revisions_iterator):
152
"""Check revision objects by decorating a generator.
154
:param revisions_iterator: An iterator of(revid, Revision-or-None).
155
:return: A generator of the contents of revisions_iterator.
157
self.planned_revisions = set()
158
for revid, revision in revisions_iterator:
159
yield revid, revision
160
self._check_one_rev(revid, revision)
161
# Flatten the revisions we found to guarantee consistent later
163
self.planned_revisions = list(self.planned_revisions)
164
# TODO: extract digital signatures as items to callback on too.
166
def check_revisions(self):
167
"""Scan revisions, checking data directly available as we go."""
168
revision_iterator = self.repository._iter_revisions(None)
169
revision_iterator = self._check_revisions(revision_iterator)
170
# We read the all revisions here:
171
# - doing this allows later code to depend on the revision index.
172
# - we can fill out existence flags at this point
173
# - we can read the revision inventory sha at this point
174
# - we can check properties and serialisers etc.
175
if not self.repository._format.revision_graph_can_have_wrong_parents:
176
# The check against the index isn't needed.
94
def check_revision_graph(self):
95
if not self.repository.revision_graph_can_have_wrong_parents():
96
# This check is not necessary.
177
97
self.revs_with_bad_parents_in_index = None
178
for thing in revision_iterator:
181
bad_revisions = self.repository._find_inconsistent_revision_parents(
183
self.revs_with_bad_parents_in_index = list(bad_revisions)
99
bad_revisions = self.repository._find_inconsistent_revision_parents()
100
self.revs_with_bad_parents_in_index = list(bad_revisions)
102
def plan_revisions(self):
103
repository = self.repository
104
self.planned_revisions = repository.all_revision_ids()
105
self.progress.clear()
106
inventoried = set(key[-1] for key in self.inventory_weave.keys())
107
awol = set(self.planned_revisions) - inventoried
109
raise BzrCheckError('Stored revisions missing from inventory'
110
'{%s}' % ','.join([f for f in awol]))
185
112
def report_results(self, verbose):
187
self._report_repo_results(verbose)
188
for result in self.other_results:
189
result.report_results(verbose)
191
def _report_repo_results(self, verbose):
192
113
note('checked repository %s format %s',
193
self.repository.user_url,
194
self.repository._format)
114
self.repository.bzrdir.root_transport,
115
self.repository._format)
195
116
note('%6d revisions', self.checked_rev_cnt)
196
117
note('%6d file-ids', len(self.checked_weaves))
198
note('%6d unreferenced text versions',
199
len(self.unreferenced_versions))
200
if verbose and len(self.unreferenced_versions):
201
for file_id, revision_id in self.unreferenced_versions:
202
note('unreferenced version: {%s} in %s', revision_id,
118
note('%6d unique file texts', self.checked_text_cnt)
119
note('%6d repeated file texts', self.repeated_text_cnt)
120
note('%6d unreferenced text versions',
121
len(self.unreferenced_versions))
204
122
if self.missing_inventory_sha_cnt:
205
123
note('%6d revisions are missing inventory_sha1',
206
124
self.missing_inventory_sha_cnt)
239
161
' %s has wrong parents in index: '
240
162
'%r should be %r',
241
163
revision_id, index_parents, actual_parents)
242
for item in self._report_items:
245
def _check_one_rev(self, rev_id, rev):
246
"""Cross-check one revision.
248
:param rev_id: A revision id to check.
249
:param rev: A revision or None to indicate a missing revision.
165
def check_one_rev(self, rev_id):
166
"""Check one revision.
168
rev_id - the one to check
170
rev = self.repository.get_revision(rev_id)
251
172
if rev.revision_id != rev_id:
252
self._report_items.append(
253
'Mismatched internal revid {%s} and index revid {%s}' % (
254
rev.revision_id, rev_id))
255
rev_id = rev.revision_id
256
# Check this revision tree etc, and count as seen when we encounter a
258
self.planned_revisions.add(rev_id)
260
self.ghosts.discard(rev_id)
261
# Count all parents as ghosts if we haven't seen them yet.
173
raise BzrCheckError('wrong internal revision id in revision {%s}'
262
176
for parent in rev.parent_ids:
263
177
if not parent in self.planned_revisions:
264
self.ghosts.add(parent)
266
self.ancestors[rev_id] = tuple(rev.parent_ids) or (NULL_REVISION,)
267
self.add_pending_item(rev_id, ('inventories', rev_id), 'inventory',
178
# rev has a parent we didn't know about.
179
missing_links = self.missing_parent_links.get(parent, [])
180
missing_links.append(rev_id)
181
self.missing_parent_links[parent] = missing_links
182
# list based so somewhat slow,
183
# TODO have a planned_revisions list and set.
184
if self.repository.has_revision(parent):
185
missing_ancestry = self.repository.get_ancestry(parent)
186
for missing in missing_ancestry:
187
if (missing is not None
188
and missing not in self.planned_revisions):
189
self.planned_revisions.append(missing)
191
self.ghosts.append(rev_id)
193
if rev.inventory_sha1:
194
# Loopback - this is currently circular logic as the
195
# knit get_inventory_sha1 call returns rev.inventory_sha1.
196
# Repository.py's get_inventory_sha1 should instead return
197
# inventories.get_record_stream([(revid,)]).next().sha1 or
199
inv_sha1 = self.repository.get_inventory_sha1(rev_id)
200
if inv_sha1 != rev.inventory_sha1:
201
raise BzrCheckError('Inventory sha1 hash doesn\'t match'
202
' value in revision {%s}' % rev_id)
203
self._check_revision_tree(rev_id)
269
204
self.checked_rev_cnt += 1
271
def add_pending_item(self, referer, key, kind, sha1):
272
"""Add a reference to a sha1 to be cross checked against a key.
274
:param referer: The referer that expects key to have sha1.
275
:param key: A storage key e.g. ('texts', 'foo@bar-20040504-1234')
276
:param kind: revision/inventory/text/map/signature
277
:param sha1: A hex sha1 or None if no sha1 is known.
279
existing = self.pending_keys.get(key)
281
if sha1 != existing[1]:
282
self._report_items.append('Multiple expected sha1s for %s. {%s}'
283
' expects {%s}, {%s} expects {%s}', (
284
key, referer, sha1, existing[1], existing[0]))
286
self.pending_keys[key] = (kind, sha1, referer)
288
206
def check_weaves(self):
289
207
"""Check all the weaves we can get our hands on.
292
storebar = ui.ui_factory.nested_progress_bar()
294
self._check_weaves(storebar)
298
def _check_weaves(self, storebar):
299
storebar.update('text-index', 0, 2)
300
if self.repository._format.fast_deltas:
301
# We haven't considered every fileid instance so far.
302
weave_checker = self.repository._get_versioned_file_checker(
303
ancestors=self.ancestors)
305
weave_checker = self.repository._get_versioned_file_checker(
306
text_key_references=self.text_key_references,
307
ancestors=self.ancestors)
308
storebar.update('file-graph', 1)
210
self.progress.update('checking inventory', 0, 2)
211
self.inventory_weave.check(progress_bar=self.progress)
212
self.progress.update('checking text storage', 1, 2)
213
self.repository.texts.check(progress_bar=self.progress)
214
weave_checker = self.repository._get_versioned_file_checker(
215
text_key_references=self.text_key_references)
309
216
result = weave_checker.check_file_version_parents(
310
self.repository.texts)
217
self.repository.texts, progress_bar=self.progress)
311
218
self.checked_weaves = weave_checker.file_ids
312
219
bad_parents, unused_versions = result
313
220
bad_parents = bad_parents.items()
330
257
self.text_key_references[key] = True
333
def scan_branch(branch, needed_refs, to_unlock):
334
"""Scan a branch for refs.
336
:param branch: The branch to schedule for checking.
337
:param needed_refs: Refs we are accumulating.
338
:param to_unlock: The unlock list accumulating.
340
note("Checking branch at '%s'." % (branch.base,))
260
@deprecated_function(deprecated_in((1,6,0)))
261
def check(branch, verbose):
262
"""Run consistency checks on a branch.
264
Results are reported through logging.
266
Deprecated in 1.6. Please use check_branch instead.
268
:raise BzrCheckError: if there's a consistency error.
270
check_branch(branch, verbose)
273
def check_branch(branch, verbose):
274
"""Run consistency checks on a branch.
276
Results are reported through logging.
278
:raise BzrCheckError: if there's a consistency error.
341
280
branch.lock_read()
342
to_unlock.append(branch)
343
branch_refs = branch._get_check_refs()
344
for ref in branch_refs:
345
reflist = needed_refs.setdefault(ref, [])
346
reflist.append(branch)
349
def scan_tree(base_tree, tree, needed_refs, to_unlock):
350
"""Scan a tree for refs.
352
:param base_tree: The original tree check opened, used to detect duplicate
354
:param tree: The tree to schedule for checking.
355
:param needed_refs: Refs we are accumulating.
356
:param to_unlock: The unlock list accumulating.
358
if base_tree is not None and tree.basedir == base_tree.basedir:
360
note("Checking working tree at '%s'." % (tree.basedir,))
362
to_unlock.append(tree)
363
tree_refs = tree._get_check_refs()
364
for ref in tree_refs:
365
reflist = needed_refs.setdefault(ref, [])
282
branch_result = branch.check()
285
branch_result.report_results(verbose)
369
288
def check_dwim(path, verbose, do_branch=False, do_repo=False, do_tree=False):
370
"""Check multiple objects.
372
If errors occur they are accumulated and reported as far as possible, and
373
an exception raised at the end of the process.
376
base_tree, branch, repo, relpath = \
290
tree, branch, repo, relpath = \
377
291
BzrDir.open_containing_tree_branch_or_repository(path)
378
292
except errors.NotBranchError:
379
base_tree = branch = repo = None
384
if base_tree is not None:
385
# If the tree is a lightweight checkout we won't see it in
386
# repo.find_branches - add now.
388
scan_tree(None, base_tree, needed_refs, to_unlock)
389
branch = base_tree.branch
390
if branch is not None:
393
# The branch is in a shared repository
394
repo = branch.repository
397
to_unlock.append(repo)
398
branches = repo.find_branches(using=True)
400
if do_branch or do_tree:
401
for branch in branches:
404
tree = branch.bzrdir.open_workingtree()
406
except (errors.NotLocalUrl, errors.NoWorkingTree):
409
scan_tree(base_tree, tree, needed_refs, to_unlock)
411
scan_branch(branch, needed_refs, to_unlock)
412
if do_branch and not branches:
413
note("No branch found at specified location.")
414
if do_tree and base_tree is None and not saw_tree:
415
note("No working tree found at specified location.")
416
if do_repo or do_branch or do_tree:
418
note("Checking repository at '%s'."
420
result = repo.check(None, callback_refs=needed_refs,
293
tree = branch = repo = None
297
note("Checking working tree at '%s'."
298
% (tree.bzrdir.root_transport.base,))
301
log_error("No working tree found at specified location.")
303
if branch is not None:
306
# The branch is in a shared repository
307
repo = branch.repository
309
elif repo is not None:
310
branches = repo.find_branches(using=True)
316
note("Checking repository at '%s'."
317
% (repo.bzrdir.root_transport.base,))
318
result = repo.check()
422
319
result.report_results(verbose)
425
note("No working tree found at specified location.")
427
note("No branch found at specified location.")
429
note("No repository found at specified location.")
431
for thing in to_unlock:
322
log_error("No branch found at specified location.")
324
for branch in branches:
325
note("Checking branch at '%s'."
326
% (branch.bzrdir.root_transport.base,))
327
check_branch(branch, verbose)
332
log_error("No branch found at specified location.")
334
log_error("No repository found at specified location.")
added
removed
bzrlib/check.py
