32
32
# raising them. If there's more than one exception it'd be good to see them
35
from bzrlib import errors, osutils
36
from bzrlib import repository as _mod_repository
37
from bzrlib import revision
35
"""Checking of bzr objects.
37
check_refs is a concept used for optimising check. Objects that depend on other
38
objects (e.g. tree on repository) can list the objects they would be requesting
39
so that when the dependent object is checked, matches can be pulled out and
40
evaluated in-line rather than re-reading the same data many times.
41
check_refs are tuples (kind, value). Currently defined kinds are:
43
* 'trees', where value is a revid and the looked up objects are revision trees.
44
* 'lefthand-distance', where value is a revid and the looked up objects are the
45
distance along the lefthand path to NULL for that revid.
46
* 'revision-existence', where value is a revid, and the result is True or False
47
indicating that the revision was found/not found.
50
from __future__ import absolute_import
38
56
from bzrlib.branch import Branch
39
from bzrlib.bzrdir import BzrDir
40
from bzrlib.errors import BzrCheckError
41
from bzrlib.repository import Repository
42
from bzrlib.symbol_versioning import deprecated_function, deprecated_in
43
from bzrlib.trace import log_error, note
57
from bzrlib.controldir import ControlDir
58
from bzrlib.revision import NULL_REVISION
59
from bzrlib.trace import note
45
60
from bzrlib.workingtree import WorkingTree
61
from bzrlib.i18n import gettext
47
63
class Check(object):
48
64
"""Check a repository"""
66
def __init__(self, repository, check_repo=True):
67
self.repository = repository
69
def report_results(self, verbose):
70
raise NotImplementedError(self.report_results)
73
class VersionedFileCheck(Check):
74
"""Check a versioned file repository"""
50
76
# The Check object interacts with InventoryEntry.check, etc.
52
def __init__(self, repository):
78
def __init__(self, repository, check_repo=True):
53
79
self.repository = repository
54
self.checked_text_cnt = 0
55
80
self.checked_rev_cnt = 0
57
self.repeated_text_cnt = 0
58
82
self.missing_parent_links = {}
59
83
self.missing_inventory_sha_cnt = 0
60
84
self.missing_revision_cnt = 0
61
# maps (file-id, version) -> sha1; used by InventoryFile._check
62
self.checked_texts = {}
63
85
self.checked_weaves = set()
64
86
self.unreferenced_versions = set()
65
87
self.inconsistent_parents = []
66
88
self.rich_roots = repository.supports_rich_root()
67
89
self.text_key_references = {}
90
self.check_repo = check_repo
91
self.other_results = []
92
# Plain text lines to include in the report
93
self._report_items = []
94
# Keys we are looking for; may be large and need spilling to disk.
95
# key->(type(revision/inventory/text/signature/map), sha1, first-referer)
96
self.pending_keys = {}
97
# Ancestors map for all of revisions being checked; while large helper
98
# functions we call would create it anyway, so better to have once and
102
def check(self, callback_refs=None, check_repo=True):
103
if callback_refs is None:
70
105
self.repository.lock_read()
71
self.progress = bzrlib.ui.ui_factory.nested_progress_bar()
106
self.progress = ui.ui_factory.nested_progress_bar()
73
self.progress.update('retrieving inventory', 0, 2)
74
# do not put in init, as it should be done with progess,
75
# and inside the lock.
76
self.inventory_weave = self.repository.inventories
77
self.progress.update('checking revision graph', 1)
78
self.check_revision_graph()
81
while revno < len(self.planned_revisions):
82
rev_id = self.planned_revisions[revno]
83
self.progress.update('checking revision', revno,
84
len(self.planned_revisions))
86
self.check_one_rev(rev_id)
87
# check_weaves is done after the revision scan so that
88
# revision index is known to be valid.
108
self.progress.update(gettext('check'), 0, 4)
110
self.progress.update(gettext('checking revisions'), 0)
111
self.check_revisions()
112
self.progress.update(gettext('checking commit contents'), 1)
113
self.repository._check_inventories(self)
114
self.progress.update(gettext('checking file graphs'), 2)
115
# check_weaves is done after the revision scan so that
116
# revision index is known to be valid.
118
self.progress.update(gettext('checking branches and trees'), 3)
120
repo = self.repository
121
# calculate all refs, and callback the objects requesting them.
123
wanting_items = set()
124
# Current crude version calculates everything and calls
125
# everything at once. Doing a queue and popping as things are
126
# satisfied would be cheaper on memory [but few people have
127
# huge numbers of working trees today. TODO: fix before
131
for ref, wantlist in callback_refs.iteritems():
132
wanting_items.update(wantlist)
135
refs[ref] = repo.revision_tree(value)
136
elif kind == 'lefthand-distance':
138
elif kind == 'revision-existence':
139
existences.add(value)
141
raise AssertionError(
142
'unknown ref kind for ref %s' % ref)
143
node_distances = repo.get_graph().find_lefthand_distances(distances)
144
for key, distance in node_distances.iteritems():
145
refs[('lefthand-distance', key)] = distance
146
if key in existences and distance > 0:
147
refs[('revision-existence', key)] = True
148
existences.remove(key)
149
parent_map = repo.get_graph().get_parent_map(existences)
150
for key in parent_map:
151
refs[('revision-existence', key)] = True
152
existences.remove(key)
153
for key in existences:
154
refs[('revision-existence', key)] = False
155
for item in wanting_items:
156
if isinstance(item, WorkingTree):
158
if isinstance(item, Branch):
159
self.other_results.append(item.check(refs))
91
161
self.progress.finished()
92
162
self.repository.unlock()
94
def check_revision_graph(self):
95
if not self.repository.revision_graph_can_have_wrong_parents():
96
# This check is not necessary.
164
def _check_revisions(self, revisions_iterator):
165
"""Check revision objects by decorating a generator.
167
:param revisions_iterator: An iterator of(revid, Revision-or-None).
168
:return: A generator of the contents of revisions_iterator.
170
self.planned_revisions = set()
171
for revid, revision in revisions_iterator:
172
yield revid, revision
173
self._check_one_rev(revid, revision)
174
# Flatten the revisions we found to guarantee consistent later
176
self.planned_revisions = list(self.planned_revisions)
177
# TODO: extract digital signatures as items to callback on too.
179
def check_revisions(self):
180
"""Scan revisions, checking data directly available as we go."""
181
revision_iterator = self.repository._iter_revisions(None)
182
revision_iterator = self._check_revisions(revision_iterator)
183
# We read the all revisions here:
184
# - doing this allows later code to depend on the revision index.
185
# - we can fill out existence flags at this point
186
# - we can read the revision inventory sha at this point
187
# - we can check properties and serialisers etc.
188
if not self.repository._format.revision_graph_can_have_wrong_parents:
189
# The check against the index isn't needed.
97
190
self.revs_with_bad_parents_in_index = None
99
bad_revisions = self.repository._find_inconsistent_revision_parents()
100
self.revs_with_bad_parents_in_index = list(bad_revisions)
102
def plan_revisions(self):
103
repository = self.repository
104
self.planned_revisions = repository.all_revision_ids()
105
self.progress.clear()
106
inventoried = set(key[-1] for key in self.inventory_weave.keys())
107
awol = set(self.planned_revisions) - inventoried
109
raise BzrCheckError('Stored revisions missing from inventory'
110
'{%s}' % ','.join([f for f in awol]))
191
for thing in revision_iterator:
194
bad_revisions = self.repository._find_inconsistent_revision_parents(
196
self.revs_with_bad_parents_in_index = list(bad_revisions)
112
198
def report_results(self, verbose):
113
note('checked repository %s format %s',
114
self.repository.bzrdir.root_transport,
115
self.repository._format)
116
note('%6d revisions', self.checked_rev_cnt)
117
note('%6d file-ids', len(self.checked_weaves))
118
note('%6d unique file texts', self.checked_text_cnt)
119
note('%6d repeated file texts', self.repeated_text_cnt)
120
note('%6d unreferenced text versions',
121
len(self.unreferenced_versions))
200
self._report_repo_results(verbose)
201
for result in self.other_results:
202
result.report_results(verbose)
204
def _report_repo_results(self, verbose):
205
note(gettext('checked repository {0} format {1}').format(
206
self.repository.user_url,
207
self.repository._format))
208
note(gettext('%6d revisions'), self.checked_rev_cnt)
209
note(gettext('%6d file-ids'), len(self.checked_weaves))
211
note(gettext('%6d unreferenced text versions'),
212
len(self.unreferenced_versions))
213
if verbose and len(self.unreferenced_versions):
214
for file_id, revision_id in self.unreferenced_versions:
215
note(gettext('unreferenced version: {{{0}}} in {1}').format(revision_id,
122
217
if self.missing_inventory_sha_cnt:
123
note('%6d revisions are missing inventory_sha1',
218
note(gettext('%6d revisions are missing inventory_sha1'),
124
219
self.missing_inventory_sha_cnt)
125
220
if self.missing_revision_cnt:
126
note('%6d revisions are mentioned but not present',
221
note(gettext('%6d revisions are mentioned but not present'),
127
222
self.missing_revision_cnt)
128
223
if len(self.ghosts):
129
note('%6d ghost revisions', len(self.ghosts))
224
note(gettext('%6d ghost revisions'), len(self.ghosts))
131
226
for ghost in self.ghosts:
132
227
note(' %s', ghost)
133
228
if len(self.missing_parent_links):
134
note('%6d revisions missing parents in ancestry',
229
note(gettext('%6d revisions missing parents in ancestry'),
135
230
len(self.missing_parent_links))
137
232
for link, linkers in self.missing_parent_links.items():
138
note(' %s should be in the ancestry for:', link)
233
note(gettext(' %s should be in the ancestry for:'), link)
139
234
for linker in linkers:
140
235
note(' * %s', linker)
142
for file_id, revision_id in self.unreferenced_versions:
143
log_error('unreferenced version: {%s} in %s', revision_id,
145
236
if len(self.inconsistent_parents):
146
note('%6d inconsistent parents', len(self.inconsistent_parents))
237
note(gettext('%6d inconsistent parents'), len(self.inconsistent_parents))
148
239
for info in self.inconsistent_parents:
149
240
revision_id, file_id, found_parents, correct_parents = info
150
note(' * %s version %s has parents %r '
152
% (file_id, revision_id, found_parents,
241
note(gettext(' * {0} version {1} has parents {2!r} '
242
'but should have {3!r}').format(
243
file_id, revision_id, found_parents,
153
244
correct_parents))
154
245
if self.revs_with_bad_parents_in_index:
155
note('%6d revisions have incorrect parents in the revision index',
247
'%6d revisions have incorrect parents in the revision index'),
156
248
len(self.revs_with_bad_parents_in_index))
158
250
for item in self.revs_with_bad_parents_in_index:
159
251
revision_id, index_parents, actual_parents = item
161
' %s has wrong parents in index: '
163
revision_id, index_parents, actual_parents)
165
def check_one_rev(self, rev_id):
166
"""Check one revision.
168
rev_id - the one to check
253
' {0} has wrong parents in index: '
254
'{1!r} should be {2!r}').format(
255
revision_id, index_parents, actual_parents))
256
for item in self._report_items:
259
def _check_one_rev(self, rev_id, rev):
260
"""Cross-check one revision.
262
:param rev_id: A revision id to check.
263
:param rev: A revision or None to indicate a missing revision.
170
rev = self.repository.get_revision(rev_id)
172
265
if rev.revision_id != rev_id:
173
raise BzrCheckError('wrong internal revision id in revision {%s}'
266
self._report_items.append(gettext(
267
'Mismatched internal revid {{{0}}} and index revid {{{1}}}').format(
268
rev.revision_id, rev_id))
269
rev_id = rev.revision_id
270
# Check this revision tree etc, and count as seen when we encounter a
272
self.planned_revisions.add(rev_id)
274
self.ghosts.discard(rev_id)
275
# Count all parents as ghosts if we haven't seen them yet.
176
276
for parent in rev.parent_ids:
177
277
if not parent in self.planned_revisions:
178
# rev has a parent we didn't know about.
179
missing_links = self.missing_parent_links.get(parent, [])
180
missing_links.append(rev_id)
181
self.missing_parent_links[parent] = missing_links
182
# list based so somewhat slow,
183
# TODO have a planned_revisions list and set.
184
if self.repository.has_revision(parent):
185
missing_ancestry = self.repository.get_ancestry(parent)
186
for missing in missing_ancestry:
187
if (missing is not None
188
and missing not in self.planned_revisions):
189
self.planned_revisions.append(missing)
191
self.ghosts.append(rev_id)
193
if rev.inventory_sha1:
194
# Loopback - this is currently circular logic as the
195
# knit get_inventory_sha1 call returns rev.inventory_sha1.
196
# Repository.py's get_inventory_sha1 should instead return
197
# inventories.get_record_stream([(revid,)]).next().sha1 or
199
inv_sha1 = self.repository.get_inventory_sha1(rev_id)
200
if inv_sha1 != rev.inventory_sha1:
201
raise BzrCheckError('Inventory sha1 hash doesn\'t match'
202
' value in revision {%s}' % rev_id)
203
self._check_revision_tree(rev_id)
278
self.ghosts.add(parent)
280
self.ancestors[rev_id] = tuple(rev.parent_ids) or (NULL_REVISION,)
281
self.add_pending_item(rev_id, ('inventories', rev_id), 'inventory',
204
283
self.checked_rev_cnt += 1
285
def add_pending_item(self, referer, key, kind, sha1):
286
"""Add a reference to a sha1 to be cross checked against a key.
288
:param referer: The referer that expects key to have sha1.
289
:param key: A storage key e.g. ('texts', 'foo@bar-20040504-1234')
290
:param kind: revision/inventory/text/map/signature
291
:param sha1: A hex sha1 or None if no sha1 is known.
293
existing = self.pending_keys.get(key)
295
if sha1 != existing[1]:
296
self._report_items.append(gettext('Multiple expected sha1s for {0}. {{{1}}}'
297
' expects {{{2}}}, {{{3}}} expects {{{4}}}').format(
298
key, referer, sha1, existing[1], existing[0]))
300
self.pending_keys[key] = (kind, sha1, referer)
206
302
def check_weaves(self):
207
303
"""Check all the weaves we can get our hands on.
210
self.progress.update('checking inventory', 0, 2)
211
self.inventory_weave.check(progress_bar=self.progress)
212
self.progress.update('checking text storage', 1, 2)
213
self.repository.texts.check(progress_bar=self.progress)
214
weave_checker = self.repository._get_versioned_file_checker(
215
text_key_references=self.text_key_references)
306
storebar = ui.ui_factory.nested_progress_bar()
308
self._check_weaves(storebar)
312
def _check_weaves(self, storebar):
313
storebar.update('text-index', 0, 2)
314
if self.repository._format.fast_deltas:
315
# We haven't considered every fileid instance so far.
316
weave_checker = self.repository._get_versioned_file_checker(
317
ancestors=self.ancestors)
319
weave_checker = self.repository._get_versioned_file_checker(
320
text_key_references=self.text_key_references,
321
ancestors=self.ancestors)
322
storebar.update('file-graph', 1)
216
323
result = weave_checker.check_file_version_parents(
217
self.repository.texts, progress_bar=self.progress)
324
self.repository.texts)
218
325
self.checked_weaves = weave_checker.file_ids
219
326
bad_parents, unused_versions = result
220
327
bad_parents = bad_parents.items()
257
344
self.text_key_references[key] = True
260
@deprecated_function(deprecated_in((1,6,0)))
261
def check(branch, verbose):
262
"""Run consistency checks on a branch.
264
Results are reported through logging.
266
Deprecated in 1.6. Please use check_branch instead.
268
:raise BzrCheckError: if there's a consistency error.
270
check_branch(branch, verbose)
273
def check_branch(branch, verbose):
274
"""Run consistency checks on a branch.
276
Results are reported through logging.
278
:raise BzrCheckError: if there's a consistency error.
347
def scan_branch(branch, needed_refs, to_unlock):
348
"""Scan a branch for refs.
350
:param branch: The branch to schedule for checking.
351
:param needed_refs: Refs we are accumulating.
352
:param to_unlock: The unlock list accumulating.
354
note(gettext("Checking branch at '%s'.") % (branch.base,))
280
355
branch.lock_read()
282
branch_result = branch.check()
285
branch_result.report_results(verbose)
356
to_unlock.append(branch)
357
branch_refs = branch._get_check_refs()
358
for ref in branch_refs:
359
reflist = needed_refs.setdefault(ref, [])
360
reflist.append(branch)
363
def scan_tree(base_tree, tree, needed_refs, to_unlock):
364
"""Scan a tree for refs.
366
:param base_tree: The original tree check opened, used to detect duplicate
368
:param tree: The tree to schedule for checking.
369
:param needed_refs: Refs we are accumulating.
370
:param to_unlock: The unlock list accumulating.
372
if base_tree is not None and tree.basedir == base_tree.basedir:
374
note(gettext("Checking working tree at '%s'.") % (tree.basedir,))
376
to_unlock.append(tree)
377
tree_refs = tree._get_check_refs()
378
for ref in tree_refs:
379
reflist = needed_refs.setdefault(ref, [])
288
383
def check_dwim(path, verbose, do_branch=False, do_repo=False, do_tree=False):
384
"""Check multiple objects.
386
If errors occur they are accumulated and reported as far as possible, and
387
an exception raised at the end of the process.
290
tree, branch, repo, relpath = \
291
BzrDir.open_containing_tree_branch_or_repository(path)
390
base_tree, branch, repo, relpath = \
391
ControlDir.open_containing_tree_branch_or_repository(path)
292
392
except errors.NotBranchError:
293
tree = branch = repo = None
297
note("Checking working tree at '%s'."
298
% (tree.bzrdir.root_transport.base,))
301
log_error("No working tree found at specified location.")
303
if branch is not None:
306
# The branch is in a shared repository
307
repo = branch.repository
309
elif repo is not None:
310
branches = repo.find_branches(using=True)
316
note("Checking repository at '%s'."
317
% (repo.bzrdir.root_transport.base,))
318
result = repo.check()
393
base_tree = branch = repo = None
398
if base_tree is not None:
399
# If the tree is a lightweight checkout we won't see it in
400
# repo.find_branches - add now.
402
scan_tree(None, base_tree, needed_refs, to_unlock)
403
branch = base_tree.branch
404
if branch is not None:
407
# The branch is in a shared repository
408
repo = branch.repository
411
to_unlock.append(repo)
412
branches = repo.find_branches(using=True)
414
if do_branch or do_tree:
415
for branch in branches:
418
tree = branch.bzrdir.open_workingtree()
420
except (errors.NotLocalUrl, errors.NoWorkingTree):
423
scan_tree(base_tree, tree, needed_refs, to_unlock)
425
scan_branch(branch, needed_refs, to_unlock)
426
if do_branch and not branches:
427
note(gettext("No branch found at specified location."))
428
if do_tree and base_tree is None and not saw_tree:
429
note(gettext("No working tree found at specified location."))
430
if do_repo or do_branch or do_tree:
432
note(gettext("Checking repository at '%s'.")
434
result = repo.check(None, callback_refs=needed_refs,
319
436
result.report_results(verbose)
439
note(gettext("No working tree found at specified location."))
322
log_error("No branch found at specified location.")
324
for branch in branches:
325
note("Checking branch at '%s'."
326
% (branch.bzrdir.root_transport.base,))
327
check_branch(branch, verbose)
332
log_error("No branch found at specified location.")
334
log_error("No repository found at specified location.")
441
note(gettext("No branch found at specified location."))
443
note(gettext("No repository found at specified location."))
445
for thing in to_unlock:
added
removed
bzrlib/check.py
