~bzr-pqm/bzr/bzr.dev

2052.3.2 by John Arbash Meinel
Change Copyright .. by Canonical to Copyright ... Canonical
1
# Copyright (C) 2005, 2006 Canonical Ltd
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
2
#
1 by mbp at sourcefrog
import from baz patch-364
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
7
#
1 by mbp at sourcefrog
import from baz patch-364
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
12
#
1 by mbp at sourcefrog
import from baz patch-364
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
1335 by Martin Pool
doc
17
# TODO: Check ancestries are correct for every revision: includes
18
# every committed so far, and in a reasonable order.
19
1347 by Martin Pool
- refactor check code into method object
20
# TODO: Also check non-mainline revisions mentioned as parents.
21
22
# TODO: Check for extra files in the control directory.
23
1348 by Martin Pool
- more refactoring of check code
24
# TODO: Check revision, inventory and entry objects have all 
25
# required fields.
26
1185.16.101 by mbp at sourcefrog
todo
27
# TODO: Get every revision in the revision-store even if they're not
28
# referenced by history and make sure they're all valid.
1347 by Martin Pool
- refactor check code into method object
29
1616.1.5 by Martin Pool
Cleanup and document some check code
30
# TODO: Perhaps have a way to record errors other than by raising exceptions;
31
# would perhaps be enough to accumulate exception objects in a list without
32
# raising them.  If there's more than one exception it'd be good to see them
33
# all.
34
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
35
from bzrlib import errors
2745.6.16 by Aaron Bentley
Update from review
36
from bzrlib import repository as _mod_repository
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
37
from bzrlib import revision
1773.4.1 by Martin Pool
Add pyflakes makefile target; fix many warnings
38
from bzrlib.errors import BzrCheckError
1104 by Martin Pool
- Add a simple UIFactory
39
import bzrlib.ui
2745.6.61 by Andrew Bennetts
Remove some useless mutters, and cope with a file_id that isn't present in a revision's inventory.
40
from bzrlib.trace import log_error, note
1104 by Martin Pool
- Add a simple UIFactory
41
1347 by Martin Pool
- refactor check code into method object
42
class Check(object):
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
43
    """Check a repository"""
1449 by Robert Collins
teach check about ghosts
44
1616.1.5 by Martin Pool
Cleanup and document some check code
45
    # The Check object interacts with InventoryEntry.check, etc.
46
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
47
    def __init__(self, repository):
48
        self.repository = repository
1383 by Martin Pool
- untabify only
49
        self.checked_text_cnt = 0
50
        self.checked_rev_cnt = 0
1449 by Robert Collins
teach check about ghosts
51
        self.ghosts = []
1365 by Martin Pool
- try to avoid checking texts repeatedly
52
        self.repeated_text_cnt = 0
1449 by Robert Collins
teach check about ghosts
53
        self.missing_parent_links = {}
1348 by Martin Pool
- more refactoring of check code
54
        self.missing_inventory_sha_cnt = 0
55
        self.missing_revision_cnt = 0
1616.1.5 by Martin Pool
Cleanup and document some check code
56
        # maps (file-id, version) -> sha1; used by InventoryFile._check
1365 by Martin Pool
- try to avoid checking texts repeatedly
57
        self.checked_texts = {}
1185.50.28 by John Arbash Meinel
Lots of updates for 'bzr check'
58
        self.checked_weaves = {}
2745.6.16 by Aaron Bentley
Update from review
59
        self.revision_versions = _mod_repository._RevisionTextVersionCache(
60
            self.repository)
2745.6.6 by Aaron Bentley
Add unreferenced ancestors to check output
61
        self.unreferenced_ancestors = set()
2745.6.33 by Andrew Bennetts
Add VersionedFile.check_parents, and use it instead of find_bad_ancestors in reconcile.
62
        self.inconsistent_parents = []
2927.2.13 by Andrew Bennetts
Display number of file versions not referenced by their corresponding inventory (i.e. dangling versions) in bzr check output.
63
        self.dangling_versions = set()
676 by Martin Pool
- lock branch while checking
64
1449 by Robert Collins
teach check about ghosts
65
    def check(self):
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
66
        self.repository.lock_read()
1594.1.3 by Robert Collins
Fixup pb usage to use nested_progress_bar.
67
        self.progress = bzrlib.ui.ui_factory.nested_progress_bar()
1449 by Robert Collins
teach check about ghosts
68
        try:
2819.2.3 by Andrew Bennetts
Add test that repo.check will report on wrong parents in the revision graph.
69
            self.progress.update('retrieving inventory', 0, 2)
1510 by Robert Collins
Merge from mpool, adjusting check to retain HTTP support.
70
            # do not put in init, as it should be done with progess,
71
            # and inside the lock.
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
72
            self.inventory_weave = self.repository.get_inventory_weave()
2819.2.3 by Andrew Bennetts
Add test that repo.check will report on wrong parents in the revision graph.
73
            self.progress.update('checking revision graph', 1)
74
            self.check_revision_graph()
1510 by Robert Collins
Merge from mpool, adjusting check to retain HTTP support.
75
            self.plan_revisions()
76
            revno = 0
77
            while revno < len(self.planned_revisions):
78
                rev_id = self.planned_revisions[revno]
79
                self.progress.update('checking revision', revno,
1449 by Robert Collins
teach check about ghosts
80
                                     len(self.planned_revisions))
1510 by Robert Collins
Merge from mpool, adjusting check to retain HTTP support.
81
                revno += 1
1449 by Robert Collins
teach check about ghosts
82
                self.check_one_rev(rev_id)
2745.6.16 by Aaron Bentley
Update from review
83
            # check_weaves is done after the revision scan so that
84
            # revision_versions is pre-populated
2745.6.3 by Aaron Bentley
Implement versionedfile checking for bzr check
85
            self.check_weaves()
1185.35.34 by Aaron Bentley
Made bzr check for stored revisions missing from ancestry
86
        finally:
1594.1.3 by Robert Collins
Fixup pb usage to use nested_progress_bar.
87
            self.progress.finished()
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
88
            self.repository.unlock()
1449 by Robert Collins
teach check about ghosts
89
2819.2.3 by Andrew Bennetts
Add test that repo.check will report on wrong parents in the revision graph.
90
    def check_revision_graph(self):
2819.2.4 by Andrew Bennetts
Add a 'revision_graph_can_have_wrong_parents' method to repository.
91
        if not self.repository.revision_graph_can_have_wrong_parents():
92
            # This check is not necessary.
2819.2.3 by Andrew Bennetts
Add test that repo.check will report on wrong parents in the revision graph.
93
            self.revs_with_bad_parents_in_index = None
94
            return
95
        bad_revisions = self.repository._find_inconsistent_revision_parents()
96
        self.revs_with_bad_parents_in_index = list(bad_revisions)
97
1510 by Robert Collins
Merge from mpool, adjusting check to retain HTTP support.
98
    def plan_revisions(self):
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
99
        repository = self.repository
2745.6.3 by Aaron Bentley
Implement versionedfile checking for bzr check
100
        self.planned_revisions = repository.all_revision_ids()
1563.2.22 by Robert Collins
Move responsibility for repository.has_revision into RevisionStore
101
        self.progress.clear()
1563.2.35 by Robert Collins
cleanup deprecation warnings and finish conversion so the inventory is knit based too.
102
        inventoried = set(self.inventory_weave.versions())
2745.6.3 by Aaron Bentley
Implement versionedfile checking for bzr check
103
        awol = set(self.planned_revisions) - inventoried
1510 by Robert Collins
Merge from mpool, adjusting check to retain HTTP support.
104
        if len(awol) > 0:
105
            raise BzrCheckError('Stored revisions missing from inventory'
106
                '{%s}' % ','.join([f for f in awol]))
107
1449 by Robert Collins
teach check about ghosts
108
    def report_results(self, verbose):
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
109
        note('checked repository %s format %s',
110
             self.repository.bzrdir.root_transport,
111
             self.repository._format)
1365 by Martin Pool
- try to avoid checking texts repeatedly
112
        note('%6d revisions', self.checked_rev_cnt)
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
113
        note('%6d file-ids', len(self.checked_weaves))
1365 by Martin Pool
- try to avoid checking texts repeatedly
114
        note('%6d unique file texts', self.checked_text_cnt)
115
        note('%6d repeated file texts', self.repeated_text_cnt)
2745.6.6 by Aaron Bentley
Add unreferenced ancestors to check output
116
        note('%6d unreferenced text ancestors',
117
             len(self.unreferenced_ancestors))
1348 by Martin Pool
- more refactoring of check code
118
        if self.missing_inventory_sha_cnt:
1449 by Robert Collins
teach check about ghosts
119
            note('%6d revisions are missing inventory_sha1',
1383 by Martin Pool
- untabify only
120
                 self.missing_inventory_sha_cnt)
1348 by Martin Pool
- more refactoring of check code
121
        if self.missing_revision_cnt:
1449 by Robert Collins
teach check about ghosts
122
            note('%6d revisions are mentioned but not present',
1383 by Martin Pool
- untabify only
123
                 self.missing_revision_cnt)
1449 by Robert Collins
teach check about ghosts
124
        if len(self.ghosts):
125
            note('%6d ghost revisions', len(self.ghosts))
126
            if verbose:
127
                for ghost in self.ghosts:
128
                    note('      %s', ghost)
129
        if len(self.missing_parent_links):
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
130
            note('%6d revisions missing parents in ancestry',
1449 by Robert Collins
teach check about ghosts
131
                 len(self.missing_parent_links))
132
            if verbose:
133
                for link, linkers in self.missing_parent_links.items():
134
                    note('      %s should be in the ancestry for:', link)
135
                    for linker in linkers:
136
                        note('       * %s', linker)
2745.6.6 by Aaron Bentley
Add unreferenced ancestors to check output
137
            if verbose:
138
                for file_id, revision_id in self.unreferenced_ancestors:
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
139
                    log_error('unreferenced ancestor: {%s} in %s', revision_id,
2745.6.6 by Aaron Bentley
Add unreferenced ancestors to check output
140
                        file_id)
2745.6.39 by Andrew Bennetts
Use scenario in test_check too, and make check actually report inconsistent parents to the end user.
141
        if len(self.inconsistent_parents):
142
            note('%6d inconsistent parents', len(self.inconsistent_parents))
143
            if verbose:
144
                for info in self.inconsistent_parents:
145
                    revision_id, file_id, found_parents, correct_parents = info
146
                    note('      * %s version %s has parents %r '
147
                         'but should have %r'
148
                         % (file_id, revision_id, found_parents,
149
                             correct_parents))
2819.2.3 by Andrew Bennetts
Add test that repo.check will report on wrong parents in the revision graph.
150
        if self.revs_with_bad_parents_in_index:
151
            note('%6d revisions have incorrect parents in the revision index',
152
                 len(self.revs_with_bad_parents_in_index))
153
            if verbose:
154
                for item in self.revs_with_bad_parents_in_index:
155
                    revision_id, index_parents, actual_parents = item
156
                    note(
157
                        '       %s has wrong parents in index: '
158
                        '%r should be %r',
159
                        revision_id, index_parents, actual_parents)
2927.2.13 by Andrew Bennetts
Display number of file versions not referenced by their corresponding inventory (i.e. dangling versions) in bzr check output.
160
        if self.dangling_versions:
161
            note('%6d file versions are not referenced by their inventory',
162
                 len(self.dangling_versions))
1449 by Robert Collins
teach check about ghosts
163
164
    def check_one_rev(self, rev_id):
1383 by Martin Pool
- untabify only
165
        """Check one revision.
166
167
        rev_id - the one to check
168
        """
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
169
        rev = self.repository.get_revision(rev_id)
1449 by Robert Collins
teach check about ghosts
170
                
1383 by Martin Pool
- untabify only
171
        if rev.revision_id != rev_id:
172
            raise BzrCheckError('wrong internal revision id in revision {%s}'
173
                                % rev_id)
174
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
175
        for parent in rev.parent_ids:
176
            if not parent in self.planned_revisions:
177
                missing_links = self.missing_parent_links.get(parent, [])
178
                missing_links.append(rev_id)
179
                self.missing_parent_links[parent] = missing_links
180
                # list based so somewhat slow,
181
                # TODO have a planned_revisions list and set.
182
                if self.repository.has_revision(parent):
183
                    missing_ancestry = self.repository.get_ancestry(parent)
184
                    for missing in missing_ancestry:
185
                        if (missing is not None 
186
                            and missing not in self.planned_revisions):
187
                            self.planned_revisions.append(missing)
1449 by Robert Collins
teach check about ghosts
188
                else:
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
189
                    self.ghosts.append(rev_id)
1383 by Martin Pool
- untabify only
190
191
        if rev.inventory_sha1:
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
192
            inv_sha1 = self.repository.get_inventory_sha1(rev_id)
1383 by Martin Pool
- untabify only
193
            if inv_sha1 != rev.inventory_sha1:
194
                raise BzrCheckError('Inventory sha1 hash doesn\'t match'
195
                    ' value in revision {%s}' % rev_id)
196
        self._check_revision_tree(rev_id)
1362 by Martin Pool
- keep track of number of checked revisions
197
        self.checked_rev_cnt += 1
1349 by Martin Pool
- more refactoring of check code
198
1185.50.28 by John Arbash Meinel
Lots of updates for 'bzr check'
199
    def check_weaves(self):
200
        """Check all the weaves we can get our hands on.
201
        """
202
        n_weaves = 1
203
        weave_ids = []
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
204
        if self.repository.weave_store.listable():
205
            weave_ids = list(self.repository.weave_store)
2592.3.63 by Robert Collins
Supply weave_store.__iter__ for compatibility with check.
206
            n_weaves = len(weave_ids) + 1
2745.6.8 by Aaron Bentley
Clean up text
207
        self.progress.update('checking versionedfile', 0, n_weaves)
1185.50.28 by John Arbash Meinel
Lots of updates for 'bzr check'
208
        self.inventory_weave.check(progress_bar=self.progress)
2745.6.33 by Andrew Bennetts
Add VersionedFile.check_parents, and use it instead of find_bad_ancestors in reconcile.
209
        files_in_revisions = {}
210
        revisions_of_files = {}
1185.50.28 by John Arbash Meinel
Lots of updates for 'bzr check'
211
        for i, weave_id in enumerate(weave_ids):
2745.6.8 by Aaron Bentley
Clean up text
212
            self.progress.update('checking versionedfile', i, n_weaves)
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
213
            w = self.repository.weave_store.get_weave(weave_id,
214
                    self.repository.get_transaction())
1185.50.28 by John Arbash Meinel
Lots of updates for 'bzr check'
215
            # No progress here, because it looks ugly.
216
            w.check()
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
217
218
            weave_checker = self.repository.get_versioned_file_checker(
219
                self.planned_revisions, self.revision_versions)
2745.6.49 by Andrew Bennetts
Get rid of bzrlib.repository._RevisionParentsProvider.
220
            result = weave_checker.check_file_version_parents(w, weave_id)
2927.2.6 by Andrew Bennetts
Make some more check tests pass.
221
            bad_parents, dangling_versions = result
222
            bad_parents = bad_parents.items()
223
            for revision_id, (weave_parents,correct_parents) in bad_parents:
2745.6.33 by Andrew Bennetts
Add VersionedFile.check_parents, and use it instead of find_bad_ancestors in reconcile.
224
                self.inconsistent_parents.append(
225
                    (revision_id, weave_id, weave_parents, correct_parents))
2927.2.6 by Andrew Bennetts
Make some more check tests pass.
226
                if weave_parents is None:
227
                    weave_parents = []
2745.6.50 by Andrew Bennetts
Remove find_bad_ancestors; it's not needed anymore.
228
                unreferenced_parents = set(weave_parents)-set(correct_parents)
229
                for unreferenced_parent in unreferenced_parents:
230
                    self.unreferenced_ancestors.add(
231
                        (weave_id, unreferenced_parent))
2927.2.13 by Andrew Bennetts
Display number of file versions not referenced by their corresponding inventory (i.e. dangling versions) in bzr check output.
232
            self.dangling_versions.update(dangling_versions)
1185.50.28 by John Arbash Meinel
Lots of updates for 'bzr check'
233
            self.checked_weaves[weave_id] = True
234
1349 by Martin Pool
- more refactoring of check code
235
    def _check_revision_tree(self, rev_id):
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
236
        tree = self.repository.revision_tree(rev_id)
2745.6.53 by Andrew Bennetts
Some more changes suggested by review.
237
        self.revision_versions.add_revision_text_versions(tree)
1383 by Martin Pool
- untabify only
238
        inv = tree.inventory
239
        seen_ids = {}
240
        for file_id in inv:
241
            if file_id in seen_ids:
242
                raise BzrCheckError('duplicated file_id {%s} '
243
                                    'in inventory for revision {%s}'
244
                                    % (file_id, rev_id))
245
            seen_ids[file_id] = True
246
        for file_id in inv:
1092.2.20 by Robert Collins
symlink and weaves, whaddya know
247
            ie = inv[file_id]
248
            ie.check(self, rev_id, inv, tree)
1383 by Martin Pool
- untabify only
249
        seen_names = {}
250
        for path, ie in inv.iter_entries():
251
            if path in seen_names:
252
                raise BzrCheckError('duplicated path %s '
253
                                    'in inventory for revision {%s}'
254
                                    % (path, rev_id))
255
            seen_names[path] = True
1349 by Martin Pool
- more refactoring of check code
256
1347 by Martin Pool
- refactor check code into method object
257
1449 by Robert Collins
teach check about ghosts
258
def check(branch, verbose):
1732.2.4 by Martin Pool
Split check into Branch.check and Repository.check
259
    """Run consistency checks on a branch.
260
    
261
    Results are reported through logging.
262
    
263
    :raise BzrCheckError: if there's a consistency error.
264
    """
265
    branch.lock_read()
266
    try:
267
        branch_result = branch.check()
268
        repo_result = branch.repository.check([branch.last_revision()])
269
    finally:
270
        branch.unlock()
271
    branch_result.report_results(verbose)
272
    repo_result.report_results(verbose)
2745.6.47 by Andrew Bennetts
Move check_parents out of VersionedFile.
273
274
275