← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/reconcile.py

  • Committer: John Arbash Meinel
  • Author(s): Mark Hammond
  • Date: 2008-09-09 17:02:21 UTC
  • mto: This revision was merged to the branch mainline in revision 3697.
  • Revision ID: john@arbash-meinel.com-20080909170221-svim3jw2mrz0amp3
An updated transparent icon for bzr.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/reconcile.py
1
 
# Copyright (C) 2006-2010 Canonical Ltd
 
1
# Copyright (C) 2005, 2006 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
16
 
17
17
"""Reconcilers are able to fix some potential data errors in a branch."""
18
18
 
19
 
from __future__ import absolute_import
20
19
 
21
20
__all__ = [
22
21
    'KnitReconciler',
28
27
 
29
28
 
30
29
from bzrlib import (
31
 
    cleanup,
32
30
    errors,
33
 
    revision as _mod_revision,
34
31
    ui,
 
32
    repository,
 
33
    repofmt,
35
34
    )
36
 
from bzrlib.trace import mutter
37
 
from bzrlib.tsort import topo_sort
 
35
from bzrlib.trace import mutter, note
 
36
from bzrlib.tsort import TopoSorter
38
37
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
39
 
from bzrlib.i18n import gettext
40
 
 
41
 
 
42
 
def reconcile(dir, canonicalize_chks=False):
 
38
 
 
39
 
 
40
def reconcile(dir, other=None):
43
41
    """Reconcile the data in dir.
44
42
 
45
43
    Currently this is limited to a inventory 'reweave'.
49
47
    Directly using Reconciler is recommended for library users that
50
48
    desire fine grained control or analysis of the found issues.
51
49
 
52
 
    :param canonicalize_chks: Make sure CHKs are in canonical form.
 
50
    :param other: another bzrdir to reconcile against.
53
51
    """
54
 
    reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
 
52
    reconciler = Reconciler(dir, other=other)
55
53
    reconciler.reconcile()
56
54
 
57
55
 
58
56
class Reconciler(object):
59
57
    """Reconcilers are used to reconcile existing data."""
60
58
 
61
 
    def __init__(self, dir, other=None, canonicalize_chks=False):
 
59
    def __init__(self, dir, other=None):
62
60
        """Create a Reconciler."""
63
61
        self.bzrdir = dir
64
 
        self.canonicalize_chks = canonicalize_chks
65
62
 
66
63
    def reconcile(self):
67
64
        """Perform reconciliation.
68
 
 
 
65
        
69
66
        After reconciliation the following attributes document found issues:
70
 
 
71
 
        * `inconsistent_parents`: The number of revisions in the repository
72
 
          whose ancestry was being reported incorrectly.
73
 
        * `garbage_inventories`: The number of inventory objects without
74
 
          revisions that were garbage collected.
75
 
        * `fixed_branch_history`: None if there was no branch, False if the
76
 
          branch history was correct, True if the branch history needed to be
77
 
          re-normalized.
 
67
        inconsistent_parents: The number of revisions in the repository whose
 
68
                              ancestry was being reported incorrectly.
 
69
        garbage_inventories: The number of inventory objects without revisions
 
70
                             that were garbage collected.
 
71
        fixed_branch_history: None if there was no branch, False if the branch
 
72
                              history was correct, True if the branch history
 
73
                              needed to be re-normalized.
78
74
        """
79
75
        self.pb = ui.ui_factory.nested_progress_bar()
80
76
        try:
94
90
            # Nothing to check here
95
91
            self.fixed_branch_history = None
96
92
            return
97
 
        ui.ui_factory.note(gettext('Reconciling branch %s') % self.branch.base)
 
93
        self.pb.note('Reconciling branch %s',
 
94
                     self.branch.base)
98
95
        branch_reconciler = self.branch.reconcile(thorough=True)
99
96
        self.fixed_branch_history = branch_reconciler.fixed_history
100
97
 
101
98
    def _reconcile_repository(self):
102
99
        self.repo = self.bzrdir.find_repository()
103
 
        ui.ui_factory.note(gettext('Reconciling repository %s') %
104
 
            self.repo.user_url)
105
 
        self.pb.update(gettext("Reconciling repository"), 0, 1)
106
 
        if self.canonicalize_chks:
107
 
            try:
108
 
                self.repo.reconcile_canonicalize_chks
109
 
            except AttributeError:
110
 
                raise errors.BzrError(
111
 
                    gettext("%s cannot canonicalize CHKs.") % (self.repo,))
112
 
            repo_reconciler = self.repo.reconcile_canonicalize_chks()
113
 
        else:
114
 
            repo_reconciler = self.repo.reconcile(thorough=True)
 
100
        self.pb.note('Reconciling repository %s',
 
101
                     self.repo.bzrdir.root_transport.base)
 
102
        self.pb.update("Reconciling repository", 0, 1)
 
103
        repo_reconciler = self.repo.reconcile(thorough=True)
115
104
        self.inconsistent_parents = repo_reconciler.inconsistent_parents
116
105
        self.garbage_inventories = repo_reconciler.garbage_inventories
117
106
        if repo_reconciler.aborted:
118
 
            ui.ui_factory.note(gettext(
119
 
                'Reconcile aborted: revision index has inconsistent parents.'))
120
 
            ui.ui_factory.note(gettext(
121
 
                'Run "bzr check" for more details.'))
 
107
            self.pb.note(
 
108
                'Reconcile aborted: revision index has inconsistent parents.')
 
109
            self.pb.note(
 
110
                'Run "bzr check" for more details.')
122
111
        else:
123
 
            ui.ui_factory.note(gettext('Reconciliation complete.'))
 
112
            self.pb.note('Reconciliation complete.')
124
113
 
125
114
 
126
115
class BranchReconciler(object):
132
121
        self.branch = a_branch
133
122
 
134
123
    def reconcile(self):
135
 
        operation = cleanup.OperationWithCleanups(self._reconcile)
136
 
        self.add_cleanup = operation.add_cleanup
137
 
        operation.run_simple()
138
 
 
139
 
    def _reconcile(self):
140
124
        self.branch.lock_write()
141
 
        self.add_cleanup(self.branch.unlock)
142
 
        self.pb = ui.ui_factory.nested_progress_bar()
143
 
        self.add_cleanup(self.pb.finished)
144
 
        self._reconcile_steps()
 
125
        try:
 
126
            self.pb = ui.ui_factory.nested_progress_bar()
 
127
            try:
 
128
                self._reconcile_steps()
 
129
            finally:
 
130
                self.pb.finished()
 
131
        finally:
 
132
            self.branch.unlock()
145
133
 
146
134
    def _reconcile_steps(self):
147
135
        self._reconcile_revision_history()
148
136
 
149
137
    def _reconcile_revision_history(self):
 
138
        repo = self.branch.repository
150
139
        last_revno, last_revision_id = self.branch.last_revision_info()
151
 
        real_history = []
152
 
        graph = self.branch.repository.get_graph()
153
 
        try:
154
 
            for revid in graph.iter_lefthand_ancestry(
155
 
                    last_revision_id, (_mod_revision.NULL_REVISION,)):
156
 
                real_history.append(revid)
157
 
        except errors.RevisionNotPresent:
158
 
            pass # Hit a ghost left hand parent
 
140
        real_history = list(repo.iter_reverse_revision_history(
 
141
                                last_revision_id))
159
142
        real_history.reverse()
160
143
        if last_revno != len(real_history):
161
144
            self.fixed_history = True
163
146
            # set_revision_history, as this will regenerate it again.
164
147
            # Not really worth a whole BranchReconciler class just for this,
165
148
            # though.
166
 
            ui.ui_factory.note(gettext('Fixing last revision info {0} '\
167
 
                                       ' => {1}').format(
168
 
                                       last_revno, len(real_history)))
 
149
            self.pb.note('Fixing last revision info %s => %s',
 
150
                         last_revno, len(real_history))
169
151
            self.branch.set_last_revision_info(len(real_history),
170
152
                                               last_revision_id)
171
153
        else:
172
154
            self.fixed_history = False
173
 
            ui.ui_factory.note(gettext('revision_history ok.'))
 
155
            self.pb.note('revision_history ok.')
174
156
 
175
157
 
176
158
class RepoReconciler(object):
177
159
    """Reconciler that reconciles a repository.
178
160
 
179
161
    The goal of repository reconciliation is to make any derived data
180
 
    consistent with the core data committed by a user. This can involve
 
162
    consistent with the core data committed by a user. This can involve 
181
163
    reindexing, or removing unreferenced data if that can interfere with
182
164
    queries in a given repository.
183
165
 
199
181
 
200
182
    def reconcile(self):
201
183
        """Perform reconciliation.
202
 
 
 
184
        
203
185
        After reconciliation the following attributes document found issues:
204
 
 
205
 
        * `inconsistent_parents`: The number of revisions in the repository
206
 
          whose ancestry was being reported incorrectly.
207
 
        * `garbage_inventories`: The number of inventory objects without
208
 
          revisions that were garbage collected.
 
186
        inconsistent_parents: The number of revisions in the repository whose
 
187
                              ancestry was being reported incorrectly.
 
188
        garbage_inventories: The number of inventory objects without revisions
 
189
                             that were garbage collected.
209
190
        """
210
 
        operation = cleanup.OperationWithCleanups(self._reconcile)
211
 
        self.add_cleanup = operation.add_cleanup
212
 
        operation.run_simple()
213
 
 
214
 
    def _reconcile(self):
215
191
        self.repo.lock_write()
216
 
        self.add_cleanup(self.repo.unlock)
217
 
        self.pb = ui.ui_factory.nested_progress_bar()
218
 
        self.add_cleanup(self.pb.finished)
219
 
        self._reconcile_steps()
 
192
        try:
 
193
            self.pb = ui.ui_factory.nested_progress_bar()
 
194
            try:
 
195
                self._reconcile_steps()
 
196
            finally:
 
197
                self.pb.finished()
 
198
        finally:
 
199
            self.repo.unlock()
220
200
 
221
201
    def _reconcile_steps(self):
222
202
        """Perform the steps to reconcile this repository."""
224
204
 
225
205
    def _reweave_inventory(self):
226
206
        """Regenerate the inventory weave for the repository from scratch.
227
 
 
228
 
        This is a smart function: it will only do the reweave if doing it
 
207
        
 
208
        This is a smart function: it will only do the reweave if doing it 
229
209
        will correct data issues. The self.thorough flag controls whether
230
210
        only data-loss causing issues (!self.thorough) or all issues
231
211
        (self.thorough) are treated as requiring the reweave.
232
212
        """
 
213
        # local because needing to know about WeaveFile is a wart we want to hide
 
214
        from bzrlib.weave import WeaveFile, Weave
233
215
        transaction = self.repo.get_transaction()
234
 
        self.pb.update(gettext('Reading inventory data'))
 
216
        self.pb.update('Reading inventory data.')
235
217
        self.inventory = self.repo.inventories
236
218
        self.revisions = self.repo.revisions
237
219
        # the total set of revisions to process
247
229
            # put a revision into the graph.
248
230
            self._graph_revision(rev_id)
249
231
        self._check_garbage_inventories()
250
 
        # if there are no inconsistent_parents and
 
232
        # if there are no inconsistent_parents and 
251
233
        # (no garbage inventories or we are not doing a thorough check)
252
 
        if (not self.inconsistent_parents and
 
234
        if (not self.inconsistent_parents and 
253
235
            (not self.garbage_inventories or not self.thorough)):
254
 
            ui.ui_factory.note(gettext('Inventory ok.'))
 
236
            self.pb.note('Inventory ok.')
255
237
            return
256
 
        self.pb.update(gettext('Backing up inventory'), 0, 0)
 
238
        self.pb.update('Backing up inventory...', 0, 0)
257
239
        self.repo._backup_inventory()
258
 
        ui.ui_factory.note(gettext('Backup inventory created.'))
 
240
        self.pb.note('Backup Inventory created.')
259
241
        new_inventories = self.repo._temp_inventories()
260
242
 
261
243
        # we have topological order of revisions and non ghost parents ready.
262
244
        self._setup_steps(len(self._rev_graph))
263
 
        revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
 
245
        revision_keys = [(rev_id,) for rev_id in
 
246
            TopoSorter(self._rev_graph.items()).iter_topo_order()]
264
247
        stream = self._change_inv_parents(
265
248
            self.inventory.get_record_stream(revision_keys, 'unordered', True),
266
249
            self._new_inv_parents,
271
254
        if not (set(new_inventories.keys()) ==
272
255
            set([(revid,) for revid in self.pending])):
273
256
            raise AssertionError()
274
 
        self.pb.update(gettext('Writing weave'))
 
257
        self.pb.update('Writing weave')
275
258
        self.repo._activate_new_inventory()
276
259
        self.inventory = None
277
 
        ui.ui_factory.note(gettext('Inventory regenerated.'))
 
260
        self.pb.note('Inventory regenerated.')
278
261
 
279
262
    def _new_inv_parents(self, revision_key):
280
263
        """Lookup ghost-filtered parents for revision_key."""
368
351
    def _load_indexes(self):
369
352
        """Load indexes for the reconciliation."""
370
353
        self.transaction = self.repo.get_transaction()
371
 
        self.pb.update(gettext('Reading indexes'), 0, 2)
 
354
        self.pb.update('Reading indexes.', 0, 2)
372
355
        self.inventory = self.repo.inventories
373
 
        self.pb.update(gettext('Reading indexes'), 1, 2)
 
356
        self.pb.update('Reading indexes.', 1, 2)
374
357
        self.repo._check_for_inconsistent_revision_parents()
375
358
        self.revisions = self.repo.revisions
376
 
        self.pb.update(gettext('Reading indexes'), 2, 2)
 
359
        self.pb.update('Reading indexes.', 2, 2)
377
360
 
378
361
    def _gc_inventory(self):
379
362
        """Remove inventories that are not referenced from the revision store."""
380
 
        self.pb.update(gettext('Checking unused inventories'), 0, 1)
 
363
        self.pb.update('Checking unused inventories.', 0, 1)
381
364
        self._check_garbage_inventories()
382
 
        self.pb.update(gettext('Checking unused inventories'), 1, 3)
 
365
        self.pb.update('Checking unused inventories.', 1, 3)
383
366
        if not self.garbage_inventories:
384
 
            ui.ui_factory.note(gettext('Inventory ok.'))
 
367
            self.pb.note('Inventory ok.')
385
368
            return
386
 
        self.pb.update(gettext('Backing up inventory'), 0, 0)
 
369
        self.pb.update('Backing up inventory...', 0, 0)
387
370
        self.repo._backup_inventory()
388
 
        ui.ui_factory.note(gettext('Backup Inventory created'))
 
371
        self.pb.note('Backup Inventory created.')
389
372
        # asking for '' should never return a non-empty weave
390
373
        new_inventories = self.repo._temp_inventories()
391
374
        # we have topological order of revisions and non ghost parents ready.
392
375
        graph = self.revisions.get_parent_map(self.revisions.keys())
393
 
        revision_keys = topo_sort(graph)
 
376
        revision_keys = list(TopoSorter(graph).iter_topo_order())
394
377
        revision_ids = [key[-1] for key in revision_keys]
395
378
        self._setup_steps(len(revision_keys))
396
379
        stream = self._change_inv_parents(
402
385
        # the revisionds list
403
386
        if not(set(new_inventories.keys()) == set(revision_keys)):
404
387
            raise AssertionError()
405
 
        self.pb.update(gettext('Writing weave'))
 
388
        self.pb.update('Writing weave')
406
389
        self.repo._activate_new_inventory()
407
390
        self.inventory = None
408
 
        ui.ui_factory.note(gettext('Inventory regenerated.'))
 
391
        self.pb.note('Inventory regenerated.')
409
392
 
410
393
    def _fix_text_parents(self):
411
394
        """Fix bad versionedfile parent entries.
443
426
            versions_list.append(text_key[1])
444
427
        # Do the reconcile of individual weaves.
445
428
        for num, file_id in enumerate(per_id_bad_parents):
446
 
            self.pb.update(gettext('Fixing text parents'), num,
 
429
            self.pb.update('Fixing text parents', num,
447
430
                           len(per_id_bad_parents))
448
431
            versions_with_bad_parents = per_id_bad_parents[file_id]
449
432
            id_unused_versions = set(key[-1] for key in unused_versions
507
490
    #  - lock the names list
508
491
    #  - perform a customised pack() that regenerates data as needed
509
492
    #  - unlock the names list
510
 
    # https://bugs.launchpad.net/bzr/+bug/154173
511
 
 
512
 
    def __init__(self, repo, other=None, thorough=False,
513
 
            canonicalize_chks=False):
514
 
        super(PackReconciler, self).__init__(repo, other=other,
515
 
            thorough=thorough)
516
 
        self.canonicalize_chks = canonicalize_chks
 
493
    # https://bugs.edge.launchpad.net/bzr/+bug/154173
517
494
 
518
495
    def _reconcile_steps(self):
519
496
        """Perform the steps to reconcile this repository."""
522
499
        collection = self.repo._pack_collection
523
500
        collection.ensure_loaded()
524
501
        collection.lock_names()
525
 
        self.add_cleanup(collection._unlock_names)
526
 
        packs = collection.all_packs()
527
 
        all_revisions = self.repo.all_revision_ids()
528
 
        total_inventories = len(list(
529
 
            collection.inventory_index.combined_index.iter_all_entries()))
530
 
        if len(all_revisions):
531
 
            if self.canonicalize_chks:
532
 
                reconcile_meth = self.repo._canonicalize_chks_pack
 
502
        try:
 
503
            packs = collection.all_packs()
 
504
            all_revisions = self.repo.all_revision_ids()
 
505
            total_inventories = len(list(
 
506
                collection.inventory_index.combined_index.iter_all_entries()))
 
507
            if len(all_revisions):
 
508
                self._packer = repofmt.pack_repo.ReconcilePacker(
 
509
                    collection, packs, ".reconcile", all_revisions)
 
510
                new_pack = self._packer.pack(pb=self.pb)
 
511
                if new_pack is not None:
 
512
                    self._discard_and_save(packs)
533
513
            else:
534
 
                reconcile_meth = self.repo._reconcile_pack
535
 
            new_pack = reconcile_meth(collection, packs, ".reconcile",
536
 
                all_revisions, self.pb)
537
 
            if new_pack is not None:
 
514
                # only make a new pack when there is data to copy.
538
515
                self._discard_and_save(packs)
539
 
        else:
540
 
            # only make a new pack when there is data to copy.
541
 
            self._discard_and_save(packs)
542
 
        self.garbage_inventories = total_inventories - len(list(
543
 
            collection.inventory_index.combined_index.iter_all_entries()))
 
516
            self.garbage_inventories = total_inventories - len(list(
 
517
                collection.inventory_index.combined_index.iter_all_entries()))
 
518
        finally:
 
519
            collection._unlock_names()
544
520
 
545
521
    def _discard_and_save(self, packs):
546
522
        """Discard some packs from the repository.