← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/reconcile.py

  • Committer: Tarmac
  • Author(s): Vincent Ladeuil
  • Date: 2017-01-30 14:42:05 UTC
  • mfrom: (6620.1.1 trunk)
  • Revision ID: tarmac-20170130144205-r8fh2xpmiuxyozpv
Merge  2.7 into trunk including fix for bug #1657238 [r=vila]

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/reconcile.py
1
 
# Copyright (C) 2005, 2006 Canonical Ltd
 
1
# Copyright (C) 2006-2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
 
17
17
"""Reconcilers are able to fix some potential data errors in a branch."""
18
18
 
 
19
from __future__ import absolute_import
19
20
 
20
21
__all__ = [
21
22
    'KnitReconciler',
27
28
 
28
29
 
29
30
from bzrlib import (
 
31
    cleanup,
30
32
    errors,
 
33
    revision as _mod_revision,
31
34
    ui,
32
 
    repository,
33
 
    repofmt,
34
35
    )
35
 
from bzrlib.trace import mutter, note
36
 
from bzrlib.tsort import TopoSorter
 
36
from bzrlib.trace import mutter
 
37
from bzrlib.tsort import topo_sort
37
38
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
38
 
 
39
 
 
40
 
def reconcile(dir, other=None):
 
39
from bzrlib.i18n import gettext
 
40
 
 
41
 
 
42
def reconcile(dir, canonicalize_chks=False):
41
43
    """Reconcile the data in dir.
42
44
 
43
45
    Currently this is limited to a inventory 'reweave'.
47
49
    Directly using Reconciler is recommended for library users that
48
50
    desire fine grained control or analysis of the found issues.
49
51
 
50
 
    :param other: another bzrdir to reconcile against.
 
52
    :param canonicalize_chks: Make sure CHKs are in canonical form.
51
53
    """
52
 
    reconciler = Reconciler(dir, other=other)
 
54
    reconciler = Reconciler(dir, canonicalize_chks=canonicalize_chks)
53
55
    reconciler.reconcile()
54
56
 
55
57
 
56
58
class Reconciler(object):
57
59
    """Reconcilers are used to reconcile existing data."""
58
60
 
59
 
    def __init__(self, dir, other=None):
 
61
    def __init__(self, dir, other=None, canonicalize_chks=False):
60
62
        """Create a Reconciler."""
61
63
        self.bzrdir = dir
 
64
        self.canonicalize_chks = canonicalize_chks
62
65
 
63
66
    def reconcile(self):
64
67
        """Perform reconciliation.
65
 
        
 
68
 
66
69
        After reconciliation the following attributes document found issues:
67
 
        inconsistent_parents: The number of revisions in the repository whose
68
 
                              ancestry was being reported incorrectly.
69
 
        garbage_inventories: The number of inventory objects without revisions
70
 
                             that were garbage collected.
71
 
        fixed_branch_history: None if there was no branch, False if the branch
72
 
                              history was correct, True if the branch history
73
 
                              needed to be re-normalized.
 
70
 
 
71
        * `inconsistent_parents`: The number of revisions in the repository
 
72
          whose ancestry was being reported incorrectly.
 
73
        * `garbage_inventories`: The number of inventory objects without
 
74
          revisions that were garbage collected.
 
75
        * `fixed_branch_history`: None if there was no branch, False if the
 
76
          branch history was correct, True if the branch history needed to be
 
77
          re-normalized.
74
78
        """
75
79
        self.pb = ui.ui_factory.nested_progress_bar()
76
80
        try:
90
94
            # Nothing to check here
91
95
            self.fixed_branch_history = None
92
96
            return
93
 
        self.pb.note('Reconciling branch %s',
94
 
                     self.branch.base)
 
97
        ui.ui_factory.note(gettext('Reconciling branch %s') % self.branch.base)
95
98
        branch_reconciler = self.branch.reconcile(thorough=True)
96
99
        self.fixed_branch_history = branch_reconciler.fixed_history
97
100
 
98
101
    def _reconcile_repository(self):
99
102
        self.repo = self.bzrdir.find_repository()
100
 
        self.pb.note('Reconciling repository %s',
101
 
                     self.repo.bzrdir.root_transport.base)
102
 
        self.pb.update("Reconciling repository", 0, 1)
103
 
        repo_reconciler = self.repo.reconcile(thorough=True)
 
103
        ui.ui_factory.note(gettext('Reconciling repository %s') %
 
104
            self.repo.user_url)
 
105
        self.pb.update(gettext("Reconciling repository"), 0, 1)
 
106
        if self.canonicalize_chks:
 
107
            try:
 
108
                self.repo.reconcile_canonicalize_chks
 
109
            except AttributeError:
 
110
                raise errors.BzrError(
 
111
                    gettext("%s cannot canonicalize CHKs.") % (self.repo,))
 
112
            repo_reconciler = self.repo.reconcile_canonicalize_chks()
 
113
        else:
 
114
            repo_reconciler = self.repo.reconcile(thorough=True)
104
115
        self.inconsistent_parents = repo_reconciler.inconsistent_parents
105
116
        self.garbage_inventories = repo_reconciler.garbage_inventories
106
117
        if repo_reconciler.aborted:
107
 
            self.pb.note(
108
 
                'Reconcile aborted: revision index has inconsistent parents.')
109
 
            self.pb.note(
110
 
                'Run "bzr check" for more details.')
 
118
            ui.ui_factory.note(gettext(
 
119
                'Reconcile aborted: revision index has inconsistent parents.'))
 
120
            ui.ui_factory.note(gettext(
 
121
                'Run "bzr check" for more details.'))
111
122
        else:
112
 
            self.pb.note('Reconciliation complete.')
 
123
            ui.ui_factory.note(gettext('Reconciliation complete.'))
113
124
 
114
125
 
115
126
class BranchReconciler(object):
121
132
        self.branch = a_branch
122
133
 
123
134
    def reconcile(self):
 
135
        operation = cleanup.OperationWithCleanups(self._reconcile)
 
136
        self.add_cleanup = operation.add_cleanup
 
137
        operation.run_simple()
 
138
 
 
139
    def _reconcile(self):
124
140
        self.branch.lock_write()
125
 
        try:
126
 
            self.pb = ui.ui_factory.nested_progress_bar()
127
 
            try:
128
 
                self._reconcile_steps()
129
 
            finally:
130
 
                self.pb.finished()
131
 
        finally:
132
 
            self.branch.unlock()
 
141
        self.add_cleanup(self.branch.unlock)
 
142
        self.pb = ui.ui_factory.nested_progress_bar()
 
143
        self.add_cleanup(self.pb.finished)
 
144
        self._reconcile_steps()
133
145
 
134
146
    def _reconcile_steps(self):
135
147
        self._reconcile_revision_history()
136
148
 
137
149
    def _reconcile_revision_history(self):
138
 
        repo = self.branch.repository
139
150
        last_revno, last_revision_id = self.branch.last_revision_info()
140
 
        real_history = list(repo.iter_reverse_revision_history(
141
 
                                last_revision_id))
 
151
        real_history = []
 
152
        graph = self.branch.repository.get_graph()
 
153
        try:
 
154
            for revid in graph.iter_lefthand_ancestry(
 
155
                    last_revision_id, (_mod_revision.NULL_REVISION,)):
 
156
                real_history.append(revid)
 
157
        except errors.RevisionNotPresent:
 
158
            pass # Hit a ghost left hand parent
142
159
        real_history.reverse()
143
160
        if last_revno != len(real_history):
144
161
            self.fixed_history = True
146
163
            # set_revision_history, as this will regenerate it again.
147
164
            # Not really worth a whole BranchReconciler class just for this,
148
165
            # though.
149
 
            self.pb.note('Fixing last revision info %s => %s',
150
 
                         last_revno, len(real_history))
 
166
            ui.ui_factory.note(gettext('Fixing last revision info {0} '\
 
167
                                       ' => {1}').format(
 
168
                                       last_revno, len(real_history)))
151
169
            self.branch.set_last_revision_info(len(real_history),
152
170
                                               last_revision_id)
153
171
        else:
154
172
            self.fixed_history = False
155
 
            self.pb.note('revision_history ok.')
 
173
            ui.ui_factory.note(gettext('revision_history ok.'))
156
174
 
157
175
 
158
176
class RepoReconciler(object):
159
177
    """Reconciler that reconciles a repository.
160
178
 
161
179
    The goal of repository reconciliation is to make any derived data
162
 
    consistent with the core data committed by a user. This can involve 
 
180
    consistent with the core data committed by a user. This can involve
163
181
    reindexing, or removing unreferenced data if that can interfere with
164
182
    queries in a given repository.
165
183
 
181
199
 
182
200
    def reconcile(self):
183
201
        """Perform reconciliation.
184
 
        
 
202
 
185
203
        After reconciliation the following attributes document found issues:
186
 
        inconsistent_parents: The number of revisions in the repository whose
187
 
                              ancestry was being reported incorrectly.
188
 
        garbage_inventories: The number of inventory objects without revisions
189
 
                             that were garbage collected.
 
204
 
 
205
        * `inconsistent_parents`: The number of revisions in the repository
 
206
          whose ancestry was being reported incorrectly.
 
207
        * `garbage_inventories`: The number of inventory objects without
 
208
          revisions that were garbage collected.
190
209
        """
 
210
        operation = cleanup.OperationWithCleanups(self._reconcile)
 
211
        self.add_cleanup = operation.add_cleanup
 
212
        operation.run_simple()
 
213
 
 
214
    def _reconcile(self):
191
215
        self.repo.lock_write()
192
 
        try:
193
 
            self.pb = ui.ui_factory.nested_progress_bar()
194
 
            try:
195
 
                self._reconcile_steps()
196
 
            finally:
197
 
                self.pb.finished()
198
 
        finally:
199
 
            self.repo.unlock()
 
216
        self.add_cleanup(self.repo.unlock)
 
217
        self.pb = ui.ui_factory.nested_progress_bar()
 
218
        self.add_cleanup(self.pb.finished)
 
219
        self._reconcile_steps()
200
220
 
201
221
    def _reconcile_steps(self):
202
222
        """Perform the steps to reconcile this repository."""
204
224
 
205
225
    def _reweave_inventory(self):
206
226
        """Regenerate the inventory weave for the repository from scratch.
207
 
        
208
 
        This is a smart function: it will only do the reweave if doing it 
 
227
 
 
228
        This is a smart function: it will only do the reweave if doing it
209
229
        will correct data issues. The self.thorough flag controls whether
210
230
        only data-loss causing issues (!self.thorough) or all issues
211
231
        (self.thorough) are treated as requiring the reweave.
212
232
        """
213
 
        # local because needing to know about WeaveFile is a wart we want to hide
214
 
        from bzrlib.weave import WeaveFile, Weave
215
233
        transaction = self.repo.get_transaction()
216
 
        self.pb.update('Reading inventory data.')
 
234
        self.pb.update(gettext('Reading inventory data'))
217
235
        self.inventory = self.repo.inventories
218
236
        self.revisions = self.repo.revisions
219
237
        # the total set of revisions to process
229
247
            # put a revision into the graph.
230
248
            self._graph_revision(rev_id)
231
249
        self._check_garbage_inventories()
232
 
        # if there are no inconsistent_parents and 
 
250
        # if there are no inconsistent_parents and
233
251
        # (no garbage inventories or we are not doing a thorough check)
234
 
        if (not self.inconsistent_parents and 
 
252
        if (not self.inconsistent_parents and
235
253
            (not self.garbage_inventories or not self.thorough)):
236
 
            self.pb.note('Inventory ok.')
 
254
            ui.ui_factory.note(gettext('Inventory ok.'))
237
255
            return
238
 
        self.pb.update('Backing up inventory...', 0, 0)
 
256
        self.pb.update(gettext('Backing up inventory'), 0, 0)
239
257
        self.repo._backup_inventory()
240
 
        self.pb.note('Backup Inventory created.')
 
258
        ui.ui_factory.note(gettext('Backup inventory created.'))
241
259
        new_inventories = self.repo._temp_inventories()
242
260
 
243
261
        # we have topological order of revisions and non ghost parents ready.
244
262
        self._setup_steps(len(self._rev_graph))
245
 
        revision_keys = [(rev_id,) for rev_id in
246
 
            TopoSorter(self._rev_graph.items()).iter_topo_order()]
 
263
        revision_keys = [(rev_id,) for rev_id in topo_sort(self._rev_graph)]
247
264
        stream = self._change_inv_parents(
248
265
            self.inventory.get_record_stream(revision_keys, 'unordered', True),
249
266
            self._new_inv_parents,
254
271
        if not (set(new_inventories.keys()) ==
255
272
            set([(revid,) for revid in self.pending])):
256
273
            raise AssertionError()
257
 
        self.pb.update('Writing weave')
 
274
        self.pb.update(gettext('Writing weave'))
258
275
        self.repo._activate_new_inventory()
259
276
        self.inventory = None
260
 
        self.pb.note('Inventory regenerated.')
 
277
        ui.ui_factory.note(gettext('Inventory regenerated.'))
261
278
 
262
279
    def _new_inv_parents(self, revision_key):
263
280
        """Lookup ghost-filtered parents for revision_key."""
351
368
    def _load_indexes(self):
352
369
        """Load indexes for the reconciliation."""
353
370
        self.transaction = self.repo.get_transaction()
354
 
        self.pb.update('Reading indexes.', 0, 2)
 
371
        self.pb.update(gettext('Reading indexes'), 0, 2)
355
372
        self.inventory = self.repo.inventories
356
 
        self.pb.update('Reading indexes.', 1, 2)
 
373
        self.pb.update(gettext('Reading indexes'), 1, 2)
357
374
        self.repo._check_for_inconsistent_revision_parents()
358
375
        self.revisions = self.repo.revisions
359
 
        self.pb.update('Reading indexes.', 2, 2)
 
376
        self.pb.update(gettext('Reading indexes'), 2, 2)
360
377
 
361
378
    def _gc_inventory(self):
362
379
        """Remove inventories that are not referenced from the revision store."""
363
 
        self.pb.update('Checking unused inventories.', 0, 1)
 
380
        self.pb.update(gettext('Checking unused inventories'), 0, 1)
364
381
        self._check_garbage_inventories()
365
 
        self.pb.update('Checking unused inventories.', 1, 3)
 
382
        self.pb.update(gettext('Checking unused inventories'), 1, 3)
366
383
        if not self.garbage_inventories:
367
 
            self.pb.note('Inventory ok.')
 
384
            ui.ui_factory.note(gettext('Inventory ok.'))
368
385
            return
369
 
        self.pb.update('Backing up inventory...', 0, 0)
 
386
        self.pb.update(gettext('Backing up inventory'), 0, 0)
370
387
        self.repo._backup_inventory()
371
 
        self.pb.note('Backup Inventory created.')
 
388
        ui.ui_factory.note(gettext('Backup Inventory created'))
372
389
        # asking for '' should never return a non-empty weave
373
390
        new_inventories = self.repo._temp_inventories()
374
391
        # we have topological order of revisions and non ghost parents ready.
375
392
        graph = self.revisions.get_parent_map(self.revisions.keys())
376
 
        revision_keys = list(TopoSorter(graph).iter_topo_order())
 
393
        revision_keys = topo_sort(graph)
377
394
        revision_ids = [key[-1] for key in revision_keys]
378
395
        self._setup_steps(len(revision_keys))
379
396
        stream = self._change_inv_parents(
385
402
        # the revisionds list
386
403
        if not(set(new_inventories.keys()) == set(revision_keys)):
387
404
            raise AssertionError()
388
 
        self.pb.update('Writing weave')
 
405
        self.pb.update(gettext('Writing weave'))
389
406
        self.repo._activate_new_inventory()
390
407
        self.inventory = None
391
 
        self.pb.note('Inventory regenerated.')
 
408
        ui.ui_factory.note(gettext('Inventory regenerated.'))
392
409
 
393
410
    def _fix_text_parents(self):
394
411
        """Fix bad versionedfile parent entries.
426
443
            versions_list.append(text_key[1])
427
444
        # Do the reconcile of individual weaves.
428
445
        for num, file_id in enumerate(per_id_bad_parents):
429
 
            self.pb.update('Fixing text parents', num,
 
446
            self.pb.update(gettext('Fixing text parents'), num,
430
447
                           len(per_id_bad_parents))
431
448
            versions_with_bad_parents = per_id_bad_parents[file_id]
432
449
            id_unused_versions = set(key[-1] for key in unused_versions
490
507
    #  - lock the names list
491
508
    #  - perform a customised pack() that regenerates data as needed
492
509
    #  - unlock the names list
493
 
    # https://bugs.edge.launchpad.net/bzr/+bug/154173
 
510
    # https://bugs.launchpad.net/bzr/+bug/154173
 
511
 
 
512
    def __init__(self, repo, other=None, thorough=False,
 
513
            canonicalize_chks=False):
 
514
        super(PackReconciler, self).__init__(repo, other=other,
 
515
            thorough=thorough)
 
516
        self.canonicalize_chks = canonicalize_chks
494
517
 
495
518
    def _reconcile_steps(self):
496
519
        """Perform the steps to reconcile this repository."""
499
522
        collection = self.repo._pack_collection
500
523
        collection.ensure_loaded()
501
524
        collection.lock_names()
502
 
        try:
503
 
            packs = collection.all_packs()
504
 
            all_revisions = self.repo.all_revision_ids()
505
 
            total_inventories = len(list(
506
 
                collection.inventory_index.combined_index.iter_all_entries()))
507
 
            if len(all_revisions):
508
 
                self._packer = repofmt.pack_repo.ReconcilePacker(
509
 
                    collection, packs, ".reconcile", all_revisions)
510
 
                new_pack = self._packer.pack(pb=self.pb)
511
 
                if new_pack is not None:
512
 
                    self._discard_and_save(packs)
 
525
        self.add_cleanup(collection._unlock_names)
 
526
        packs = collection.all_packs()
 
527
        all_revisions = self.repo.all_revision_ids()
 
528
        total_inventories = len(list(
 
529
            collection.inventory_index.combined_index.iter_all_entries()))
 
530
        if len(all_revisions):
 
531
            if self.canonicalize_chks:
 
532
                reconcile_meth = self.repo._canonicalize_chks_pack
513
533
            else:
514
 
                # only make a new pack when there is data to copy.
 
534
                reconcile_meth = self.repo._reconcile_pack
 
535
            new_pack = reconcile_meth(collection, packs, ".reconcile",
 
536
                all_revisions, self.pb)
 
537
            if new_pack is not None:
515
538
                self._discard_and_save(packs)
516
 
            self.garbage_inventories = total_inventories - len(list(
517
 
                collection.inventory_index.combined_index.iter_all_entries()))
518
 
        finally:
519
 
            collection._unlock_names()
 
539
        else:
 
540
            # only make a new pack when there is data to copy.
 
541
            self._discard_and_save(packs)
 
542
        self.garbage_inventories = total_inventories - len(list(
 
543
            collection.inventory_index.combined_index.iter_all_entries()))
520
544
 
521
545
    def _discard_and_save(self, packs):
522
546
        """Discard some packs from the repository.