← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/reconcile.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2007-10-25 08:29:08 UTC
  • mfrom: (2940.1.2 ianc-integration)
  • Revision ID: pqm@pqm.ubuntu.com-20071025082908-abn3kunrb2ivdvth
renaming of experimental pack formats to include knitpack in their name (Ian Clatworthy)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/reconcile.py
30
30
    errors,
31
31
    ui,
32
32
    repository,
33
 
    repofmt,
34
33
    )
35
34
from bzrlib.trace import mutter, note
36
35
from bzrlib.tsort import TopoSorter
37
 
from bzrlib.versionedfile import AdapterFactory, FulltextContentFactory
38
36
 
39
37
 
40
38
def reconcile(dir, other=None):
68
66
                              ancestry was being reported incorrectly.
69
67
        garbage_inventories: The number of inventory objects without revisions
70
68
                             that were garbage collected.
71
 
        fixed_branch_history: None if there was no branch, False if the branch
72
 
                              history was correct, True if the branch history
73
 
                              needed to be re-normalized.
74
69
        """
75
70
        self.pb = ui.ui_factory.nested_progress_bar()
76
71
        try:
80
75
 
81
76
    def _reconcile(self):
82
77
        """Helper function for performing reconciliation."""
83
 
        self._reconcile_branch()
84
 
        self._reconcile_repository()
85
 
 
86
 
    def _reconcile_branch(self):
87
 
        try:
88
 
            self.branch = self.bzrdir.open_branch()
89
 
        except errors.NotBranchError:
90
 
            # Nothing to check here
91
 
            self.fixed_branch_history = None
92
 
            return
93
 
        self.pb.note('Reconciling branch %s',
94
 
                     self.branch.base)
95
 
        branch_reconciler = self.branch.reconcile(thorough=True)
96
 
        self.fixed_branch_history = branch_reconciler.fixed_history
97
 
 
98
 
    def _reconcile_repository(self):
99
78
        self.repo = self.bzrdir.find_repository()
100
79
        self.pb.note('Reconciling repository %s',
101
80
                     self.repo.bzrdir.root_transport.base)
102
 
        self.pb.update("Reconciling repository", 0, 1)
103
81
        repo_reconciler = self.repo.reconcile(thorough=True)
104
82
        self.inconsistent_parents = repo_reconciler.inconsistent_parents
105
83
        self.garbage_inventories = repo_reconciler.garbage_inventories
112
90
            self.pb.note('Reconciliation complete.')
113
91
 
114
92
 
115
 
class BranchReconciler(object):
116
 
    """Reconciler that works on a branch."""
117
 
 
118
 
    def __init__(self, a_branch, thorough=False):
119
 
        self.fixed_history = None
120
 
        self.thorough = thorough
121
 
        self.branch = a_branch
122
 
 
123
 
    def reconcile(self):
124
 
        self.branch.lock_write()
125
 
        try:
126
 
            self.pb = ui.ui_factory.nested_progress_bar()
127
 
            try:
128
 
                self._reconcile_steps()
129
 
            finally:
130
 
                self.pb.finished()
131
 
        finally:
132
 
            self.branch.unlock()
133
 
 
134
 
    def _reconcile_steps(self):
135
 
        self._reconcile_revision_history()
136
 
 
137
 
    def _reconcile_revision_history(self):
138
 
        repo = self.branch.repository
139
 
        last_revno, last_revision_id = self.branch.last_revision_info()
140
 
        real_history = list(repo.iter_reverse_revision_history(
141
 
                                last_revision_id))
142
 
        real_history.reverse()
143
 
        if last_revno != len(real_history):
144
 
            self.fixed_history = True
145
 
            # Technically for Branch5 formats, it is more efficient to use
146
 
            # set_revision_history, as this will regenerate it again.
147
 
            # Not really worth a whole BranchReconciler class just for this,
148
 
            # though.
149
 
            self.pb.note('Fixing last revision info %s => %s',
150
 
                         last_revno, len(real_history))
151
 
            self.branch.set_last_revision_info(len(real_history),
152
 
                                               last_revision_id)
153
 
        else:
154
 
            self.fixed_history = False
155
 
            self.pb.note('revision_history ok.')
156
 
 
157
 
 
158
93
class RepoReconciler(object):
159
94
    """Reconciler that reconciles a repository.
160
95
 
214
149
        from bzrlib.weave import WeaveFile, Weave
215
150
        transaction = self.repo.get_transaction()
216
151
        self.pb.update('Reading inventory data.')
217
 
        self.inventory = self.repo.inventories
218
 
        self.revisions = self.repo.revisions
 
152
        self.inventory = self.repo.get_inventory_weave()
219
153
        # the total set of revisions to process
220
 
        self.pending = set([key[-1] for key in self.revisions.keys()])
 
154
        self.pending = set([rev_id for rev_id in self.repo._revision_store.all_revision_ids(transaction)])
221
155
 
222
156
        # mapping from revision_id to parents
223
157
        self._rev_graph = {}
236
170
            self.pb.note('Inventory ok.')
237
171
            return
238
172
        self.pb.update('Backing up inventory...', 0, 0)
239
 
        self.repo._backup_inventory()
 
173
        self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.repo.get_transaction())
240
174
        self.pb.note('Backup Inventory created.')
241
 
        new_inventories = self.repo._temp_inventories()
 
175
        # asking for '' should never return a non-empty weave
 
176
        new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
 
177
            self.repo.get_transaction())
242
178
 
243
179
        # we have topological order of revisions and non ghost parents ready.
244
180
        self._setup_steps(len(self._rev_graph))
245
 
        revision_keys = [(rev_id,) for rev_id in
246
 
            TopoSorter(self._rev_graph.items()).iter_topo_order()]
247
 
        stream = self._change_inv_parents(
248
 
            self.inventory.get_record_stream(revision_keys, 'unordered', True),
249
 
            self._new_inv_parents,
250
 
            set(revision_keys))
251
 
        new_inventories.insert_record_stream(stream)
252
 
        # if this worked, the set of new_inventories.keys should equal
 
181
        for rev_id in TopoSorter(self._rev_graph.items()).iter_topo_order():
 
182
            parents = self._rev_graph[rev_id]
 
183
            # double check this really is in topological order.
 
184
            unavailable = [p for p in parents if p not in new_inventory_vf]
 
185
            assert len(unavailable) == 0
 
186
            # this entry has all the non ghost parents in the inventory
 
187
            # file already.
 
188
            self._reweave_step('adding inventories')
 
189
            if isinstance(new_inventory_vf, WeaveFile):
 
190
                # It's really a WeaveFile, but we call straight into the
 
191
                # Weave's add method to disable the auto-write-out behaviour.
 
192
                # This is done to avoid a revision_count * time-to-write additional overhead on 
 
193
                # reconcile.
 
194
                new_inventory_vf._check_write_ok()
 
195
                Weave._add_lines(new_inventory_vf, rev_id, parents,
 
196
                    self.inventory.get_lines(rev_id), None, None, None, False, True)
 
197
            else:
 
198
                new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
 
199
 
 
200
        if isinstance(new_inventory_vf, WeaveFile):
 
201
            new_inventory_vf._save()
 
202
        # if this worked, the set of new_inventory_vf.names should equal
253
203
        # self.pending
254
 
        if not (set(new_inventories.keys()) ==
255
 
            set([(revid,) for revid in self.pending])):
256
 
            raise AssertionError()
 
204
        assert set(new_inventory_vf.versions()) == self.pending
257
205
        self.pb.update('Writing weave')
258
 
        self.repo._activate_new_inventory()
 
206
        self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.repo.get_transaction())
 
207
        self.repo.control_weaves.delete('inventory.new', self.repo.get_transaction())
259
208
        self.inventory = None
260
209
        self.pb.note('Inventory regenerated.')
261
210
 
262
 
    def _new_inv_parents(self, revision_key):
263
 
        """Lookup ghost-filtered parents for revision_key."""
264
 
        # Use the filtered ghostless parents list:
265
 
        return tuple([(revid,) for revid in self._rev_graph[revision_key[-1]]])
266
 
 
267
 
    def _change_inv_parents(self, stream, get_parents, all_revision_keys):
268
 
        """Adapt a record stream to reconcile the parents."""
269
 
        for record in stream:
270
 
            wanted_parents = get_parents(record.key)
271
 
            if wanted_parents and wanted_parents[0] not in all_revision_keys:
272
 
                # The check for the left most parent only handles knit
273
 
                # compressors, but this code only applies to knit and weave
274
 
                # repositories anyway.
275
 
                bytes = record.get_bytes_as('fulltext')
276
 
                yield FulltextContentFactory(record.key, wanted_parents, record.sha1, bytes)
277
 
            else:
278
 
                adapted_record = AdapterFactory(record.key, wanted_parents, record)
279
 
                yield adapted_record
280
 
            self._reweave_step('adding inventories')
281
 
 
282
211
    def _setup_steps(self, new_total):
283
212
        """Setup the markers we need to control the progress bar."""
284
213
        self.total = new_total
290
219
        # analyse revision id rev_id and put it in the stack.
291
220
        self._reweave_step('loading revisions')
292
221
        rev = self.repo.get_revision_reconcile(rev_id)
 
222
        assert rev.revision_id == rev_id
293
223
        parents = []
294
224
        for parent in rev.parent_ids:
295
225
            if self._parent_is_available(parent):
296
226
                parents.append(parent)
297
227
            else:
298
228
                mutter('found ghost %s', parent)
299
 
        self._rev_graph[rev_id] = parents
 
229
        self._rev_graph[rev_id] = parents   
 
230
        if self._parents_are_inconsistent(rev_id, parents):
 
231
            self.inconsistent_parents += 1
 
232
            mutter('Inconsistent inventory parents: id {%s} '
 
233
                   'inventory claims %r, '
 
234
                   'available parents are %r, '
 
235
                   'unavailable parents are %r',
 
236
                   rev_id, 
 
237
                   set(self.inventory.get_parents(rev_id)),
 
238
                   set(parents),
 
239
                   set(rev.parent_ids).difference(set(parents)))
 
240
 
 
241
    def _parents_are_inconsistent(self, rev_id, parents):
 
242
        """Return True if the parents list of rev_id does not match the weave.
 
243
 
 
244
        This detects inconsistencies based on the self.thorough value:
 
245
        if thorough is on, the first parent value is checked as well as ghost
 
246
        differences.
 
247
        Otherwise only the ghost differences are evaluated.
 
248
        """
 
249
        weave_parents = self.inventory.get_parents(rev_id)
 
250
        weave_missing_old_ghosts = set(weave_parents) != set(parents)
 
251
        first_parent_is_wrong = (
 
252
            len(weave_parents) and len(parents) and
 
253
            parents[0] != weave_parents[0])
 
254
        if self.thorough:
 
255
            return weave_missing_old_ghosts or first_parent_is_wrong
 
256
        else:
 
257
            return weave_missing_old_ghosts
300
258
 
301
259
    def _check_garbage_inventories(self):
302
260
        """Check for garbage inventories which we cannot trust
306
264
        """
307
265
        if not self.thorough:
308
266
            return
309
 
        inventories = set(self.inventory.keys())
310
 
        revisions = set(self.revisions.keys())
 
267
        inventories = set(self.inventory.versions())
 
268
        revisions = set(self._rev_graph.keys())
311
269
        garbage = inventories.difference(revisions)
312
270
        self.garbage_inventories = len(garbage)
313
 
        for revision_key in garbage:
314
 
            mutter('Garbage inventory {%s} found.', revision_key[-1])
 
271
        for revision_id in garbage:
 
272
            mutter('Garbage inventory {%s} found.', revision_id)
315
273
 
316
274
    def _parent_is_available(self, parent):
317
275
        """True if parent is a fully available revision
319
277
        A fully available revision has a inventory and a revision object in the
320
278
        repository.
321
279
        """
322
 
        if parent in self._rev_graph:
323
 
            return True
324
 
        inv_present = (1 == len(self.inventory.get_parent_map([(parent,)])))
325
 
        return (inv_present and self.repo.has_revision(parent))
 
280
        return (parent in self._rev_graph or 
 
281
                (parent in self.inventory and self.repo.has_revision(parent)))
326
282
 
327
283
    def _reweave_step(self, message):
328
284
        """Mark a single step of regeneration complete."""
352
308
        """Load indexes for the reconciliation."""
353
309
        self.transaction = self.repo.get_transaction()
354
310
        self.pb.update('Reading indexes.', 0, 2)
355
 
        self.inventory = self.repo.inventories
 
311
        self.inventory = self.repo.get_inventory_weave()
356
312
        self.pb.update('Reading indexes.', 1, 2)
357
313
        self.repo._check_for_inconsistent_revision_parents()
358
 
        self.revisions = self.repo.revisions
 
314
        self.revisions = self.repo._revision_store.get_revision_file(self.transaction)
359
315
        self.pb.update('Reading indexes.', 2, 2)
360
316
 
361
317
    def _gc_inventory(self):
367
323
            self.pb.note('Inventory ok.')
368
324
            return
369
325
        self.pb.update('Backing up inventory...', 0, 0)
370
 
        self.repo._backup_inventory()
 
326
        self.repo.control_weaves.copy(self.inventory, 'inventory.backup', self.transaction)
371
327
        self.pb.note('Backup Inventory created.')
372
328
        # asking for '' should never return a non-empty weave
373
 
        new_inventories = self.repo._temp_inventories()
 
329
        new_inventory_vf = self.repo.control_weaves.get_empty('inventory.new',
 
330
            self.transaction)
 
331
 
374
332
        # we have topological order of revisions and non ghost parents ready.
375
 
        graph = self.revisions.get_parent_map(self.revisions.keys())
376
 
        revision_keys = list(TopoSorter(graph).iter_topo_order())
377
 
        revision_ids = [key[-1] for key in revision_keys]
378
 
        self._setup_steps(len(revision_keys))
379
 
        stream = self._change_inv_parents(
380
 
            self.inventory.get_record_stream(revision_keys, 'unordered', True),
381
 
            graph.__getitem__,
382
 
            set(revision_keys))
383
 
        new_inventories.insert_record_stream(stream)
 
333
        self._setup_steps(len(self.revisions))
 
334
        for rev_id in TopoSorter(self.revisions.get_graph().items()).iter_topo_order():
 
335
            parents = self.revisions.get_parents(rev_id)
 
336
            # double check this really is in topological order.
 
337
            unavailable = [p for p in parents if p not in new_inventory_vf]
 
338
            assert len(unavailable) == 0
 
339
            # this entry has all the non ghost parents in the inventory
 
340
            # file already.
 
341
            self._reweave_step('adding inventories')
 
342
            # ugly but needed, weaves are just way tooooo slow else.
 
343
            new_inventory_vf.add_lines(rev_id, parents, self.inventory.get_lines(rev_id))
 
344
 
384
345
        # if this worked, the set of new_inventory_vf.names should equal
385
 
        # the revisionds list
386
 
        if not(set(new_inventories.keys()) == set(revision_keys)):
387
 
            raise AssertionError()
 
346
        # self.pending
 
347
        assert set(new_inventory_vf.versions()) == set(self.revisions.versions())
388
348
        self.pb.update('Writing weave')
389
 
        self.repo._activate_new_inventory()
 
349
        self.repo.control_weaves.copy(new_inventory_vf, 'inventory', self.transaction)
 
350
        self.repo.control_weaves.delete('inventory.new', self.transaction)
390
351
        self.inventory = None
391
352
        self.pb.note('Inventory regenerated.')
392
353
 
 
354
    def _check_garbage_inventories(self):
 
355
        """Check for garbage inventories which we cannot trust
 
356
 
 
357
        We cant trust them because their pre-requisite file data may not
 
358
        be present - all we know is that their revision was not installed.
 
359
        """
 
360
        inventories = set(self.inventory.versions())
 
361
        revisions = set(self.revisions.versions())
 
362
        garbage = inventories.difference(revisions)
 
363
        self.garbage_inventories = len(garbage)
 
364
        for revision_id in garbage:
 
365
            mutter('Garbage inventory {%s} found.', revision_id)
 
366
 
393
367
    def _fix_text_parents(self):
394
368
        """Fix bad versionedfile parent entries.
395
369
 
400
374
        parent lists, and replaces the versionedfile with a corrected version.
401
375
        """
402
376
        transaction = self.repo.get_transaction()
403
 
        versions = [key[-1] for key in self.revisions.keys()]
404
 
        mutter('Prepopulating revision text cache with %d revisions',
405
 
                len(versions))
406
 
        vf_checker = self.repo._get_versioned_file_checker()
407
 
        bad_parents, unused_versions = vf_checker.check_file_version_parents(
408
 
            self.repo.texts, self.pb)
409
 
        text_index = vf_checker.text_index
410
 
        per_id_bad_parents = {}
411
 
        for key in unused_versions:
412
 
            # Ensure that every file with unused versions gets rewritten.
413
 
            # NB: This is really not needed, reconcile != pack.
414
 
            per_id_bad_parents[key[0]] = {}
415
 
        # Generate per-knit/weave data.
416
 
        for key, details in bad_parents.iteritems():
417
 
            file_id = key[0]
418
 
            rev_id = key[1]
419
 
            knit_parents = tuple([parent[-1] for parent in details[0]])
420
 
            correct_parents = tuple([parent[-1] for parent in details[1]])
421
 
            file_details = per_id_bad_parents.setdefault(file_id, {})
422
 
            file_details[rev_id] = (knit_parents, correct_parents)
423
 
        file_id_versions = {}
424
 
        for text_key in text_index:
425
 
            versions_list = file_id_versions.setdefault(text_key[0], [])
426
 
            versions_list.append(text_key[1])
427
 
        # Do the reconcile of individual weaves.
428
 
        for num, file_id in enumerate(per_id_bad_parents):
 
377
        revision_versions = repository._RevisionTextVersionCache(self.repo)
 
378
        versions = self.revisions.versions()
 
379
        revision_versions.prepopulate_revs(versions)
 
380
        for num, file_id in enumerate(self.repo.weave_store):
429
381
            self.pb.update('Fixing text parents', num,
430
 
                           len(per_id_bad_parents))
431
 
            versions_with_bad_parents = per_id_bad_parents[file_id]
432
 
            id_unused_versions = set(key[-1] for key in unused_versions
433
 
                if key[0] == file_id)
434
 
            if file_id in file_id_versions:
435
 
                file_versions = file_id_versions[file_id]
436
 
            else:
437
 
                # This id was present in the disk store but is not referenced
438
 
                # by any revision at all.
439
 
                file_versions = []
440
 
            self._fix_text_parent(file_id, versions_with_bad_parents,
441
 
                 id_unused_versions, file_versions)
 
382
                           len(self.repo.weave_store))
 
383
            vf = self.repo.weave_store.get_weave(file_id, transaction)
 
384
            vf_checker = self.repo.get_versioned_file_checker(
 
385
                versions, revision_versions)
 
386
            versions_with_bad_parents = vf_checker.check_file_version_parents(
 
387
                vf, file_id)
 
388
            if len(versions_with_bad_parents) == 0:
 
389
                continue
 
390
            self._fix_text_parent(file_id, vf, versions_with_bad_parents)
442
391
 
443
 
    def _fix_text_parent(self, file_id, versions_with_bad_parents,
444
 
            unused_versions, all_versions):
 
392
    def _fix_text_parent(self, file_id, vf, versions_with_bad_parents):
445
393
        """Fix bad versionedfile entries in a single versioned file."""
446
 
        mutter('fixing text parent: %r (%d versions)', file_id,
447
 
                len(versions_with_bad_parents))
448
 
        mutter('(%d are unused)', len(unused_versions))
449
 
        new_file_id = 'temp:%s' % file_id
 
394
        new_vf = self.repo.weave_store.get_empty('temp:%s' % file_id,
 
395
            self.transaction)
450
396
        new_parents = {}
451
 
        needed_keys = set()
452
 
        for version in all_versions:
453
 
            if version in unused_versions:
454
 
                continue
455
 
            elif version in versions_with_bad_parents:
 
397
        for version in vf.versions():
 
398
            if version in versions_with_bad_parents:
456
399
                parents = versions_with_bad_parents[version][1]
457
400
            else:
458
 
                pmap = self.repo.texts.get_parent_map([(file_id, version)])
459
 
                parents = [key[-1] for key in pmap[(file_id, version)]]
460
 
            new_parents[(new_file_id, version)] = [
461
 
                (new_file_id, parent) for parent in parents]
462
 
            needed_keys.add((file_id, version))
463
 
        def fix_parents(stream):
464
 
            for record in stream:
465
 
                bytes = record.get_bytes_as('fulltext')
466
 
                new_key = (new_file_id, record.key[-1])
467
 
                parents = new_parents[new_key]
468
 
                yield FulltextContentFactory(new_key, parents, record.sha1, bytes)
469
 
        stream = self.repo.texts.get_record_stream(needed_keys, 'topological', True)
470
 
        self.repo._remove_file_id(new_file_id)
471
 
        self.repo.texts.insert_record_stream(fix_parents(stream))
472
 
        self.repo._remove_file_id(file_id)
473
 
        if len(new_parents):
474
 
            self.repo._move_file_id(new_file_id, file_id)
 
401
                parents = vf.get_parents(version)
 
402
            new_parents[version] = parents
 
403
        for version in TopoSorter(new_parents.items()).iter_topo_order():
 
404
            new_vf.add_lines(version, new_parents[version],
 
405
                             vf.get_lines(version))
 
406
        self.repo.weave_store.copy(new_vf, file_id, self.transaction)
 
407
        self.repo.weave_store.delete('temp:%s' % file_id, self.transaction)
475
408
 
476
409
 
477
410
class PackReconciler(RepoReconciler):
494
427
 
495
428
    def _reconcile_steps(self):
496
429
        """Perform the steps to reconcile this repository."""
497
 
        if not self.thorough:
498
 
            return
499
 
        collection = self.repo._pack_collection
500
 
        collection.ensure_loaded()
501
 
        collection.lock_names()
502
 
        try:
503
 
            packs = collection.all_packs()
504
 
            all_revisions = self.repo.all_revision_ids()
505
 
            total_inventories = len(list(
506
 
                collection.inventory_index.combined_index.iter_all_entries()))
507
 
            if len(all_revisions):
508
 
                self._packer = repofmt.pack_repo.ReconcilePacker(
509
 
                    collection, packs, ".reconcile", all_revisions)
510
 
                new_pack = self._packer.pack(pb=self.pb)
511
 
                if new_pack is not None:
512
 
                    self._discard_and_save(packs)
513
 
            else:
514
 
                # only make a new pack when there is data to copy.
515
 
                self._discard_and_save(packs)
516
 
            self.garbage_inventories = total_inventories - len(list(
517
 
                collection.inventory_index.combined_index.iter_all_entries()))
518
 
        finally:
519
 
            collection._unlock_names()
520
 
 
521
 
    def _discard_and_save(self, packs):
522
 
        """Discard some packs from the repository.
523
 
 
524
 
        This removes them from the memory index, saves the in-memory index
525
 
        which makes the newly reconciled pack visible and hides the packs to be
526
 
        discarded, and finally renames the packs being discarded into the
527
 
        obsolete packs directory.
528
 
 
529
 
        :param packs: The packs to discard.
530
 
        """
531
 
        for pack in packs:
532
 
            self.repo._pack_collection._remove_pack_from_memory(pack)
533
 
        self.repo._pack_collection._save_pack_names()
534
 
        self.repo._pack_collection._obsolete_packs(packs)