← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/pack_repo.py

  • Committer: Martin Pool
  • Date: 2009-07-01 07:22:00 UTC
  • mto: This revision was merged to the branch mainline in revision 4502.
  • Revision ID: mbp@sourcefrog.net-20090701072200-hh21x94g1g11dll7
ReadVFile copes if readv result isn't an iter; also better errors

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/pack_repo.py
36
36
    )
37
37
from bzrlib.index import (
38
38
    CombinedGraphIndex,
 
39
    GraphIndex,
 
40
    GraphIndexBuilder,
39
41
    GraphIndexPrefixAdapter,
 
42
    InMemoryGraphIndex,
40
43
    )
41
44
from bzrlib.knit import (
42
45
    KnitPlainFactory,
52
55
    lockable_files,
53
56
    lockdir,
54
57
    revision as _mod_revision,
 
58
    symbol_versioning,
55
59
    )
56
60
 
57
61
from bzrlib.decorators import needs_write_lock
71
75
    RootCommitBuilder,
72
76
    StreamSource,
73
77
    )
 
78
import bzrlib.revision as _mod_revision
74
79
from bzrlib.trace import (
75
80
    mutter,
76
81
    warning,
224
229
        return self.index_name('text', name)
225
230
 
226
231
    def _replace_index_with_readonly(self, index_type):
227
 
        unlimited_cache = False
228
 
        if index_type == 'chk':
229
 
            unlimited_cache = True
230
232
        setattr(self, index_type + '_index',
231
233
            self.index_class(self.index_transport,
232
234
                self.index_name(index_type, self.name),
233
 
                self.index_sizes[self.index_offset(index_type)],
234
 
                unlimited_cache=unlimited_cache))
 
235
                self.index_sizes[self.index_offset(index_type)]))
235
236
 
236
237
 
237
238
class ExistingPack(Pack):
426
427
        self._writer.begin()
427
428
        # what state is the pack in? (open, finished, aborted)
428
429
        self._state = 'open'
429
 
        # no name until we finish writing the content
430
 
        self.name = None
431
430
 
432
431
    def abort(self):
433
432
        """Cancel creating this pack."""
454
453
            self.signature_index.key_count() or
455
454
            (self.chk_index is not None and self.chk_index.key_count()))
456
455
 
457
 
    def finish_content(self):
458
 
        if self.name is not None:
459
 
            return
460
 
        self._writer.end()
461
 
        if self._buffer[1]:
462
 
            self._write_data('', flush=True)
463
 
        self.name = self._hash.hexdigest()
464
 
 
465
456
    def finish(self, suspend=False):
466
457
        """Finish the new pack.
467
458
 
473
464
         - stores the index size tuple for the pack in the index_sizes
474
465
           attribute.
475
466
        """
476
 
        self.finish_content()
 
467
        self._writer.end()
 
468
        if self._buffer[1]:
 
469
            self._write_data('', flush=True)
 
470
        self.name = self._hash.hexdigest()
477
471
        if not suspend:
478
472
            self._check_references()
479
473
        # write indices
1578
1572
        # determine which packs need changing
1579
1573
        pack_operations = [[0, []]]
1580
1574
        for pack in self.all_packs():
1581
 
            if hint is None or pack.name in hint:
1582
 
                # Either no hint was provided (so we are packing everything),
1583
 
                # or this pack was included in the hint.
 
1575
            if not hint or pack.name in hint:
1584
1576
                pack_operations[-1][0] += pack.get_revision_count()
1585
1577
                pack_operations[-1][1].append(pack)
1586
1578
        self._execute_pack_operations(pack_operations, OptimisingPacker)
1678
1670
            txt_index = self._make_index(name, '.tix')
1679
1671
            sig_index = self._make_index(name, '.six')
1680
1672
            if self.chk_index is not None:
1681
 
                chk_index = self._make_index(name, '.cix', unlimited_cache=True)
 
1673
                chk_index = self._make_index(name, '.cix')
1682
1674
            else:
1683
1675
                chk_index = None
1684
1676
            result = ExistingPack(self._pack_transport, name, rev_index,
1703
1695
            txt_index = self._make_index(name, '.tix', resume=True)
1704
1696
            sig_index = self._make_index(name, '.six', resume=True)
1705
1697
            if self.chk_index is not None:
1706
 
                chk_index = self._make_index(name, '.cix', resume=True,
1707
 
                                             unlimited_cache=True)
 
1698
                chk_index = self._make_index(name, '.cix', resume=True)
1708
1699
            else:
1709
1700
                chk_index = None
1710
1701
            result = self.resumed_pack_factory(name, rev_index, inv_index,
1740
1731
        return self._index_class(self.transport, 'pack-names', None
1741
1732
                ).iter_all_entries()
1742
1733
 
1743
 
    def _make_index(self, name, suffix, resume=False, unlimited_cache=False):
 
1734
    def _make_index(self, name, suffix, resume=False):
1744
1735
        size_offset = self._suffix_offsets[suffix]
1745
1736
        index_name = name + suffix
1746
1737
        if resume:
1749
1740
        else:
1750
1741
            transport = self._index_transport
1751
1742
            index_size = self._names[name][size_offset]
1752
 
        return self._index_class(transport, index_name, index_size,
1753
 
                                 unlimited_cache=unlimited_cache)
 
1743
        return self._index_class(transport, index_name, index_size)
1754
1744
 
1755
1745
    def _max_pack_count(self, total_revisions):
1756
1746
        """Return the maximum number of packs to use for total revisions.
2069
2059
            self._remove_pack_indices(resumed_pack)
2070
2060
        del self._resumed_packs[:]
2071
2061
 
2072
 
    def _check_new_inventories(self):
2073
 
        """Detect missing inventories in this write group.
2074
 
 
2075
 
        :returns: list of strs, summarising any problems found.  If the list is
2076
 
            empty no problems were found.
2077
 
        """
2078
 
        # The base implementation does no checks.  GCRepositoryPackCollection
2079
 
        # overrides this.
2080
 
        return []
2081
 
        
2082
2062
    def _commit_write_group(self):
2083
2063
        all_missing = set()
2084
2064
        for prefix, versioned_file in (
2093
2073
            raise errors.BzrCheckError(
2094
2074
                "Repository %s has missing compression parent(s) %r "
2095
2075
                 % (self.repo, sorted(all_missing)))
2096
 
        problems = self._check_new_inventories()
2097
 
        if problems:
2098
 
            problems_summary = '\n'.join(problems)
2099
 
            raise errors.BzrCheckError(
2100
 
                "Cannot add revision(s) to repository: " + problems_summary)
2101
2076
        self._remove_pack_indices(self._new_pack)
2102
 
        any_new_content = False
 
2077
        should_autopack = False
2103
2078
        if self._new_pack.data_inserted():
2104
2079
            # get all the data to disk and read to use
2105
2080
            self._new_pack.finish()
2106
2081
            self.allocate(self._new_pack)
2107
2082
            self._new_pack = None
2108
 
            any_new_content = True
 
2083
            should_autopack = True
2109
2084
        else:
2110
2085
            self._new_pack.abort()
2111
2086
            self._new_pack = None
2116
2091
            self._remove_pack_from_memory(resumed_pack)
2117
2092
            resumed_pack.finish()
2118
2093
            self.allocate(resumed_pack)
2119
 
            any_new_content = True
 
2094
            should_autopack = True
2120
2095
        del self._resumed_packs[:]
2121
 
        if any_new_content:
2122
 
            result = self.autopack()
2123
 
            if not result:
 
2096
        if should_autopack:
 
2097
            if not self.autopack():
2124
2098
                # when autopack takes no steps, the names list is still
2125
2099
                # unsaved.
2126
2100
                return self._save_pack_names()
2127
 
            return result
2128
 
        return []
2129
2101
 
2130
2102
    def _suspend_write_group(self):
2131
2103
        tokens = [pack.name for pack in self._resumed_packs]
2245
2217
                    % (self._format, self.bzrdir.transport.base))
2246
2218
 
2247
2219
    def _abort_write_group(self):
2248
 
        self.revisions._index._key_dependencies.clear()
 
2220
        self.revisions._index._key_dependencies.refs.clear()
2249
2221
        self._pack_collection._abort_write_group()
2250
2222
 
 
2223
    def _find_inconsistent_revision_parents(self):
 
2224
        """Find revisions with incorrectly cached parents.
 
2225
 
 
2226
        :returns: an iterator yielding tuples of (revison-id, parents-in-index,
 
2227
            parents-in-revision).
 
2228
        """
 
2229
        if not self.is_locked():
 
2230
            raise errors.ObjectNotLocked(self)
 
2231
        pb = ui.ui_factory.nested_progress_bar()
 
2232
        result = []
 
2233
        try:
 
2234
            revision_nodes = self._pack_collection.revision_index \
 
2235
                .combined_index.iter_all_entries()
 
2236
            index_positions = []
 
2237
            # Get the cached index values for all revisions, and also the
 
2238
            # location in each index of the revision text so we can perform
 
2239
            # linear IO.
 
2240
            for index, key, value, refs in revision_nodes:
 
2241
                node = (index, key, value, refs)
 
2242
                index_memo = self.revisions._index._node_to_position(node)
 
2243
                if index_memo[0] != index:
 
2244
                    raise AssertionError('%r != %r' % (index_memo[0], index))
 
2245
                index_positions.append((index_memo, key[0],
 
2246
                                       tuple(parent[0] for parent in refs[0])))
 
2247
                pb.update("Reading revision index", 0, 0)
 
2248
            index_positions.sort()
 
2249
            batch_size = 1000
 
2250
            pb.update("Checking cached revision graph", 0,
 
2251
                      len(index_positions))
 
2252
            for offset in xrange(0, len(index_positions), 1000):
 
2253
                pb.update("Checking cached revision graph", offset)
 
2254
                to_query = index_positions[offset:offset + batch_size]
 
2255
                if not to_query:
 
2256
                    break
 
2257
                rev_ids = [item[1] for item in to_query]
 
2258
                revs = self.get_revisions(rev_ids)
 
2259
                for revision, item in zip(revs, to_query):
 
2260
                    index_parents = item[2]
 
2261
                    rev_parents = tuple(revision.parent_ids)
 
2262
                    if index_parents != rev_parents:
 
2263
                        result.append((revision.revision_id, index_parents,
 
2264
                                       rev_parents))
 
2265
        finally:
 
2266
            pb.finished()
 
2267
        return result
 
2268
 
2251
2269
    def _get_source(self, to_format):
2252
2270
        if to_format.network_name() == self._format.network_name():
2253
2271
            return KnitPackStreamSource(self, to_format)
2265
2283
        self._pack_collection._start_write_group()
2266
2284
 
2267
2285
    def _commit_write_group(self):
2268
 
        hint = self._pack_collection._commit_write_group()
2269
 
        self.revisions._index._key_dependencies.clear()
2270
 
        return hint
 
2286
        self.revisions._index._key_dependencies.refs.clear()
 
2287
        return self._pack_collection._commit_write_group()
2271
2288
 
2272
2289
    def suspend_write_group(self):
2273
2290
        # XXX check self._write_group is self.get_transaction()?
2274
2291
        tokens = self._pack_collection._suspend_write_group()
2275
 
        self.revisions._index._key_dependencies.clear()
 
2292
        self.revisions._index._key_dependencies.refs.clear()
2276
2293
        self._write_group = None
2277
2294
        return tokens
2278
2295
 
2553
2570
        """See RepositoryFormat.get_format_description()."""
2554
2571
        return "Packs containing knits without subtree support"
2555
2572
 
 
2573
    def check_conversion_target(self, target_format):
 
2574
        pass
 
2575
 
2556
2576
 
2557
2577
class RepositoryFormatKnitPack3(RepositoryFormatPack):
2558
2578
    """A subtrees parameterized Pack repository.
2584
2604
 
2585
2605
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2586
2606
 
 
2607
    def check_conversion_target(self, target_format):
 
2608
        if not target_format.rich_root_data:
 
2609
            raise errors.BadConversionTarget(
 
2610
                'Does not support rich root data.', target_format)
 
2611
        if not getattr(target_format, 'supports_tree_reference', False):
 
2612
            raise errors.BadConversionTarget(
 
2613
                'Does not support nested trees', target_format)
 
2614
 
2587
2615
    def get_format_string(self):
2588
2616
        """See RepositoryFormat.get_format_string()."""
2589
2617
        return "Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n"
2622
2650
 
2623
2651
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2624
2652
 
 
2653
    def check_conversion_target(self, target_format):
 
2654
        if not target_format.rich_root_data:
 
2655
            raise errors.BadConversionTarget(
 
2656
                'Does not support rich root data.', target_format)
 
2657
 
2625
2658
    def get_format_string(self):
2626
2659
        """See RepositoryFormat.get_format_string()."""
2627
2660
        return ("Bazaar pack repository format 1 with rich root"
2668
2701
        """See RepositoryFormat.get_format_description()."""
2669
2702
        return "Packs 5 (adds stacking support, requires bzr 1.6)"
2670
2703
 
 
2704
    def check_conversion_target(self, target_format):
 
2705
        pass
 
2706
 
2671
2707
 
2672
2708
class RepositoryFormatKnitPack5RichRoot(RepositoryFormatPack):
2673
2709
    """A repository with rich roots and stacking.
2700
2736
 
2701
2737
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2702
2738
 
 
2739
    def check_conversion_target(self, target_format):
 
2740
        if not target_format.rich_root_data:
 
2741
            raise errors.BadConversionTarget(
 
2742
                'Does not support rich root data.', target_format)
 
2743
 
2703
2744
    def get_format_string(self):
2704
2745
        """See RepositoryFormat.get_format_string()."""
2705
2746
        return "Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6.1)\n"
2746
2787
 
2747
2788
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2748
2789
 
 
2790
    def check_conversion_target(self, target_format):
 
2791
        if not target_format.rich_root_data:
 
2792
            raise errors.BadConversionTarget(
 
2793
                'Does not support rich root data.', target_format)
 
2794
 
2749
2795
    def get_format_string(self):
2750
2796
        """See RepositoryFormat.get_format_string()."""
2751
2797
        return "Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6)\n"
2789
2835
        """See RepositoryFormat.get_format_description()."""
2790
2836
        return "Packs 6 (uses btree indexes, requires bzr 1.9)"
2791
2837
 
 
2838
    def check_conversion_target(self, target_format):
 
2839
        pass
 
2840
 
2792
2841
 
2793
2842
class RepositoryFormatKnitPack6RichRoot(RepositoryFormatPack):
2794
2843
    """A repository with rich roots, no subtrees, stacking and btree indexes.
2818
2867
 
2819
2868
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2820
2869
 
 
2870
    def check_conversion_target(self, target_format):
 
2871
        if not target_format.rich_root_data:
 
2872
            raise errors.BadConversionTarget(
 
2873
                'Does not support rich root data.', target_format)
 
2874
 
2821
2875
    def get_format_string(self):
2822
2876
        """See RepositoryFormat.get_format_string()."""
2823
2877
        return "Bazaar RepositoryFormatKnitPack6RichRoot (bzr 1.9)\n"
2859
2913
 
2860
2914
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2861
2915
 
 
2916
    def check_conversion_target(self, target_format):
 
2917
        if not target_format.rich_root_data:
 
2918
            raise errors.BadConversionTarget(
 
2919
                'Does not support rich root data.', target_format)
 
2920
        if not getattr(target_format, 'supports_tree_reference', False):
 
2921
            raise errors.BadConversionTarget(
 
2922
                'Does not support nested trees', target_format)
 
2923
 
2862
2924
    def get_format_string(self):
2863
2925
        """See RepositoryFormat.get_format_string()."""
2864
2926
        return ("Bazaar development format 2 with subtree support "