~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/pack_repo.py

  • Committer: John Arbash Meinel
  • Date: 2010-08-30 21:23:49 UTC
  • mto: This revision was merged to the branch mainline in revision 5398.
  • Revision ID: john@arbash-meinel.com-20100830212349-figt9yz2cic6hy68
Remove the 'false' invocation.

Show diffs side-by-side

added added

removed removed

Lines of Context:
1
 
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
 
1
# Copyright (C) 2007-2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
24
24
 
25
25
from bzrlib import (
26
26
    chk_map,
 
27
    cleanup,
27
28
    debug,
28
29
    graph,
29
30
    osutils,
48
49
""")
49
50
from bzrlib import (
50
51
    bzrdir,
 
52
    btree_index,
51
53
    errors,
52
54
    lockable_files,
53
55
    lockdir,
54
56
    revision as _mod_revision,
55
57
    )
56
58
 
57
 
from bzrlib.decorators import needs_write_lock
58
 
from bzrlib.btree_index import (
59
 
    BTreeGraphIndex,
60
 
    BTreeBuilder,
61
 
    )
 
59
from bzrlib.decorators import needs_write_lock, only_raises
62
60
from bzrlib.index import (
63
61
    GraphIndex,
64
62
    InMemoryGraphIndex,
65
63
    )
 
64
from bzrlib.lock import LogicalLockResult
66
65
from bzrlib.repofmt.knitrepo import KnitRepository
67
66
from bzrlib.repository import (
68
67
    CommitBuilder,
69
68
    MetaDirRepositoryFormat,
70
69
    RepositoryFormat,
 
70
    RepositoryWriteLockResult,
71
71
    RootCommitBuilder,
72
72
    StreamSource,
73
73
    )
74
74
from bzrlib.trace import (
75
75
    mutter,
 
76
    note,
76
77
    warning,
77
78
    )
78
79
 
224
225
        return self.index_name('text', name)
225
226
 
226
227
    def _replace_index_with_readonly(self, index_type):
227
 
        setattr(self, index_type + '_index',
228
 
            self.index_class(self.index_transport,
229
 
                self.index_name(index_type, self.name),
230
 
                self.index_sizes[self.index_offset(index_type)]))
 
228
        unlimited_cache = False
 
229
        if index_type == 'chk':
 
230
            unlimited_cache = True
 
231
        index = self.index_class(self.index_transport,
 
232
                    self.index_name(index_type, self.name),
 
233
                    self.index_sizes[self.index_offset(index_type)],
 
234
                    unlimited_cache=unlimited_cache)
 
235
        if index_type == 'chk':
 
236
            index._leaf_factory = btree_index._gcchk_factory
 
237
        setattr(self, index_type + '_index', index)
231
238
 
232
239
 
233
240
class ExistingPack(Pack):
422
429
        self._writer.begin()
423
430
        # what state is the pack in? (open, finished, aborted)
424
431
        self._state = 'open'
 
432
        # no name until we finish writing the content
 
433
        self.name = None
425
434
 
426
435
    def abort(self):
427
436
        """Cancel creating this pack."""
448
457
            self.signature_index.key_count() or
449
458
            (self.chk_index is not None and self.chk_index.key_count()))
450
459
 
 
460
    def finish_content(self):
 
461
        if self.name is not None:
 
462
            return
 
463
        self._writer.end()
 
464
        if self._buffer[1]:
 
465
            self._write_data('', flush=True)
 
466
        self.name = self._hash.hexdigest()
 
467
 
451
468
    def finish(self, suspend=False):
452
469
        """Finish the new pack.
453
470
 
459
476
         - stores the index size tuple for the pack in the index_sizes
460
477
           attribute.
461
478
        """
462
 
        self._writer.end()
463
 
        if self._buffer[1]:
464
 
            self._write_data('', flush=True)
465
 
        self.name = self._hash.hexdigest()
 
479
        self.finish_content()
466
480
        if not suspend:
467
481
            self._check_references()
468
482
        # write indices
574
588
                                             flush_func=flush_func)
575
589
        self.add_callback = None
576
590
 
577
 
    def replace_indices(self, index_to_pack, indices):
578
 
        """Replace the current mappings with fresh ones.
579
 
 
580
 
        This should probably not be used eventually, rather incremental add and
581
 
        removal of indices. It has been added during refactoring of existing
582
 
        code.
583
 
 
584
 
        :param index_to_pack: A mapping from index objects to
585
 
            (transport, name) tuples for the pack file data.
586
 
        :param indices: A list of indices.
587
 
        """
588
 
        # refresh the revision pack map dict without replacing the instance.
589
 
        self.index_to_pack.clear()
590
 
        self.index_to_pack.update(index_to_pack)
591
 
        # XXX: API break - clearly a 'replace' method would be good?
592
 
        self.combined_index._indices[:] = indices
593
 
        # the current add nodes callback for the current writable index if
594
 
        # there is one.
595
 
        self.add_callback = None
596
 
 
597
591
    def add_index(self, index, pack):
598
592
        """Add index to the aggregate, which is an index for Pack pack.
599
593
 
606
600
        # expose it to the index map
607
601
        self.index_to_pack[index] = pack.access_tuple()
608
602
        # put it at the front of the linear index list
609
 
        self.combined_index.insert_index(0, index)
 
603
        self.combined_index.insert_index(0, index, pack.name)
610
604
 
611
605
    def add_writable_index(self, index, pack):
612
606
        """Add an index which is able to have data added to it.
632
626
        self.data_access.set_writer(None, None, (None, None))
633
627
        self.index_to_pack.clear()
634
628
        del self.combined_index._indices[:]
 
629
        del self.combined_index._index_names[:]
635
630
        self.add_callback = None
636
631
 
637
 
    def remove_index(self, index, pack):
 
632
    def remove_index(self, index):
638
633
        """Remove index from the indices used to answer queries.
639
634
 
640
635
        :param index: An index from the pack parameter.
641
 
        :param pack: A Pack instance.
642
636
        """
643
637
        del self.index_to_pack[index]
644
 
        self.combined_index._indices.remove(index)
 
638
        pos = self.combined_index._indices.index(index)
 
639
        del self.combined_index._indices[pos]
 
640
        del self.combined_index._index_names[pos]
645
641
        if (self.add_callback is not None and
646
642
            getattr(index, 'add_nodes', None) == self.add_callback):
647
643
            self.add_callback = None
1105
1101
            iterator is a tuple with:
1106
1102
            index, readv_vector, node_vector. readv_vector is a list ready to
1107
1103
            hand to the transport readv method, and node_vector is a list of
1108
 
            (key, eol_flag, references) for the the node retrieved by the
 
1104
            (key, eol_flag, references) for the node retrieved by the
1109
1105
            matching readv_vector.
1110
1106
        """
1111
1107
        # group by pack so we do one readv per pack
1403
1399
        self.inventory_index = AggregateIndex(self.reload_pack_names, flush)
1404
1400
        self.text_index = AggregateIndex(self.reload_pack_names, flush)
1405
1401
        self.signature_index = AggregateIndex(self.reload_pack_names, flush)
 
1402
        all_indices = [self.revision_index, self.inventory_index,
 
1403
                self.text_index, self.signature_index]
1406
1404
        if use_chk_index:
1407
1405
            self.chk_index = AggregateIndex(self.reload_pack_names, flush)
 
1406
            all_indices.append(self.chk_index)
1408
1407
        else:
1409
1408
            # used to determine if we're using a chk_index elsewhere.
1410
1409
            self.chk_index = None
 
1410
        # Tell all the CombinedGraphIndex objects about each other, so they can
 
1411
        # share hints about which pack names to search first.
 
1412
        all_combined = [agg_idx.combined_index for agg_idx in all_indices]
 
1413
        for combined_idx in all_combined:
 
1414
            combined_idx.set_sibling_indices(
 
1415
                set(all_combined).difference([combined_idx]))
1411
1416
        # resumed packs
1412
1417
        self._resumed_packs = []
1413
1418
 
 
1419
    def __repr__(self):
 
1420
        return '%s(%r)' % (self.__class__.__name__, self.repo)
 
1421
 
1414
1422
    def add_pack_to_memory(self, pack):
1415
1423
        """Make a Pack object available to the repository to satisfy queries.
1416
1424
 
1530
1538
                self._remove_pack_from_memory(pack)
1531
1539
        # record the newly available packs and stop advertising the old
1532
1540
        # packs
1533
 
        result = self._save_pack_names(clear_obsolete_packs=True)
1534
 
        # Move the old packs out of the way now they are no longer referenced.
1535
 
        for revision_count, packs in pack_operations:
1536
 
            self._obsolete_packs(packs)
 
1541
        to_be_obsoleted = []
 
1542
        for _, packs in pack_operations:
 
1543
            to_be_obsoleted.extend(packs)
 
1544
        result = self._save_pack_names(clear_obsolete_packs=True,
 
1545
                                       obsolete_packs=to_be_obsoleted)
1537
1546
        return result
1538
1547
 
1539
1548
    def _flush_new_pack(self):
1552
1561
        """Is the collection already packed?"""
1553
1562
        return not (self.repo._format.pack_compresses or (len(self._names) > 1))
1554
1563
 
1555
 
    def pack(self, hint=None):
 
1564
    def pack(self, hint=None, clean_obsolete_packs=False):
1556
1565
        """Pack the pack collection totally."""
1557
1566
        self.ensure_loaded()
1558
1567
        total_packs = len(self._names)
1567
1576
        # determine which packs need changing
1568
1577
        pack_operations = [[0, []]]
1569
1578
        for pack in self.all_packs():
1570
 
            if not hint or pack.name in hint:
 
1579
            if hint is None or pack.name in hint:
 
1580
                # Either no hint was provided (so we are packing everything),
 
1581
                # or this pack was included in the hint.
1571
1582
                pack_operations[-1][0] += pack.get_revision_count()
1572
1583
                pack_operations[-1][1].append(pack)
1573
1584
        self._execute_pack_operations(pack_operations, OptimisingPacker)
1574
1585
 
 
1586
        if clean_obsolete_packs:
 
1587
            self._clear_obsolete_packs()
 
1588
 
1575
1589
    def plan_autopack_combinations(self, existing_packs, pack_distribution):
1576
1590
        """Plan a pack operation.
1577
1591
 
1665
1679
            txt_index = self._make_index(name, '.tix')
1666
1680
            sig_index = self._make_index(name, '.six')
1667
1681
            if self.chk_index is not None:
1668
 
                chk_index = self._make_index(name, '.cix')
 
1682
                chk_index = self._make_index(name, '.cix', is_chk=True)
1669
1683
            else:
1670
1684
                chk_index = None
1671
1685
            result = ExistingPack(self._pack_transport, name, rev_index,
1690
1704
            txt_index = self._make_index(name, '.tix', resume=True)
1691
1705
            sig_index = self._make_index(name, '.six', resume=True)
1692
1706
            if self.chk_index is not None:
1693
 
                chk_index = self._make_index(name, '.cix', resume=True)
 
1707
                chk_index = self._make_index(name, '.cix', resume=True,
 
1708
                                             is_chk=True)
1694
1709
            else:
1695
1710
                chk_index = None
1696
1711
            result = self.resumed_pack_factory(name, rev_index, inv_index,
1726
1741
        return self._index_class(self.transport, 'pack-names', None
1727
1742
                ).iter_all_entries()
1728
1743
 
1729
 
    def _make_index(self, name, suffix, resume=False):
 
1744
    def _make_index(self, name, suffix, resume=False, is_chk=False):
1730
1745
        size_offset = self._suffix_offsets[suffix]
1731
1746
        index_name = name + suffix
1732
1747
        if resume:
1735
1750
        else:
1736
1751
            transport = self._index_transport
1737
1752
            index_size = self._names[name][size_offset]
1738
 
        return self._index_class(transport, index_name, index_size)
 
1753
        index = self._index_class(transport, index_name, index_size,
 
1754
                                  unlimited_cache=is_chk)
 
1755
        if is_chk and self._index_class is btree_index.BTreeGraphIndex: 
 
1756
            index._leaf_factory = btree_index._gcchk_factory
 
1757
        return index
1739
1758
 
1740
1759
    def _max_pack_count(self, total_revisions):
1741
1760
        """Return the maximum number of packs to use for total revisions.
1769
1788
        :param return: None.
1770
1789
        """
1771
1790
        for pack in packs:
1772
 
            pack.pack_transport.rename(pack.file_name(),
1773
 
                '../obsolete_packs/' + pack.file_name())
 
1791
            try:
 
1792
                pack.pack_transport.rename(pack.file_name(),
 
1793
                    '../obsolete_packs/' + pack.file_name())
 
1794
            except (errors.PathError, errors.TransportError), e:
 
1795
                # TODO: Should these be warnings or mutters?
 
1796
                mutter("couldn't rename obsolete pack, skipping it:\n%s"
 
1797
                       % (e,))
1774
1798
            # TODO: Probably needs to know all possible indices for this pack
1775
1799
            # - or maybe list the directory and move all indices matching this
1776
1800
            # name whether we recognize it or not?
1778
1802
            if self.chk_index is not None:
1779
1803
                suffixes.append('.cix')
1780
1804
            for suffix in suffixes:
1781
 
                self._index_transport.rename(pack.name + suffix,
1782
 
                    '../obsolete_packs/' + pack.name + suffix)
 
1805
                try:
 
1806
                    self._index_transport.rename(pack.name + suffix,
 
1807
                        '../obsolete_packs/' + pack.name + suffix)
 
1808
                except (errors.PathError, errors.TransportError), e:
 
1809
                    mutter("couldn't rename obsolete index, skipping it:\n%s"
 
1810
                           % (e,))
1783
1811
 
1784
1812
    def pack_distribution(self, total_revisions):
1785
1813
        """Generate a list of the number of revisions to put in each pack.
1811
1839
        self._remove_pack_indices(pack)
1812
1840
        self.packs.remove(pack)
1813
1841
 
1814
 
    def _remove_pack_indices(self, pack):
1815
 
        """Remove the indices for pack from the aggregated indices."""
1816
 
        self.revision_index.remove_index(pack.revision_index, pack)
1817
 
        self.inventory_index.remove_index(pack.inventory_index, pack)
1818
 
        self.text_index.remove_index(pack.text_index, pack)
1819
 
        self.signature_index.remove_index(pack.signature_index, pack)
1820
 
        if self.chk_index is not None:
1821
 
            self.chk_index.remove_index(pack.chk_index, pack)
 
1842
    def _remove_pack_indices(self, pack, ignore_missing=False):
 
1843
        """Remove the indices for pack from the aggregated indices.
 
1844
        
 
1845
        :param ignore_missing: Suppress KeyErrors from calling remove_index.
 
1846
        """
 
1847
        for index_type in Pack.index_definitions.keys():
 
1848
            attr_name = index_type + '_index'
 
1849
            aggregate_index = getattr(self, attr_name)
 
1850
            if aggregate_index is not None:
 
1851
                pack_index = getattr(pack, attr_name)
 
1852
                try:
 
1853
                    aggregate_index.remove_index(pack_index)
 
1854
                except KeyError:
 
1855
                    if ignore_missing:
 
1856
                        continue
 
1857
                    raise
1822
1858
 
1823
1859
    def reset(self):
1824
1860
        """Clear all cached data."""
1857
1893
        disk_nodes = set()
1858
1894
        for index, key, value in self._iter_disk_pack_index():
1859
1895
            disk_nodes.add((key, value))
 
1896
        orig_disk_nodes = set(disk_nodes)
1860
1897
 
1861
1898
        # do a two-way diff against our original content
1862
1899
        current_nodes = set()
1875
1912
        disk_nodes.difference_update(deleted_nodes)
1876
1913
        disk_nodes.update(new_nodes)
1877
1914
 
1878
 
        return disk_nodes, deleted_nodes, new_nodes
 
1915
        return disk_nodes, deleted_nodes, new_nodes, orig_disk_nodes
1879
1916
 
1880
1917
    def _syncronize_pack_names_from_disk_nodes(self, disk_nodes):
1881
1918
        """Given the correct set of pack files, update our saved info.
1921
1958
                added.append(name)
1922
1959
        return removed, added, modified
1923
1960
 
1924
 
    def _save_pack_names(self, clear_obsolete_packs=False):
 
1961
    def _save_pack_names(self, clear_obsolete_packs=False, obsolete_packs=None):
1925
1962
        """Save the list of packs.
1926
1963
 
1927
1964
        This will take out the mutex around the pack names list for the
1931
1968
 
1932
1969
        :param clear_obsolete_packs: If True, clear out the contents of the
1933
1970
            obsolete_packs directory.
 
1971
        :param obsolete_packs: Packs that are obsolete once the new pack-names
 
1972
            file has been written.
1934
1973
        :return: A list of the names saved that were not previously on disk.
1935
1974
        """
 
1975
        already_obsolete = []
1936
1976
        self.lock_names()
1937
1977
        try:
1938
1978
            builder = self._index_builder_class()
1939
 
            disk_nodes, deleted_nodes, new_nodes = self._diff_pack_names()
 
1979
            (disk_nodes, deleted_nodes, new_nodes,
 
1980
             orig_disk_nodes) = self._diff_pack_names()
1940
1981
            # TODO: handle same-name, index-size-changes here -
1941
1982
            # e.g. use the value from disk, not ours, *unless* we're the one
1942
1983
            # changing it.
1944
1985
                builder.add_node(key, value)
1945
1986
            self.transport.put_file('pack-names', builder.finish(),
1946
1987
                mode=self.repo.bzrdir._get_file_mode())
1947
 
            # move the baseline forward
1948
1988
            self._packs_at_load = disk_nodes
1949
1989
            if clear_obsolete_packs:
1950
 
                self._clear_obsolete_packs()
 
1990
                to_preserve = None
 
1991
                if obsolete_packs:
 
1992
                    to_preserve = set([o.name for o in obsolete_packs])
 
1993
                already_obsolete = self._clear_obsolete_packs(to_preserve)
1951
1994
        finally:
1952
1995
            self._unlock_names()
1953
1996
        # synchronise the memory packs list with what we just wrote:
1954
1997
        self._syncronize_pack_names_from_disk_nodes(disk_nodes)
 
1998
        if obsolete_packs:
 
1999
            # TODO: We could add one more condition here. "if o.name not in
 
2000
            #       orig_disk_nodes and o != the new_pack we haven't written to
 
2001
            #       disk yet. However, the new pack object is not easily
 
2002
            #       accessible here (it would have to be passed through the
 
2003
            #       autopacking code, etc.)
 
2004
            obsolete_packs = [o for o in obsolete_packs
 
2005
                              if o.name not in already_obsolete]
 
2006
            self._obsolete_packs(obsolete_packs)
1955
2007
        return [new_node[0][0] for new_node in new_nodes]
1956
2008
 
1957
2009
    def reload_pack_names(self):
1972
2024
        if first_read:
1973
2025
            return True
1974
2026
        # out the new value.
1975
 
        disk_nodes, _, _ = self._diff_pack_names()
1976
 
        self._packs_at_load = disk_nodes
 
2027
        (disk_nodes, deleted_nodes, new_nodes,
 
2028
         orig_disk_nodes) = self._diff_pack_names()
 
2029
        # _packs_at_load is meant to be the explicit list of names in
 
2030
        # 'pack-names' at then start. As such, it should not contain any
 
2031
        # pending names that haven't been written out yet.
 
2032
        self._packs_at_load = orig_disk_nodes
1977
2033
        (removed, added,
1978
2034
         modified) = self._syncronize_pack_names_from_disk_nodes(disk_nodes)
1979
2035
        if removed or added or modified:
1988
2044
            raise
1989
2045
        raise errors.RetryAutopack(self.repo, False, sys.exc_info())
1990
2046
 
1991
 
    def _clear_obsolete_packs(self):
 
2047
    def _clear_obsolete_packs(self, preserve=None):
1992
2048
        """Delete everything from the obsolete-packs directory.
 
2049
 
 
2050
        :return: A list of pack identifiers (the filename without '.pack') that
 
2051
            were found in obsolete_packs.
1993
2052
        """
 
2053
        found = []
1994
2054
        obsolete_pack_transport = self.transport.clone('obsolete_packs')
 
2055
        if preserve is None:
 
2056
            preserve = set()
1995
2057
        for filename in obsolete_pack_transport.list_dir('.'):
 
2058
            name, ext = osutils.splitext(filename)
 
2059
            if ext == '.pack':
 
2060
                found.append(name)
 
2061
            if name in preserve:
 
2062
                continue
1996
2063
            try:
1997
2064
                obsolete_pack_transport.delete(filename)
1998
2065
            except (errors.PathError, errors.TransportError), e:
1999
 
                warning("couldn't delete obsolete pack, skipping it:\n%s" % (e,))
 
2066
                warning("couldn't delete obsolete pack, skipping it:\n%s"
 
2067
                        % (e,))
 
2068
        return found
2000
2069
 
2001
2070
    def _start_write_group(self):
2002
2071
        # Do not permit preparation for writing if we're not in a 'write lock'.
2029
2098
        # FIXME: just drop the transient index.
2030
2099
        # forget what names there are
2031
2100
        if self._new_pack is not None:
2032
 
            try:
2033
 
                self._new_pack.abort()
2034
 
            finally:
2035
 
                # XXX: If we aborted while in the middle of finishing the write
2036
 
                # group, _remove_pack_indices can fail because the indexes are
2037
 
                # already gone.  If they're not there we shouldn't fail in this
2038
 
                # case.  -- mbp 20081113
2039
 
                self._remove_pack_indices(self._new_pack)
2040
 
                self._new_pack = None
 
2101
            operation = cleanup.OperationWithCleanups(self._new_pack.abort)
 
2102
            operation.add_cleanup(setattr, self, '_new_pack', None)
 
2103
            # If we aborted while in the middle of finishing the write
 
2104
            # group, _remove_pack_indices could fail because the indexes are
 
2105
            # already gone.  But they're not there we shouldn't fail in this
 
2106
            # case, so we pass ignore_missing=True.
 
2107
            operation.add_cleanup(self._remove_pack_indices, self._new_pack,
 
2108
                ignore_missing=True)
 
2109
            operation.run_simple()
2041
2110
        for resumed_pack in self._resumed_packs:
2042
 
            try:
2043
 
                resumed_pack.abort()
2044
 
            finally:
2045
 
                # See comment in previous finally block.
2046
 
                try:
2047
 
                    self._remove_pack_indices(resumed_pack)
2048
 
                except KeyError:
2049
 
                    pass
 
2111
            operation = cleanup.OperationWithCleanups(resumed_pack.abort)
 
2112
            # See comment in previous finally block.
 
2113
            operation.add_cleanup(self._remove_pack_indices, resumed_pack,
 
2114
                ignore_missing=True)
 
2115
            operation.run_simple()
2050
2116
        del self._resumed_packs[:]
2051
2117
 
2052
2118
    def _remove_resumed_pack_indices(self):
2054
2120
            self._remove_pack_indices(resumed_pack)
2055
2121
        del self._resumed_packs[:]
2056
2122
 
 
2123
    def _check_new_inventories(self):
 
2124
        """Detect missing inventories in this write group.
 
2125
 
 
2126
        :returns: list of strs, summarising any problems found.  If the list is
 
2127
            empty no problems were found.
 
2128
        """
 
2129
        # The base implementation does no checks.  GCRepositoryPackCollection
 
2130
        # overrides this.
 
2131
        return []
 
2132
        
2057
2133
    def _commit_write_group(self):
2058
2134
        all_missing = set()
2059
2135
        for prefix, versioned_file in (
2068
2144
            raise errors.BzrCheckError(
2069
2145
                "Repository %s has missing compression parent(s) %r "
2070
2146
                 % (self.repo, sorted(all_missing)))
 
2147
        problems = self._check_new_inventories()
 
2148
        if problems:
 
2149
            problems_summary = '\n'.join(problems)
 
2150
            raise errors.BzrCheckError(
 
2151
                "Cannot add revision(s) to repository: " + problems_summary)
2071
2152
        self._remove_pack_indices(self._new_pack)
2072
 
        should_autopack = False
 
2153
        any_new_content = False
2073
2154
        if self._new_pack.data_inserted():
2074
2155
            # get all the data to disk and read to use
2075
2156
            self._new_pack.finish()
2076
2157
            self.allocate(self._new_pack)
2077
2158
            self._new_pack = None
2078
 
            should_autopack = True
 
2159
            any_new_content = True
2079
2160
        else:
2080
2161
            self._new_pack.abort()
2081
2162
            self._new_pack = None
2086
2167
            self._remove_pack_from_memory(resumed_pack)
2087
2168
            resumed_pack.finish()
2088
2169
            self.allocate(resumed_pack)
2089
 
            should_autopack = True
 
2170
            any_new_content = True
2090
2171
        del self._resumed_packs[:]
2091
 
        if should_autopack:
2092
 
            if not self.autopack():
 
2172
        if any_new_content:
 
2173
            result = self.autopack()
 
2174
            if not result:
2093
2175
                # when autopack takes no steps, the names list is still
2094
2176
                # unsaved.
2095
2177
                return self._save_pack_names()
 
2178
            return result
 
2179
        return []
2096
2180
 
2097
2181
    def _suspend_write_group(self):
2098
2182
        tokens = [pack.name for pack in self._resumed_packs]
2200
2284
        self._reconcile_fixes_text_parents = True
2201
2285
        self._reconcile_backsup_inventory = False
2202
2286
 
2203
 
    def _warn_if_deprecated(self):
 
2287
    def _warn_if_deprecated(self, branch=None):
2204
2288
        # This class isn't deprecated, but one sub-format is
2205
2289
        if isinstance(self._format, RepositoryFormatKnitPack5RichRootBroken):
2206
 
            from bzrlib import repository
2207
 
            if repository._deprecation_warning_done:
2208
 
                return
2209
 
            repository._deprecation_warning_done = True
2210
 
            warning("Format %s for %s is deprecated - please use"
2211
 
                    " 'bzr upgrade --1.6.1-rich-root'"
2212
 
                    % (self._format, self.bzrdir.transport.base))
 
2290
            super(KnitPackRepository, self)._warn_if_deprecated(branch)
2213
2291
 
2214
2292
    def _abort_write_group(self):
2215
 
        self.revisions._index._key_dependencies.refs.clear()
 
2293
        self.revisions._index._key_dependencies.clear()
2216
2294
        self._pack_collection._abort_write_group()
2217
2295
 
2218
 
    def _find_inconsistent_revision_parents(self):
2219
 
        """Find revisions with incorrectly cached parents.
2220
 
 
2221
 
        :returns: an iterator yielding tuples of (revison-id, parents-in-index,
2222
 
            parents-in-revision).
2223
 
        """
2224
 
        if not self.is_locked():
2225
 
            raise errors.ObjectNotLocked(self)
2226
 
        pb = ui.ui_factory.nested_progress_bar()
2227
 
        result = []
2228
 
        try:
2229
 
            revision_nodes = self._pack_collection.revision_index \
2230
 
                .combined_index.iter_all_entries()
2231
 
            index_positions = []
2232
 
            # Get the cached index values for all revisions, and also the
2233
 
            # location in each index of the revision text so we can perform
2234
 
            # linear IO.
2235
 
            for index, key, value, refs in revision_nodes:
2236
 
                node = (index, key, value, refs)
2237
 
                index_memo = self.revisions._index._node_to_position(node)
2238
 
                if index_memo[0] != index:
2239
 
                    raise AssertionError('%r != %r' % (index_memo[0], index))
2240
 
                index_positions.append((index_memo, key[0],
2241
 
                                       tuple(parent[0] for parent in refs[0])))
2242
 
                pb.update("Reading revision index", 0, 0)
2243
 
            index_positions.sort()
2244
 
            batch_size = 1000
2245
 
            pb.update("Checking cached revision graph", 0,
2246
 
                      len(index_positions))
2247
 
            for offset in xrange(0, len(index_positions), 1000):
2248
 
                pb.update("Checking cached revision graph", offset)
2249
 
                to_query = index_positions[offset:offset + batch_size]
2250
 
                if not to_query:
2251
 
                    break
2252
 
                rev_ids = [item[1] for item in to_query]
2253
 
                revs = self.get_revisions(rev_ids)
2254
 
                for revision, item in zip(revs, to_query):
2255
 
                    index_parents = item[2]
2256
 
                    rev_parents = tuple(revision.parent_ids)
2257
 
                    if index_parents != rev_parents:
2258
 
                        result.append((revision.revision_id, index_parents,
2259
 
                                       rev_parents))
2260
 
        finally:
2261
 
            pb.finished()
2262
 
        return result
2263
 
 
2264
2296
    def _get_source(self, to_format):
2265
2297
        if to_format.network_name() == self._format.network_name():
2266
2298
            return KnitPackStreamSource(self, to_format)
2278
2310
        self._pack_collection._start_write_group()
2279
2311
 
2280
2312
    def _commit_write_group(self):
2281
 
        self.revisions._index._key_dependencies.refs.clear()
2282
 
        return self._pack_collection._commit_write_group()
 
2313
        hint = self._pack_collection._commit_write_group()
 
2314
        self.revisions._index._key_dependencies.clear()
 
2315
        return hint
2283
2316
 
2284
2317
    def suspend_write_group(self):
2285
2318
        # XXX check self._write_group is self.get_transaction()?
2286
2319
        tokens = self._pack_collection._suspend_write_group()
2287
 
        self.revisions._index._key_dependencies.refs.clear()
 
2320
        self.revisions._index._key_dependencies.clear()
2288
2321
        self._write_group = None
2289
2322
        return tokens
2290
2323
 
2311
2344
        return self._write_lock_count
2312
2345
 
2313
2346
    def lock_write(self, token=None):
 
2347
        """Lock the repository for writes.
 
2348
 
 
2349
        :return: A bzrlib.repository.RepositoryWriteLockResult.
 
2350
        """
2314
2351
        locked = self.is_locked()
2315
2352
        if not self._write_lock_count and locked:
2316
2353
            raise errors.ReadOnlyError(self)
2318
2355
        if self._write_lock_count == 1:
2319
2356
            self._transaction = transactions.WriteTransaction()
2320
2357
        if not locked:
 
2358
            if 'relock' in debug.debug_flags and self._prev_lock == 'w':
 
2359
                note('%r was write locked again', self)
 
2360
            self._prev_lock = 'w'
2321
2361
            for repo in self._fallback_repositories:
2322
2362
                # Writes don't affect fallback repos
2323
2363
                repo.lock_read()
2324
2364
            self._refresh_data()
 
2365
        return RepositoryWriteLockResult(self.unlock, None)
2325
2366
 
2326
2367
    def lock_read(self):
 
2368
        """Lock the repository for reads.
 
2369
 
 
2370
        :return: A bzrlib.lock.LogicalLockResult.
 
2371
        """
2327
2372
        locked = self.is_locked()
2328
2373
        if self._write_lock_count:
2329
2374
            self._write_lock_count += 1
2330
2375
        else:
2331
2376
            self.control_files.lock_read()
2332
2377
        if not locked:
 
2378
            if 'relock' in debug.debug_flags and self._prev_lock == 'r':
 
2379
                note('%r was read locked again', self)
 
2380
            self._prev_lock = 'r'
2333
2381
            for repo in self._fallback_repositories:
2334
2382
                repo.lock_read()
2335
2383
            self._refresh_data()
 
2384
        return LogicalLockResult(self.unlock)
2336
2385
 
2337
2386
    def leave_lock_in_place(self):
2338
2387
        # not supported - raise an error
2343
2392
        raise NotImplementedError(self.dont_leave_lock_in_place)
2344
2393
 
2345
2394
    @needs_write_lock
2346
 
    def pack(self, hint=None):
 
2395
    def pack(self, hint=None, clean_obsolete_packs=False):
2347
2396
        """Compress the data within the repository.
2348
2397
 
2349
2398
        This will pack all the data to a single pack. In future it may
2350
2399
        recompress deltas or do other such expensive operations.
2351
2400
        """
2352
 
        self._pack_collection.pack(hint=hint)
 
2401
        self._pack_collection.pack(hint=hint, clean_obsolete_packs=clean_obsolete_packs)
2353
2402
 
2354
2403
    @needs_write_lock
2355
2404
    def reconcile(self, other=None, thorough=False):
2363
2412
        packer = ReconcilePacker(collection, packs, extension, revs)
2364
2413
        return packer.pack(pb)
2365
2414
 
 
2415
    @only_raises(errors.LockNotHeld, errors.LockBroken)
2366
2416
    def unlock(self):
2367
2417
        if self._write_lock_count == 1 and self._write_group is not None:
2368
2418
            self.abort_write_group()
2510
2560
        utf8_files = [('format', self.get_format_string())]
2511
2561
 
2512
2562
        self._upload_blank_content(a_bzrdir, dirs, files, utf8_files, shared)
2513
 
        return self.open(a_bzrdir=a_bzrdir, _found=True)
 
2563
        repository = self.open(a_bzrdir=a_bzrdir, _found=True)
 
2564
        self._run_post_repo_init_hooks(repository, a_bzrdir, shared)
 
2565
        return repository
2514
2566
 
2515
2567
    def open(self, a_bzrdir, _found=False, _override_transport=None):
2516
2568
        """See RepositoryFormat.open().
2565
2617
        """See RepositoryFormat.get_format_description()."""
2566
2618
        return "Packs containing knits without subtree support"
2567
2619
 
2568
 
    def check_conversion_target(self, target_format):
2569
 
        pass
2570
 
 
2571
2620
 
2572
2621
class RepositoryFormatKnitPack3(RepositoryFormatPack):
2573
2622
    """A subtrees parameterized Pack repository.
2582
2631
    repository_class = KnitPackRepository
2583
2632
    _commit_builder_class = PackRootCommitBuilder
2584
2633
    rich_root_data = True
 
2634
    experimental = True
2585
2635
    supports_tree_reference = True
2586
2636
    @property
2587
2637
    def _serializer(self):
2599
2649
 
2600
2650
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2601
2651
 
2602
 
    def check_conversion_target(self, target_format):
2603
 
        if not target_format.rich_root_data:
2604
 
            raise errors.BadConversionTarget(
2605
 
                'Does not support rich root data.', target_format)
2606
 
        if not getattr(target_format, 'supports_tree_reference', False):
2607
 
            raise errors.BadConversionTarget(
2608
 
                'Does not support nested trees', target_format)
2609
 
 
2610
2652
    def get_format_string(self):
2611
2653
        """See RepositoryFormat.get_format_string()."""
2612
2654
        return "Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n"
2645
2687
 
2646
2688
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2647
2689
 
2648
 
    def check_conversion_target(self, target_format):
2649
 
        if not target_format.rich_root_data:
2650
 
            raise errors.BadConversionTarget(
2651
 
                'Does not support rich root data.', target_format)
2652
 
 
2653
2690
    def get_format_string(self):
2654
2691
        """See RepositoryFormat.get_format_string()."""
2655
2692
        return ("Bazaar pack repository format 1 with rich root"
2696
2733
        """See RepositoryFormat.get_format_description()."""
2697
2734
        return "Packs 5 (adds stacking support, requires bzr 1.6)"
2698
2735
 
2699
 
    def check_conversion_target(self, target_format):
2700
 
        pass
2701
 
 
2702
2736
 
2703
2737
class RepositoryFormatKnitPack5RichRoot(RepositoryFormatPack):
2704
2738
    """A repository with rich roots and stacking.
2731
2765
 
2732
2766
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2733
2767
 
2734
 
    def check_conversion_target(self, target_format):
2735
 
        if not target_format.rich_root_data:
2736
 
            raise errors.BadConversionTarget(
2737
 
                'Does not support rich root data.', target_format)
2738
 
 
2739
2768
    def get_format_string(self):
2740
2769
        """See RepositoryFormat.get_format_string()."""
2741
2770
        return "Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6.1)\n"
2782
2811
 
2783
2812
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2784
2813
 
2785
 
    def check_conversion_target(self, target_format):
2786
 
        if not target_format.rich_root_data:
2787
 
            raise errors.BadConversionTarget(
2788
 
                'Does not support rich root data.', target_format)
2789
 
 
2790
2814
    def get_format_string(self):
2791
2815
        """See RepositoryFormat.get_format_string()."""
2792
2816
        return "Bazaar RepositoryFormatKnitPack5RichRoot (bzr 1.6)\n"
2807
2831
    _commit_builder_class = PackCommitBuilder
2808
2832
    supports_external_lookups = True
2809
2833
    # What index classes to use
2810
 
    index_builder_class = BTreeBuilder
2811
 
    index_class = BTreeGraphIndex
 
2834
    index_builder_class = btree_index.BTreeBuilder
 
2835
    index_class = btree_index.BTreeGraphIndex
2812
2836
 
2813
2837
    @property
2814
2838
    def _serializer(self):
2830
2854
        """See RepositoryFormat.get_format_description()."""
2831
2855
        return "Packs 6 (uses btree indexes, requires bzr 1.9)"
2832
2856
 
2833
 
    def check_conversion_target(self, target_format):
2834
 
        pass
2835
 
 
2836
2857
 
2837
2858
class RepositoryFormatKnitPack6RichRoot(RepositoryFormatPack):
2838
2859
    """A repository with rich roots, no subtrees, stacking and btree indexes.
2846
2867
    supports_tree_reference = False # no subtrees
2847
2868
    supports_external_lookups = True
2848
2869
    # What index classes to use
2849
 
    index_builder_class = BTreeBuilder
2850
 
    index_class = BTreeGraphIndex
 
2870
    index_builder_class = btree_index.BTreeBuilder
 
2871
    index_class = btree_index.BTreeGraphIndex
2851
2872
 
2852
2873
    @property
2853
2874
    def _serializer(self):
2862
2883
 
2863
2884
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2864
2885
 
2865
 
    def check_conversion_target(self, target_format):
2866
 
        if not target_format.rich_root_data:
2867
 
            raise errors.BadConversionTarget(
2868
 
                'Does not support rich root data.', target_format)
2869
 
 
2870
2886
    def get_format_string(self):
2871
2887
        """See RepositoryFormat.get_format_string()."""
2872
2888
        return "Bazaar RepositoryFormatKnitPack6RichRoot (bzr 1.9)\n"
2889
2905
    repository_class = KnitPackRepository
2890
2906
    _commit_builder_class = PackRootCommitBuilder
2891
2907
    rich_root_data = True
 
2908
    experimental = True
2892
2909
    supports_tree_reference = True
2893
2910
    supports_external_lookups = True
2894
2911
    # What index classes to use
2895
 
    index_builder_class = BTreeBuilder
2896
 
    index_class = BTreeGraphIndex
 
2912
    index_builder_class = btree_index.BTreeBuilder
 
2913
    index_class = btree_index.BTreeGraphIndex
2897
2914
 
2898
2915
    @property
2899
2916
    def _serializer(self):
2908
2925
 
2909
2926
    _matchingbzrdir = property(_get_matching_bzrdir, _ignore_setting_bzrdir)
2910
2927
 
2911
 
    def check_conversion_target(self, target_format):
2912
 
        if not target_format.rich_root_data:
2913
 
            raise errors.BadConversionTarget(
2914
 
                'Does not support rich root data.', target_format)
2915
 
        if not getattr(target_format, 'supports_tree_reference', False):
2916
 
            raise errors.BadConversionTarget(
2917
 
                'Does not support nested trees', target_format)
2918
 
 
2919
2928
    def get_format_string(self):
2920
2929
        """See RepositoryFormat.get_format_string()."""
2921
2930
        return ("Bazaar development format 2 with subtree support "