← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/repofmt/pack_repo.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2010-10-07 12:45:51 UTC
  • mfrom: (5459.4.1 561061-ssh-protocol)
  • Revision ID: pqm@pqm.ubuntu.com-20101007124551-zfqhlkqdegiy7otp
(vila) Don't force openssh to use protocol=2 (Neil Martinsen-Burrell)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/repofmt/pack_repo.py
49
49
""")
50
50
from bzrlib import (
51
51
    bzrdir,
 
52
    btree_index,
52
53
    errors,
53
54
    lockable_files,
54
55
    lockdir,
56
57
    )
57
58
 
58
59
from bzrlib.decorators import needs_write_lock, only_raises
59
 
from bzrlib.btree_index import (
60
 
    BTreeGraphIndex,
61
 
    BTreeBuilder,
62
 
    )
63
60
from bzrlib.index import (
64
61
    GraphIndex,
65
62
    InMemoryGraphIndex,
66
63
    )
 
64
from bzrlib.lock import LogicalLockResult
67
65
from bzrlib.repofmt.knitrepo import KnitRepository
68
66
from bzrlib.repository import (
69
67
    CommitBuilder,
70
68
    MetaDirRepositoryFormat,
71
69
    RepositoryFormat,
 
70
    RepositoryWriteLockResult,
72
71
    RootCommitBuilder,
73
72
    StreamSource,
74
73
    )
229
228
        unlimited_cache = False
230
229
        if index_type == 'chk':
231
230
            unlimited_cache = True
232
 
        setattr(self, index_type + '_index',
233
 
            self.index_class(self.index_transport,
234
 
                self.index_name(index_type, self.name),
235
 
                self.index_sizes[self.index_offset(index_type)],
236
 
                unlimited_cache=unlimited_cache))
 
231
        index = self.index_class(self.index_transport,
 
232
                    self.index_name(index_type, self.name),
 
233
                    self.index_sizes[self.index_offset(index_type)],
 
234
                    unlimited_cache=unlimited_cache)
 
235
        if index_type == 'chk':
 
236
            index._leaf_factory = btree_index._gcchk_factory
 
237
        setattr(self, index_type + '_index', index)
237
238
 
238
239
 
239
240
class ExistingPack(Pack):
587
588
                                             flush_func=flush_func)
588
589
        self.add_callback = None
589
590
 
590
 
    def replace_indices(self, index_to_pack, indices):
591
 
        """Replace the current mappings with fresh ones.
592
 
 
593
 
        This should probably not be used eventually, rather incremental add and
594
 
        removal of indices. It has been added during refactoring of existing
595
 
        code.
596
 
 
597
 
        :param index_to_pack: A mapping from index objects to
598
 
            (transport, name) tuples for the pack file data.
599
 
        :param indices: A list of indices.
600
 
        """
601
 
        # refresh the revision pack map dict without replacing the instance.
602
 
        self.index_to_pack.clear()
603
 
        self.index_to_pack.update(index_to_pack)
604
 
        # XXX: API break - clearly a 'replace' method would be good?
605
 
        self.combined_index._indices[:] = indices
606
 
        # the current add nodes callback for the current writable index if
607
 
        # there is one.
608
 
        self.add_callback = None
609
 
 
610
591
    def add_index(self, index, pack):
611
592
        """Add index to the aggregate, which is an index for Pack pack.
612
593
 
619
600
        # expose it to the index map
620
601
        self.index_to_pack[index] = pack.access_tuple()
621
602
        # put it at the front of the linear index list
622
 
        self.combined_index.insert_index(0, index)
 
603
        self.combined_index.insert_index(0, index, pack.name)
623
604
 
624
605
    def add_writable_index(self, index, pack):
625
606
        """Add an index which is able to have data added to it.
645
626
        self.data_access.set_writer(None, None, (None, None))
646
627
        self.index_to_pack.clear()
647
628
        del self.combined_index._indices[:]
 
629
        del self.combined_index._index_names[:]
648
630
        self.add_callback = None
649
631
 
650
632
    def remove_index(self, index):
653
635
        :param index: An index from the pack parameter.
654
636
        """
655
637
        del self.index_to_pack[index]
656
 
        self.combined_index._indices.remove(index)
 
638
        pos = self.combined_index._indices.index(index)
 
639
        del self.combined_index._indices[pos]
 
640
        del self.combined_index._index_names[pos]
657
641
        if (self.add_callback is not None and
658
642
            getattr(index, 'add_nodes', None) == self.add_callback):
659
643
            self.add_callback = None
738
722
        :return: A Pack object, or None if nothing was copied.
739
723
        """
740
724
        # open a pack - using the same name as the last temporary file
741
 
        # - which has already been flushed, so its safe.
 
725
        # - which has already been flushed, so it's safe.
742
726
        # XXX: - duplicate code warning with start_write_group; fix before
743
727
        #      considering 'done'.
744
728
        if self._pack_collection._new_pack is not None:
1308
1292
        # reinserted, and if d3 has incorrect parents it will also be
1309
1293
        # reinserted. If we insert d3 first, d2 is present (as it was bulk
1310
1294
        # copied), so we will try to delta, but d2 is not currently able to be
1311
 
        # extracted because it's basis d1 is not present. Topologically sorting
 
1295
        # extracted because its basis d1 is not present. Topologically sorting
1312
1296
        # addresses this. The following generates a sort for all the texts that
1313
1297
        # are being inserted without having to reference the entire text key
1314
1298
        # space (we only topo sort the revisions, which is smaller).
1415
1399
        self.inventory_index = AggregateIndex(self.reload_pack_names, flush)
1416
1400
        self.text_index = AggregateIndex(self.reload_pack_names, flush)
1417
1401
        self.signature_index = AggregateIndex(self.reload_pack_names, flush)
 
1402
        all_indices = [self.revision_index, self.inventory_index,
 
1403
                self.text_index, self.signature_index]
1418
1404
        if use_chk_index:
1419
1405
            self.chk_index = AggregateIndex(self.reload_pack_names, flush)
 
1406
            all_indices.append(self.chk_index)
1420
1407
        else:
1421
1408
            # used to determine if we're using a chk_index elsewhere.
1422
1409
            self.chk_index = None
 
1410
        # Tell all the CombinedGraphIndex objects about each other, so they can
 
1411
        # share hints about which pack names to search first.
 
1412
        all_combined = [agg_idx.combined_index for agg_idx in all_indices]
 
1413
        for combined_idx in all_combined:
 
1414
            combined_idx.set_sibling_indices(
 
1415
                set(all_combined).difference([combined_idx]))
1423
1416
        # resumed packs
1424
1417
        self._resumed_packs = []
1425
1418
 
1568
1561
        """Is the collection already packed?"""
1569
1562
        return not (self.repo._format.pack_compresses or (len(self._names) > 1))
1570
1563
 
1571
 
    def pack(self, hint=None):
 
1564
    def pack(self, hint=None, clean_obsolete_packs=False):
1572
1565
        """Pack the pack collection totally."""
1573
1566
        self.ensure_loaded()
1574
1567
        total_packs = len(self._names)
1590
1583
                pack_operations[-1][1].append(pack)
1591
1584
        self._execute_pack_operations(pack_operations, OptimisingPacker)
1592
1585
 
 
1586
        if clean_obsolete_packs:
 
1587
            self._clear_obsolete_packs()
 
1588
 
1593
1589
    def plan_autopack_combinations(self, existing_packs, pack_distribution):
1594
1590
        """Plan a pack operation.
1595
1591
 
1604
1600
        pack_operations = [[0, []]]
1605
1601
        # plan out what packs to keep, and what to reorganise
1606
1602
        while len(existing_packs):
1607
 
            # take the largest pack, and if its less than the head of the
 
1603
            # take the largest pack, and if it's less than the head of the
1608
1604
            # distribution chart we will include its contents in the new pack
1609
 
            # for that position. If its larger, we remove its size from the
 
1605
            # for that position. If it's larger, we remove its size from the
1610
1606
            # distribution chart
1611
1607
            next_pack_rev_count, next_pack = existing_packs.pop(0)
1612
1608
            if next_pack_rev_count >= pack_distribution[0]:
1647
1643
 
1648
1644
        :return: True if the disk names had not been previously read.
1649
1645
        """
1650
 
        # NB: if you see an assertion error here, its probably access against
 
1646
        # NB: if you see an assertion error here, it's probably access against
1651
1647
        # an unlocked repo. Naughty.
1652
1648
        if not self.repo.is_locked():
1653
1649
            raise errors.ObjectNotLocked(self.repo)
1683
1679
            txt_index = self._make_index(name, '.tix')
1684
1680
            sig_index = self._make_index(name, '.six')
1685
1681
            if self.chk_index is not None:
1686
 
                chk_index = self._make_index(name, '.cix', unlimited_cache=True)
 
1682
                chk_index = self._make_index(name, '.cix', is_chk=True)
1687
1683
            else:
1688
1684
                chk_index = None
1689
1685
            result = ExistingPack(self._pack_transport, name, rev_index,
1709
1705
            sig_index = self._make_index(name, '.six', resume=True)
1710
1706
            if self.chk_index is not None:
1711
1707
                chk_index = self._make_index(name, '.cix', resume=True,
1712
 
                                             unlimited_cache=True)
 
1708
                                             is_chk=True)
1713
1709
            else:
1714
1710
                chk_index = None
1715
1711
            result = self.resumed_pack_factory(name, rev_index, inv_index,
1745
1741
        return self._index_class(self.transport, 'pack-names', None
1746
1742
                ).iter_all_entries()
1747
1743
 
1748
 
    def _make_index(self, name, suffix, resume=False, unlimited_cache=False):
 
1744
    def _make_index(self, name, suffix, resume=False, is_chk=False):
1749
1745
        size_offset = self._suffix_offsets[suffix]
1750
1746
        index_name = name + suffix
1751
1747
        if resume:
1754
1750
        else:
1755
1751
            transport = self._index_transport
1756
1752
            index_size = self._names[name][size_offset]
1757
 
        return self._index_class(transport, index_name, index_size,
1758
 
                                 unlimited_cache=unlimited_cache)
 
1753
        index = self._index_class(transport, index_name, index_size,
 
1754
                                  unlimited_cache=is_chk)
 
1755
        if is_chk and self._index_class is btree_index.BTreeGraphIndex: 
 
1756
            index._leaf_factory = btree_index._gcchk_factory
 
1757
        return index
1759
1758
 
1760
1759
    def _max_pack_count(self, total_revisions):
1761
1760
        """Return the maximum number of packs to use for total revisions.
1947
1946
                    # disk index because the set values are the same, unless
1948
1947
                    # the only index shows up as deleted by the set difference
1949
1948
                    # - which it may. Until there is a specific test for this,
1950
 
                    # assume its broken. RBC 20071017.
 
1949
                    # assume it's broken. RBC 20071017.
1951
1950
                    self._remove_pack_from_memory(self.get_pack_by_name(name))
1952
1951
                    self._names[name] = sizes
1953
1952
                    self.get_pack_by_name(name)
2018
2017
        """
2019
2018
        # The ensure_loaded call is to handle the case where the first call
2020
2019
        # made involving the collection was to reload_pack_names, where we 
2021
 
        # don't have a view of disk contents. Its a bit of a bandaid, and
2022
 
        # causes two reads of pack-names, but its a rare corner case not struck
2023
 
        # with regular push/pull etc.
 
2020
        # don't have a view of disk contents. It's a bit of a bandaid, and
 
2021
        # causes two reads of pack-names, but it's a rare corner case not
 
2022
        # struck with regular push/pull etc.
2024
2023
        first_read = self.ensure_loaded()
2025
2024
        if first_read:
2026
2025
            return True
2345
2344
        return self._write_lock_count
2346
2345
 
2347
2346
    def lock_write(self, token=None):
 
2347
        """Lock the repository for writes.
 
2348
 
 
2349
        :return: A bzrlib.repository.RepositoryWriteLockResult.
 
2350
        """
2348
2351
        locked = self.is_locked()
2349
2352
        if not self._write_lock_count and locked:
2350
2353
            raise errors.ReadOnlyError(self)
2359
2362
                # Writes don't affect fallback repos
2360
2363
                repo.lock_read()
2361
2364
            self._refresh_data()
 
2365
        return RepositoryWriteLockResult(self.unlock, None)
2362
2366
 
2363
2367
    def lock_read(self):
 
2368
        """Lock the repository for reads.
 
2369
 
 
2370
        :return: A bzrlib.lock.LogicalLockResult.
 
2371
        """
2364
2372
        locked = self.is_locked()
2365
2373
        if self._write_lock_count:
2366
2374
            self._write_lock_count += 1
2373
2381
            for repo in self._fallback_repositories:
2374
2382
                repo.lock_read()
2375
2383
            self._refresh_data()
 
2384
        return LogicalLockResult(self.unlock)
2376
2385
 
2377
2386
    def leave_lock_in_place(self):
2378
2387
        # not supported - raise an error
2383
2392
        raise NotImplementedError(self.dont_leave_lock_in_place)
2384
2393
 
2385
2394
    @needs_write_lock
2386
 
    def pack(self, hint=None):
 
2395
    def pack(self, hint=None, clean_obsolete_packs=False):
2387
2396
        """Compress the data within the repository.
2388
2397
 
2389
2398
        This will pack all the data to a single pack. In future it may
2390
2399
        recompress deltas or do other such expensive operations.
2391
2400
        """
2392
 
        self._pack_collection.pack(hint=hint)
 
2401
        self._pack_collection.pack(hint=hint, clean_obsolete_packs=clean_obsolete_packs)
2393
2402
 
2394
2403
    @needs_write_lock
2395
2404
    def reconcile(self, other=None, thorough=False):
2551
2560
        utf8_files = [('format', self.get_format_string())]
2552
2561
 
2553
2562
        self._upload_blank_content(a_bzrdir, dirs, files, utf8_files, shared)
2554
 
        return self.open(a_bzrdir=a_bzrdir, _found=True)
 
2563
        repository = self.open(a_bzrdir=a_bzrdir, _found=True)
 
2564
        self._run_post_repo_init_hooks(repository, a_bzrdir, shared)
 
2565
        return repository
2555
2566
 
2556
2567
    def open(self, a_bzrdir, _found=False, _override_transport=None):
2557
2568
        """See RepositoryFormat.open().
2820
2831
    _commit_builder_class = PackCommitBuilder
2821
2832
    supports_external_lookups = True
2822
2833
    # What index classes to use
2823
 
    index_builder_class = BTreeBuilder
2824
 
    index_class = BTreeGraphIndex
 
2834
    index_builder_class = btree_index.BTreeBuilder
 
2835
    index_class = btree_index.BTreeGraphIndex
2825
2836
 
2826
2837
    @property
2827
2838
    def _serializer(self):
2856
2867
    supports_tree_reference = False # no subtrees
2857
2868
    supports_external_lookups = True
2858
2869
    # What index classes to use
2859
 
    index_builder_class = BTreeBuilder
2860
 
    index_class = BTreeGraphIndex
 
2870
    index_builder_class = btree_index.BTreeBuilder
 
2871
    index_class = btree_index.BTreeGraphIndex
2861
2872
 
2862
2873
    @property
2863
2874
    def _serializer(self):
2898
2909
    supports_tree_reference = True
2899
2910
    supports_external_lookups = True
2900
2911
    # What index classes to use
2901
 
    index_builder_class = BTreeBuilder
2902
 
    index_class = BTreeGraphIndex
 
2912
    index_builder_class = btree_index.BTreeBuilder
 
2913
    index_class = btree_index.BTreeGraphIndex
2903
2914
 
2904
2915
    @property
2905
2916
    def _serializer(self):
2907
2918
 
2908
2919
    def _get_matching_bzrdir(self):
2909
2920
        return bzrdir.format_registry.make_bzrdir(
2910
 
            'development-subtree')
 
2921
            'development5-subtree')
2911
2922
 
2912
2923
    def _ignore_setting_bzrdir(self, format):
2913
2924
        pass