← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/knit.py

  • Committer: Jelmer Vernooij
  • Date: 2009-09-08 13:16:32 UTC
  • mto: This revision was merged to the branch mainline in revision 4757.
  • Revision ID: jelmer@samba.org-20090908131632-c5c6nyzf79im6fhs
Some review comments from John.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/knit.py
1
 
# Copyright (C) 2006-2010 Canonical Ltd
 
1
# Copyright (C) 2005, 2006, 2007, 2008 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
69
69
    lru_cache,
70
70
    pack,
71
71
    progress,
72
 
    static_tuple,
73
72
    trace,
74
73
    tsort,
75
74
    tuned_gzip,
76
 
    ui,
77
75
    )
78
76
""")
79
77
from bzrlib import (
1520
1518
                if source is parent_maps[0]:
1521
1519
                    # this KnitVersionedFiles
1522
1520
                    records = [(key, positions[key][1]) for key in keys]
1523
 
                    for key, raw_data in self._read_records_iter_unchecked(records):
 
1521
                    for key, raw_data, sha1 in self._read_records_iter_raw(records):
1524
1522
                        (record_details, index_memo, _) = positions[key]
1525
1523
                        yield KnitContentFactory(key, global_map[key],
1526
 
                            record_details, None, raw_data, self._factory.annotated, None)
 
1524
                            record_details, sha1, raw_data, self._factory.annotated, None)
1527
1525
                else:
1528
1526
                    vf = self._fallback_vfs[parent_maps.index(source) - 1]
1529
1527
                    for record in vf.get_record_stream(keys, ordering,
1598
1596
        # key = basis_parent, value = index entry to add
1599
1597
        buffered_index_entries = {}
1600
1598
        for record in stream:
1601
 
            kind = record.storage_kind
1602
 
            if kind.startswith('knit-') and kind.endswith('-gz'):
1603
 
                # Check that the ID in the header of the raw knit bytes matches
1604
 
                # the record metadata.
1605
 
                raw_data = record._raw_record
1606
 
                df, rec = self._parse_record_header(record.key, raw_data)
1607
 
                df.close()
1608
1599
            buffered = False
1609
1600
            parents = record.parents
1610
1601
            if record.storage_kind in delta_types:
1712
1703
            # There were index entries buffered at the end of the stream,
1713
1704
            # So these need to be added (if the index supports holding such
1714
1705
            # entries for later insertion)
1715
 
            all_entries = []
1716
1706
            for key in buffered_index_entries:
1717
1707
                index_entries = buffered_index_entries[key]
1718
 
                all_entries.extend(index_entries)
1719
 
            self._index.add_records(
1720
 
                all_entries, missing_compression_parents=True)
 
1708
                self._index.add_records(index_entries,
 
1709
                    missing_compression_parents=True)
1721
1710
 
1722
1711
    def get_missing_compression_parent_keys(self):
1723
1712
        """Return an iterable of keys of missing compression parents.
1756
1745
        :return: An iterator over (line, key).
1757
1746
        """
1758
1747
        if pb is None:
1759
 
            pb = ui.ui_factory.nested_progress_bar()
 
1748
            pb = progress.DummyProgress()
1760
1749
        keys = set(keys)
1761
1750
        total = len(keys)
1762
1751
        done = False
2369
2358
    FLAGS is a comma separated list of flags about the record. Values include
2370
2359
        no-eol, line-delta, fulltext.
2371
2360
    BYTE_OFFSET is the ascii representation of the byte offset in the data file
2372
 
        that the compressed data starts at.
 
2361
        that the the compressed data starts at.
2373
2362
    LENGTH is the ascii representation of the length of the data file.
2374
2363
    PARENT_ID a utf-8 revision id prefixed by a '.' that is a parent of
2375
2364
        REVISION_ID.
2788
2777
 
2789
2778
class _KeyRefs(object):
2790
2779
 
2791
 
    def __init__(self, track_new_keys=False):
 
2780
    def __init__(self):
2792
2781
        # dict mapping 'key' to 'set of keys referring to that key'
2793
2782
        self.refs = {}
2794
 
        if track_new_keys:
2795
 
            # set remembering all new keys
2796
 
            self.new_keys = set()
2797
 
        else:
2798
 
            self.new_keys = None
2799
 
 
2800
 
    def clear(self):
2801
 
        if self.refs:
2802
 
            self.refs.clear()
2803
 
        if self.new_keys:
2804
 
            self.new_keys.clear()
2805
2783
 
2806
2784
    def add_references(self, key, refs):
2807
2785
        # Record the new references
2814
2792
        # Discard references satisfied by the new key
2815
2793
        self.add_key(key)
2816
2794
 
2817
 
    def get_new_keys(self):
2818
 
        return self.new_keys
2819
 
    
2820
2795
    def get_unsatisfied_refs(self):
2821
2796
        return self.refs.iterkeys()
2822
2797
 
2823
 
    def _satisfy_refs_for_key(self, key):
 
2798
    def add_key(self, key):
2824
2799
        try:
2825
2800
            del self.refs[key]
2826
2801
        except KeyError:
2827
2802
            # No keys depended on this key.  That's ok.
2828
2803
            pass
2829
2804
 
2830
 
    def add_key(self, key):
2831
 
        # satisfy refs for key, and remember that we've seen this key.
2832
 
        self._satisfy_refs_for_key(key)
2833
 
        if self.new_keys is not None:
2834
 
            self.new_keys.add(key)
2835
 
 
2836
 
    def satisfy_refs_for_keys(self, keys):
 
2805
    def add_keys(self, keys):
2837
2806
        for key in keys:
2838
 
            self._satisfy_refs_for_key(key)
 
2807
            self.add_key(key)
2839
2808
 
2840
2809
    def get_referrers(self):
2841
2810
        result = set()
2946
2915
        if not random_id:
2947
2916
            present_nodes = self._get_entries(keys)
2948
2917
            for (index, key, value, node_refs) in present_nodes:
2949
 
                parents = node_refs[:1]
2950
 
                # Sometimes these are passed as a list rather than a tuple
2951
 
                passed = static_tuple.as_tuples(keys[key])
2952
 
                passed_parents = passed[1][:1]
2953
2918
                if (value[0] != keys[key][0][0] or
2954
 
                    parents != passed_parents):
2955
 
                    node_refs = static_tuple.as_tuples(node_refs)
 
2919
                    node_refs[:1] != keys[key][1][:1]):
2956
2920
                    raise KnitCorrupt(self, "inconsistent details in add_records"
2957
 
                        ": %s %s" % ((value, node_refs), passed))
 
2921
                        ": %s %s" % ((value, node_refs), keys[key]))
2958
2922
                del keys[key]
2959
2923
        result = []
2960
2924
        if self._parents:
3008
2972
        # If updating this, you should also update
3009
2973
        # groupcompress._GCGraphIndex.get_missing_parents
3010
2974
        # We may have false positives, so filter those out.
3011
 
        self._key_dependencies.satisfy_refs_for_keys(
 
2975
        self._key_dependencies.add_keys(
3012
2976
            self.get_parent_map(self._key_dependencies.get_unsatisfied_refs()))
3013
2977
        return frozenset(self._key_dependencies.get_unsatisfied_refs())
3014
2978
 
3709
3673
 
3710
3674
try:
3711
3675
    from bzrlib._knit_load_data_pyx import _load_data_c as _load_data
3712
 
except ImportError, e:
3713
 
    osutils.failed_to_load_extension(e)
 
3676
except ImportError:
3714
3677
    from bzrlib._knit_load_data_py import _load_data_py as _load_data