← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/knit.py

  • Committer: Martin Pool
  • Date: 2010-01-29 14:09:05 UTC
  • mto: This revision was merged to the branch mainline in revision 4992.
  • Revision ID: mbp@sourcefrog.net-20100129140905-2uiarb6p8di1ywsr
Correction to url

from review: https://code.edge.launchpad.net/~mbp/bzr/doc/+merge/18250

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/knit.py
69
69
    lru_cache,
70
70
    pack,
71
71
    progress,
 
72
    static_tuple,
72
73
    trace,
73
74
    tsort,
74
75
    tuned_gzip,
1190
1191
        generator = _VFContentMapGenerator(self, [key])
1191
1192
        return generator._get_content(key)
1192
1193
 
 
1194
    def get_known_graph_ancestry(self, keys):
 
1195
        """Get a KnownGraph instance with the ancestry of keys."""
 
1196
        parent_map, missing_keys = self._index.find_ancestry(keys)
 
1197
        for fallback in self._fallback_vfs:
 
1198
            if not missing_keys:
 
1199
                break
 
1200
            (f_parent_map, f_missing_keys) = fallback._index.find_ancestry(
 
1201
                                                missing_keys)
 
1202
            parent_map.update(f_parent_map)
 
1203
            missing_keys = f_missing_keys
 
1204
        kg = _mod_graph.KnownGraph(parent_map)
 
1205
        return kg
 
1206
 
1193
1207
    def get_parent_map(self, keys):
1194
1208
        """Get a map of the graph parents of keys.
1195
1209
 
1505
1519
                if source is parent_maps[0]:
1506
1520
                    # this KnitVersionedFiles
1507
1521
                    records = [(key, positions[key][1]) for key in keys]
1508
 
                    for key, raw_data, sha1 in self._read_records_iter_raw(records):
 
1522
                    for key, raw_data in self._read_records_iter_unchecked(records):
1509
1523
                        (record_details, index_memo, _) = positions[key]
1510
1524
                        yield KnitContentFactory(key, global_map[key],
1511
 
                            record_details, sha1, raw_data, self._factory.annotated, None)
 
1525
                            record_details, None, raw_data, self._factory.annotated, None)
1512
1526
                else:
1513
1527
                    vf = self._fallback_vfs[parent_maps.index(source) - 1]
1514
1528
                    for record in vf.get_record_stream(keys, ordering,
1583
1597
        # key = basis_parent, value = index entry to add
1584
1598
        buffered_index_entries = {}
1585
1599
        for record in stream:
 
1600
            kind = record.storage_kind
 
1601
            if kind.startswith('knit-') and kind.endswith('-gz'):
 
1602
                # Check that the ID in the header of the raw knit bytes matches
 
1603
                # the record metadata.
 
1604
                raw_data = record._raw_record
 
1605
                df, rec = self._parse_record_header(record.key, raw_data)
 
1606
                df.close()
1586
1607
            buffered = False
1587
1608
            parents = record.parents
1588
1609
            if record.storage_kind in delta_types:
1690
1711
            # There were index entries buffered at the end of the stream,
1691
1712
            # So these need to be added (if the index supports holding such
1692
1713
            # entries for later insertion)
 
1714
            all_entries = []
1693
1715
            for key in buffered_index_entries:
1694
1716
                index_entries = buffered_index_entries[key]
1695
 
                self._index.add_records(index_entries,
1696
 
                    missing_compression_parents=True)
 
1717
                all_entries.extend(index_entries)
 
1718
            self._index.add_records(
 
1719
                all_entries, missing_compression_parents=True)
1697
1720
 
1698
1721
    def get_missing_compression_parent_keys(self):
1699
1722
        """Return an iterable of keys of missing compression parents.
2345
2368
    FLAGS is a comma separated list of flags about the record. Values include
2346
2369
        no-eol, line-delta, fulltext.
2347
2370
    BYTE_OFFSET is the ascii representation of the byte offset in the data file
2348
 
        that the the compressed data starts at.
 
2371
        that the compressed data starts at.
2349
2372
    LENGTH is the ascii representation of the length of the data file.
2350
2373
    PARENT_ID a utf-8 revision id prefixed by a '.' that is a parent of
2351
2374
        REVISION_ID.
2560
2583
        except KeyError:
2561
2584
            raise RevisionNotPresent(key, self)
2562
2585
 
 
2586
    def find_ancestry(self, keys):
 
2587
        """See CombinedGraphIndex.find_ancestry()"""
 
2588
        prefixes = set(key[:-1] for key in keys)
 
2589
        self._load_prefixes(prefixes)
 
2590
        result = {}
 
2591
        parent_map = {}
 
2592
        missing_keys = set()
 
2593
        pending_keys = list(keys)
 
2594
        # This assumes that keys will not reference parents in a different
 
2595
        # prefix, which is accurate so far.
 
2596
        while pending_keys:
 
2597
            key = pending_keys.pop()
 
2598
            if key in parent_map:
 
2599
                continue
 
2600
            prefix = key[:-1]
 
2601
            try:
 
2602
                suffix_parents = self._kndx_cache[prefix][0][key[-1]][4]
 
2603
            except KeyError:
 
2604
                missing_keys.add(key)
 
2605
            else:
 
2606
                parent_keys = tuple([prefix + (suffix,)
 
2607
                                     for suffix in suffix_parents])
 
2608
                parent_map[key] = parent_keys
 
2609
                pending_keys.extend([p for p in parent_keys
 
2610
                                        if p not in parent_map])
 
2611
        return parent_map, missing_keys
 
2612
 
2563
2613
    def get_parent_map(self, keys):
2564
2614
        """Get a map of the parents of keys.
2565
2615
 
2737
2787
 
2738
2788
class _KeyRefs(object):
2739
2789
 
2740
 
    def __init__(self):
 
2790
    def __init__(self, track_new_keys=False):
2741
2791
        # dict mapping 'key' to 'set of keys referring to that key'
2742
2792
        self.refs = {}
 
2793
        if track_new_keys:
 
2794
            # set remembering all new keys
 
2795
            self.new_keys = set()
 
2796
        else:
 
2797
            self.new_keys = None
 
2798
 
 
2799
    def clear(self):
 
2800
        if self.refs:
 
2801
            self.refs.clear()
 
2802
        if self.new_keys:
 
2803
            self.new_keys.clear()
2743
2804
 
2744
2805
    def add_references(self, key, refs):
2745
2806
        # Record the new references
2752
2813
        # Discard references satisfied by the new key
2753
2814
        self.add_key(key)
2754
2815
 
 
2816
    def get_new_keys(self):
 
2817
        return self.new_keys
 
2818
    
2755
2819
    def get_unsatisfied_refs(self):
2756
2820
        return self.refs.iterkeys()
2757
2821
 
2758
 
    def add_key(self, key):
 
2822
    def _satisfy_refs_for_key(self, key):
2759
2823
        try:
2760
2824
            del self.refs[key]
2761
2825
        except KeyError:
2762
2826
            # No keys depended on this key.  That's ok.
2763
2827
            pass
2764
2828
 
2765
 
    def add_keys(self, keys):
 
2829
    def add_key(self, key):
 
2830
        # satisfy refs for key, and remember that we've seen this key.
 
2831
        self._satisfy_refs_for_key(key)
 
2832
        if self.new_keys is not None:
 
2833
            self.new_keys.add(key)
 
2834
 
 
2835
    def satisfy_refs_for_keys(self, keys):
2766
2836
        for key in keys:
2767
 
            self.add_key(key)
 
2837
            self._satisfy_refs_for_key(key)
2768
2838
 
2769
2839
    def get_referrers(self):
2770
2840
        result = set()
2875
2945
        if not random_id:
2876
2946
            present_nodes = self._get_entries(keys)
2877
2947
            for (index, key, value, node_refs) in present_nodes:
 
2948
                parents = node_refs[:1]
 
2949
                # Sometimes these are passed as a list rather than a tuple
 
2950
                passed = static_tuple.as_tuples(keys[key])
 
2951
                passed_parents = passed[1][:1]
2878
2952
                if (value[0] != keys[key][0][0] or
2879
 
                    node_refs[:1] != keys[key][1][:1]):
 
2953
                    parents != passed_parents):
 
2954
                    node_refs = static_tuple.as_tuples(node_refs)
2880
2955
                    raise KnitCorrupt(self, "inconsistent details in add_records"
2881
 
                        ": %s %s" % ((value, node_refs), keys[key]))
 
2956
                        ": %s %s" % ((value, node_refs), passed))
2882
2957
                del keys[key]
2883
2958
        result = []
2884
2959
        if self._parents:
2932
3007
        # If updating this, you should also update
2933
3008
        # groupcompress._GCGraphIndex.get_missing_parents
2934
3009
        # We may have false positives, so filter those out.
2935
 
        self._key_dependencies.add_keys(
 
3010
        self._key_dependencies.satisfy_refs_for_keys(
2936
3011
            self.get_parent_map(self._key_dependencies.get_unsatisfied_refs()))
2937
3012
        return frozenset(self._key_dependencies.get_unsatisfied_refs())
2938
3013
 
3049
3124
            options.append('no-eol')
3050
3125
        return options
3051
3126
 
 
3127
    def find_ancestry(self, keys):
 
3128
        """See CombinedGraphIndex.find_ancestry()"""
 
3129
        return self._graph_index.find_ancestry(keys, 0)
 
3130
 
3052
3131
    def get_parent_map(self, keys):
3053
3132
        """Get a map of the parents of keys.
3054
3133
 
3629
3708
 
3630
3709
try:
3631
3710
    from bzrlib._knit_load_data_pyx import _load_data_c as _load_data
3632
 
except ImportError:
 
3711
except ImportError, e:
 
3712
    osutils.failed_to_load_extension(e)
3633
3713
    from bzrlib._knit_load_data_py import _load_data_py as _load_data