~bzr-pqm/bzr/bzr.dev

5557.1.7 by John Arbash Meinel
Merge in the bzr.dev 5582
1
# Copyright (C) 2007-2011 Canonical Ltd
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
16
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
17
import time
18
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
19
from bzrlib import (
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
20
    debug,
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
21
    errors,
4574.3.6 by Martin Pool
More warnings when failing to load extensions
22
    osutils,
3052.1.3 by John Arbash Meinel
deprecate revision.is_ancestor, update the callers and the tests.
23
    revision,
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
24
    trace,
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
25
    )
2490.2.1 by Aaron Bentley
Start work on GraphWalker
26
3377.4.9 by John Arbash Meinel
STEP every 5
27
STEP_UNIQUE_SEARCHER_EVERY = 5
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
28
2490.2.25 by Aaron Bentley
Update from review
29
# DIAGRAM of terminology
30
#       A
31
#       /\
32
#      B  C
33
#      |  |\
34
#      D  E F
35
#      |\/| |
36
#      |/\|/
37
#      G  H
38
#
39
# In this diagram, relative to G and H:
40
# A, B, C, D, E are common ancestors.
41
# C, D and E are border ancestors, because each has a non-common descendant.
42
# D and E are least common ancestors because none of their descendants are
43
# common ancestors.
44
# C is not a least common ancestor because its descendant, E, is a common
45
# ancestor.
46
#
47
# The find_unique_lca algorithm will pick A in two steps:
48
# 1. find_lca('G', 'H') => ['D', 'E']
49
# 2. Since len(['D', 'E']) > 1, find_lca('D', 'E') => ['A']
50
51
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
52
class DictParentsProvider(object):
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
53
    """A parents provider for Graph objects."""
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
54
55
    def __init__(self, ancestry):
56
        self.ancestry = ancestry
57
58
    def __repr__(self):
59
        return 'DictParentsProvider(%r)' % self.ancestry
60
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
61
    def get_parent_map(self, keys):
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
62
        """See StackedParentsProvider.get_parent_map"""
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
63
        ancestry = self.ancestry
64
        return dict((k, ancestry[k]) for k in keys if k in ancestry)
65
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
66
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
67
class StackedParentsProvider(object):
68
    """A parents provider which stacks (or unions) multiple providers.
69
    
70
    The providers are queries in the order of the provided parent_providers.
71
    """
72
    
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
73
    def __init__(self, parent_providers):
74
        self._parent_providers = parent_providers
75
2490.2.28 by Aaron Bentley
Fix handling of null revision
76
    def __repr__(self):
4379.3.4 by Gary van der Merwe
Make StackedParentsProvider.__repr__ more dynamic.
77
        return "%s(%r)" % (self.__class__.__name__, self._parent_providers)
2490.2.28 by Aaron Bentley
Fix handling of null revision
78
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
79
    def get_parent_map(self, keys):
80
        """Get a mapping of keys => parents
81
82
        A dictionary is returned with an entry for each key present in this
83
        source. If this source doesn't have information about a key, it should
84
        not include an entry.
85
86
        [NULL_REVISION] is used as the parent of the first user-committed
87
        revision.  Its parent list is empty.
88
89
        :param keys: An iterable returning keys to check (eg revision_ids)
90
        :return: A dictionary mapping each key to its parents
91
        """
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
92
        found = {}
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
93
        remaining = set(keys)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
94
        for parents_provider in self._parent_providers:
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
95
            new_found = parents_provider.get_parent_map(remaining)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
96
            found.update(new_found)
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
97
            remaining.difference_update(new_found)
98
            if not remaining:
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
99
                break
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
100
        return found
101
102
103
class CachingParentsProvider(object):
3835.1.13 by Aaron Bentley
Update documentation
104
    """A parents provider which will cache the revision => parents as a dict.
105
106
    This is useful for providers which have an expensive look up.
107
108
    Either a ParentsProvider or a get_parent_map-like callback may be
109
    supplied.  If it provides extra un-asked-for parents, they will be cached,
110
    but filtered out of get_parent_map.
3835.1.16 by Aaron Bentley
Updates from review
111
112
    The cache is enabled by default, but may be disabled and re-enabled.
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
113
    """
3896.1.1 by Andrew Bennetts
Remove broken debugging cruft, and some unused imports.
114
    def __init__(self, parent_provider=None, get_parent_map=None):
3835.1.13 by Aaron Bentley
Update documentation
115
        """Constructor.
116
117
        :param parent_provider: The ParentProvider to use.  It or
118
            get_parent_map must be supplied.
119
        :param get_parent_map: The get_parent_map callback to use.  It or
120
            parent_provider must be supplied.
121
        """
3835.1.12 by Aaron Bentley
Unify CachingExtraParentsProvider and CachingParentsProvider.
122
        self._real_provider = parent_provider
123
        if get_parent_map is None:
124
            self._get_parent_map = self._real_provider.get_parent_map
125
        else:
126
            self._get_parent_map = get_parent_map
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
127
        self._cache = None
128
        self.enable_cache(True)
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
129
3835.1.12 by Aaron Bentley
Unify CachingExtraParentsProvider and CachingParentsProvider.
130
    def __repr__(self):
131
        return "%s(%r)" % (self.__class__.__name__, self._real_provider)
132
3835.1.15 by Aaron Bentley
Allow miss caching to be disabled.
133
    def enable_cache(self, cache_misses=True):
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
134
        """Enable cache."""
3835.1.19 by Aaron Bentley
Raise exception when caching is enabled twice.
135
        if self._cache is not None:
3835.1.20 by Aaron Bentley
Change custom error to an AssertionError.
136
            raise AssertionError('Cache enabled when already enabled.')
3835.1.16 by Aaron Bentley
Updates from review
137
        self._cache = {}
3835.1.15 by Aaron Bentley
Allow miss caching to be disabled.
138
        self._cache_misses = cache_misses
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
139
        self.missing_keys = set()
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
140
141
    def disable_cache(self):
3835.1.16 by Aaron Bentley
Updates from review
142
        """Disable and clear the cache."""
143
        self._cache = None
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
144
        self._cache_misses = None
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
145
        self.missing_keys = set()
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
146
147
    def get_cached_map(self):
148
        """Return any cached get_parent_map values."""
3835.1.16 by Aaron Bentley
Updates from review
149
        if self._cache is None:
3835.1.12 by Aaron Bentley
Unify CachingExtraParentsProvider and CachingParentsProvider.
150
            return None
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
151
        return dict(self._cache)
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
152
153
    def get_parent_map(self, keys):
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
154
        """See StackedParentsProvider.get_parent_map."""
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
155
        cache = self._cache
156
        if cache is None:
157
            cache = self._get_parent_map(keys)
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
158
        else:
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
159
            needed_revisions = set(key for key in keys if key not in cache)
160
            # Do not ask for negatively cached keys
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
161
            needed_revisions.difference_update(self.missing_keys)
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
162
            if needed_revisions:
163
                parent_map = self._get_parent_map(needed_revisions)
164
                cache.update(parent_map)
165
                if self._cache_misses:
166
                    for key in needed_revisions:
167
                        if key not in parent_map:
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
168
                            self.note_missing_key(key)
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
169
        result = {}
170
        for key in keys:
171
            value = cache.get(key)
172
            if value is not None:
173
                result[key] = value
174
        return result
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
175
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
176
    def note_missing_key(self, key):
177
        """Note that key is a missing key."""
178
        if self._cache_misses:
179
            self.missing_keys.add(key)
180
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
181
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
182
class Graph(object):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
183
    """Provide incremental access to revision graphs.
184
185
    This is the generic implementation; it is intended to be subclassed to
186
    specialize it for other repository types.
187
    """
2490.2.1 by Aaron Bentley
Start work on GraphWalker
188
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
189
    def __init__(self, parents_provider):
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
190
        """Construct a Graph that uses several graphs as its input
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
191
192
        This should not normally be invoked directly, because there may be
193
        specialized implementations for particular repository types.  See
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
194
        Repository.get_graph().
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
195
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
196
        :param parents_provider: An object providing a get_parent_map call
197
            conforming to the behavior of
198
            StackedParentsProvider.get_parent_map.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
199
        """
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
200
        if getattr(parents_provider, 'get_parents', None) is not None:
201
            self.get_parents = parents_provider.get_parents
202
        if getattr(parents_provider, 'get_parent_map', None) is not None:
203
            self.get_parent_map = parents_provider.get_parent_map
2490.2.29 by Aaron Bentley
Make parents provider private
204
        self._parents_provider = parents_provider
2490.2.28 by Aaron Bentley
Fix handling of null revision
205
206
    def __repr__(self):
2490.2.29 by Aaron Bentley
Make parents provider private
207
        return 'Graph(%r)' % self._parents_provider
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
208
209
    def find_lca(self, *revisions):
210
        """Determine the lowest common ancestors of the provided revisions
211
212
        A lowest common ancestor is a common ancestor none of whose
213
        descendants are common ancestors.  In graphs, unlike trees, there may
214
        be multiple lowest common ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
215
216
        This algorithm has two phases.  Phase 1 identifies border ancestors,
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
217
        and phase 2 filters border ancestors to determine lowest common
218
        ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
219
220
        In phase 1, border ancestors are identified, using a breadth-first
221
        search starting at the bottom of the graph.  Searches are stopped
222
        whenever a node or one of its descendants is determined to be common
223
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
224
        In phase 2, the border ancestors are filtered to find the least
2490.2.12 by Aaron Bentley
Improve documentation
225
        common ancestors.  This is done by searching the ancestries of each
226
        border ancestor.
227
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
228
        Phase 2 is perfomed on the principle that a border ancestor that is
229
        not an ancestor of any other border ancestor is a least common
230
        ancestor.
2490.2.12 by Aaron Bentley
Improve documentation
231
232
        Searches are stopped when they find a node that is determined to be a
233
        common ancestor of all border ancestors, because this shows that it
234
        cannot be a descendant of any border ancestor.
235
236
        The scaling of this operation should be proportional to
237
        1. The number of uncommon ancestors
238
        2. The number of border ancestors
239
        3. The length of the shortest path between a border ancestor and an
240
           ancestor of all border ancestors.
2490.2.3 by Aaron Bentley
Implement new merge base picker
241
        """
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
242
        border_common, common, sides = self._find_border_ancestors(revisions)
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
243
        # We may have common ancestors that can be reached from each other.
244
        # - ask for the heads of them to filter it down to only ones that
245
        # cannot be reached from each other - phase 2.
246
        return self.heads(border_common)
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
247
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
248
    def find_difference(self, left_revision, right_revision):
2490.2.25 by Aaron Bentley
Update from review
249
        """Determine the graph difference between two revisions"""
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
250
        border, common, searchers = self._find_border_ancestors(
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
251
            [left_revision, right_revision])
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
252
        self._search_for_extra_common(common, searchers)
253
        left = searchers[0].seen
254
        right = searchers[1].seen
255
        return (left.difference(right), right.difference(left))
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
256
5365.6.1 by Aaron Bentley
Implement find_descendants.
257
    def find_descendants(self, old_key, new_key):
258
        """Find descendants of old_key that are ancestors of new_key."""
259
        child_map = self.get_child_map(self._find_descendant_ancestors(
260
            old_key, new_key))
261
        graph = Graph(DictParentsProvider(child_map))
262
        searcher = graph._make_breadth_first_searcher([old_key])
263
        list(searcher)
264
        return searcher.seen
265
266
    def _find_descendant_ancestors(self, old_key, new_key):
267
        """Find ancestors of new_key that may be descendants of old_key."""
268
        stop = self._make_breadth_first_searcher([old_key])
269
        descendants = self._make_breadth_first_searcher([new_key])
270
        for revisions in descendants:
271
            old_stop = stop.seen.intersection(revisions)
272
            descendants.stop_searching_any(old_stop)
273
            seen_stop = descendants.find_seen_ancestors(stop.step())
274
            descendants.stop_searching_any(seen_stop)
275
        return descendants.seen.difference(stop.seen)
276
277
    def get_child_map(self, keys):
278
        """Get a mapping from parents to children of the specified keys.
279
280
        This is simply the inversion of get_parent_map.  Only supplied keys
281
        will be discovered as children.
282
        :return: a dict of key:child_list for keys.
283
        """
284
        parent_map = self._parents_provider.get_parent_map(keys)
285
        parent_child = {}
286
        for child, parents in sorted(parent_map.items()):
287
            for parent in parents:
288
                parent_child.setdefault(parent, []).append(child)
289
        return parent_child
290
3445.1.4 by John Arbash Meinel
Change the function to be called 'find_distance_to_null'
291
    def find_distance_to_null(self, target_revision_id, known_revision_ids):
292
        """Find the left-hand distance to the NULL_REVISION.
293
294
        (This can also be considered the revno of a branch at
295
        target_revision_id.)
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
296
297
        :param target_revision_id: A revision_id which we would like to know
298
            the revno for.
299
        :param known_revision_ids: [(revision_id, revno)] A list of known
300
            revno, revision_id tuples. We'll use this to seed the search.
301
        """
302
        # Map from revision_ids to a known value for their revno
303
        known_revnos = dict(known_revision_ids)
304
        cur_tip = target_revision_id
305
        num_steps = 0
306
        NULL_REVISION = revision.NULL_REVISION
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
307
        known_revnos[NULL_REVISION] = 0
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
308
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
309
        searching_known_tips = list(known_revnos.keys())
310
311
        unknown_searched = {}
312
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
313
        while cur_tip not in known_revnos:
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
314
            unknown_searched[cur_tip] = num_steps
315
            num_steps += 1
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
316
            to_search = set([cur_tip])
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
317
            to_search.update(searching_known_tips)
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
318
            parent_map = self.get_parent_map(to_search)
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
319
            parents = parent_map.get(cur_tip, None)
3445.1.8 by John Arbash Meinel
Clarity tweaks recommended by Ian
320
            if not parents: # An empty list or None is a ghost
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
321
                raise errors.GhostRevisionsHaveNoRevno(target_revision_id,
322
                                                       cur_tip)
323
            cur_tip = parents[0]
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
324
            next_known_tips = []
325
            for revision_id in searching_known_tips:
326
                parents = parent_map.get(revision_id, None)
327
                if not parents:
328
                    continue
329
                next = parents[0]
330
                next_revno = known_revnos[revision_id] - 1
331
                if next in unknown_searched:
332
                    # We have enough information to return a value right now
333
                    return next_revno + unknown_searched[next]
334
                if next in known_revnos:
335
                    continue
336
                known_revnos[next] = next_revno
337
                next_known_tips.append(next)
338
            searching_known_tips = next_known_tips
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
339
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
340
        # We reached a known revision, so just add in how many steps it took to
341
        # get there.
342
        return known_revnos[cur_tip] + num_steps
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
343
4332.3.4 by Robert Collins
Add a graph API for getting multiple distances to NULL at once.
344
    def find_lefthand_distances(self, keys):
345
        """Find the distance to null for all the keys in keys.
346
347
        :param keys: keys to lookup.
348
        :return: A dict key->distance for all of keys.
349
        """
350
        # Optimisable by concurrent searching, but a random spread should get
351
        # some sort of hit rate.
352
        result = {}
353
        known_revnos = []
4332.3.6 by Robert Collins
Teach graph.find_lefthand_distances about ghosts.
354
        ghosts = []
4332.3.4 by Robert Collins
Add a graph API for getting multiple distances to NULL at once.
355
        for key in keys:
4332.3.6 by Robert Collins
Teach graph.find_lefthand_distances about ghosts.
356
            try:
357
                known_revnos.append(
358
                    (key, self.find_distance_to_null(key, known_revnos)))
359
            except errors.GhostRevisionsHaveNoRevno:
360
                ghosts.append(key)
361
        for key in ghosts:
362
            known_revnos.append((key, -1))
4332.3.4 by Robert Collins
Add a graph API for getting multiple distances to NULL at once.
363
        return dict(known_revnos)
364
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
365
    def find_unique_ancestors(self, unique_revision, common_revisions):
366
        """Find the unique ancestors for a revision versus others.
367
368
        This returns the ancestry of unique_revision, excluding all revisions
369
        in the ancestry of common_revisions. If unique_revision is in the
370
        ancestry, then the empty set will be returned.
371
372
        :param unique_revision: The revision_id whose ancestry we are
373
            interested in.
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
374
            XXX: Would this API be better if we allowed multiple revisions on
375
                 to be searched here?
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
376
        :param common_revisions: Revision_ids of ancestries to exclude.
377
        :return: A set of revisions in the ancestry of unique_revision
378
        """
379
        if unique_revision in common_revisions:
380
            return set()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
381
382
        # Algorithm description
383
        # 1) Walk backwards from the unique node and all common nodes.
384
        # 2) When a node is seen by both sides, stop searching it in the unique
385
        #    walker, include it in the common walker.
386
        # 3) Stop searching when there are no nodes left for the unique walker.
387
        #    At this point, you have a maximal set of unique nodes. Some of
388
        #    them may actually be common, and you haven't reached them yet.
389
        # 4) Start new searchers for the unique nodes, seeded with the
390
        #    information you have so far.
391
        # 5) Continue searching, stopping the common searches when the search
392
        #    tip is an ancestor of all unique nodes.
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
393
        # 6) Aggregate together unique searchers when they are searching the
394
        #    same tips. When all unique searchers are searching the same node,
395
        #    stop move it to a single 'all_unique_searcher'.
396
        # 7) The 'all_unique_searcher' represents the very 'tip' of searching.
397
        #    Most of the time this produces very little important information.
398
        #    So don't step it as quickly as the other searchers.
399
        # 8) Search is done when all common searchers have completed.
400
401
        unique_searcher, common_searcher = self._find_initial_unique_nodes(
402
            [unique_revision], common_revisions)
403
404
        unique_nodes = unique_searcher.seen.difference(common_searcher.seen)
405
        if not unique_nodes:
406
            return unique_nodes
407
408
        (all_unique_searcher,
409
         unique_tip_searchers) = self._make_unique_searchers(unique_nodes,
410
                                    unique_searcher, common_searcher)
411
412
        self._refine_unique_nodes(unique_searcher, all_unique_searcher,
413
                                  unique_tip_searchers, common_searcher)
414
        true_unique_nodes = unique_nodes.difference(common_searcher.seen)
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
415
        if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
416
            trace.mutter('Found %d truly unique nodes out of %d',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
417
                         len(true_unique_nodes), len(unique_nodes))
418
        return true_unique_nodes
419
420
    def _find_initial_unique_nodes(self, unique_revisions, common_revisions):
421
        """Steps 1-3 of find_unique_ancestors.
422
423
        Find the maximal set of unique nodes. Some of these might actually
424
        still be common, but we are sure that there are no other unique nodes.
425
426
        :return: (unique_searcher, common_searcher)
427
        """
428
429
        unique_searcher = self._make_breadth_first_searcher(unique_revisions)
430
        # we know that unique_revisions aren't in common_revisions, so skip
431
        # past them.
3377.3.27 by John Arbash Meinel
some simple updates
432
        unique_searcher.next()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
433
        common_searcher = self._make_breadth_first_searcher(common_revisions)
434
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
435
        # As long as we are still finding unique nodes, keep searching
3377.3.27 by John Arbash Meinel
some simple updates
436
        while unique_searcher._next_query:
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
437
            next_unique_nodes = set(unique_searcher.step())
438
            next_common_nodes = set(common_searcher.step())
439
440
            # Check if either searcher encounters new nodes seen by the other
441
            # side.
442
            unique_are_common_nodes = next_unique_nodes.intersection(
443
                common_searcher.seen)
444
            unique_are_common_nodes.update(
445
                next_common_nodes.intersection(unique_searcher.seen))
446
            if unique_are_common_nodes:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
447
                ancestors = unique_searcher.find_seen_ancestors(
448
                                unique_are_common_nodes)
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
449
                # TODO: This is a bit overboard, we only really care about
450
                #       the ancestors of the tips because the rest we
451
                #       already know. This is *correct* but causes us to
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
452
                #       search too much ancestry.
3377.3.29 by John Arbash Meinel
Revert the _find_any_seen change.
453
                ancestors.update(common_searcher.find_seen_ancestors(ancestors))
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
454
                unique_searcher.stop_searching_any(ancestors)
455
                common_searcher.start_searching(ancestors)
456
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
457
        return unique_searcher, common_searcher
458
459
    def _make_unique_searchers(self, unique_nodes, unique_searcher,
460
                               common_searcher):
461
        """Create a searcher for all the unique search tips (step 4).
462
463
        As a side effect, the common_searcher will stop searching any nodes
464
        that are ancestors of the unique searcher tips.
465
466
        :return: (all_unique_searcher, unique_tip_searchers)
467
        """
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
468
        unique_tips = self._remove_simple_descendants(unique_nodes,
469
                        self.get_parent_map(unique_nodes))
470
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
471
        if len(unique_tips) == 1:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
472
            unique_tip_searchers = []
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
473
            ancestor_all_unique = unique_searcher.find_seen_ancestors(unique_tips)
474
        else:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
475
            unique_tip_searchers = []
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
476
            for tip in unique_tips:
477
                revs_to_search = unique_searcher.find_seen_ancestors([tip])
478
                revs_to_search.update(
479
                    common_searcher.find_seen_ancestors(revs_to_search))
480
                searcher = self._make_breadth_first_searcher(revs_to_search)
481
                # We don't care about the starting nodes.
482
                searcher._label = tip
483
                searcher.step()
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
484
                unique_tip_searchers.append(searcher)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
485
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
486
            ancestor_all_unique = None
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
487
            for searcher in unique_tip_searchers:
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
488
                if ancestor_all_unique is None:
489
                    ancestor_all_unique = set(searcher.seen)
490
                else:
491
                    ancestor_all_unique = ancestor_all_unique.intersection(
492
                                                searcher.seen)
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
493
        # Collapse all the common nodes into a single searcher
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
494
        all_unique_searcher = self._make_breadth_first_searcher(
495
                                ancestor_all_unique)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
496
        if ancestor_all_unique:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
497
            # We've seen these nodes in all the searchers, so we'll just go to
498
            # the next
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
499
            all_unique_searcher.step()
500
501
            # Stop any search tips that are already known as ancestors of the
502
            # unique nodes
503
            stopped_common = common_searcher.stop_searching_any(
504
                common_searcher.find_seen_ancestors(ancestor_all_unique))
505
506
            total_stopped = 0
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
507
            for searcher in unique_tip_searchers:
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
508
                total_stopped += len(searcher.stop_searching_any(
509
                    searcher.find_seen_ancestors(ancestor_all_unique)))
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
510
        if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
511
            trace.mutter('For %d unique nodes, created %d + 1 unique searchers'
512
                         ' (%d stopped search tips, %d common ancestors'
513
                         ' (%d stopped common)',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
514
                         len(unique_nodes), len(unique_tip_searchers),
515
                         total_stopped, len(ancestor_all_unique),
516
                         len(stopped_common))
517
        return all_unique_searcher, unique_tip_searchers
518
519
    def _step_unique_and_common_searchers(self, common_searcher,
520
                                          unique_tip_searchers,
521
                                          unique_searcher):
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
522
        """Step all the searchers"""
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
523
        newly_seen_common = set(common_searcher.step())
524
        newly_seen_unique = set()
525
        for searcher in unique_tip_searchers:
526
            next = set(searcher.step())
527
            next.update(unique_searcher.find_seen_ancestors(next))
528
            next.update(common_searcher.find_seen_ancestors(next))
529
            for alt_searcher in unique_tip_searchers:
530
                if alt_searcher is searcher:
531
                    continue
532
                next.update(alt_searcher.find_seen_ancestors(next))
533
            searcher.start_searching(next)
534
            newly_seen_unique.update(next)
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
535
        return newly_seen_common, newly_seen_unique
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
536
537
    def _find_nodes_common_to_all_unique(self, unique_tip_searchers,
538
                                         all_unique_searcher,
539
                                         newly_seen_unique, step_all_unique):
540
        """Find nodes that are common to all unique_tip_searchers.
541
542
        If it is time, step the all_unique_searcher, and add its nodes to the
543
        result.
544
        """
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
545
        common_to_all_unique_nodes = newly_seen_unique.copy()
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
546
        for searcher in unique_tip_searchers:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
547
            common_to_all_unique_nodes.intersection_update(searcher.seen)
548
        common_to_all_unique_nodes.intersection_update(
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
549
                                    all_unique_searcher.seen)
550
        # Step all-unique less frequently than the other searchers.
551
        # In the common case, we don't need to spider out far here, so
552
        # avoid doing extra work.
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
553
        if step_all_unique:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
554
            tstart = time.clock()
555
            nodes = all_unique_searcher.step()
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
556
            common_to_all_unique_nodes.update(nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
557
            if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
558
                tdelta = time.clock() - tstart
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
559
                trace.mutter('all_unique_searcher step() took %.3fs'
560
                             'for %d nodes (%d total), iteration: %s',
561
                             tdelta, len(nodes), len(all_unique_searcher.seen),
562
                             all_unique_searcher._iterations)
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
563
        return common_to_all_unique_nodes
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
564
565
    def _collapse_unique_searchers(self, unique_tip_searchers,
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
566
                                   common_to_all_unique_nodes):
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
567
        """Combine searchers that are searching the same tips.
568
569
        When two searchers are searching the same tips, we can stop one of the
570
        searchers. We also know that the maximal set of common ancestors is the
571
        intersection of the two original searchers.
572
573
        :return: A list of searchers that are searching unique nodes.
574
        """
575
        # Filter out searchers that don't actually search different
576
        # nodes. We already have the ancestry intersection for them
577
        unique_search_tips = {}
578
        for searcher in unique_tip_searchers:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
579
            stopped = searcher.stop_searching_any(common_to_all_unique_nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
580
            will_search_set = frozenset(searcher._next_query)
581
            if not will_search_set:
582
                if 'graph' in debug.debug_flags:
583
                    trace.mutter('Unique searcher %s was stopped.'
584
                                 ' (%s iterations) %d nodes stopped',
585
                                 searcher._label,
586
                                 searcher._iterations,
587
                                 len(stopped))
588
            elif will_search_set not in unique_search_tips:
589
                # This searcher is searching a unique set of nodes, let it
590
                unique_search_tips[will_search_set] = [searcher]
591
            else:
592
                unique_search_tips[will_search_set].append(searcher)
593
        # TODO: it might be possible to collapse searchers faster when they
594
        #       only have *some* search tips in common.
595
        next_unique_searchers = []
596
        for searchers in unique_search_tips.itervalues():
597
            if len(searchers) == 1:
598
                # Searching unique tips, go for it
599
                next_unique_searchers.append(searchers[0])
600
            else:
601
                # These searchers have started searching the same tips, we
602
                # don't need them to cover the same ground. The
603
                # intersection of their ancestry won't change, so create a
604
                # new searcher, combining their histories.
605
                next_searcher = searchers[0]
606
                for searcher in searchers[1:]:
607
                    next_searcher.seen.intersection_update(searcher.seen)
608
                if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
609
                    trace.mutter('Combining %d searchers into a single'
610
                                 ' searcher searching %d nodes with'
611
                                 ' %d ancestry',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
612
                                 len(searchers),
613
                                 len(next_searcher._next_query),
614
                                 len(next_searcher.seen))
615
                next_unique_searchers.append(next_searcher)
616
        return next_unique_searchers
617
618
    def _refine_unique_nodes(self, unique_searcher, all_unique_searcher,
619
                             unique_tip_searchers, common_searcher):
620
        """Steps 5-8 of find_unique_ancestors.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
621
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
622
        This function returns when common_searcher has stopped searching for
623
        more nodes.
624
        """
625
        # We step the ancestor_all_unique searcher only every
626
        # STEP_UNIQUE_SEARCHER_EVERY steps.
627
        step_all_unique_counter = 0
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
628
        # While we still have common nodes to search
629
        while common_searcher._next_query:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
630
            (newly_seen_common,
631
             newly_seen_unique) = self._step_unique_and_common_searchers(
632
                common_searcher, unique_tip_searchers, unique_searcher)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
633
            # These nodes are common ancestors of all unique nodes
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
634
            common_to_all_unique_nodes = self._find_nodes_common_to_all_unique(
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
635
                unique_tip_searchers, all_unique_searcher, newly_seen_unique,
636
                step_all_unique_counter==0)
637
            step_all_unique_counter = ((step_all_unique_counter + 1)
638
                                       % STEP_UNIQUE_SEARCHER_EVERY)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
639
640
            if newly_seen_common:
641
                # If a 'common' node is an ancestor of all unique searchers, we
642
                # can stop searching it.
643
                common_searcher.stop_searching_any(
644
                    all_unique_searcher.seen.intersection(newly_seen_common))
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
645
            if common_to_all_unique_nodes:
646
                common_to_all_unique_nodes.update(
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
647
                    common_searcher.find_seen_ancestors(
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
648
                        common_to_all_unique_nodes))
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
649
                # The all_unique searcher can start searching the common nodes
650
                # but everyone else can stop.
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
651
                # This is the sort of thing where we would like to not have it
652
                # start_searching all of the nodes, but only mark all of them
653
                # as seen, and have it search only the actual tips. Otherwise
654
                # it is another get_parent_map() traversal for it to figure out
655
                # what we already should know.
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
656
                all_unique_searcher.start_searching(common_to_all_unique_nodes)
657
                common_searcher.stop_searching_any(common_to_all_unique_nodes)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
658
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
659
            next_unique_searchers = self._collapse_unique_searchers(
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
660
                unique_tip_searchers, common_to_all_unique_nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
661
            if len(unique_tip_searchers) != len(next_unique_searchers):
662
                if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
663
                    trace.mutter('Collapsed %d unique searchers => %d'
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
664
                                 ' at %s iterations',
665
                                 len(unique_tip_searchers),
666
                                 len(next_unique_searchers),
667
                                 all_unique_searcher._iterations)
668
            unique_tip_searchers = next_unique_searchers
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
669
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
670
    def get_parent_map(self, revisions):
671
        """Get a map of key:parent_list for revisions.
672
673
        This implementation delegates to get_parents, for old parent_providers
674
        that do not supply get_parent_map.
675
        """
676
        result = {}
677
        for rev, parents in self.get_parents(revisions):
678
            if parents is not None:
679
                result[rev] = parents
680
        return result
681
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
682
    def _make_breadth_first_searcher(self, revisions):
683
        return _BreadthFirstSearcher(revisions, self)
684
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
685
    def _find_border_ancestors(self, revisions):
2490.2.12 by Aaron Bentley
Improve documentation
686
        """Find common ancestors with at least one uncommon descendant.
687
688
        Border ancestors are identified using a breadth-first
689
        search starting at the bottom of the graph.  Searches are stopped
690
        whenever a node or one of its descendants is determined to be common.
691
692
        This will scale with the number of uncommon ancestors.
2490.2.25 by Aaron Bentley
Update from review
693
694
        As well as the border ancestors, a set of seen common ancestors and a
695
        list of sets of seen ancestors for each input revision is returned.
696
        This allows calculation of graph difference from the results of this
697
        operation.
2490.2.12 by Aaron Bentley
Improve documentation
698
        """
2490.2.28 by Aaron Bentley
Fix handling of null revision
699
        if None in revisions:
700
            raise errors.InvalidRevisionId(None, self)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
701
        common_ancestors = set()
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
702
        searchers = [self._make_breadth_first_searcher([r])
703
                     for r in revisions]
704
        active_searchers = searchers[:]
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
705
        border_ancestors = set()
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
706
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
707
        while True:
708
            newly_seen = set()
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
709
            for searcher in searchers:
710
                new_ancestors = searcher.step()
711
                if new_ancestors:
712
                    newly_seen.update(new_ancestors)
713
            new_common = set()
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
714
            for revision in newly_seen:
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
715
                if revision in common_ancestors:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
716
                    # Not a border ancestor because it was seen as common
717
                    # already
718
                    new_common.add(revision)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
719
                    continue
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
720
                for searcher in searchers:
721
                    if revision not in searcher.seen:
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
722
                        break
723
                else:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
724
                    # This is a border because it is a first common that we see
725
                    # after walking for a while.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
726
                    border_ancestors.add(revision)
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
727
                    new_common.add(revision)
728
            if new_common:
729
                for searcher in searchers:
730
                    new_common.update(searcher.find_seen_ancestors(new_common))
731
                for searcher in searchers:
732
                    searcher.start_searching(new_common)
733
                common_ancestors.update(new_common)
734
735
            # Figure out what the searchers will be searching next, and if
736
            # there is only 1 set being searched, then we are done searching,
737
            # since all searchers would have to be searching the same data,
738
            # thus it *must* be in common.
739
            unique_search_sets = set()
740
            for searcher in searchers:
741
                will_search_set = frozenset(searcher._next_query)
742
                if will_search_set not in unique_search_sets:
743
                    # This searcher is searching a unique set of nodes, let it
744
                    unique_search_sets.add(will_search_set)
745
746
            if len(unique_search_sets) == 1:
747
                nodes = unique_search_sets.pop()
748
                uncommon_nodes = nodes.difference(common_ancestors)
3376.2.14 by Martin Pool
Remove recently-introduced assert statements
749
                if uncommon_nodes:
750
                    raise AssertionError("Somehow we ended up converging"
751
                                         " without actually marking them as"
752
                                         " in common."
753
                                         "\nStart_nodes: %s"
754
                                         "\nuncommon_nodes: %s"
755
                                         % (revisions, uncommon_nodes))
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
756
                break
757
        return border_ancestors, common_ancestors, searchers
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
758
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
759
    def heads(self, keys):
760
        """Return the heads from amongst keys.
761
762
        This is done by searching the ancestries of each key.  Any key that is
763
        reachable from another key is not returned; all the others are.
764
765
        This operation scales with the relative depth between any two keys. If
766
        any two keys are completely disconnected all ancestry of both sides
767
        will be retrieved.
768
769
        :param keys: An iterable of keys.
2776.1.4 by Robert Collins
Trivial review feedback changes.
770
        :return: A set of the heads. Note that as a set there is no ordering
771
            information. Callers will need to filter their input to create
772
            order if they need it.
2490.2.12 by Aaron Bentley
Improve documentation
773
        """
2776.1.4 by Robert Collins
Trivial review feedback changes.
774
        candidate_heads = set(keys)
3052.5.5 by John Arbash Meinel
Special case Graph.heads() for NULL_REVISION rather than is_ancestor.
775
        if revision.NULL_REVISION in candidate_heads:
776
            # NULL_REVISION is only a head if it is the only entry
777
            candidate_heads.remove(revision.NULL_REVISION)
778
            if not candidate_heads:
779
                return set([revision.NULL_REVISION])
2850.2.1 by Robert Collins
(robertc) Special case the zero-or-no-heads case for Graph.heads(). (Robert Collins)
780
        if len(candidate_heads) < 2:
781
            return candidate_heads
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
782
        searchers = dict((c, self._make_breadth_first_searcher([c]))
2776.1.4 by Robert Collins
Trivial review feedback changes.
783
                          for c in candidate_heads)
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
784
        active_searchers = dict(searchers)
785
        # skip over the actual candidate for each searcher
786
        for searcher in active_searchers.itervalues():
1551.15.81 by Aaron Bentley
Remove testing code
787
            searcher.next()
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
788
        # The common walker finds nodes that are common to two or more of the
789
        # input keys, so that we don't access all history when a currently
790
        # uncommon search point actually meets up with something behind a
791
        # common search point. Common search points do not keep searches
792
        # active; they just allow us to make searches inactive without
793
        # accessing all history.
794
        common_walker = self._make_breadth_first_searcher([])
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
795
        while len(active_searchers) > 0:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
796
            ancestors = set()
797
            # advance searches
798
            try:
799
                common_walker.next()
800
            except StopIteration:
2921.3.4 by Robert Collins
Review feedback.
801
                # No common points being searched at this time.
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
802
                pass
1551.15.78 by Aaron Bentley
Fix KeyError in filter_candidate_lca
803
            for candidate in active_searchers.keys():
804
                try:
805
                    searcher = active_searchers[candidate]
806
                except KeyError:
807
                    # rare case: we deleted candidate in a previous iteration
808
                    # through this for loop, because it was determined to be
809
                    # a descendant of another candidate.
810
                    continue
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
811
                try:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
812
                    ancestors.update(searcher.next())
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
813
                except StopIteration:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
814
                    del active_searchers[candidate]
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
815
                    continue
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
816
            # process found nodes
817
            new_common = set()
818
            for ancestor in ancestors:
819
                if ancestor in candidate_heads:
820
                    candidate_heads.remove(ancestor)
821
                    del searchers[ancestor]
822
                    if ancestor in active_searchers:
823
                        del active_searchers[ancestor]
824
                # it may meet up with a known common node
2921.3.4 by Robert Collins
Review feedback.
825
                if ancestor in common_walker.seen:
826
                    # some searcher has encountered our known common nodes:
827
                    # just stop it
828
                    ancestor_set = set([ancestor])
829
                    for searcher in searchers.itervalues():
830
                        searcher.stop_searching_any(ancestor_set)
831
                else:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
832
                    # or it may have been just reached by all the searchers:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
833
                    for searcher in searchers.itervalues():
834
                        if ancestor not in searcher.seen:
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
835
                            break
836
                    else:
2921.3.4 by Robert Collins
Review feedback.
837
                        # The final active searcher has just reached this node,
838
                        # making it be known as a descendant of all candidates,
839
                        # so we can stop searching it, and any seen ancestors
840
                        new_common.add(ancestor)
841
                        for searcher in searchers.itervalues():
842
                            seen_ancestors =\
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
843
                                searcher.find_seen_ancestors([ancestor])
2921.3.4 by Robert Collins
Review feedback.
844
                            searcher.stop_searching_any(seen_ancestors)
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
845
            common_walker.start_searching(new_common)
2776.1.4 by Robert Collins
Trivial review feedback changes.
846
        return candidate_heads
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
847
3514.2.8 by John Arbash Meinel
The insertion ordering into the weave has an impact on conflicts.
848
    def find_merge_order(self, tip_revision_id, lca_revision_ids):
849
        """Find the order that each revision was merged into tip.
850
851
        This basically just walks backwards with a stack, and walks left-first
852
        until it finds a node to stop.
853
        """
854
        if len(lca_revision_ids) == 1:
855
            return list(lca_revision_ids)
856
        looking_for = set(lca_revision_ids)
857
        # TODO: Is there a way we could do this "faster" by batching up the
858
        # get_parent_map requests?
859
        # TODO: Should we also be culling the ancestry search right away? We
860
        # could add looking_for to the "stop" list, and walk their
861
        # ancestry in batched mode. The flip side is it might mean we walk a
862
        # lot of "stop" nodes, rather than only the minimum.
863
        # Then again, without it we may trace back into ancestry we could have
864
        # stopped early.
865
        stack = [tip_revision_id]
866
        found = []
867
        stop = set()
868
        while stack and looking_for:
869
            next = stack.pop()
870
            stop.add(next)
871
            if next in looking_for:
872
                found.append(next)
873
                looking_for.remove(next)
874
                if len(looking_for) == 1:
875
                    found.append(looking_for.pop())
876
                    break
877
                continue
878
            parent_ids = self.get_parent_map([next]).get(next, None)
879
            if not parent_ids: # Ghost, nothing to search here
880
                continue
881
            for parent_id in reversed(parent_ids):
882
                # TODO: (performance) We see the parent at this point, but we
883
                #       wait to mark it until later to make sure we get left
884
                #       parents before right parents. However, instead of
885
                #       waiting until we have traversed enough parents, we
886
                #       could instead note that we've found it, and once all
887
                #       parents are in the stack, just reverse iterate the
888
                #       stack for them.
889
                if parent_id not in stop:
890
                    # this will need to be searched
891
                    stack.append(parent_id)
892
                stop.add(parent_id)
893
        return found
894
5365.6.3 by Aaron Bentley
Implement find_lefthand_merger.
895
    def find_lefthand_merger(self, merged_key, tip_key):
896
        """Find the first lefthand ancestor of tip_key that merged merged_key.
897
898
        We do this by first finding the descendants of merged_key, then
899
        walking through the lefthand ancestry of tip_key until we find a key
900
        that doesn't descend from merged_key.  Its child is the key that
901
        merged merged_key.
902
903
        :return: The first lefthand ancestor of tip_key to merge merged_key.
904
            merged_key if it is a lefthand ancestor of tip_key.
905
            None if no ancestor of tip_key merged merged_key.
906
        """
907
        descendants = self.find_descendants(merged_key, tip_key)
908
        candidate_iterator = self.iter_lefthand_ancestry(tip_key)
909
        last_candidate = None
910
        for candidate in candidate_iterator:
911
            if candidate not in descendants:
912
                return last_candidate
913
            last_candidate = candidate
914
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
915
    def find_unique_lca(self, left_revision, right_revision,
916
                        count_steps=False):
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
917
        """Find a unique LCA.
918
919
        Find lowest common ancestors.  If there is no unique  common
920
        ancestor, find the lowest common ancestors of those ancestors.
921
922
        Iteration stops when a unique lowest common ancestor is found.
923
        The graph origin is necessarily a unique lowest common ancestor.
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
924
925
        Note that None is not an acceptable substitute for NULL_REVISION.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
926
        in the input for this method.
1551.19.12 by Aaron Bentley
Add documentation for the count_steps parameter of Graph.find_unique_lca
927
928
        :param count_steps: If True, the return value will be a tuple of
929
            (unique_lca, steps) where steps is the number of times that
930
            find_lca was run.  If False, only unique_lca is returned.
2490.2.3 by Aaron Bentley
Implement new merge base picker
931
        """
932
        revisions = [left_revision, right_revision]
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
933
        steps = 0
2490.2.3 by Aaron Bentley
Implement new merge base picker
934
        while True:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
935
            steps += 1
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
936
            lca = self.find_lca(*revisions)
937
            if len(lca) == 1:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
938
                result = lca.pop()
939
                if count_steps:
940
                    return result, steps
941
                else:
942
                    return result
2520.4.104 by Aaron Bentley
Avoid infinite loop when there is no unique lca
943
            if len(lca) == 0:
944
                raise errors.NoCommonAncestor(left_revision, right_revision)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
945
            revisions = lca
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
946
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
947
    def iter_ancestry(self, revision_ids):
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
948
        """Iterate the ancestry of this revision.
949
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
950
        :param revision_ids: Nodes to start the search
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
951
        :return: Yield tuples mapping a revision_id to its parents for the
952
            ancestry of revision_id.
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
953
            Ghosts will be returned with None as their parents, and nodes
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
954
            with no parents will have NULL_REVISION as their only parent. (As
955
            defined by get_parent_map.)
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
956
            There will also be a node for (NULL_REVISION, ())
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
957
        """
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
958
        pending = set(revision_ids)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
959
        processed = set()
960
        while pending:
961
            processed.update(pending)
962
            next_map = self.get_parent_map(pending)
963
            next_pending = set()
964
            for item in next_map.iteritems():
965
                yield item
966
                next_pending.update(p for p in item[1] if p not in processed)
967
            ghosts = pending.difference(next_map)
968
            for ghost in ghosts:
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
969
                yield (ghost, None)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
970
            pending = next_pending
971
5365.6.2 by Aaron Bentley
Extract iter_lefthand_ancestry from Repository.iter_ancestry.
972
    def iter_lefthand_ancestry(self, start_key, stop_keys=None):
973
        if stop_keys is None:
974
            stop_keys = ()
975
        next_key = start_key
976
        def get_parents(key):
977
            try:
978
                return self._parents_provider.get_parent_map([key])[key]
979
            except KeyError:
980
                raise errors.RevisionNotPresent(next_key, self)
981
        while True:
982
            if next_key in stop_keys:
983
                return
984
            parents = get_parents(next_key)
985
            yield next_key
986
            if len(parents) == 0:
987
                return
988
            else:
989
                next_key = parents[0]
990
2490.2.31 by Aaron Bentley
Fix iter_topo_order to permit un-included parents
991
    def iter_topo_order(self, revisions):
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
992
        """Iterate through the input revisions in topological order.
993
994
        This sorting only ensures that parents come before their children.
995
        An ancestor may sort after a descendant if the relationship is not
996
        visible in the supplied list of revisions.
997
        """
4593.5.30 by John Arbash Meinel
Move the topo_sort implementation into KnownGraph, rather than calling back to it.
998
        from bzrlib import tsort
3099.3.3 by John Arbash Meinel
Deprecate get_parents() in favor of get_parent_map()
999
        sorter = tsort.TopoSorter(self.get_parent_map(revisions))
2490.2.34 by Aaron Bentley
Update NEWS and change implementation to return an iterator
1000
        return sorter.iter_topo_order()
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
1001
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
1002
    def is_ancestor(self, candidate_ancestor, candidate_descendant):
2653.2.5 by Aaron Bentley
Update to clarify algorithm
1003
        """Determine whether a revision is an ancestor of another.
1004
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
1005
        We answer this using heads() as heads() has the logic to perform the
3078.2.6 by Ian Clatworthy
fix efficiency of local commit detection as recommended by jameinel's review
1006
        smallest number of parent lookups to determine the ancestral
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
1007
        relationship between N revisions.
2653.2.5 by Aaron Bentley
Update to clarify algorithm
1008
        """
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
1009
        return set([candidate_descendant]) == self.heads(
1010
            [candidate_ancestor, candidate_descendant])
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
1011
3921.3.5 by Marius Kruger
extract graph.is_between from builtins.cmd_tags.run, and test it
1012
    def is_between(self, revid, lower_bound_revid, upper_bound_revid):
1013
        """Determine whether a revision is between two others.
1014
1015
        returns true if and only if:
1016
        lower_bound_revid <= revid <= upper_bound_revid
1017
        """
1018
        return ((upper_bound_revid is None or
1019
                    self.is_ancestor(revid, upper_bound_revid)) and
1020
               (lower_bound_revid is None or
1021
                    self.is_ancestor(lower_bound_revid, revid)))
1022
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1023
    def _search_for_extra_common(self, common, searchers):
1024
        """Make sure that unique nodes are genuinely unique.
1025
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1026
        After _find_border_ancestors, all nodes marked "common" are indeed
1027
        common. Some of the nodes considered unique are not, due to history
1028
        shortcuts stopping the searches early.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1029
1030
        We know that we have searched enough when all common search tips are
1031
        descended from all unique (uncommon) nodes because we know that a node
1032
        cannot be an ancestor of its own ancestor.
1033
1034
        :param common: A set of common nodes
1035
        :param searchers: The searchers returned from _find_border_ancestors
1036
        :return: None
1037
        """
1038
        # Basic algorithm...
1039
        #   A) The passed in searchers should all be on the same tips, thus
1040
        #      they should be considered the "common" searchers.
1041
        #   B) We find the difference between the searchers, these are the
1042
        #      "unique" nodes for each side.
1043
        #   C) We do a quick culling so that we only start searching from the
1044
        #      more interesting unique nodes. (A unique ancestor is more
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1045
        #      interesting than any of its children.)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1046
        #   D) We start searching for ancestors common to all unique nodes.
1047
        #   E) We have the common searchers stop searching any ancestors of
1048
        #      nodes found by (D)
1049
        #   F) When there are no more common search tips, we stop
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1050
1051
        # TODO: We need a way to remove unique_searchers when they overlap with
1052
        #       other unique searchers.
3376.2.14 by Martin Pool
Remove recently-introduced assert statements
1053
        if len(searchers) != 2:
1054
            raise NotImplementedError(
1055
                "Algorithm not yet implemented for > 2 searchers")
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1056
        common_searchers = searchers
1057
        left_searcher = searchers[0]
1058
        right_searcher = searchers[1]
3377.3.15 by John Arbash Meinel
minor update
1059
        unique = left_searcher.seen.symmetric_difference(right_searcher.seen)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1060
        if not unique: # No unique nodes, nothing to do
1061
            return
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1062
        total_unique = len(unique)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1063
        unique = self._remove_simple_descendants(unique,
1064
                    self.get_parent_map(unique))
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1065
        simple_unique = len(unique)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
1066
1067
        unique_searchers = []
1068
        for revision_id in unique:
3377.3.15 by John Arbash Meinel
minor update
1069
            if revision_id in left_searcher.seen:
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
1070
                parent_searcher = left_searcher
1071
            else:
1072
                parent_searcher = right_searcher
1073
            revs_to_search = parent_searcher.find_seen_ancestors([revision_id])
1074
            if not revs_to_search: # XXX: This shouldn't be possible
1075
                revs_to_search = [revision_id]
3377.3.15 by John Arbash Meinel
minor update
1076
            searcher = self._make_breadth_first_searcher(revs_to_search)
1077
            # We don't care about the starting nodes.
1078
            searcher.step()
1079
            unique_searchers.append(searcher)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
1080
3377.3.16 by John Arbash Meinel
small cleanups
1081
        # possible todo: aggregate the common searchers into a single common
1082
        #   searcher, just make sure that we include the nodes into the .seen
1083
        #   properties of the original searchers
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1084
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1085
        ancestor_all_unique = None
1086
        for searcher in unique_searchers:
1087
            if ancestor_all_unique is None:
1088
                ancestor_all_unique = set(searcher.seen)
1089
            else:
1090
                ancestor_all_unique = ancestor_all_unique.intersection(
1091
                                            searcher.seen)
1092
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
1093
        trace.mutter('Started %s unique searchers for %s unique revisions',
1094
                     simple_unique, total_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
1095
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1096
        while True: # If we have no more nodes we have nothing to do
1097
            newly_seen_common = set()
1098
            for searcher in common_searchers:
1099
                newly_seen_common.update(searcher.step())
1100
            newly_seen_unique = set()
1101
            for searcher in unique_searchers:
1102
                newly_seen_unique.update(searcher.step())
1103
            new_common_unique = set()
1104
            for revision in newly_seen_unique:
1105
                for searcher in unique_searchers:
1106
                    if revision not in searcher.seen:
1107
                        break
1108
                else:
1109
                    # This is a border because it is a first common that we see
1110
                    # after walking for a while.
1111
                    new_common_unique.add(revision)
1112
            if newly_seen_common:
1113
                # These are nodes descended from one of the 'common' searchers.
1114
                # Make sure all searchers are on the same page
1115
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
1116
                    newly_seen_common.update(
1117
                        searcher.find_seen_ancestors(newly_seen_common))
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
1118
                # We start searching the whole ancestry. It is a bit wasteful,
1119
                # though. We really just want to mark all of these nodes as
1120
                # 'seen' and then start just the tips. However, it requires a
1121
                # get_parent_map() call to figure out the tips anyway, and all
1122
                # redundant requests should be fairly fast.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1123
                for searcher in common_searchers:
1124
                    searcher.start_searching(newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
1125
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1126
                # If a 'common' node is an ancestor of all unique searchers, we
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
1127
                # can stop searching it.
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1128
                stop_searching_common = ancestor_all_unique.intersection(
1129
                                            newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
1130
                if stop_searching_common:
1131
                    for searcher in common_searchers:
1132
                        searcher.stop_searching_any(stop_searching_common)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1133
            if new_common_unique:
3377.3.20 by John Arbash Meinel
comment cleanups.
1134
                # We found some ancestors that are common
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1135
                for searcher in unique_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
1136
                    new_common_unique.update(
1137
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1138
                # Since these are common, we can grab another set of ancestors
1139
                # that we have seen
1140
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
1141
                    new_common_unique.update(
1142
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1143
1144
                # We can tell all of the unique searchers to start at these
1145
                # nodes, and tell all of the common searchers to *stop*
1146
                # searching these nodes
1147
                for searcher in unique_searchers:
1148
                    searcher.start_searching(new_common_unique)
1149
                for searcher in common_searchers:
1150
                    searcher.stop_searching_any(new_common_unique)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1151
                ancestor_all_unique.update(new_common_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
1152
3377.3.20 by John Arbash Meinel
comment cleanups.
1153
                # Filter out searchers that don't actually search different
1154
                # nodes. We already have the ancestry intersection for them
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
1155
                next_unique_searchers = []
1156
                unique_search_sets = set()
1157
                for searcher in unique_searchers:
1158
                    will_search_set = frozenset(searcher._next_query)
1159
                    if will_search_set not in unique_search_sets:
1160
                        # This searcher is searching a unique set of nodes, let it
1161
                        unique_search_sets.add(will_search_set)
1162
                        next_unique_searchers.append(searcher)
1163
                unique_searchers = next_unique_searchers
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
1164
            for searcher in common_searchers:
1165
                if searcher._next_query:
1166
                    break
1167
            else:
1168
                # All common searcher have stopped searching
3377.3.16 by John Arbash Meinel
small cleanups
1169
                return
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1170
1171
    def _remove_simple_descendants(self, revisions, parent_map):
1172
        """remove revisions which are children of other ones in the set
1173
1174
        This doesn't do any graph searching, it just checks the immediate
1175
        parent_map to find if there are any children which can be removed.
1176
1177
        :param revisions: A set of revision_ids
1178
        :return: A set of revision_ids with the children removed
1179
        """
1180
        simple_ancestors = revisions.copy()
1181
        # TODO: jam 20071214 we *could* restrict it to searching only the
1182
        #       parent_map of revisions already present in 'revisions', but
1183
        #       considering the general use case, I think this is actually
1184
        #       better.
1185
1186
        # This is the same as the following loop. I don't know that it is any
1187
        # faster.
1188
        ## simple_ancestors.difference_update(r for r, p_ids in parent_map.iteritems()
1189
        ##     if p_ids is not None and revisions.intersection(p_ids))
1190
        ## return simple_ancestors
1191
1192
        # Yet Another Way, invert the parent map (which can be cached)
1193
        ## descendants = {}
1194
        ## for revision_id, parent_ids in parent_map.iteritems():
1195
        ##   for p_id in parent_ids:
1196
        ##       descendants.setdefault(p_id, []).append(revision_id)
1197
        ## for revision in revisions.intersection(descendants):
1198
        ##   simple_ancestors.difference_update(descendants[revision])
1199
        ## return simple_ancestors
1200
        for revision, parent_ids in parent_map.iteritems():
1201
            if parent_ids is None:
1202
                continue
1203
            for parent_id in parent_ids:
1204
                if parent_id in revisions:
1205
                    # This node has a parent present in the set, so we can
1206
                    # remove it
1207
                    simple_ancestors.discard(revision)
1208
                    break
1209
        return simple_ancestors
1210
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1211
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1212
class HeadsCache(object):
1213
    """A cache of results for graph heads calls."""
1214
1215
    def __init__(self, graph):
1216
        self.graph = graph
1217
        self._heads = {}
1218
1219
    def heads(self, keys):
1220
        """Return the heads of keys.
1221
2911.4.3 by Robert Collins
Make the contract of HeadsCache.heads() more clear.
1222
        This matches the API of Graph.heads(), specifically the return value is
1223
        a set which can be mutated, and ordering of the input is not preserved
1224
        in the output.
1225
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1226
        :see also: Graph.heads.
1227
        :param keys: The keys to calculate heads for.
1228
        :return: A set containing the heads, which may be mutated without
1229
            affecting future lookups.
1230
        """
2911.4.2 by Robert Collins
Make HeadsCache actually work.
1231
        keys = frozenset(keys)
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1232
        try:
1233
            return set(self._heads[keys])
1234
        except KeyError:
1235
            heads = self.graph.heads(keys)
1236
            self._heads[keys] = heads
1237
            return set(heads)
1238
1239
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1240
class FrozenHeadsCache(object):
1241
    """Cache heads() calls, assuming the caller won't modify them."""
1242
1243
    def __init__(self, graph):
1244
        self.graph = graph
1245
        self._heads = {}
1246
1247
    def heads(self, keys):
1248
        """Return the heads of keys.
1249
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
1250
        Similar to Graph.heads(). The main difference is that the return value
1251
        is a frozen set which cannot be mutated.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1252
1253
        :see also: Graph.heads.
1254
        :param keys: The keys to calculate heads for.
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
1255
        :return: A frozenset containing the heads.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1256
        """
1257
        keys = frozenset(keys)
1258
        try:
1259
            return self._heads[keys]
1260
        except KeyError:
1261
            heads = frozenset(self.graph.heads(keys))
1262
            self._heads[keys] = heads
1263
            return heads
1264
1265
    def cache(self, keys, heads):
1266
        """Store a known value."""
1267
        self._heads[frozenset(keys)] = frozenset(heads)
1268
1269
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
1270
class _BreadthFirstSearcher(object):
2921.3.4 by Robert Collins
Review feedback.
1271
    """Parallel search breadth-first the ancestry of revisions.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1272
1273
    This class implements the iterator protocol, but additionally
1274
    1. provides a set of seen ancestors, and
1275
    2. allows some ancestries to be unsearched, via stop_searching_any
1276
    """
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1277
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
1278
    def __init__(self, revisions, parents_provider):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1279
        self._iterations = 0
1280
        self._next_query = set(revisions)
1281
        self.seen = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1282
        self._started_keys = set(self._next_query)
1283
        self._stopped_keys = set()
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
1284
        self._parents_provider = parents_provider
3177.3.3 by Robert Collins
Review feedback.
1285
        self._returning = 'next_with_ghosts'
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1286
        self._current_present = set()
1287
        self._current_ghosts = set()
1288
        self._current_parents = {}
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1289
1290
    def __repr__(self):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1291
        if self._iterations:
1292
            prefix = "searching"
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
1293
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1294
            prefix = "starting"
1295
        search = '%s=%r' % (prefix, list(self._next_query))
1296
        return ('_BreadthFirstSearcher(iterations=%d, %s,'
1297
                ' seen=%r)' % (self._iterations, search, list(self.seen)))
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1298
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1299
    def get_result(self):
1300
        """Get a SearchResult for the current state of this searcher.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1301
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1302
        :return: A SearchResult for this search so far. The SearchResult is
1303
            static - the search can be advanced and the search result will not
1304
            be invalidated or altered.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1305
        """
1306
        if self._returning == 'next':
1307
            # We have to know the current nodes children to be able to list the
1308
            # exclude keys for them. However, while we could have a second
1309
            # look-ahead result buffer and shuffle things around, this method
1310
            # is typically only called once per search - when memoising the
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1311
            # results of the search.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1312
            found, ghosts, next, parents = self._do_query(self._next_query)
1313
            # pretend we didn't query: perhaps we should tweak _do_query to be
1314
            # entirely stateless?
1315
            self.seen.difference_update(next)
3184.1.3 by Robert Collins
Automatically exclude ghosts.
1316
            next_query = next.union(ghosts)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1317
        else:
1318
            next_query = self._next_query
3184.1.5 by Robert Collins
Record the number of found revisions for cross checking.
1319
        excludes = self._stopped_keys.union(next_query)
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1320
        included_keys = self.seen.difference(excludes)
1321
        return SearchResult(self._started_keys, excludes, len(included_keys),
1322
            included_keys)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1323
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1324
    def step(self):
1325
        try:
1326
            return self.next()
1327
        except StopIteration:
1328
            return ()
1329
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1330
    def next(self):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1331
        """Return the next ancestors of this revision.
1332
2490.2.12 by Aaron Bentley
Improve documentation
1333
        Ancestors are returned in the order they are seen in a breadth-first
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1334
        traversal.  No ancestor will be returned more than once. Ancestors are
1335
        returned before their parentage is queried, so ghosts and missing
1336
        revisions (including the start revisions) are included in the result.
1337
        This can save a round trip in LCA style calculation by allowing
1338
        convergence to be detected without reading the data for the revision
1339
        the convergence occurs on.
1340
1341
        :return: A set of revision_ids.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1342
        """
3177.3.3 by Robert Collins
Review feedback.
1343
        if self._returning != 'next':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1344
            # switch to returning the query, not the results.
3177.3.3 by Robert Collins
Review feedback.
1345
            self._returning = 'next'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1346
            self._iterations += 1
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1347
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1348
            self._advance()
1349
        if len(self._next_query) == 0:
1350
            raise StopIteration()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1351
        # We have seen what we're querying at this point as we are returning
1352
        # the query, not the results.
1353
        self.seen.update(self._next_query)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1354
        return self._next_query
1355
1356
    def next_with_ghosts(self):
1357
        """Return the next found ancestors, with ghosts split out.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1358
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1359
        Ancestors are returned in the order they are seen in a breadth-first
1360
        traversal.  No ancestor will be returned more than once. Ancestors are
3177.3.3 by Robert Collins
Review feedback.
1361
        returned only after asking for their parents, which allows us to detect
1362
        which revisions are ghosts and which are not.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1363
1364
        :return: A tuple with (present ancestors, ghost ancestors) sets.
1365
        """
3177.3.3 by Robert Collins
Review feedback.
1366
        if self._returning != 'next_with_ghosts':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1367
            # switch to returning the results, not the current query.
3177.3.3 by Robert Collins
Review feedback.
1368
            self._returning = 'next_with_ghosts'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1369
            self._advance()
1370
        if len(self._next_query) == 0:
1371
            raise StopIteration()
1372
        self._advance()
1373
        return self._current_present, self._current_ghosts
1374
1375
    def _advance(self):
1376
        """Advance the search.
1377
1378
        Updates self.seen, self._next_query, self._current_present,
3177.3.3 by Robert Collins
Review feedback.
1379
        self._current_ghosts, self._current_parents and self._iterations.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1380
        """
1381
        self._iterations += 1
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1382
        found, ghosts, next, parents = self._do_query(self._next_query)
1383
        self._current_present = found
1384
        self._current_ghosts = ghosts
1385
        self._next_query = next
1386
        self._current_parents = parents
3184.1.3 by Robert Collins
Automatically exclude ghosts.
1387
        # ghosts are implicit stop points, otherwise the search cannot be
1388
        # repeated when ghosts are filled.
1389
        self._stopped_keys.update(ghosts)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1390
1391
    def _do_query(self, revisions):
1392
        """Query for revisions.
1393
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1394
        Adds revisions to the seen set.
1395
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1396
        :param revisions: Revisions to query.
1397
        :return: A tuple: (set(found_revisions), set(ghost_revisions),
1398
           set(parents_of_found_revisions), dict(found_revisions:parents)).
1399
        """
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1400
        found_revisions = set()
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1401
        parents_of_found = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1402
        # revisions may contain nodes that point to other nodes in revisions:
1403
        # we want to filter them out.
1404
        self.seen.update(revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1405
        parent_map = self._parents_provider.get_parent_map(revisions)
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1406
        found_revisions.update(parent_map)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1407
        for rev_id, parents in parent_map.iteritems():
3517.4.2 by Martin Pool
Make simple-annotation and graph code more tolerant of knits with no graph
1408
            if parents is None:
1409
                continue
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1410
            new_found_parents = [p for p in parents if p not in self.seen]
1411
            if new_found_parents:
1412
                # Calling set.update() with an empty generator is actually
1413
                # rather expensive.
1414
                parents_of_found.update(new_found_parents)
1415
        ghost_revisions = revisions - found_revisions
1416
        return found_revisions, ghost_revisions, parents_of_found, parent_map
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1417
2490.2.8 by Aaron Bentley
fix iteration stuff
1418
    def __iter__(self):
1419
        return self
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1420
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1421
    def find_seen_ancestors(self, revisions):
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
1422
        """Find ancestors of these revisions that have already been seen.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1423
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
1424
        This function generally makes the assumption that querying for the
1425
        parents of a node that has already been queried is reasonably cheap.
1426
        (eg, not a round trip to a remote host).
1427
        """
1428
        # TODO: Often we might ask one searcher for its seen ancestors, and
1429
        #       then ask another searcher the same question. This can result in
1430
        #       searching the same revisions repeatedly if the two searchers
1431
        #       have a lot of overlap.
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1432
        all_seen = self.seen
1433
        pending = set(revisions).intersection(all_seen)
1434
        seen_ancestors = set(pending)
1435
1436
        if self._returning == 'next':
1437
            # self.seen contains what nodes have been returned, not what nodes
1438
            # have been queried. We don't want to probe for nodes that haven't
1439
            # been searched yet.
1440
            not_searched_yet = self._next_query
1441
        else:
1442
            not_searched_yet = ()
3377.3.11 by John Arbash Meinel
Committing a debug thunk that was very helpful
1443
        pending.difference_update(not_searched_yet)
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1444
        get_parent_map = self._parents_provider.get_parent_map
3377.3.12 by John Arbash Meinel
Remove the helpful but ugly thunk
1445
        while pending:
1446
            parent_map = get_parent_map(pending)
1447
            all_parents = []
1448
            # We don't care if it is a ghost, since it can't be seen if it is
1449
            # a ghost
1450
            for parent_ids in parent_map.itervalues():
1451
                all_parents.extend(parent_ids)
1452
            next_pending = all_seen.intersection(all_parents).difference(seen_ancestors)
1453
            seen_ancestors.update(next_pending)
1454
            next_pending.difference_update(not_searched_yet)
1455
            pending = next_pending
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1456
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1457
        return seen_ancestors
1458
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1459
    def stop_searching_any(self, revisions):
1460
        """
1461
        Remove any of the specified revisions from the search list.
1462
1463
        None of the specified revisions are required to be present in the
3808.1.4 by John Arbash Meinel
make _walk_to_common responsible for stopping ancestors
1464
        search list.
3808.1.1 by Andrew Bennetts
Possible fix for bug in new _walk_to_common_revisions.
1465
3808.1.4 by John Arbash Meinel
make _walk_to_common responsible for stopping ancestors
1466
        It is okay to call stop_searching_any() for revisions which were seen
1467
        in previous iterations. It is the callers responsibility to call
1468
        find_seen_ancestors() to make sure that current search tips that are
1469
        ancestors of those revisions are also stopped.  All explicitly stopped
1470
        revisions will be excluded from the search result's get_keys(), though.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1471
        """
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
1472
        # TODO: does this help performance?
1473
        # if not revisions:
1474
        #     return set()
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1475
        revisions = frozenset(revisions)
3177.3.3 by Robert Collins
Review feedback.
1476
        if self._returning == 'next':
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1477
            stopped = self._next_query.intersection(revisions)
1478
            self._next_query = self._next_query.difference(revisions)
1479
        else:
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1480
            stopped_present = self._current_present.intersection(revisions)
1481
            stopped = stopped_present.union(
1482
                self._current_ghosts.intersection(revisions))
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1483
            self._current_present.difference_update(stopped)
1484
            self._current_ghosts.difference_update(stopped)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1485
            # stopping 'x' should stop returning parents of 'x', but
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1486
            # not if 'y' always references those same parents
1487
            stop_rev_references = {}
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1488
            for rev in stopped_present:
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1489
                for parent_id in self._current_parents[rev]:
1490
                    if parent_id not in stop_rev_references:
1491
                        stop_rev_references[parent_id] = 0
1492
                    stop_rev_references[parent_id] += 1
1493
            # if only the stopped revisions reference it, the ref count will be
1494
            # 0 after this loop
3177.3.3 by Robert Collins
Review feedback.
1495
            for parents in self._current_parents.itervalues():
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1496
                for parent_id in parents:
1497
                    try:
1498
                        stop_rev_references[parent_id] -= 1
1499
                    except KeyError:
1500
                        pass
1501
            stop_parents = set()
1502
            for rev_id, refs in stop_rev_references.iteritems():
1503
                if refs == 0:
1504
                    stop_parents.add(rev_id)
1505
            self._next_query.difference_update(stop_parents)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1506
        self._stopped_keys.update(stopped)
4053.2.2 by Andrew Bennetts
Better fix, with test.
1507
        self._stopped_keys.update(revisions)
2490.2.25 by Aaron Bentley
Update from review
1508
        return stopped
2490.2.17 by Aaron Bentley
Add start_searching, tweak stop_searching_any
1509
1510
    def start_searching(self, revisions):
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1511
        """Add revisions to the search.
1512
1513
        The parents of revisions will be returned from the next call to next()
1514
        or next_with_ghosts(). If next_with_ghosts was the most recently used
1515
        next* call then the return value is the result of looking up the
1516
        ghost/not ghost status of revisions. (A tuple (present, ghosted)).
1517
        """
1518
        revisions = frozenset(revisions)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1519
        self._started_keys.update(revisions)
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1520
        new_revisions = revisions.difference(self.seen)
3177.3.3 by Robert Collins
Review feedback.
1521
        if self._returning == 'next':
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1522
            self._next_query.update(new_revisions)
3377.3.30 by John Arbash Meinel
Can we avoid the extra _do_query in start_searching?
1523
            self.seen.update(new_revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1524
        else:
1525
            # perform a query on revisions
3377.3.30 by John Arbash Meinel
Can we avoid the extra _do_query in start_searching?
1526
            revs, ghosts, query, parents = self._do_query(revisions)
1527
            self._stopped_keys.update(ghosts)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1528
            self._current_present.update(revs)
1529
            self._current_ghosts.update(ghosts)
1530
            self._next_query.update(query)
1531
            self._current_parents.update(parents)
1532
            return revs, ghosts
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1533
1534
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1535
class AbstractSearchResult(object):
5535.3.50 by Andrew Bennetts
Add docstrings to AbstractSearch and AbstractSearchResult.
1536
    """The result of a search, describing a set of keys.
1537
    
1538
    Search results are typically used as the 'fetch_spec' parameter when
1539
    fetching revisions.
1540
1541
    :seealso: AbstractSearch
1542
    """
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1543
1544
    def get_recipe(self):
1545
        """Return a recipe that can be used to replay this search.
1546
1547
        The recipe allows reconstruction of the same results at a later date.
1548
1549
        :return: A tuple of (search_kind_str, *details).  The details vary by
1550
            kind of search result.
1551
        """
1552
        raise NotImplementedError(self.get_recipe)
1553
1554
    def get_network_struct(self):
1555
        """Return a tuple that can be transmitted via the HPSS protocol."""
1556
        raise NotImplementedError(self.get_network_struct)
1557
1558
    def get_keys(self):
1559
        """Return the keys found in this search.
1560
1561
        :return: A set of keys.
1562
        """
1563
        raise NotImplementedError(self.get_keys)
1564
1565
    def is_empty(self):
1566
        """Return false if the search lists 1 or more revisions."""
1567
        raise NotImplementedError(self.is_empty)
1568
1569
    def refine(self, seen, referenced):
1570
        """Create a new search by refining this search.
1571
1572
        :param seen: Revisions that have been satisfied.
1573
        :param referenced: Revision references observed while satisfying some
1574
            of this search.
1575
        :return: A search result.
1576
        """
1577
        raise NotImplementedError(self.refine)
1578
1579
1580
class AbstractSearch(object):
5535.3.50 by Andrew Bennetts
Add docstrings to AbstractSearch and AbstractSearchResult.
1581
    """A search that can be executed, producing a search result.
1582
1583
    :seealso: AbstractSearchResult
1584
    """
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1585
5536.3.1 by Andrew Bennetts
Rename get_search_result to execute, require a SearchResult (and not a search) be passed to fetch.
1586
    def execute(self):
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1587
        """Construct a network-ready search result from this search description.
1588
1589
        This may take some time to search repositories, etc.
1590
5536.3.1 by Andrew Bennetts
Rename get_search_result to execute, require a SearchResult (and not a search) be passed to fetch.
1591
        :return: A search result (an object that implements
1592
            AbstractSearchResult's API).
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1593
        """
5536.3.1 by Andrew Bennetts
Rename get_search_result to execute, require a SearchResult (and not a search) be passed to fetch.
1594
        raise NotImplementedError(self.execute)
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1595
1596
1597
class SearchResult(AbstractSearchResult):
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1598
    """The result of a breadth first search.
1599
1600
    A SearchResult provides the ability to reconstruct the search or access a
1601
    set of the keys the search found.
1602
    """
1603
1604
    def __init__(self, start_keys, exclude_keys, key_count, keys):
1605
        """Create a SearchResult.
1606
1607
        :param start_keys: The keys the search started at.
1608
        :param exclude_keys: The keys the search excludes.
1609
        :param key_count: The total number of keys (from start to but not
1610
            including exclude).
1611
        :param keys: The keys the search found. Note that in future we may get
1612
            a SearchResult from a smart server, in which case the keys list is
1613
            not necessarily immediately available.
1614
        """
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1615
        self._recipe = ('search', start_keys, exclude_keys, key_count)
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1616
        self._keys = frozenset(keys)
1617
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1618
    def __repr__(self):
1619
        kind, start_keys, exclude_keys, key_count = self._recipe
1620
        if len(start_keys) > 5:
1621
            start_keys_repr = repr(list(start_keys)[:5])[:-1] + ', ...]'
1622
        else:
1623
            start_keys_repr = repr(start_keys)
1624
        if len(exclude_keys) > 5:
1625
            exclude_keys_repr = repr(list(exclude_keys)[:5])[:-1] + ', ...]'
1626
        else:
1627
            exclude_keys_repr = repr(exclude_keys)
1628
        return '<%s %s:(%s, %s, %d)>' % (self.__class__.__name__,
1629
            kind, start_keys_repr, exclude_keys_repr, key_count)
1630
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1631
    def get_recipe(self):
1632
        """Return a recipe that can be used to replay this search.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1633
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1634
        The recipe allows reconstruction of the same results at a later date
1635
        without knowing all the found keys. The essential elements are a list
4031.3.1 by Frank Aspell
Fixing various typos
1636
        of keys to start and to stop at. In order to give reproducible
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1637
        results when ghosts are encountered by a search they are automatically
1638
        added to the exclude list (or else ghost filling may alter the
1639
        results).
1640
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1641
        :return: A tuple ('search', start_keys_set, exclude_keys_set,
1642
            revision_count). To recreate the results of this search, create a
1643
            breadth first searcher on the same graph starting at start_keys.
1644
            Then call next() (or next_with_ghosts()) repeatedly, and on every
1645
            result, call stop_searching_any on any keys from the exclude_keys
1646
            set. The revision_count value acts as a trivial cross-check - the
1647
            found revisions of the new search should have as many elements as
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1648
            revision_count. If it does not, then additional revisions have been
1649
            ghosted since the search was executed the first time and the second
1650
            time.
1651
        """
1652
        return self._recipe
1653
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1654
    def get_network_struct(self):
1655
        start_keys = ' '.join(self._recipe[1])
1656
        stop_keys = ' '.join(self._recipe[2])
1657
        count = str(self._recipe[3])
1658
        return (self._recipe[0], '\n'.join((start_keys, stop_keys, count)))
1659
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1660
    def get_keys(self):
1661
        """Return the keys found in this search.
1662
1663
        :return: A set of keys.
1664
        """
1665
        return self._keys
1666
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1667
    def is_empty(self):
4204.2.2 by Matt Nordhoff
Fix docstrings on graph.py's is_empty methods that said they returned true when they were *not* empty.
1668
        """Return false if the search lists 1 or more revisions."""
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1669
        return self._recipe[3] == 0
1670
1671
    def refine(self, seen, referenced):
1672
        """Create a new search by refining this search.
1673
1674
        :param seen: Revisions that have been satisfied.
1675
        :param referenced: Revision references observed while satisfying some
1676
            of this search.
1677
        """
1678
        start = self._recipe[1]
1679
        exclude = self._recipe[2]
1680
        count = self._recipe[3]
1681
        keys = self.get_keys()
1682
        # New heads = referenced + old heads - seen things - exclude
1683
        pending_refs = set(referenced)
1684
        pending_refs.update(start)
1685
        pending_refs.difference_update(seen)
1686
        pending_refs.difference_update(exclude)
1687
        # New exclude = old exclude + satisfied heads
1688
        seen_heads = start.intersection(seen)
1689
        exclude.update(seen_heads)
1690
        # keys gets seen removed
1691
        keys = keys - seen
1692
        # length is reduced by len(seen)
1693
        count -= len(seen)
1694
        return SearchResult(pending_refs, exclude, count, keys)
1695
3514.2.14 by John Arbash Meinel
Bring in the code to collapse linear portions of the graph.
1696
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1697
class PendingAncestryResult(AbstractSearchResult):
4070.9.14 by Andrew Bennetts
Tweaks requested by Robert's review.
1698
    """A search result that will reconstruct the ancestry for some graph heads.
4086.1.4 by Andrew Bennetts
Fix whitespace nit.
1699
4070.9.14 by Andrew Bennetts
Tweaks requested by Robert's review.
1700
    Unlike SearchResult, this doesn't hold the complete search result in
1701
    memory, it just holds a description of how to generate it.
1702
    """
1703
1704
    def __init__(self, heads, repo):
1705
        """Constructor.
1706
1707
        :param heads: an iterable of graph heads.
1708
        :param repo: a repository to use to generate the ancestry for the given
1709
            heads.
1710
        """
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1711
        self.heads = frozenset(heads)
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1712
        self.repo = repo
1713
5539.2.20 by Andrew Bennetts
Add PendingAncestryResult.__repr__.
1714
    def __repr__(self):
1715
        if len(self.heads) > 5:
5535.4.21 by Andrew Bennetts
Slightly more informative __repr__.
1716
            heads_repr = repr(list(self.heads)[:5])[:-1]
1717
            heads_repr += ', <%d more>...]' % (len(self.heads) - 5,)
5539.2.20 by Andrew Bennetts
Add PendingAncestryResult.__repr__.
1718
        else:
1719
            heads_repr = repr(self.heads)
1720
        return '<%s heads:%s repo:%r>' % (
1721
            self.__class__.__name__, heads_repr, self.repo)
1722
4070.9.5 by Andrew Bennetts
Better wire protocol: don't shoehorn MiniSearchResult serialisation into previous serialisation format.
1723
    def get_recipe(self):
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1724
        """Return a recipe that can be used to replay this search.
1725
1726
        The recipe allows reconstruction of the same results at a later date.
1727
1728
        :seealso SearchResult.get_recipe:
1729
1730
        :return: A tuple ('proxy-search', start_keys_set, set(), -1)
1731
            To recreate this result, create a PendingAncestryResult with the
1732
            start_keys_set.
1733
        """
1734
        return ('proxy-search', self.heads, set(), -1)
4070.9.5 by Andrew Bennetts
Better wire protocol: don't shoehorn MiniSearchResult serialisation into previous serialisation format.
1735
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1736
    def get_network_struct(self):
1737
        parts = ['ancestry-of']
1738
        parts.extend(self.heads)
1739
        return parts
1740
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1741
    def get_keys(self):
4098.1.1 by Andrew Bennetts
Fix a bug with how PendingAncestryResult.get_keys handles NULL_REVISION.
1742
        """See SearchResult.get_keys.
4098.1.2 by Andrew Bennetts
Fix 'trailing' whitespace (actually just a blank line in an indented docstring).
1743
4098.1.1 by Andrew Bennetts
Fix a bug with how PendingAncestryResult.get_keys handles NULL_REVISION.
1744
        Returns all the keys for the ancestry of the heads, excluding
1745
        NULL_REVISION.
1746
        """
1747
        return self._get_keys(self.repo.get_graph())
4098.1.3 by Andrew Bennetts
Fix 'trailing' whitespace (actually just a blank line between methods).
1748
4098.1.1 by Andrew Bennetts
Fix a bug with how PendingAncestryResult.get_keys handles NULL_REVISION.
1749
    def _get_keys(self, graph):
1750
        NULL_REVISION = revision.NULL_REVISION
1751
        keys = [key for (key, parents) in graph.iter_ancestry(self.heads)
4343.3.11 by John Arbash Meinel
Change PendingAncestryResult to strip ghosts from .get_keys()
1752
                if key != NULL_REVISION and parents is not None]
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1753
        return keys
1754
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1755
    def is_empty(self):
4204.2.2 by Matt Nordhoff
Fix docstrings on graph.py's is_empty methods that said they returned true when they were *not* empty.
1756
        """Return false if the search lists 1 or more revisions."""
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1757
        if revision.NULL_REVISION in self.heads:
1758
            return len(self.heads) == 1
1759
        else:
1760
            return len(self.heads) == 0
1761
1762
    def refine(self, seen, referenced):
1763
        """Create a new search by refining this search.
1764
1765
        :param seen: Revisions that have been satisfied.
1766
        :param referenced: Revision references observed while satisfying some
1767
            of this search.
1768
        """
1769
        referenced = self.heads.union(referenced)
1770
        return PendingAncestryResult(referenced - seen, self.repo)
1771
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1772
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1773
class EmptySearchResult(AbstractSearchResult):
5539.2.15 by Andrew Bennetts
Add a docstring.
1774
    """An empty search result."""
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1775
1776
    def is_empty(self):
1777
        return True
1778
    
1779
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1780
class EverythingResult(AbstractSearchResult):
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1781
    """A search result that simply requests everything in the repository."""
1782
1783
    def __init__(self, repo):
1784
        self._repo = repo
1785
5535.3.33 by Andrew Bennetts
Fix a bug.
1786
    def __repr__(self):
1787
        return '%s(%r)' % (self.__class__.__name__, self._repo)
1788
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1789
    def get_recipe(self):
1790
        raise NotImplementedError(self.get_recipe)
1791
1792
    def get_network_struct(self):
1793
        return ('everything',)
1794
1795
    def get_keys(self):
1796
        if 'evil' in debug.debug_flags:
1797
            from bzrlib import remote
1798
            if isinstance(self._repo, remote.RemoteRepository):
1799
                # warn developers (not users) not to do this
1800
                trace.mutter_callsite(
1801
                    2, "EverythingResult(RemoteRepository).get_keys() is slow.")
5539.2.4 by Andrew Bennetts
Add some basic tests for the new verb, fix some shallow bugs.
1802
        return self._repo.all_revision_ids()
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1803
1804
    def is_empty(self):
1805
        # It's ok for this to wrongly return False: the worst that can happen
1806
        # is that RemoteStreamSource will initiate a get_stream on an empty
1807
        # repository.  And almost all repositories are non-empty.
1808
        return False
1809
1810
    def refine(self, seen, referenced):
5539.3.3 by Andrew Bennetts
Implement EverythingResult.refine.
1811
        heads = set(self._repo.all_revision_ids())
1812
        heads.difference_update(seen)
1813
        heads.update(referenced)
1814
        return PendingAncestryResult(heads, self._repo)
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1815
1816
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1817
class EverythingNotInOther(AbstractSearch):
1818
    """Find all revisions in that are in one repo but not the other."""
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1819
1820
    def __init__(self, to_repo, from_repo, find_ghosts=False):
1821
        self.to_repo = to_repo
1822
        self.from_repo = from_repo
1823
        self.find_ghosts = find_ghosts
1824
5536.3.1 by Andrew Bennetts
Rename get_search_result to execute, require a SearchResult (and not a search) be passed to fetch.
1825
    def execute(self):
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1826
        return self.to_repo.search_missing_revision_ids(
1827
            self.from_repo, find_ghosts=self.find_ghosts)
1828
1829
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1830
class NotInOtherForRevs(AbstractSearch):
1831
    """Find all revisions missing in one repo for a some specific heads."""
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1832
5535.3.31 by Andrew Bennetts
Cope with tags that reference missing revisions.
1833
    def __init__(self, to_repo, from_repo, required_ids, if_present_ids=None,
1834
            find_ghosts=False):
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1835
        """Constructor.
1836
1837
        :param required_ids: revision IDs of heads that must be found, or else
1838
            the search will fail with NoSuchRevision.  All revisions in their
1839
            ancestry not already in the other repository will be included in
1840
            the search result.
1841
        :param if_present_ids: revision IDs of heads that may be absent in the
1842
            source repository.  If present, then their ancestry not already
1843
            found in other will be included in the search result.
1844
        """
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1845
        self.to_repo = to_repo
1846
        self.from_repo = from_repo
1847
        self.find_ghosts = find_ghosts
5535.3.31 by Andrew Bennetts
Cope with tags that reference missing revisions.
1848
        self.required_ids = required_ids
1849
        self.if_present_ids = if_present_ids
1850
1851
    def __repr__(self):
1852
        if len(self.required_ids) > 5:
1853
            reqd_revs_repr = repr(list(self.required_ids)[:5])[:-1] + ', ...]'
1854
        else:
1855
            reqd_revs_repr = repr(self.required_ids)
1856
        if self.if_present_ids and len(self.if_present_ids) > 5:
1857
            ifp_revs_repr = repr(list(self.if_present_ids)[:5])[:-1] + ', ...]'
1858
        else:
1859
            ifp_revs_repr = repr(self.if_present_ids)
1860
1861
        return "<%s from:%r to:%r find_ghosts:%r req'd:%r if-present:%r>" % (
1862
            self.__class__.__name__, self.from_repo, self.to_repo,
1863
            self.find_ghosts, reqd_revs_repr, ifp_revs_repr)
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1864
5536.3.1 by Andrew Bennetts
Rename get_search_result to execute, require a SearchResult (and not a search) be passed to fetch.
1865
    def execute(self):
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1866
        return self.to_repo.search_missing_revision_ids(
5535.3.31 by Andrew Bennetts
Cope with tags that reference missing revisions.
1867
            self.from_repo, revision_ids=self.required_ids,
1868
            if_present_ids=self.if_present_ids, find_ghosts=self.find_ghosts)
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1869
1870
3514.2.14 by John Arbash Meinel
Bring in the code to collapse linear portions of the graph.
1871
def collapse_linear_regions(parent_map):
1872
    """Collapse regions of the graph that are 'linear'.
1873
1874
    For example::
1875
1876
      A:[B], B:[C]
1877
1878
    can be collapsed by removing B and getting::
1879
1880
      A:[C]
1881
1882
    :param parent_map: A dictionary mapping children to their parents
1883
    :return: Another dictionary with 'linear' chains collapsed
1884
    """
1885
    # Note: this isn't a strictly minimal collapse. For example:
1886
    #   A
1887
    #  / \
1888
    # B   C
1889
    #  \ /
1890
    #   D
1891
    #   |
1892
    #   E
1893
    # Will not have 'D' removed, even though 'E' could fit. Also:
1894
    #   A
1895
    #   |    A
1896
    #   B => |
1897
    #   |    C
1898
    #   C
1899
    # A and C are both kept because they are edges of the graph. We *could* get
1900
    # rid of A if we wanted.
1901
    #   A
1902
    #  / \
1903
    # B   C
1904
    # |   |
1905
    # D   E
1906
    #  \ /
1907
    #   F
1908
    # Will not have any nodes removed, even though you do have an
1909
    # 'uninteresting' linear D->B and E->C
1910
    children = {}
1911
    for child, parents in parent_map.iteritems():
1912
        children.setdefault(child, [])
1913
        for p in parents:
1914
            children.setdefault(p, []).append(child)
1915
1916
    orig_children = dict(children)
1917
    removed = set()
1918
    result = dict(parent_map)
1919
    for node in parent_map:
1920
        parents = result[node]
1921
        if len(parents) == 1:
1922
            parent_children = children[parents[0]]
1923
            if len(parent_children) != 1:
1924
                # This is not the only child
1925
                continue
1926
            node_children = children[node]
1927
            if len(node_children) != 1:
1928
                continue
1929
            child_parents = result.get(node_children[0], None)
1930
            if len(child_parents) != 1:
1931
                # This is not its only parent
1932
                continue
1933
            # The child of this node only points at it, and the parent only has
1934
            # this as a child. remove this node, and join the others together
1935
            result[node_children[0]] = parents
1936
            children[parents[0]] = node_children
1937
            del result[node]
1938
            del children[node]
1939
            removed.add(node)
1940
1941
    return result
4371.3.18 by John Arbash Meinel
Change VF.annotate to use the new KnownGraph code.
1942
1943
4819.2.3 by John Arbash Meinel
Add a GraphThunkIdsToKeys as a tested class.
1944
class GraphThunkIdsToKeys(object):
1945
    """Forwards calls about 'ids' to be about keys internally."""
1946
1947
    def __init__(self, graph):
1948
        self._graph = graph
1949
4913.4.4 by Jelmer Vernooij
Add test for Repository.get_known_graph_ancestry().
1950
    def topo_sort(self):
1951
        return [r for (r,) in self._graph.topo_sort()]
1952
4819.2.3 by John Arbash Meinel
Add a GraphThunkIdsToKeys as a tested class.
1953
    def heads(self, ids):
1954
        """See Graph.heads()"""
1955
        as_keys = [(i,) for i in ids]
1956
        head_keys = self._graph.heads(as_keys)
1957
        return set([h[0] for h in head_keys])
1958
4913.4.2 by Jelmer Vernooij
Add Repository.get_known_graph_ancestry.
1959
    def merge_sort(self, tip_revision):
1960
        return self._graph.merge_sort((tip_revision,))
1961
5559.3.1 by Jelmer Vernooij
Add GraphThunkIdsToKeys.add_node.
1962
    def add_node(self, revision, parents):
1963
        self._graph.add_node((revision,), [(p,) for p in parents])
1964
4819.2.3 by John Arbash Meinel
Add a GraphThunkIdsToKeys as a tested class.
1965
4371.3.38 by John Arbash Meinel
Add a failing test for handling nodes that are in the same linear chain.
1966
_counters = [0,0,0,0,0,0,0]
4371.3.18 by John Arbash Meinel
Change VF.annotate to use the new KnownGraph code.
1967
try:
1968
    from bzrlib._known_graph_pyx import KnownGraph
4574.3.6 by Martin Pool
More warnings when failing to load extensions
1969
except ImportError, e:
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
1970
    osutils.failed_to_load_extension(e)
4371.3.18 by John Arbash Meinel
Change VF.annotate to use the new KnownGraph code.
1971
    from bzrlib._known_graph_py import KnownGraph