~bzr-pqm/bzr/bzr.dev

4371.3.6 by John Arbash Meinel
Remove the debug timing for _find_gdfo
1
# Copyright (C) 2007, 2008, 2009 Canonical Ltd
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
16
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
17
import time
18
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
19
from bzrlib import (
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
20
    debug,
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
21
    errors,
3052.1.3 by John Arbash Meinel
deprecate revision.is_ancestor, update the callers and the tests.
22
    revision,
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
23
    trace,
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
24
    )
4379.3.5 by Gary van der Merwe
Still have _StackedParentsProvider, but mark it as depreciated.
25
from bzrlib.symbol_versioning import deprecated_function, deprecated_in
2490.2.1 by Aaron Bentley
Start work on GraphWalker
26
3377.4.9 by John Arbash Meinel
STEP every 5
27
STEP_UNIQUE_SEARCHER_EVERY = 5
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
28
2490.2.25 by Aaron Bentley
Update from review
29
# DIAGRAM of terminology
30
#       A
31
#       /\
32
#      B  C
33
#      |  |\
34
#      D  E F
35
#      |\/| |
36
#      |/\|/
37
#      G  H
38
#
39
# In this diagram, relative to G and H:
40
# A, B, C, D, E are common ancestors.
41
# C, D and E are border ancestors, because each has a non-common descendant.
42
# D and E are least common ancestors because none of their descendants are
43
# common ancestors.
44
# C is not a least common ancestor because its descendant, E, is a common
45
# ancestor.
46
#
47
# The find_unique_lca algorithm will pick A in two steps:
48
# 1. find_lca('G', 'H') => ['D', 'E']
49
# 2. Since len(['D', 'E']) > 1, find_lca('D', 'E') => ['A']
50
51
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
52
class DictParentsProvider(object):
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
53
    """A parents provider for Graph objects."""
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
54
55
    def __init__(self, ancestry):
56
        self.ancestry = ancestry
57
58
    def __repr__(self):
59
        return 'DictParentsProvider(%r)' % self.ancestry
60
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
61
    def get_parent_map(self, keys):
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
62
        """See StackedParentsProvider.get_parent_map"""
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
63
        ancestry = self.ancestry
64
        return dict((k, ancestry[k]) for k in keys if k in ancestry)
65
4379.3.6 by Gary van der Merwe
Test that _StackedParentsProvider is deprecated and still works. The deprecated version was also incorrect.
66
@deprecated_function(deprecated_in((1, 16, 0)))
4379.3.5 by Gary van der Merwe
Still have _StackedParentsProvider, but mark it as depreciated.
67
def _StackedParentsProvider(*args, **kwargs):
68
    return StackedParentsProvider(*args, **kwargs)
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
69
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
70
class StackedParentsProvider(object):
71
    """A parents provider which stacks (or unions) multiple providers.
72
    
73
    The providers are queries in the order of the provided parent_providers.
74
    """
75
    
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
76
    def __init__(self, parent_providers):
77
        self._parent_providers = parent_providers
78
2490.2.28 by Aaron Bentley
Fix handling of null revision
79
    def __repr__(self):
4379.3.4 by Gary van der Merwe
Make StackedParentsProvider.__repr__ more dynamic.
80
        return "%s(%r)" % (self.__class__.__name__, self._parent_providers)
2490.2.28 by Aaron Bentley
Fix handling of null revision
81
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
82
    def get_parent_map(self, keys):
83
        """Get a mapping of keys => parents
84
85
        A dictionary is returned with an entry for each key present in this
86
        source. If this source doesn't have information about a key, it should
87
        not include an entry.
88
89
        [NULL_REVISION] is used as the parent of the first user-committed
90
        revision.  Its parent list is empty.
91
92
        :param keys: An iterable returning keys to check (eg revision_ids)
93
        :return: A dictionary mapping each key to its parents
94
        """
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
95
        found = {}
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
96
        remaining = set(keys)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
97
        for parents_provider in self._parent_providers:
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
98
            new_found = parents_provider.get_parent_map(remaining)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
99
            found.update(new_found)
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
100
            remaining.difference_update(new_found)
101
            if not remaining:
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
102
                break
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
103
        return found
104
105
106
class CachingParentsProvider(object):
3835.1.13 by Aaron Bentley
Update documentation
107
    """A parents provider which will cache the revision => parents as a dict.
108
109
    This is useful for providers which have an expensive look up.
110
111
    Either a ParentsProvider or a get_parent_map-like callback may be
112
    supplied.  If it provides extra un-asked-for parents, they will be cached,
113
    but filtered out of get_parent_map.
3835.1.16 by Aaron Bentley
Updates from review
114
115
    The cache is enabled by default, but may be disabled and re-enabled.
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
116
    """
3896.1.1 by Andrew Bennetts
Remove broken debugging cruft, and some unused imports.
117
    def __init__(self, parent_provider=None, get_parent_map=None):
3835.1.13 by Aaron Bentley
Update documentation
118
        """Constructor.
119
120
        :param parent_provider: The ParentProvider to use.  It or
121
            get_parent_map must be supplied.
122
        :param get_parent_map: The get_parent_map callback to use.  It or
123
            parent_provider must be supplied.
124
        """
3835.1.12 by Aaron Bentley
Unify CachingExtraParentsProvider and CachingParentsProvider.
125
        self._real_provider = parent_provider
126
        if get_parent_map is None:
127
            self._get_parent_map = self._real_provider.get_parent_map
128
        else:
129
            self._get_parent_map = get_parent_map
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
130
        self._cache = None
131
        self.enable_cache(True)
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
132
3835.1.12 by Aaron Bentley
Unify CachingExtraParentsProvider and CachingParentsProvider.
133
    def __repr__(self):
134
        return "%s(%r)" % (self.__class__.__name__, self._real_provider)
135
3835.1.15 by Aaron Bentley
Allow miss caching to be disabled.
136
    def enable_cache(self, cache_misses=True):
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
137
        """Enable cache."""
3835.1.19 by Aaron Bentley
Raise exception when caching is enabled twice.
138
        if self._cache is not None:
3835.1.20 by Aaron Bentley
Change custom error to an AssertionError.
139
            raise AssertionError('Cache enabled when already enabled.')
3835.1.16 by Aaron Bentley
Updates from review
140
        self._cache = {}
3835.1.15 by Aaron Bentley
Allow miss caching to be disabled.
141
        self._cache_misses = cache_misses
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
142
        self.missing_keys = set()
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
143
144
    def disable_cache(self):
3835.1.16 by Aaron Bentley
Updates from review
145
        """Disable and clear the cache."""
146
        self._cache = None
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
147
        self._cache_misses = None
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
148
        self.missing_keys = set()
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
149
150
    def get_cached_map(self):
151
        """Return any cached get_parent_map values."""
3835.1.16 by Aaron Bentley
Updates from review
152
        if self._cache is None:
3835.1.12 by Aaron Bentley
Unify CachingExtraParentsProvider and CachingParentsProvider.
153
            return None
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
154
        return dict(self._cache)
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
155
156
    def get_parent_map(self, keys):
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
157
        """See StackedParentsProvider.get_parent_map."""
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
158
        cache = self._cache
159
        if cache is None:
160
            cache = self._get_parent_map(keys)
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
161
        else:
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
162
            needed_revisions = set(key for key in keys if key not in cache)
163
            # Do not ask for negatively cached keys
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
164
            needed_revisions.difference_update(self.missing_keys)
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
165
            if needed_revisions:
166
                parent_map = self._get_parent_map(needed_revisions)
167
                cache.update(parent_map)
168
                if self._cache_misses:
169
                    for key in needed_revisions:
170
                        if key not in parent_map:
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
171
                            self.note_missing_key(key)
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
172
        result = {}
173
        for key in keys:
174
            value = cache.get(key)
175
            if value is not None:
176
                result[key] = value
177
        return result
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
178
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
179
    def note_missing_key(self, key):
180
        """Note that key is a missing key."""
181
        if self._cache_misses:
182
            self.missing_keys.add(key)
183
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
184
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
185
class Graph(object):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
186
    """Provide incremental access to revision graphs.
187
188
    This is the generic implementation; it is intended to be subclassed to
189
    specialize it for other repository types.
190
    """
2490.2.1 by Aaron Bentley
Start work on GraphWalker
191
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
192
    def __init__(self, parents_provider):
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
193
        """Construct a Graph that uses several graphs as its input
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
194
195
        This should not normally be invoked directly, because there may be
196
        specialized implementations for particular repository types.  See
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
197
        Repository.get_graph().
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
198
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
199
        :param parents_provider: An object providing a get_parent_map call
200
            conforming to the behavior of
201
            StackedParentsProvider.get_parent_map.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
202
        """
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
203
        if getattr(parents_provider, 'get_parents', None) is not None:
204
            self.get_parents = parents_provider.get_parents
205
        if getattr(parents_provider, 'get_parent_map', None) is not None:
206
            self.get_parent_map = parents_provider.get_parent_map
2490.2.29 by Aaron Bentley
Make parents provider private
207
        self._parents_provider = parents_provider
2490.2.28 by Aaron Bentley
Fix handling of null revision
208
209
    def __repr__(self):
2490.2.29 by Aaron Bentley
Make parents provider private
210
        return 'Graph(%r)' % self._parents_provider
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
211
212
    def find_lca(self, *revisions):
213
        """Determine the lowest common ancestors of the provided revisions
214
215
        A lowest common ancestor is a common ancestor none of whose
216
        descendants are common ancestors.  In graphs, unlike trees, there may
217
        be multiple lowest common ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
218
219
        This algorithm has two phases.  Phase 1 identifies border ancestors,
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
220
        and phase 2 filters border ancestors to determine lowest common
221
        ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
222
223
        In phase 1, border ancestors are identified, using a breadth-first
224
        search starting at the bottom of the graph.  Searches are stopped
225
        whenever a node or one of its descendants is determined to be common
226
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
227
        In phase 2, the border ancestors are filtered to find the least
2490.2.12 by Aaron Bentley
Improve documentation
228
        common ancestors.  This is done by searching the ancestries of each
229
        border ancestor.
230
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
231
        Phase 2 is perfomed on the principle that a border ancestor that is
232
        not an ancestor of any other border ancestor is a least common
233
        ancestor.
2490.2.12 by Aaron Bentley
Improve documentation
234
235
        Searches are stopped when they find a node that is determined to be a
236
        common ancestor of all border ancestors, because this shows that it
237
        cannot be a descendant of any border ancestor.
238
239
        The scaling of this operation should be proportional to
240
        1. The number of uncommon ancestors
241
        2. The number of border ancestors
242
        3. The length of the shortest path between a border ancestor and an
243
           ancestor of all border ancestors.
2490.2.3 by Aaron Bentley
Implement new merge base picker
244
        """
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
245
        border_common, common, sides = self._find_border_ancestors(revisions)
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
246
        # We may have common ancestors that can be reached from each other.
247
        # - ask for the heads of them to filter it down to only ones that
248
        # cannot be reached from each other - phase 2.
249
        return self.heads(border_common)
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
250
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
251
    def find_difference(self, left_revision, right_revision):
2490.2.25 by Aaron Bentley
Update from review
252
        """Determine the graph difference between two revisions"""
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
253
        border, common, searchers = self._find_border_ancestors(
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
254
            [left_revision, right_revision])
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
255
        self._search_for_extra_common(common, searchers)
256
        left = searchers[0].seen
257
        right = searchers[1].seen
258
        return (left.difference(right), right.difference(left))
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
259
3445.1.4 by John Arbash Meinel
Change the function to be called 'find_distance_to_null'
260
    def find_distance_to_null(self, target_revision_id, known_revision_ids):
261
        """Find the left-hand distance to the NULL_REVISION.
262
263
        (This can also be considered the revno of a branch at
264
        target_revision_id.)
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
265
266
        :param target_revision_id: A revision_id which we would like to know
267
            the revno for.
268
        :param known_revision_ids: [(revision_id, revno)] A list of known
269
            revno, revision_id tuples. We'll use this to seed the search.
270
        """
271
        # Map from revision_ids to a known value for their revno
272
        known_revnos = dict(known_revision_ids)
273
        cur_tip = target_revision_id
274
        num_steps = 0
275
        NULL_REVISION = revision.NULL_REVISION
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
276
        known_revnos[NULL_REVISION] = 0
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
277
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
278
        searching_known_tips = list(known_revnos.keys())
279
280
        unknown_searched = {}
281
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
282
        while cur_tip not in known_revnos:
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
283
            unknown_searched[cur_tip] = num_steps
284
            num_steps += 1
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
285
            to_search = set([cur_tip])
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
286
            to_search.update(searching_known_tips)
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
287
            parent_map = self.get_parent_map(to_search)
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
288
            parents = parent_map.get(cur_tip, None)
3445.1.8 by John Arbash Meinel
Clarity tweaks recommended by Ian
289
            if not parents: # An empty list or None is a ghost
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
290
                raise errors.GhostRevisionsHaveNoRevno(target_revision_id,
291
                                                       cur_tip)
292
            cur_tip = parents[0]
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
293
            next_known_tips = []
294
            for revision_id in searching_known_tips:
295
                parents = parent_map.get(revision_id, None)
296
                if not parents:
297
                    continue
298
                next = parents[0]
299
                next_revno = known_revnos[revision_id] - 1
300
                if next in unknown_searched:
301
                    # We have enough information to return a value right now
302
                    return next_revno + unknown_searched[next]
303
                if next in known_revnos:
304
                    continue
305
                known_revnos[next] = next_revno
306
                next_known_tips.append(next)
307
            searching_known_tips = next_known_tips
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
308
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
309
        # We reached a known revision, so just add in how many steps it took to
310
        # get there.
311
        return known_revnos[cur_tip] + num_steps
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
312
4332.3.4 by Robert Collins
Add a graph API for getting multiple distances to NULL at once.
313
    def find_lefthand_distances(self, keys):
314
        """Find the distance to null for all the keys in keys.
315
316
        :param keys: keys to lookup.
317
        :return: A dict key->distance for all of keys.
318
        """
319
        # Optimisable by concurrent searching, but a random spread should get
320
        # some sort of hit rate.
321
        result = {}
322
        known_revnos = []
4332.3.6 by Robert Collins
Teach graph.find_lefthand_distances about ghosts.
323
        ghosts = []
4332.3.4 by Robert Collins
Add a graph API for getting multiple distances to NULL at once.
324
        for key in keys:
4332.3.6 by Robert Collins
Teach graph.find_lefthand_distances about ghosts.
325
            try:
326
                known_revnos.append(
327
                    (key, self.find_distance_to_null(key, known_revnos)))
328
            except errors.GhostRevisionsHaveNoRevno:
329
                ghosts.append(key)
330
        for key in ghosts:
331
            known_revnos.append((key, -1))
4332.3.4 by Robert Collins
Add a graph API for getting multiple distances to NULL at once.
332
        return dict(known_revnos)
333
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
334
    def find_unique_ancestors(self, unique_revision, common_revisions):
335
        """Find the unique ancestors for a revision versus others.
336
337
        This returns the ancestry of unique_revision, excluding all revisions
338
        in the ancestry of common_revisions. If unique_revision is in the
339
        ancestry, then the empty set will be returned.
340
341
        :param unique_revision: The revision_id whose ancestry we are
342
            interested in.
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
343
            XXX: Would this API be better if we allowed multiple revisions on
344
                 to be searched here?
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
345
        :param common_revisions: Revision_ids of ancestries to exclude.
346
        :return: A set of revisions in the ancestry of unique_revision
347
        """
348
        if unique_revision in common_revisions:
349
            return set()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
350
351
        # Algorithm description
352
        # 1) Walk backwards from the unique node and all common nodes.
353
        # 2) When a node is seen by both sides, stop searching it in the unique
354
        #    walker, include it in the common walker.
355
        # 3) Stop searching when there are no nodes left for the unique walker.
356
        #    At this point, you have a maximal set of unique nodes. Some of
357
        #    them may actually be common, and you haven't reached them yet.
358
        # 4) Start new searchers for the unique nodes, seeded with the
359
        #    information you have so far.
360
        # 5) Continue searching, stopping the common searches when the search
361
        #    tip is an ancestor of all unique nodes.
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
362
        # 6) Aggregate together unique searchers when they are searching the
363
        #    same tips. When all unique searchers are searching the same node,
364
        #    stop move it to a single 'all_unique_searcher'.
365
        # 7) The 'all_unique_searcher' represents the very 'tip' of searching.
366
        #    Most of the time this produces very little important information.
367
        #    So don't step it as quickly as the other searchers.
368
        # 8) Search is done when all common searchers have completed.
369
370
        unique_searcher, common_searcher = self._find_initial_unique_nodes(
371
            [unique_revision], common_revisions)
372
373
        unique_nodes = unique_searcher.seen.difference(common_searcher.seen)
374
        if not unique_nodes:
375
            return unique_nodes
376
377
        (all_unique_searcher,
378
         unique_tip_searchers) = self._make_unique_searchers(unique_nodes,
379
                                    unique_searcher, common_searcher)
380
381
        self._refine_unique_nodes(unique_searcher, all_unique_searcher,
382
                                  unique_tip_searchers, common_searcher)
383
        true_unique_nodes = unique_nodes.difference(common_searcher.seen)
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
384
        if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
385
            trace.mutter('Found %d truly unique nodes out of %d',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
386
                         len(true_unique_nodes), len(unique_nodes))
387
        return true_unique_nodes
388
389
    def _find_initial_unique_nodes(self, unique_revisions, common_revisions):
390
        """Steps 1-3 of find_unique_ancestors.
391
392
        Find the maximal set of unique nodes. Some of these might actually
393
        still be common, but we are sure that there are no other unique nodes.
394
395
        :return: (unique_searcher, common_searcher)
396
        """
397
398
        unique_searcher = self._make_breadth_first_searcher(unique_revisions)
399
        # we know that unique_revisions aren't in common_revisions, so skip
400
        # past them.
3377.3.27 by John Arbash Meinel
some simple updates
401
        unique_searcher.next()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
402
        common_searcher = self._make_breadth_first_searcher(common_revisions)
403
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
404
        # As long as we are still finding unique nodes, keep searching
3377.3.27 by John Arbash Meinel
some simple updates
405
        while unique_searcher._next_query:
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
406
            next_unique_nodes = set(unique_searcher.step())
407
            next_common_nodes = set(common_searcher.step())
408
409
            # Check if either searcher encounters new nodes seen by the other
410
            # side.
411
            unique_are_common_nodes = next_unique_nodes.intersection(
412
                common_searcher.seen)
413
            unique_are_common_nodes.update(
414
                next_common_nodes.intersection(unique_searcher.seen))
415
            if unique_are_common_nodes:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
416
                ancestors = unique_searcher.find_seen_ancestors(
417
                                unique_are_common_nodes)
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
418
                # TODO: This is a bit overboard, we only really care about
419
                #       the ancestors of the tips because the rest we
420
                #       already know. This is *correct* but causes us to
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
421
                #       search too much ancestry.
3377.3.29 by John Arbash Meinel
Revert the _find_any_seen change.
422
                ancestors.update(common_searcher.find_seen_ancestors(ancestors))
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
423
                unique_searcher.stop_searching_any(ancestors)
424
                common_searcher.start_searching(ancestors)
425
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
426
        return unique_searcher, common_searcher
427
428
    def _make_unique_searchers(self, unique_nodes, unique_searcher,
429
                               common_searcher):
430
        """Create a searcher for all the unique search tips (step 4).
431
432
        As a side effect, the common_searcher will stop searching any nodes
433
        that are ancestors of the unique searcher tips.
434
435
        :return: (all_unique_searcher, unique_tip_searchers)
436
        """
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
437
        unique_tips = self._remove_simple_descendants(unique_nodes,
438
                        self.get_parent_map(unique_nodes))
439
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
440
        if len(unique_tips) == 1:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
441
            unique_tip_searchers = []
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
442
            ancestor_all_unique = unique_searcher.find_seen_ancestors(unique_tips)
443
        else:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
444
            unique_tip_searchers = []
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
445
            for tip in unique_tips:
446
                revs_to_search = unique_searcher.find_seen_ancestors([tip])
447
                revs_to_search.update(
448
                    common_searcher.find_seen_ancestors(revs_to_search))
449
                searcher = self._make_breadth_first_searcher(revs_to_search)
450
                # We don't care about the starting nodes.
451
                searcher._label = tip
452
                searcher.step()
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
453
                unique_tip_searchers.append(searcher)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
454
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
455
            ancestor_all_unique = None
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
456
            for searcher in unique_tip_searchers:
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
457
                if ancestor_all_unique is None:
458
                    ancestor_all_unique = set(searcher.seen)
459
                else:
460
                    ancestor_all_unique = ancestor_all_unique.intersection(
461
                                                searcher.seen)
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
462
        # Collapse all the common nodes into a single searcher
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
463
        all_unique_searcher = self._make_breadth_first_searcher(
464
                                ancestor_all_unique)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
465
        if ancestor_all_unique:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
466
            # We've seen these nodes in all the searchers, so we'll just go to
467
            # the next
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
468
            all_unique_searcher.step()
469
470
            # Stop any search tips that are already known as ancestors of the
471
            # unique nodes
472
            stopped_common = common_searcher.stop_searching_any(
473
                common_searcher.find_seen_ancestors(ancestor_all_unique))
474
475
            total_stopped = 0
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
476
            for searcher in unique_tip_searchers:
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
477
                total_stopped += len(searcher.stop_searching_any(
478
                    searcher.find_seen_ancestors(ancestor_all_unique)))
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
479
        if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
480
            trace.mutter('For %d unique nodes, created %d + 1 unique searchers'
481
                         ' (%d stopped search tips, %d common ancestors'
482
                         ' (%d stopped common)',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
483
                         len(unique_nodes), len(unique_tip_searchers),
484
                         total_stopped, len(ancestor_all_unique),
485
                         len(stopped_common))
486
        return all_unique_searcher, unique_tip_searchers
487
488
    def _step_unique_and_common_searchers(self, common_searcher,
489
                                          unique_tip_searchers,
490
                                          unique_searcher):
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
491
        """Step all the searchers"""
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
492
        newly_seen_common = set(common_searcher.step())
493
        newly_seen_unique = set()
494
        for searcher in unique_tip_searchers:
495
            next = set(searcher.step())
496
            next.update(unique_searcher.find_seen_ancestors(next))
497
            next.update(common_searcher.find_seen_ancestors(next))
498
            for alt_searcher in unique_tip_searchers:
499
                if alt_searcher is searcher:
500
                    continue
501
                next.update(alt_searcher.find_seen_ancestors(next))
502
            searcher.start_searching(next)
503
            newly_seen_unique.update(next)
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
504
        return newly_seen_common, newly_seen_unique
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
505
506
    def _find_nodes_common_to_all_unique(self, unique_tip_searchers,
507
                                         all_unique_searcher,
508
                                         newly_seen_unique, step_all_unique):
509
        """Find nodes that are common to all unique_tip_searchers.
510
511
        If it is time, step the all_unique_searcher, and add its nodes to the
512
        result.
513
        """
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
514
        common_to_all_unique_nodes = newly_seen_unique.copy()
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
515
        for searcher in unique_tip_searchers:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
516
            common_to_all_unique_nodes.intersection_update(searcher.seen)
517
        common_to_all_unique_nodes.intersection_update(
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
518
                                    all_unique_searcher.seen)
519
        # Step all-unique less frequently than the other searchers.
520
        # In the common case, we don't need to spider out far here, so
521
        # avoid doing extra work.
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
522
        if step_all_unique:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
523
            tstart = time.clock()
524
            nodes = all_unique_searcher.step()
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
525
            common_to_all_unique_nodes.update(nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
526
            if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
527
                tdelta = time.clock() - tstart
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
528
                trace.mutter('all_unique_searcher step() took %.3fs'
529
                             'for %d nodes (%d total), iteration: %s',
530
                             tdelta, len(nodes), len(all_unique_searcher.seen),
531
                             all_unique_searcher._iterations)
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
532
        return common_to_all_unique_nodes
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
533
534
    def _collapse_unique_searchers(self, unique_tip_searchers,
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
535
                                   common_to_all_unique_nodes):
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
536
        """Combine searchers that are searching the same tips.
537
538
        When two searchers are searching the same tips, we can stop one of the
539
        searchers. We also know that the maximal set of common ancestors is the
540
        intersection of the two original searchers.
541
542
        :return: A list of searchers that are searching unique nodes.
543
        """
544
        # Filter out searchers that don't actually search different
545
        # nodes. We already have the ancestry intersection for them
546
        unique_search_tips = {}
547
        for searcher in unique_tip_searchers:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
548
            stopped = searcher.stop_searching_any(common_to_all_unique_nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
549
            will_search_set = frozenset(searcher._next_query)
550
            if not will_search_set:
551
                if 'graph' in debug.debug_flags:
552
                    trace.mutter('Unique searcher %s was stopped.'
553
                                 ' (%s iterations) %d nodes stopped',
554
                                 searcher._label,
555
                                 searcher._iterations,
556
                                 len(stopped))
557
            elif will_search_set not in unique_search_tips:
558
                # This searcher is searching a unique set of nodes, let it
559
                unique_search_tips[will_search_set] = [searcher]
560
            else:
561
                unique_search_tips[will_search_set].append(searcher)
562
        # TODO: it might be possible to collapse searchers faster when they
563
        #       only have *some* search tips in common.
564
        next_unique_searchers = []
565
        for searchers in unique_search_tips.itervalues():
566
            if len(searchers) == 1:
567
                # Searching unique tips, go for it
568
                next_unique_searchers.append(searchers[0])
569
            else:
570
                # These searchers have started searching the same tips, we
571
                # don't need them to cover the same ground. The
572
                # intersection of their ancestry won't change, so create a
573
                # new searcher, combining their histories.
574
                next_searcher = searchers[0]
575
                for searcher in searchers[1:]:
576
                    next_searcher.seen.intersection_update(searcher.seen)
577
                if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
578
                    trace.mutter('Combining %d searchers into a single'
579
                                 ' searcher searching %d nodes with'
580
                                 ' %d ancestry',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
581
                                 len(searchers),
582
                                 len(next_searcher._next_query),
583
                                 len(next_searcher.seen))
584
                next_unique_searchers.append(next_searcher)
585
        return next_unique_searchers
586
587
    def _refine_unique_nodes(self, unique_searcher, all_unique_searcher,
588
                             unique_tip_searchers, common_searcher):
589
        """Steps 5-8 of find_unique_ancestors.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
590
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
591
        This function returns when common_searcher has stopped searching for
592
        more nodes.
593
        """
594
        # We step the ancestor_all_unique searcher only every
595
        # STEP_UNIQUE_SEARCHER_EVERY steps.
596
        step_all_unique_counter = 0
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
597
        # While we still have common nodes to search
598
        while common_searcher._next_query:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
599
            (newly_seen_common,
600
             newly_seen_unique) = self._step_unique_and_common_searchers(
601
                common_searcher, unique_tip_searchers, unique_searcher)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
602
            # These nodes are common ancestors of all unique nodes
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
603
            common_to_all_unique_nodes = self._find_nodes_common_to_all_unique(
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
604
                unique_tip_searchers, all_unique_searcher, newly_seen_unique,
605
                step_all_unique_counter==0)
606
            step_all_unique_counter = ((step_all_unique_counter + 1)
607
                                       % STEP_UNIQUE_SEARCHER_EVERY)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
608
609
            if newly_seen_common:
610
                # If a 'common' node is an ancestor of all unique searchers, we
611
                # can stop searching it.
612
                common_searcher.stop_searching_any(
613
                    all_unique_searcher.seen.intersection(newly_seen_common))
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
614
            if common_to_all_unique_nodes:
615
                common_to_all_unique_nodes.update(
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
616
                    common_searcher.find_seen_ancestors(
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
617
                        common_to_all_unique_nodes))
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
618
                # The all_unique searcher can start searching the common nodes
619
                # but everyone else can stop.
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
620
                # This is the sort of thing where we would like to not have it
621
                # start_searching all of the nodes, but only mark all of them
622
                # as seen, and have it search only the actual tips. Otherwise
623
                # it is another get_parent_map() traversal for it to figure out
624
                # what we already should know.
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
625
                all_unique_searcher.start_searching(common_to_all_unique_nodes)
626
                common_searcher.stop_searching_any(common_to_all_unique_nodes)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
627
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
628
            next_unique_searchers = self._collapse_unique_searchers(
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
629
                unique_tip_searchers, common_to_all_unique_nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
630
            if len(unique_tip_searchers) != len(next_unique_searchers):
631
                if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
632
                    trace.mutter('Collapsed %d unique searchers => %d'
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
633
                                 ' at %s iterations',
634
                                 len(unique_tip_searchers),
635
                                 len(next_unique_searchers),
636
                                 all_unique_searcher._iterations)
637
            unique_tip_searchers = next_unique_searchers
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
638
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
639
    def get_parent_map(self, revisions):
640
        """Get a map of key:parent_list for revisions.
641
642
        This implementation delegates to get_parents, for old parent_providers
643
        that do not supply get_parent_map.
644
        """
645
        result = {}
646
        for rev, parents in self.get_parents(revisions):
647
            if parents is not None:
648
                result[rev] = parents
649
        return result
650
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
651
    def _make_breadth_first_searcher(self, revisions):
652
        return _BreadthFirstSearcher(revisions, self)
653
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
654
    def _find_border_ancestors(self, revisions):
2490.2.12 by Aaron Bentley
Improve documentation
655
        """Find common ancestors with at least one uncommon descendant.
656
657
        Border ancestors are identified using a breadth-first
658
        search starting at the bottom of the graph.  Searches are stopped
659
        whenever a node or one of its descendants is determined to be common.
660
661
        This will scale with the number of uncommon ancestors.
2490.2.25 by Aaron Bentley
Update from review
662
663
        As well as the border ancestors, a set of seen common ancestors and a
664
        list of sets of seen ancestors for each input revision is returned.
665
        This allows calculation of graph difference from the results of this
666
        operation.
2490.2.12 by Aaron Bentley
Improve documentation
667
        """
2490.2.28 by Aaron Bentley
Fix handling of null revision
668
        if None in revisions:
669
            raise errors.InvalidRevisionId(None, self)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
670
        common_ancestors = set()
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
671
        searchers = [self._make_breadth_first_searcher([r])
672
                     for r in revisions]
673
        active_searchers = searchers[:]
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
674
        border_ancestors = set()
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
675
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
676
        while True:
677
            newly_seen = set()
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
678
            for searcher in searchers:
679
                new_ancestors = searcher.step()
680
                if new_ancestors:
681
                    newly_seen.update(new_ancestors)
682
            new_common = set()
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
683
            for revision in newly_seen:
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
684
                if revision in common_ancestors:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
685
                    # Not a border ancestor because it was seen as common
686
                    # already
687
                    new_common.add(revision)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
688
                    continue
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
689
                for searcher in searchers:
690
                    if revision not in searcher.seen:
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
691
                        break
692
                else:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
693
                    # This is a border because it is a first common that we see
694
                    # after walking for a while.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
695
                    border_ancestors.add(revision)
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
696
                    new_common.add(revision)
697
            if new_common:
698
                for searcher in searchers:
699
                    new_common.update(searcher.find_seen_ancestors(new_common))
700
                for searcher in searchers:
701
                    searcher.start_searching(new_common)
702
                common_ancestors.update(new_common)
703
704
            # Figure out what the searchers will be searching next, and if
705
            # there is only 1 set being searched, then we are done searching,
706
            # since all searchers would have to be searching the same data,
707
            # thus it *must* be in common.
708
            unique_search_sets = set()
709
            for searcher in searchers:
710
                will_search_set = frozenset(searcher._next_query)
711
                if will_search_set not in unique_search_sets:
712
                    # This searcher is searching a unique set of nodes, let it
713
                    unique_search_sets.add(will_search_set)
714
715
            if len(unique_search_sets) == 1:
716
                nodes = unique_search_sets.pop()
717
                uncommon_nodes = nodes.difference(common_ancestors)
3376.2.14 by Martin Pool
Remove recently-introduced assert statements
718
                if uncommon_nodes:
719
                    raise AssertionError("Somehow we ended up converging"
720
                                         " without actually marking them as"
721
                                         " in common."
722
                                         "\nStart_nodes: %s"
723
                                         "\nuncommon_nodes: %s"
724
                                         % (revisions, uncommon_nodes))
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
725
                break
726
        return border_ancestors, common_ancestors, searchers
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
727
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
728
    def heads(self, keys):
729
        """Return the heads from amongst keys.
730
731
        This is done by searching the ancestries of each key.  Any key that is
732
        reachable from another key is not returned; all the others are.
733
734
        This operation scales with the relative depth between any two keys. If
735
        any two keys are completely disconnected all ancestry of both sides
736
        will be retrieved.
737
738
        :param keys: An iterable of keys.
2776.1.4 by Robert Collins
Trivial review feedback changes.
739
        :return: A set of the heads. Note that as a set there is no ordering
740
            information. Callers will need to filter their input to create
741
            order if they need it.
2490.2.12 by Aaron Bentley
Improve documentation
742
        """
2776.1.4 by Robert Collins
Trivial review feedback changes.
743
        candidate_heads = set(keys)
3052.5.5 by John Arbash Meinel
Special case Graph.heads() for NULL_REVISION rather than is_ancestor.
744
        if revision.NULL_REVISION in candidate_heads:
745
            # NULL_REVISION is only a head if it is the only entry
746
            candidate_heads.remove(revision.NULL_REVISION)
747
            if not candidate_heads:
748
                return set([revision.NULL_REVISION])
2850.2.1 by Robert Collins
(robertc) Special case the zero-or-no-heads case for Graph.heads(). (Robert Collins)
749
        if len(candidate_heads) < 2:
750
            return candidate_heads
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
751
        searchers = dict((c, self._make_breadth_first_searcher([c]))
2776.1.4 by Robert Collins
Trivial review feedback changes.
752
                          for c in candidate_heads)
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
753
        active_searchers = dict(searchers)
754
        # skip over the actual candidate for each searcher
755
        for searcher in active_searchers.itervalues():
1551.15.81 by Aaron Bentley
Remove testing code
756
            searcher.next()
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
757
        # The common walker finds nodes that are common to two or more of the
758
        # input keys, so that we don't access all history when a currently
759
        # uncommon search point actually meets up with something behind a
760
        # common search point. Common search points do not keep searches
761
        # active; they just allow us to make searches inactive without
762
        # accessing all history.
763
        common_walker = self._make_breadth_first_searcher([])
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
764
        while len(active_searchers) > 0:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
765
            ancestors = set()
766
            # advance searches
767
            try:
768
                common_walker.next()
769
            except StopIteration:
2921.3.4 by Robert Collins
Review feedback.
770
                # No common points being searched at this time.
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
771
                pass
1551.15.78 by Aaron Bentley
Fix KeyError in filter_candidate_lca
772
            for candidate in active_searchers.keys():
773
                try:
774
                    searcher = active_searchers[candidate]
775
                except KeyError:
776
                    # rare case: we deleted candidate in a previous iteration
777
                    # through this for loop, because it was determined to be
778
                    # a descendant of another candidate.
779
                    continue
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
780
                try:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
781
                    ancestors.update(searcher.next())
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
782
                except StopIteration:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
783
                    del active_searchers[candidate]
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
784
                    continue
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
785
            # process found nodes
786
            new_common = set()
787
            for ancestor in ancestors:
788
                if ancestor in candidate_heads:
789
                    candidate_heads.remove(ancestor)
790
                    del searchers[ancestor]
791
                    if ancestor in active_searchers:
792
                        del active_searchers[ancestor]
793
                # it may meet up with a known common node
2921.3.4 by Robert Collins
Review feedback.
794
                if ancestor in common_walker.seen:
795
                    # some searcher has encountered our known common nodes:
796
                    # just stop it
797
                    ancestor_set = set([ancestor])
798
                    for searcher in searchers.itervalues():
799
                        searcher.stop_searching_any(ancestor_set)
800
                else:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
801
                    # or it may have been just reached by all the searchers:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
802
                    for searcher in searchers.itervalues():
803
                        if ancestor not in searcher.seen:
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
804
                            break
805
                    else:
2921.3.4 by Robert Collins
Review feedback.
806
                        # The final active searcher has just reached this node,
807
                        # making it be known as a descendant of all candidates,
808
                        # so we can stop searching it, and any seen ancestors
809
                        new_common.add(ancestor)
810
                        for searcher in searchers.itervalues():
811
                            seen_ancestors =\
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
812
                                searcher.find_seen_ancestors([ancestor])
2921.3.4 by Robert Collins
Review feedback.
813
                            searcher.stop_searching_any(seen_ancestors)
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
814
            common_walker.start_searching(new_common)
2776.1.4 by Robert Collins
Trivial review feedback changes.
815
        return candidate_heads
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
816
3514.2.8 by John Arbash Meinel
The insertion ordering into the weave has an impact on conflicts.
817
    def find_merge_order(self, tip_revision_id, lca_revision_ids):
818
        """Find the order that each revision was merged into tip.
819
820
        This basically just walks backwards with a stack, and walks left-first
821
        until it finds a node to stop.
822
        """
823
        if len(lca_revision_ids) == 1:
824
            return list(lca_revision_ids)
825
        looking_for = set(lca_revision_ids)
826
        # TODO: Is there a way we could do this "faster" by batching up the
827
        # get_parent_map requests?
828
        # TODO: Should we also be culling the ancestry search right away? We
829
        # could add looking_for to the "stop" list, and walk their
830
        # ancestry in batched mode. The flip side is it might mean we walk a
831
        # lot of "stop" nodes, rather than only the minimum.
832
        # Then again, without it we may trace back into ancestry we could have
833
        # stopped early.
834
        stack = [tip_revision_id]
835
        found = []
836
        stop = set()
837
        while stack and looking_for:
838
            next = stack.pop()
839
            stop.add(next)
840
            if next in looking_for:
841
                found.append(next)
842
                looking_for.remove(next)
843
                if len(looking_for) == 1:
844
                    found.append(looking_for.pop())
845
                    break
846
                continue
847
            parent_ids = self.get_parent_map([next]).get(next, None)
848
            if not parent_ids: # Ghost, nothing to search here
849
                continue
850
            for parent_id in reversed(parent_ids):
851
                # TODO: (performance) We see the parent at this point, but we
852
                #       wait to mark it until later to make sure we get left
853
                #       parents before right parents. However, instead of
854
                #       waiting until we have traversed enough parents, we
855
                #       could instead note that we've found it, and once all
856
                #       parents are in the stack, just reverse iterate the
857
                #       stack for them.
858
                if parent_id not in stop:
859
                    # this will need to be searched
860
                    stack.append(parent_id)
861
                stop.add(parent_id)
862
        return found
863
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
864
    def find_unique_lca(self, left_revision, right_revision,
865
                        count_steps=False):
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
866
        """Find a unique LCA.
867
868
        Find lowest common ancestors.  If there is no unique  common
869
        ancestor, find the lowest common ancestors of those ancestors.
870
871
        Iteration stops when a unique lowest common ancestor is found.
872
        The graph origin is necessarily a unique lowest common ancestor.
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
873
874
        Note that None is not an acceptable substitute for NULL_REVISION.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
875
        in the input for this method.
1551.19.12 by Aaron Bentley
Add documentation for the count_steps parameter of Graph.find_unique_lca
876
877
        :param count_steps: If True, the return value will be a tuple of
878
            (unique_lca, steps) where steps is the number of times that
879
            find_lca was run.  If False, only unique_lca is returned.
2490.2.3 by Aaron Bentley
Implement new merge base picker
880
        """
881
        revisions = [left_revision, right_revision]
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
882
        steps = 0
2490.2.3 by Aaron Bentley
Implement new merge base picker
883
        while True:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
884
            steps += 1
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
885
            lca = self.find_lca(*revisions)
886
            if len(lca) == 1:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
887
                result = lca.pop()
888
                if count_steps:
889
                    return result, steps
890
                else:
891
                    return result
2520.4.104 by Aaron Bentley
Avoid infinite loop when there is no unique lca
892
            if len(lca) == 0:
893
                raise errors.NoCommonAncestor(left_revision, right_revision)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
894
            revisions = lca
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
895
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
896
    def iter_ancestry(self, revision_ids):
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
897
        """Iterate the ancestry of this revision.
898
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
899
        :param revision_ids: Nodes to start the search
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
900
        :return: Yield tuples mapping a revision_id to its parents for the
901
            ancestry of revision_id.
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
902
            Ghosts will be returned with None as their parents, and nodes
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
903
            with no parents will have NULL_REVISION as their only parent. (As
904
            defined by get_parent_map.)
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
905
            There will also be a node for (NULL_REVISION, ())
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
906
        """
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
907
        pending = set(revision_ids)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
908
        processed = set()
909
        while pending:
910
            processed.update(pending)
911
            next_map = self.get_parent_map(pending)
912
            next_pending = set()
913
            for item in next_map.iteritems():
914
                yield item
915
                next_pending.update(p for p in item[1] if p not in processed)
916
            ghosts = pending.difference(next_map)
917
            for ghost in ghosts:
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
918
                yield (ghost, None)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
919
            pending = next_pending
920
2490.2.31 by Aaron Bentley
Fix iter_topo_order to permit un-included parents
921
    def iter_topo_order(self, revisions):
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
922
        """Iterate through the input revisions in topological order.
923
924
        This sorting only ensures that parents come before their children.
925
        An ancestor may sort after a descendant if the relationship is not
926
        visible in the supplied list of revisions.
927
        """
4593.5.30 by John Arbash Meinel
Move the topo_sort implementation into KnownGraph, rather than calling back to it.
928
        from bzrlib import tsort
3099.3.3 by John Arbash Meinel
Deprecate get_parents() in favor of get_parent_map()
929
        sorter = tsort.TopoSorter(self.get_parent_map(revisions))
2490.2.34 by Aaron Bentley
Update NEWS and change implementation to return an iterator
930
        return sorter.iter_topo_order()
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
931
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
932
    def is_ancestor(self, candidate_ancestor, candidate_descendant):
2653.2.5 by Aaron Bentley
Update to clarify algorithm
933
        """Determine whether a revision is an ancestor of another.
934
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
935
        We answer this using heads() as heads() has the logic to perform the
3078.2.6 by Ian Clatworthy
fix efficiency of local commit detection as recommended by jameinel's review
936
        smallest number of parent lookups to determine the ancestral
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
937
        relationship between N revisions.
2653.2.5 by Aaron Bentley
Update to clarify algorithm
938
        """
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
939
        return set([candidate_descendant]) == self.heads(
940
            [candidate_ancestor, candidate_descendant])
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
941
3921.3.5 by Marius Kruger
extract graph.is_between from builtins.cmd_tags.run, and test it
942
    def is_between(self, revid, lower_bound_revid, upper_bound_revid):
943
        """Determine whether a revision is between two others.
944
945
        returns true if and only if:
946
        lower_bound_revid <= revid <= upper_bound_revid
947
        """
948
        return ((upper_bound_revid is None or
949
                    self.is_ancestor(revid, upper_bound_revid)) and
950
               (lower_bound_revid is None or
951
                    self.is_ancestor(lower_bound_revid, revid)))
952
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
953
    def _search_for_extra_common(self, common, searchers):
954
        """Make sure that unique nodes are genuinely unique.
955
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
956
        After _find_border_ancestors, all nodes marked "common" are indeed
957
        common. Some of the nodes considered unique are not, due to history
958
        shortcuts stopping the searches early.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
959
960
        We know that we have searched enough when all common search tips are
961
        descended from all unique (uncommon) nodes because we know that a node
962
        cannot be an ancestor of its own ancestor.
963
964
        :param common: A set of common nodes
965
        :param searchers: The searchers returned from _find_border_ancestors
966
        :return: None
967
        """
968
        # Basic algorithm...
969
        #   A) The passed in searchers should all be on the same tips, thus
970
        #      they should be considered the "common" searchers.
971
        #   B) We find the difference between the searchers, these are the
972
        #      "unique" nodes for each side.
973
        #   C) We do a quick culling so that we only start searching from the
974
        #      more interesting unique nodes. (A unique ancestor is more
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
975
        #      interesting than any of its children.)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
976
        #   D) We start searching for ancestors common to all unique nodes.
977
        #   E) We have the common searchers stop searching any ancestors of
978
        #      nodes found by (D)
979
        #   F) When there are no more common search tips, we stop
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
980
981
        # TODO: We need a way to remove unique_searchers when they overlap with
982
        #       other unique searchers.
3376.2.14 by Martin Pool
Remove recently-introduced assert statements
983
        if len(searchers) != 2:
984
            raise NotImplementedError(
985
                "Algorithm not yet implemented for > 2 searchers")
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
986
        common_searchers = searchers
987
        left_searcher = searchers[0]
988
        right_searcher = searchers[1]
3377.3.15 by John Arbash Meinel
minor update
989
        unique = left_searcher.seen.symmetric_difference(right_searcher.seen)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
990
        if not unique: # No unique nodes, nothing to do
991
            return
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
992
        total_unique = len(unique)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
993
        unique = self._remove_simple_descendants(unique,
994
                    self.get_parent_map(unique))
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
995
        simple_unique = len(unique)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
996
997
        unique_searchers = []
998
        for revision_id in unique:
3377.3.15 by John Arbash Meinel
minor update
999
            if revision_id in left_searcher.seen:
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
1000
                parent_searcher = left_searcher
1001
            else:
1002
                parent_searcher = right_searcher
1003
            revs_to_search = parent_searcher.find_seen_ancestors([revision_id])
1004
            if not revs_to_search: # XXX: This shouldn't be possible
1005
                revs_to_search = [revision_id]
3377.3.15 by John Arbash Meinel
minor update
1006
            searcher = self._make_breadth_first_searcher(revs_to_search)
1007
            # We don't care about the starting nodes.
1008
            searcher.step()
1009
            unique_searchers.append(searcher)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
1010
3377.3.16 by John Arbash Meinel
small cleanups
1011
        # possible todo: aggregate the common searchers into a single common
1012
        #   searcher, just make sure that we include the nodes into the .seen
1013
        #   properties of the original searchers
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1014
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1015
        ancestor_all_unique = None
1016
        for searcher in unique_searchers:
1017
            if ancestor_all_unique is None:
1018
                ancestor_all_unique = set(searcher.seen)
1019
            else:
1020
                ancestor_all_unique = ancestor_all_unique.intersection(
1021
                                            searcher.seen)
1022
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
1023
        trace.mutter('Started %s unique searchers for %s unique revisions',
1024
                     simple_unique, total_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
1025
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1026
        while True: # If we have no more nodes we have nothing to do
1027
            newly_seen_common = set()
1028
            for searcher in common_searchers:
1029
                newly_seen_common.update(searcher.step())
1030
            newly_seen_unique = set()
1031
            for searcher in unique_searchers:
1032
                newly_seen_unique.update(searcher.step())
1033
            new_common_unique = set()
1034
            for revision in newly_seen_unique:
1035
                for searcher in unique_searchers:
1036
                    if revision not in searcher.seen:
1037
                        break
1038
                else:
1039
                    # This is a border because it is a first common that we see
1040
                    # after walking for a while.
1041
                    new_common_unique.add(revision)
1042
            if newly_seen_common:
1043
                # These are nodes descended from one of the 'common' searchers.
1044
                # Make sure all searchers are on the same page
1045
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
1046
                    newly_seen_common.update(
1047
                        searcher.find_seen_ancestors(newly_seen_common))
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
1048
                # We start searching the whole ancestry. It is a bit wasteful,
1049
                # though. We really just want to mark all of these nodes as
1050
                # 'seen' and then start just the tips. However, it requires a
1051
                # get_parent_map() call to figure out the tips anyway, and all
1052
                # redundant requests should be fairly fast.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1053
                for searcher in common_searchers:
1054
                    searcher.start_searching(newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
1055
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1056
                # If a 'common' node is an ancestor of all unique searchers, we
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
1057
                # can stop searching it.
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1058
                stop_searching_common = ancestor_all_unique.intersection(
1059
                                            newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
1060
                if stop_searching_common:
1061
                    for searcher in common_searchers:
1062
                        searcher.stop_searching_any(stop_searching_common)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1063
            if new_common_unique:
3377.3.20 by John Arbash Meinel
comment cleanups.
1064
                # We found some ancestors that are common
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1065
                for searcher in unique_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
1066
                    new_common_unique.update(
1067
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1068
                # Since these are common, we can grab another set of ancestors
1069
                # that we have seen
1070
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
1071
                    new_common_unique.update(
1072
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1073
1074
                # We can tell all of the unique searchers to start at these
1075
                # nodes, and tell all of the common searchers to *stop*
1076
                # searching these nodes
1077
                for searcher in unique_searchers:
1078
                    searcher.start_searching(new_common_unique)
1079
                for searcher in common_searchers:
1080
                    searcher.stop_searching_any(new_common_unique)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1081
                ancestor_all_unique.update(new_common_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
1082
3377.3.20 by John Arbash Meinel
comment cleanups.
1083
                # Filter out searchers that don't actually search different
1084
                # nodes. We already have the ancestry intersection for them
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
1085
                next_unique_searchers = []
1086
                unique_search_sets = set()
1087
                for searcher in unique_searchers:
1088
                    will_search_set = frozenset(searcher._next_query)
1089
                    if will_search_set not in unique_search_sets:
1090
                        # This searcher is searching a unique set of nodes, let it
1091
                        unique_search_sets.add(will_search_set)
1092
                        next_unique_searchers.append(searcher)
1093
                unique_searchers = next_unique_searchers
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
1094
            for searcher in common_searchers:
1095
                if searcher._next_query:
1096
                    break
1097
            else:
1098
                # All common searcher have stopped searching
3377.3.16 by John Arbash Meinel
small cleanups
1099
                return
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1100
1101
    def _remove_simple_descendants(self, revisions, parent_map):
1102
        """remove revisions which are children of other ones in the set
1103
1104
        This doesn't do any graph searching, it just checks the immediate
1105
        parent_map to find if there are any children which can be removed.
1106
1107
        :param revisions: A set of revision_ids
1108
        :return: A set of revision_ids with the children removed
1109
        """
1110
        simple_ancestors = revisions.copy()
1111
        # TODO: jam 20071214 we *could* restrict it to searching only the
1112
        #       parent_map of revisions already present in 'revisions', but
1113
        #       considering the general use case, I think this is actually
1114
        #       better.
1115
1116
        # This is the same as the following loop. I don't know that it is any
1117
        # faster.
1118
        ## simple_ancestors.difference_update(r for r, p_ids in parent_map.iteritems()
1119
        ##     if p_ids is not None and revisions.intersection(p_ids))
1120
        ## return simple_ancestors
1121
1122
        # Yet Another Way, invert the parent map (which can be cached)
1123
        ## descendants = {}
1124
        ## for revision_id, parent_ids in parent_map.iteritems():
1125
        ##   for p_id in parent_ids:
1126
        ##       descendants.setdefault(p_id, []).append(revision_id)
1127
        ## for revision in revisions.intersection(descendants):
1128
        ##   simple_ancestors.difference_update(descendants[revision])
1129
        ## return simple_ancestors
1130
        for revision, parent_ids in parent_map.iteritems():
1131
            if parent_ids is None:
1132
                continue
1133
            for parent_id in parent_ids:
1134
                if parent_id in revisions:
1135
                    # This node has a parent present in the set, so we can
1136
                    # remove it
1137
                    simple_ancestors.discard(revision)
1138
                    break
1139
        return simple_ancestors
1140
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1141
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1142
class HeadsCache(object):
1143
    """A cache of results for graph heads calls."""
1144
1145
    def __init__(self, graph):
1146
        self.graph = graph
1147
        self._heads = {}
1148
1149
    def heads(self, keys):
1150
        """Return the heads of keys.
1151
2911.4.3 by Robert Collins
Make the contract of HeadsCache.heads() more clear.
1152
        This matches the API of Graph.heads(), specifically the return value is
1153
        a set which can be mutated, and ordering of the input is not preserved
1154
        in the output.
1155
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1156
        :see also: Graph.heads.
1157
        :param keys: The keys to calculate heads for.
1158
        :return: A set containing the heads, which may be mutated without
1159
            affecting future lookups.
1160
        """
2911.4.2 by Robert Collins
Make HeadsCache actually work.
1161
        keys = frozenset(keys)
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1162
        try:
1163
            return set(self._heads[keys])
1164
        except KeyError:
1165
            heads = self.graph.heads(keys)
1166
            self._heads[keys] = heads
1167
            return set(heads)
1168
1169
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1170
class FrozenHeadsCache(object):
1171
    """Cache heads() calls, assuming the caller won't modify them."""
1172
1173
    def __init__(self, graph):
1174
        self.graph = graph
1175
        self._heads = {}
1176
1177
    def heads(self, keys):
1178
        """Return the heads of keys.
1179
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
1180
        Similar to Graph.heads(). The main difference is that the return value
1181
        is a frozen set which cannot be mutated.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1182
1183
        :see also: Graph.heads.
1184
        :param keys: The keys to calculate heads for.
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
1185
        :return: A frozenset containing the heads.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1186
        """
1187
        keys = frozenset(keys)
1188
        try:
1189
            return self._heads[keys]
1190
        except KeyError:
1191
            heads = frozenset(self.graph.heads(keys))
1192
            self._heads[keys] = heads
1193
            return heads
1194
1195
    def cache(self, keys, heads):
1196
        """Store a known value."""
1197
        self._heads[frozenset(keys)] = frozenset(heads)
1198
1199
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
1200
class _BreadthFirstSearcher(object):
2921.3.4 by Robert Collins
Review feedback.
1201
    """Parallel search breadth-first the ancestry of revisions.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1202
1203
    This class implements the iterator protocol, but additionally
1204
    1. provides a set of seen ancestors, and
1205
    2. allows some ancestries to be unsearched, via stop_searching_any
1206
    """
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1207
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
1208
    def __init__(self, revisions, parents_provider):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1209
        self._iterations = 0
1210
        self._next_query = set(revisions)
1211
        self.seen = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1212
        self._started_keys = set(self._next_query)
1213
        self._stopped_keys = set()
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
1214
        self._parents_provider = parents_provider
3177.3.3 by Robert Collins
Review feedback.
1215
        self._returning = 'next_with_ghosts'
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1216
        self._current_present = set()
1217
        self._current_ghosts = set()
1218
        self._current_parents = {}
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1219
1220
    def __repr__(self):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1221
        if self._iterations:
1222
            prefix = "searching"
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
1223
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1224
            prefix = "starting"
1225
        search = '%s=%r' % (prefix, list(self._next_query))
1226
        return ('_BreadthFirstSearcher(iterations=%d, %s,'
1227
                ' seen=%r)' % (self._iterations, search, list(self.seen)))
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1228
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1229
    def get_result(self):
1230
        """Get a SearchResult for the current state of this searcher.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1231
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1232
        :return: A SearchResult for this search so far. The SearchResult is
1233
            static - the search can be advanced and the search result will not
1234
            be invalidated or altered.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1235
        """
1236
        if self._returning == 'next':
1237
            # We have to know the current nodes children to be able to list the
1238
            # exclude keys for them. However, while we could have a second
1239
            # look-ahead result buffer and shuffle things around, this method
1240
            # is typically only called once per search - when memoising the
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1241
            # results of the search.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1242
            found, ghosts, next, parents = self._do_query(self._next_query)
1243
            # pretend we didn't query: perhaps we should tweak _do_query to be
1244
            # entirely stateless?
1245
            self.seen.difference_update(next)
3184.1.3 by Robert Collins
Automatically exclude ghosts.
1246
            next_query = next.union(ghosts)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1247
        else:
1248
            next_query = self._next_query
3184.1.5 by Robert Collins
Record the number of found revisions for cross checking.
1249
        excludes = self._stopped_keys.union(next_query)
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1250
        included_keys = self.seen.difference(excludes)
1251
        return SearchResult(self._started_keys, excludes, len(included_keys),
1252
            included_keys)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1253
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1254
    def step(self):
1255
        try:
1256
            return self.next()
1257
        except StopIteration:
1258
            return ()
1259
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1260
    def next(self):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1261
        """Return the next ancestors of this revision.
1262
2490.2.12 by Aaron Bentley
Improve documentation
1263
        Ancestors are returned in the order they are seen in a breadth-first
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1264
        traversal.  No ancestor will be returned more than once. Ancestors are
1265
        returned before their parentage is queried, so ghosts and missing
1266
        revisions (including the start revisions) are included in the result.
1267
        This can save a round trip in LCA style calculation by allowing
1268
        convergence to be detected without reading the data for the revision
1269
        the convergence occurs on.
1270
1271
        :return: A set of revision_ids.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1272
        """
3177.3.3 by Robert Collins
Review feedback.
1273
        if self._returning != 'next':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1274
            # switch to returning the query, not the results.
3177.3.3 by Robert Collins
Review feedback.
1275
            self._returning = 'next'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1276
            self._iterations += 1
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1277
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1278
            self._advance()
1279
        if len(self._next_query) == 0:
1280
            raise StopIteration()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1281
        # We have seen what we're querying at this point as we are returning
1282
        # the query, not the results.
1283
        self.seen.update(self._next_query)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1284
        return self._next_query
1285
1286
    def next_with_ghosts(self):
1287
        """Return the next found ancestors, with ghosts split out.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1288
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1289
        Ancestors are returned in the order they are seen in a breadth-first
1290
        traversal.  No ancestor will be returned more than once. Ancestors are
3177.3.3 by Robert Collins
Review feedback.
1291
        returned only after asking for their parents, which allows us to detect
1292
        which revisions are ghosts and which are not.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1293
1294
        :return: A tuple with (present ancestors, ghost ancestors) sets.
1295
        """
3177.3.3 by Robert Collins
Review feedback.
1296
        if self._returning != 'next_with_ghosts':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1297
            # switch to returning the results, not the current query.
3177.3.3 by Robert Collins
Review feedback.
1298
            self._returning = 'next_with_ghosts'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1299
            self._advance()
1300
        if len(self._next_query) == 0:
1301
            raise StopIteration()
1302
        self._advance()
1303
        return self._current_present, self._current_ghosts
1304
1305
    def _advance(self):
1306
        """Advance the search.
1307
1308
        Updates self.seen, self._next_query, self._current_present,
3177.3.3 by Robert Collins
Review feedback.
1309
        self._current_ghosts, self._current_parents and self._iterations.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1310
        """
1311
        self._iterations += 1
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1312
        found, ghosts, next, parents = self._do_query(self._next_query)
1313
        self._current_present = found
1314
        self._current_ghosts = ghosts
1315
        self._next_query = next
1316
        self._current_parents = parents
3184.1.3 by Robert Collins
Automatically exclude ghosts.
1317
        # ghosts are implicit stop points, otherwise the search cannot be
1318
        # repeated when ghosts are filled.
1319
        self._stopped_keys.update(ghosts)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1320
1321
    def _do_query(self, revisions):
1322
        """Query for revisions.
1323
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1324
        Adds revisions to the seen set.
1325
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1326
        :param revisions: Revisions to query.
1327
        :return: A tuple: (set(found_revisions), set(ghost_revisions),
1328
           set(parents_of_found_revisions), dict(found_revisions:parents)).
1329
        """
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1330
        found_revisions = set()
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1331
        parents_of_found = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1332
        # revisions may contain nodes that point to other nodes in revisions:
1333
        # we want to filter them out.
1334
        self.seen.update(revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1335
        parent_map = self._parents_provider.get_parent_map(revisions)
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1336
        found_revisions.update(parent_map)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1337
        for rev_id, parents in parent_map.iteritems():
3517.4.2 by Martin Pool
Make simple-annotation and graph code more tolerant of knits with no graph
1338
            if parents is None:
1339
                continue
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1340
            new_found_parents = [p for p in parents if p not in self.seen]
1341
            if new_found_parents:
1342
                # Calling set.update() with an empty generator is actually
1343
                # rather expensive.
1344
                parents_of_found.update(new_found_parents)
1345
        ghost_revisions = revisions - found_revisions
1346
        return found_revisions, ghost_revisions, parents_of_found, parent_map
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1347
2490.2.8 by Aaron Bentley
fix iteration stuff
1348
    def __iter__(self):
1349
        return self
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1350
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1351
    def find_seen_ancestors(self, revisions):
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
1352
        """Find ancestors of these revisions that have already been seen.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1353
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
1354
        This function generally makes the assumption that querying for the
1355
        parents of a node that has already been queried is reasonably cheap.
1356
        (eg, not a round trip to a remote host).
1357
        """
1358
        # TODO: Often we might ask one searcher for its seen ancestors, and
1359
        #       then ask another searcher the same question. This can result in
1360
        #       searching the same revisions repeatedly if the two searchers
1361
        #       have a lot of overlap.
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1362
        all_seen = self.seen
1363
        pending = set(revisions).intersection(all_seen)
1364
        seen_ancestors = set(pending)
1365
1366
        if self._returning == 'next':
1367
            # self.seen contains what nodes have been returned, not what nodes
1368
            # have been queried. We don't want to probe for nodes that haven't
1369
            # been searched yet.
1370
            not_searched_yet = self._next_query
1371
        else:
1372
            not_searched_yet = ()
3377.3.11 by John Arbash Meinel
Committing a debug thunk that was very helpful
1373
        pending.difference_update(not_searched_yet)
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1374
        get_parent_map = self._parents_provider.get_parent_map
3377.3.12 by John Arbash Meinel
Remove the helpful but ugly thunk
1375
        while pending:
1376
            parent_map = get_parent_map(pending)
1377
            all_parents = []
1378
            # We don't care if it is a ghost, since it can't be seen if it is
1379
            # a ghost
1380
            for parent_ids in parent_map.itervalues():
1381
                all_parents.extend(parent_ids)
1382
            next_pending = all_seen.intersection(all_parents).difference(seen_ancestors)
1383
            seen_ancestors.update(next_pending)
1384
            next_pending.difference_update(not_searched_yet)
1385
            pending = next_pending
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1386
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1387
        return seen_ancestors
1388
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1389
    def stop_searching_any(self, revisions):
1390
        """
1391
        Remove any of the specified revisions from the search list.
1392
1393
        None of the specified revisions are required to be present in the
3808.1.4 by John Arbash Meinel
make _walk_to_common responsible for stopping ancestors
1394
        search list.
3808.1.1 by Andrew Bennetts
Possible fix for bug in new _walk_to_common_revisions.
1395
3808.1.4 by John Arbash Meinel
make _walk_to_common responsible for stopping ancestors
1396
        It is okay to call stop_searching_any() for revisions which were seen
1397
        in previous iterations. It is the callers responsibility to call
1398
        find_seen_ancestors() to make sure that current search tips that are
1399
        ancestors of those revisions are also stopped.  All explicitly stopped
1400
        revisions will be excluded from the search result's get_keys(), though.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1401
        """
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
1402
        # TODO: does this help performance?
1403
        # if not revisions:
1404
        #     return set()
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1405
        revisions = frozenset(revisions)
3177.3.3 by Robert Collins
Review feedback.
1406
        if self._returning == 'next':
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1407
            stopped = self._next_query.intersection(revisions)
1408
            self._next_query = self._next_query.difference(revisions)
1409
        else:
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1410
            stopped_present = self._current_present.intersection(revisions)
1411
            stopped = stopped_present.union(
1412
                self._current_ghosts.intersection(revisions))
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1413
            self._current_present.difference_update(stopped)
1414
            self._current_ghosts.difference_update(stopped)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1415
            # stopping 'x' should stop returning parents of 'x', but
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1416
            # not if 'y' always references those same parents
1417
            stop_rev_references = {}
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1418
            for rev in stopped_present:
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1419
                for parent_id in self._current_parents[rev]:
1420
                    if parent_id not in stop_rev_references:
1421
                        stop_rev_references[parent_id] = 0
1422
                    stop_rev_references[parent_id] += 1
1423
            # if only the stopped revisions reference it, the ref count will be
1424
            # 0 after this loop
3177.3.3 by Robert Collins
Review feedback.
1425
            for parents in self._current_parents.itervalues():
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1426
                for parent_id in parents:
1427
                    try:
1428
                        stop_rev_references[parent_id] -= 1
1429
                    except KeyError:
1430
                        pass
1431
            stop_parents = set()
1432
            for rev_id, refs in stop_rev_references.iteritems():
1433
                if refs == 0:
1434
                    stop_parents.add(rev_id)
1435
            self._next_query.difference_update(stop_parents)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1436
        self._stopped_keys.update(stopped)
4053.2.2 by Andrew Bennetts
Better fix, with test.
1437
        self._stopped_keys.update(revisions)
2490.2.25 by Aaron Bentley
Update from review
1438
        return stopped
2490.2.17 by Aaron Bentley
Add start_searching, tweak stop_searching_any
1439
1440
    def start_searching(self, revisions):
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1441
        """Add revisions to the search.
1442
1443
        The parents of revisions will be returned from the next call to next()
1444
        or next_with_ghosts(). If next_with_ghosts was the most recently used
1445
        next* call then the return value is the result of looking up the
1446
        ghost/not ghost status of revisions. (A tuple (present, ghosted)).
1447
        """
1448
        revisions = frozenset(revisions)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1449
        self._started_keys.update(revisions)
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1450
        new_revisions = revisions.difference(self.seen)
3177.3.3 by Robert Collins
Review feedback.
1451
        if self._returning == 'next':
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1452
            self._next_query.update(new_revisions)
3377.3.30 by John Arbash Meinel
Can we avoid the extra _do_query in start_searching?
1453
            self.seen.update(new_revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1454
        else:
1455
            # perform a query on revisions
3377.3.30 by John Arbash Meinel
Can we avoid the extra _do_query in start_searching?
1456
            revs, ghosts, query, parents = self._do_query(revisions)
1457
            self._stopped_keys.update(ghosts)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1458
            self._current_present.update(revs)
1459
            self._current_ghosts.update(ghosts)
1460
            self._next_query.update(query)
1461
            self._current_parents.update(parents)
1462
            return revs, ghosts
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1463
1464
1465
class SearchResult(object):
1466
    """The result of a breadth first search.
1467
1468
    A SearchResult provides the ability to reconstruct the search or access a
1469
    set of the keys the search found.
1470
    """
1471
1472
    def __init__(self, start_keys, exclude_keys, key_count, keys):
1473
        """Create a SearchResult.
1474
1475
        :param start_keys: The keys the search started at.
1476
        :param exclude_keys: The keys the search excludes.
1477
        :param key_count: The total number of keys (from start to but not
1478
            including exclude).
1479
        :param keys: The keys the search found. Note that in future we may get
1480
            a SearchResult from a smart server, in which case the keys list is
1481
            not necessarily immediately available.
1482
        """
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1483
        self._recipe = ('search', start_keys, exclude_keys, key_count)
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1484
        self._keys = frozenset(keys)
1485
1486
    def get_recipe(self):
1487
        """Return a recipe that can be used to replay this search.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1488
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1489
        The recipe allows reconstruction of the same results at a later date
1490
        without knowing all the found keys. The essential elements are a list
4031.3.1 by Frank Aspell
Fixing various typos
1491
        of keys to start and to stop at. In order to give reproducible
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1492
        results when ghosts are encountered by a search they are automatically
1493
        added to the exclude list (or else ghost filling may alter the
1494
        results).
1495
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1496
        :return: A tuple ('search', start_keys_set, exclude_keys_set,
1497
            revision_count). To recreate the results of this search, create a
1498
            breadth first searcher on the same graph starting at start_keys.
1499
            Then call next() (or next_with_ghosts()) repeatedly, and on every
1500
            result, call stop_searching_any on any keys from the exclude_keys
1501
            set. The revision_count value acts as a trivial cross-check - the
1502
            found revisions of the new search should have as many elements as
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1503
            revision_count. If it does not, then additional revisions have been
1504
            ghosted since the search was executed the first time and the second
1505
            time.
1506
        """
1507
        return self._recipe
1508
1509
    def get_keys(self):
1510
        """Return the keys found in this search.
1511
1512
        :return: A set of keys.
1513
        """
1514
        return self._keys
1515
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1516
    def is_empty(self):
4204.2.2 by Matt Nordhoff
Fix docstrings on graph.py's is_empty methods that said they returned true when they were *not* empty.
1517
        """Return false if the search lists 1 or more revisions."""
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1518
        return self._recipe[3] == 0
1519
1520
    def refine(self, seen, referenced):
1521
        """Create a new search by refining this search.
1522
1523
        :param seen: Revisions that have been satisfied.
1524
        :param referenced: Revision references observed while satisfying some
1525
            of this search.
1526
        """
1527
        start = self._recipe[1]
1528
        exclude = self._recipe[2]
1529
        count = self._recipe[3]
1530
        keys = self.get_keys()
1531
        # New heads = referenced + old heads - seen things - exclude
1532
        pending_refs = set(referenced)
1533
        pending_refs.update(start)
1534
        pending_refs.difference_update(seen)
1535
        pending_refs.difference_update(exclude)
1536
        # New exclude = old exclude + satisfied heads
1537
        seen_heads = start.intersection(seen)
1538
        exclude.update(seen_heads)
1539
        # keys gets seen removed
1540
        keys = keys - seen
1541
        # length is reduced by len(seen)
1542
        count -= len(seen)
1543
        return SearchResult(pending_refs, exclude, count, keys)
1544
3514.2.14 by John Arbash Meinel
Bring in the code to collapse linear portions of the graph.
1545
4070.9.14 by Andrew Bennetts
Tweaks requested by Robert's review.
1546
class PendingAncestryResult(object):
1547
    """A search result that will reconstruct the ancestry for some graph heads.
4086.1.4 by Andrew Bennetts
Fix whitespace nit.
1548
4070.9.14 by Andrew Bennetts
Tweaks requested by Robert's review.
1549
    Unlike SearchResult, this doesn't hold the complete search result in
1550
    memory, it just holds a description of how to generate it.
1551
    """
1552
1553
    def __init__(self, heads, repo):
1554
        """Constructor.
1555
1556
        :param heads: an iterable of graph heads.
1557
        :param repo: a repository to use to generate the ancestry for the given
1558
            heads.
1559
        """
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1560
        self.heads = frozenset(heads)
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1561
        self.repo = repo
1562
4070.9.5 by Andrew Bennetts
Better wire protocol: don't shoehorn MiniSearchResult serialisation into previous serialisation format.
1563
    def get_recipe(self):
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1564
        """Return a recipe that can be used to replay this search.
1565
1566
        The recipe allows reconstruction of the same results at a later date.
1567
1568
        :seealso SearchResult.get_recipe:
1569
1570
        :return: A tuple ('proxy-search', start_keys_set, set(), -1)
1571
            To recreate this result, create a PendingAncestryResult with the
1572
            start_keys_set.
1573
        """
1574
        return ('proxy-search', self.heads, set(), -1)
4070.9.5 by Andrew Bennetts
Better wire protocol: don't shoehorn MiniSearchResult serialisation into previous serialisation format.
1575
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1576
    def get_keys(self):
4098.1.1 by Andrew Bennetts
Fix a bug with how PendingAncestryResult.get_keys handles NULL_REVISION.
1577
        """See SearchResult.get_keys.
4098.1.2 by Andrew Bennetts
Fix 'trailing' whitespace (actually just a blank line in an indented docstring).
1578
4098.1.1 by Andrew Bennetts
Fix a bug with how PendingAncestryResult.get_keys handles NULL_REVISION.
1579
        Returns all the keys for the ancestry of the heads, excluding
1580
        NULL_REVISION.
1581
        """
1582
        return self._get_keys(self.repo.get_graph())
4098.1.3 by Andrew Bennetts
Fix 'trailing' whitespace (actually just a blank line between methods).
1583
4098.1.1 by Andrew Bennetts
Fix a bug with how PendingAncestryResult.get_keys handles NULL_REVISION.
1584
    def _get_keys(self, graph):
1585
        NULL_REVISION = revision.NULL_REVISION
1586
        keys = [key for (key, parents) in graph.iter_ancestry(self.heads)
4343.3.11 by John Arbash Meinel
Change PendingAncestryResult to strip ghosts from .get_keys()
1587
                if key != NULL_REVISION and parents is not None]
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1588
        return keys
1589
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1590
    def is_empty(self):
4204.2.2 by Matt Nordhoff
Fix docstrings on graph.py's is_empty methods that said they returned true when they were *not* empty.
1591
        """Return false if the search lists 1 or more revisions."""
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1592
        if revision.NULL_REVISION in self.heads:
1593
            return len(self.heads) == 1
1594
        else:
1595
            return len(self.heads) == 0
1596
1597
    def refine(self, seen, referenced):
1598
        """Create a new search by refining this search.
1599
1600
        :param seen: Revisions that have been satisfied.
1601
        :param referenced: Revision references observed while satisfying some
1602
            of this search.
1603
        """
1604
        referenced = self.heads.union(referenced)
1605
        return PendingAncestryResult(referenced - seen, self.repo)
1606
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1607
3514.2.14 by John Arbash Meinel
Bring in the code to collapse linear portions of the graph.
1608
def collapse_linear_regions(parent_map):
1609
    """Collapse regions of the graph that are 'linear'.
1610
1611
    For example::
1612
1613
      A:[B], B:[C]
1614
1615
    can be collapsed by removing B and getting::
1616
1617
      A:[C]
1618
1619
    :param parent_map: A dictionary mapping children to their parents
1620
    :return: Another dictionary with 'linear' chains collapsed
1621
    """
1622
    # Note: this isn't a strictly minimal collapse. For example:
1623
    #   A
1624
    #  / \
1625
    # B   C
1626
    #  \ /
1627
    #   D
1628
    #   |
1629
    #   E
1630
    # Will not have 'D' removed, even though 'E' could fit. Also:
1631
    #   A
1632
    #   |    A
1633
    #   B => |
1634
    #   |    C
1635
    #   C
1636
    # A and C are both kept because they are edges of the graph. We *could* get
1637
    # rid of A if we wanted.
1638
    #   A
1639
    #  / \
1640
    # B   C
1641
    # |   |
1642
    # D   E
1643
    #  \ /
1644
    #   F
1645
    # Will not have any nodes removed, even though you do have an
1646
    # 'uninteresting' linear D->B and E->C
1647
    children = {}
1648
    for child, parents in parent_map.iteritems():
1649
        children.setdefault(child, [])
1650
        for p in parents:
1651
            children.setdefault(p, []).append(child)
1652
1653
    orig_children = dict(children)
1654
    removed = set()
1655
    result = dict(parent_map)
1656
    for node in parent_map:
1657
        parents = result[node]
1658
        if len(parents) == 1:
1659
            parent_children = children[parents[0]]
1660
            if len(parent_children) != 1:
1661
                # This is not the only child
1662
                continue
1663
            node_children = children[node]
1664
            if len(node_children) != 1:
1665
                continue
1666
            child_parents = result.get(node_children[0], None)
1667
            if len(child_parents) != 1:
1668
                # This is not its only parent
1669
                continue
1670
            # The child of this node only points at it, and the parent only has
1671
            # this as a child. remove this node, and join the others together
1672
            result[node_children[0]] = parents
1673
            children[parents[0]] = node_children
1674
            del result[node]
1675
            del children[node]
1676
            removed.add(node)
1677
1678
    return result
4371.3.18 by John Arbash Meinel
Change VF.annotate to use the new KnownGraph code.
1679
1680
4371.3.38 by John Arbash Meinel
Add a failing test for handling nodes that are in the same linear chain.
1681
_counters = [0,0,0,0,0,0,0]
4371.3.18 by John Arbash Meinel
Change VF.annotate to use the new KnownGraph code.
1682
try:
1683
    from bzrlib._known_graph_pyx import KnownGraph
1684
except ImportError:
1685
    from bzrlib._known_graph_py import KnownGraph