~bzr-pqm/bzr/bzr.dev

5557.1.7 by John Arbash Meinel
Merge in the bzr.dev 5582
1
# Copyright (C) 2007-2011 Canonical Ltd
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
16
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
17
import time
18
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
19
from bzrlib import (
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
20
    debug,
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
21
    errors,
4574.3.6 by Martin Pool
More warnings when failing to load extensions
22
    osutils,
3052.1.3 by John Arbash Meinel
deprecate revision.is_ancestor, update the callers and the tests.
23
    revision,
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
24
    trace,
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
25
    )
2490.2.1 by Aaron Bentley
Start work on GraphWalker
26
3377.4.9 by John Arbash Meinel
STEP every 5
27
STEP_UNIQUE_SEARCHER_EVERY = 5
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
28
2490.2.25 by Aaron Bentley
Update from review
29
# DIAGRAM of terminology
30
#       A
31
#       /\
32
#      B  C
33
#      |  |\
34
#      D  E F
35
#      |\/| |
36
#      |/\|/
37
#      G  H
38
#
39
# In this diagram, relative to G and H:
40
# A, B, C, D, E are common ancestors.
41
# C, D and E are border ancestors, because each has a non-common descendant.
42
# D and E are least common ancestors because none of their descendants are
43
# common ancestors.
44
# C is not a least common ancestor because its descendant, E, is a common
45
# ancestor.
46
#
47
# The find_unique_lca algorithm will pick A in two steps:
48
# 1. find_lca('G', 'H') => ['D', 'E']
49
# 2. Since len(['D', 'E']) > 1, find_lca('D', 'E') => ['A']
50
51
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
52
class DictParentsProvider(object):
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
53
    """A parents provider for Graph objects."""
2988.1.3 by Robert Collins
Add a new repositoy method _generate_text_key_index for use by reconcile/check.
54
55
    def __init__(self, ancestry):
56
        self.ancestry = ancestry
57
58
    def __repr__(self):
59
        return 'DictParentsProvider(%r)' % self.ancestry
60
6015.24.8 by John Arbash Meinel
Some cleanups suggested by Vincent.
61
    # Note: DictParentsProvider does not implement get_cached_parent_map
62
    #       Arguably, the data is clearly cached in memory. However, this class
63
    #       is mostly used for testing, and it keeps the tests clean to not
64
    #       change it.
65
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
66
    def get_parent_map(self, keys):
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
67
        """See StackedParentsProvider.get_parent_map"""
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
68
        ancestry = self.ancestry
6015.23.10 by John Arbash Meinel
Small tweaks to search performance, though still at depth=100 the primary time
69
        return dict([(k, ancestry[k]) for k in keys if k in ancestry])
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
70
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
71
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
72
class StackedParentsProvider(object):
73
    """A parents provider which stacks (or unions) multiple providers.
6015.24.3 by John Arbash Meinel
Implement get_cached_parent_map
74
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
75
    The providers are queries in the order of the provided parent_providers.
76
    """
6015.24.3 by John Arbash Meinel
Implement get_cached_parent_map
77
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
78
    def __init__(self, parent_providers):
79
        self._parent_providers = parent_providers
80
2490.2.28 by Aaron Bentley
Fix handling of null revision
81
    def __repr__(self):
4379.3.4 by Gary van der Merwe
Make StackedParentsProvider.__repr__ more dynamic.
82
        return "%s(%r)" % (self.__class__.__name__, self._parent_providers)
2490.2.28 by Aaron Bentley
Fix handling of null revision
83
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
84
    def get_parent_map(self, keys):
85
        """Get a mapping of keys => parents
86
87
        A dictionary is returned with an entry for each key present in this
88
        source. If this source doesn't have information about a key, it should
89
        not include an entry.
90
91
        [NULL_REVISION] is used as the parent of the first user-committed
92
        revision.  Its parent list is empty.
93
94
        :param keys: An iterable returning keys to check (eg revision_ids)
95
        :return: A dictionary mapping each key to its parents
96
        """
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
97
        found = {}
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
98
        remaining = set(keys)
6015.24.9 by John Arbash Meinel
Discuss why the extra getattr() calls should be fine.
99
        # This adds getattr() overhead to each get_parent_map call. However,
100
        # this is StackedParentsProvider, which means we're dealing with I/O
101
        # (either local indexes, or remote RPCs), so CPU overhead should be
102
        # minimal.
6015.24.3 by John Arbash Meinel
Implement get_cached_parent_map
103
        for parents_provider in self._parent_providers:
104
            get_cached = getattr(parents_provider, 'get_cached_parent_map',
105
                                 None)
106
            if get_cached is None:
107
                continue
108
            new_found = get_cached(remaining)
109
            found.update(new_found)
110
            remaining.difference_update(new_found)
111
            if not remaining:
112
                break
113
        if not remaining:
114
            return found
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
115
        for parents_provider in self._parent_providers:
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
116
            new_found = parents_provider.get_parent_map(remaining)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
117
            found.update(new_found)
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
118
            remaining.difference_update(new_found)
119
            if not remaining:
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
120
                break
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
121
        return found
122
123
124
class CachingParentsProvider(object):
3835.1.13 by Aaron Bentley
Update documentation
125
    """A parents provider which will cache the revision => parents as a dict.
126
127
    This is useful for providers which have an expensive look up.
128
129
    Either a ParentsProvider or a get_parent_map-like callback may be
130
    supplied.  If it provides extra un-asked-for parents, they will be cached,
131
    but filtered out of get_parent_map.
3835.1.16 by Aaron Bentley
Updates from review
132
133
    The cache is enabled by default, but may be disabled and re-enabled.
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
134
    """
3896.1.1 by Andrew Bennetts
Remove broken debugging cruft, and some unused imports.
135
    def __init__(self, parent_provider=None, get_parent_map=None):
3835.1.13 by Aaron Bentley
Update documentation
136
        """Constructor.
137
138
        :param parent_provider: The ParentProvider to use.  It or
139
            get_parent_map must be supplied.
140
        :param get_parent_map: The get_parent_map callback to use.  It or
141
            parent_provider must be supplied.
142
        """
3835.1.12 by Aaron Bentley
Unify CachingExtraParentsProvider and CachingParentsProvider.
143
        self._real_provider = parent_provider
144
        if get_parent_map is None:
145
            self._get_parent_map = self._real_provider.get_parent_map
146
        else:
147
            self._get_parent_map = get_parent_map
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
148
        self._cache = None
149
        self.enable_cache(True)
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
150
3835.1.12 by Aaron Bentley
Unify CachingExtraParentsProvider and CachingParentsProvider.
151
    def __repr__(self):
152
        return "%s(%r)" % (self.__class__.__name__, self._real_provider)
153
3835.1.15 by Aaron Bentley
Allow miss caching to be disabled.
154
    def enable_cache(self, cache_misses=True):
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
155
        """Enable cache."""
3835.1.19 by Aaron Bentley
Raise exception when caching is enabled twice.
156
        if self._cache is not None:
3835.1.20 by Aaron Bentley
Change custom error to an AssertionError.
157
            raise AssertionError('Cache enabled when already enabled.')
3835.1.16 by Aaron Bentley
Updates from review
158
        self._cache = {}
3835.1.15 by Aaron Bentley
Allow miss caching to be disabled.
159
        self._cache_misses = cache_misses
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
160
        self.missing_keys = set()
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
161
162
    def disable_cache(self):
3835.1.16 by Aaron Bentley
Updates from review
163
        """Disable and clear the cache."""
164
        self._cache = None
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
165
        self._cache_misses = None
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
166
        self.missing_keys = set()
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
167
168
    def get_cached_map(self):
169
        """Return any cached get_parent_map values."""
3835.1.16 by Aaron Bentley
Updates from review
170
        if self._cache is None:
3835.1.12 by Aaron Bentley
Unify CachingExtraParentsProvider and CachingParentsProvider.
171
            return None
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
172
        return dict(self._cache)
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
173
6015.24.3 by John Arbash Meinel
Implement get_cached_parent_map
174
    def get_cached_parent_map(self, keys):
6015.24.7 by John Arbash Meinel
Update some comments and docstrings.
175
        """Return items from the cache.
176
177
        This returns the same info as get_parent_map, but explicitly does not
178
        invoke the supplied ParentsProvider to search for uncached values.
179
        """
6015.24.3 by John Arbash Meinel
Implement get_cached_parent_map
180
        cache = self._cache
181
        if cache is None:
182
            return {}
183
        return dict([(key, cache[key]) for key in keys if key in cache])
184
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
185
    def get_parent_map(self, keys):
4379.3.3 by Gary van der Merwe
Rename and add doc string for StackedParentsProvider.
186
        """See StackedParentsProvider.get_parent_map."""
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
187
        cache = self._cache
188
        if cache is None:
189
            cache = self._get_parent_map(keys)
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
190
        else:
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
191
            needed_revisions = set(key for key in keys if key not in cache)
192
            # Do not ask for negatively cached keys
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
193
            needed_revisions.difference_update(self.missing_keys)
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
194
            if needed_revisions:
195
                parent_map = self._get_parent_map(needed_revisions)
196
                cache.update(parent_map)
197
                if self._cache_misses:
198
                    for key in needed_revisions:
199
                        if key not in parent_map:
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
200
                            self.note_missing_key(key)
4190.1.1 by Robert Collins
Negatively cache misses during read-locks in RemoteRepository.
201
        result = {}
202
        for key in keys:
203
            value = cache.get(key)
204
            if value is not None:
205
                result[key] = value
206
        return result
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
207
4190.1.4 by Robert Collins
Cache ghosts when we can get them from a RemoteRepository in get_parent_map.
208
    def note_missing_key(self, key):
209
        """Note that key is a missing key."""
210
        if self._cache_misses:
211
            self.missing_keys.add(key)
212
3835.1.10 by Aaron Bentley
Move CachingExtraParentsProvider to Graph
213
5816.8.1 by Andrew Bennetts
Be a little more clever about constructing a parents provider for stacked repositories, so that get_parent_map with local-stacked-on-remote doesn't use HPSS VFS calls.
214
class CallableToParentsProviderAdapter(object):
215
    """A parents provider that adapts any callable to the parents provider API.
216
217
    i.e. it accepts calls to self.get_parent_map and relays them to the
218
    callable it was constructed with.
219
    """
220
221
    def __init__(self, a_callable):
222
        self.callable = a_callable
223
224
    def __repr__(self):
225
        return "%s(%r)" % (self.__class__.__name__, self.callable)
226
227
    def get_parent_map(self, keys):
228
        return self.callable(keys)
229
230
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
231
class Graph(object):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
232
    """Provide incremental access to revision graphs.
233
234
    This is the generic implementation; it is intended to be subclassed to
235
    specialize it for other repository types.
236
    """
2490.2.1 by Aaron Bentley
Start work on GraphWalker
237
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
238
    def __init__(self, parents_provider):
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
239
        """Construct a Graph that uses several graphs as its input
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
240
241
        This should not normally be invoked directly, because there may be
242
        specialized implementations for particular repository types.  See
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
243
        Repository.get_graph().
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
244
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
245
        :param parents_provider: An object providing a get_parent_map call
246
            conforming to the behavior of
247
            StackedParentsProvider.get_parent_map.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
248
        """
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
249
        if getattr(parents_provider, 'get_parents', None) is not None:
250
            self.get_parents = parents_provider.get_parents
251
        if getattr(parents_provider, 'get_parent_map', None) is not None:
252
            self.get_parent_map = parents_provider.get_parent_map
2490.2.29 by Aaron Bentley
Make parents provider private
253
        self._parents_provider = parents_provider
2490.2.28 by Aaron Bentley
Fix handling of null revision
254
255
    def __repr__(self):
2490.2.29 by Aaron Bentley
Make parents provider private
256
        return 'Graph(%r)' % self._parents_provider
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
257
258
    def find_lca(self, *revisions):
259
        """Determine the lowest common ancestors of the provided revisions
260
261
        A lowest common ancestor is a common ancestor none of whose
262
        descendants are common ancestors.  In graphs, unlike trees, there may
263
        be multiple lowest common ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
264
265
        This algorithm has two phases.  Phase 1 identifies border ancestors,
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
266
        and phase 2 filters border ancestors to determine lowest common
267
        ancestors.
2490.2.12 by Aaron Bentley
Improve documentation
268
269
        In phase 1, border ancestors are identified, using a breadth-first
270
        search starting at the bottom of the graph.  Searches are stopped
271
        whenever a node or one of its descendants is determined to be common
272
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
273
        In phase 2, the border ancestors are filtered to find the least
2490.2.12 by Aaron Bentley
Improve documentation
274
        common ancestors.  This is done by searching the ancestries of each
275
        border ancestor.
276
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
277
        Phase 2 is perfomed on the principle that a border ancestor that is
278
        not an ancestor of any other border ancestor is a least common
279
        ancestor.
2490.2.12 by Aaron Bentley
Improve documentation
280
281
        Searches are stopped when they find a node that is determined to be a
282
        common ancestor of all border ancestors, because this shows that it
283
        cannot be a descendant of any border ancestor.
284
5891.1.2 by Andrew Bennetts
Fix a bunch of docstring formatting nits, making pydoctor a bit happier.
285
        The scaling of this operation should be proportional to:
286
2490.2.12 by Aaron Bentley
Improve documentation
287
        1. The number of uncommon ancestors
288
        2. The number of border ancestors
289
        3. The length of the shortest path between a border ancestor and an
290
           ancestor of all border ancestors.
2490.2.3 by Aaron Bentley
Implement new merge base picker
291
        """
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
292
        border_common, common, sides = self._find_border_ancestors(revisions)
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
293
        # We may have common ancestors that can be reached from each other.
294
        # - ask for the heads of them to filter it down to only ones that
295
        # cannot be reached from each other - phase 2.
296
        return self.heads(border_common)
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
297
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
298
    def find_difference(self, left_revision, right_revision):
2490.2.25 by Aaron Bentley
Update from review
299
        """Determine the graph difference between two revisions"""
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
300
        border, common, searchers = self._find_border_ancestors(
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
301
            [left_revision, right_revision])
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
302
        self._search_for_extra_common(common, searchers)
303
        left = searchers[0].seen
304
        right = searchers[1].seen
305
        return (left.difference(right), right.difference(left))
2490.2.23 by Aaron Bentley
Adapt find_borders to produce a graph difference
306
5365.6.1 by Aaron Bentley
Implement find_descendants.
307
    def find_descendants(self, old_key, new_key):
308
        """Find descendants of old_key that are ancestors of new_key."""
309
        child_map = self.get_child_map(self._find_descendant_ancestors(
310
            old_key, new_key))
311
        graph = Graph(DictParentsProvider(child_map))
312
        searcher = graph._make_breadth_first_searcher([old_key])
313
        list(searcher)
314
        return searcher.seen
315
316
    def _find_descendant_ancestors(self, old_key, new_key):
317
        """Find ancestors of new_key that may be descendants of old_key."""
318
        stop = self._make_breadth_first_searcher([old_key])
319
        descendants = self._make_breadth_first_searcher([new_key])
320
        for revisions in descendants:
321
            old_stop = stop.seen.intersection(revisions)
322
            descendants.stop_searching_any(old_stop)
323
            seen_stop = descendants.find_seen_ancestors(stop.step())
324
            descendants.stop_searching_any(seen_stop)
325
        return descendants.seen.difference(stop.seen)
326
327
    def get_child_map(self, keys):
328
        """Get a mapping from parents to children of the specified keys.
329
330
        This is simply the inversion of get_parent_map.  Only supplied keys
331
        will be discovered as children.
332
        :return: a dict of key:child_list for keys.
333
        """
334
        parent_map = self._parents_provider.get_parent_map(keys)
335
        parent_child = {}
336
        for child, parents in sorted(parent_map.items()):
337
            for parent in parents:
338
                parent_child.setdefault(parent, []).append(child)
339
        return parent_child
340
3445.1.4 by John Arbash Meinel
Change the function to be called 'find_distance_to_null'
341
    def find_distance_to_null(self, target_revision_id, known_revision_ids):
342
        """Find the left-hand distance to the NULL_REVISION.
343
344
        (This can also be considered the revno of a branch at
345
        target_revision_id.)
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
346
347
        :param target_revision_id: A revision_id which we would like to know
348
            the revno for.
349
        :param known_revision_ids: [(revision_id, revno)] A list of known
350
            revno, revision_id tuples. We'll use this to seed the search.
351
        """
352
        # Map from revision_ids to a known value for their revno
353
        known_revnos = dict(known_revision_ids)
354
        cur_tip = target_revision_id
355
        num_steps = 0
356
        NULL_REVISION = revision.NULL_REVISION
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
357
        known_revnos[NULL_REVISION] = 0
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
358
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
359
        searching_known_tips = list(known_revnos.keys())
360
361
        unknown_searched = {}
362
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
363
        while cur_tip not in known_revnos:
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
364
            unknown_searched[cur_tip] = num_steps
365
            num_steps += 1
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
366
            to_search = set([cur_tip])
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
367
            to_search.update(searching_known_tips)
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
368
            parent_map = self.get_parent_map(to_search)
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
369
            parents = parent_map.get(cur_tip, None)
3445.1.8 by John Arbash Meinel
Clarity tweaks recommended by Ian
370
            if not parents: # An empty list or None is a ghost
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
371
                raise errors.GhostRevisionsHaveNoRevno(target_revision_id,
372
                                                       cur_tip)
373
            cur_tip = parents[0]
3445.1.3 by John Arbash Meinel
Search from all of the known revisions.
374
            next_known_tips = []
375
            for revision_id in searching_known_tips:
376
                parents = parent_map.get(revision_id, None)
377
                if not parents:
378
                    continue
379
                next = parents[0]
380
                next_revno = known_revnos[revision_id] - 1
381
                if next in unknown_searched:
382
                    # We have enough information to return a value right now
383
                    return next_revno + unknown_searched[next]
384
                if next in known_revnos:
385
                    continue
386
                known_revnos[next] = next_revno
387
                next_known_tips.append(next)
388
            searching_known_tips = next_known_tips
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
389
3445.1.2 by John Arbash Meinel
Handle when a known revision is an ancestor.
390
        # We reached a known revision, so just add in how many steps it took to
391
        # get there.
392
        return known_revnos[cur_tip] + num_steps
3445.1.1 by John Arbash Meinel
Start working on a new Graph api to make finding revision numbers faster.
393
4332.3.4 by Robert Collins
Add a graph API for getting multiple distances to NULL at once.
394
    def find_lefthand_distances(self, keys):
395
        """Find the distance to null for all the keys in keys.
396
397
        :param keys: keys to lookup.
398
        :return: A dict key->distance for all of keys.
399
        """
400
        # Optimisable by concurrent searching, but a random spread should get
401
        # some sort of hit rate.
402
        result = {}
403
        known_revnos = []
4332.3.6 by Robert Collins
Teach graph.find_lefthand_distances about ghosts.
404
        ghosts = []
4332.3.4 by Robert Collins
Add a graph API for getting multiple distances to NULL at once.
405
        for key in keys:
4332.3.6 by Robert Collins
Teach graph.find_lefthand_distances about ghosts.
406
            try:
407
                known_revnos.append(
408
                    (key, self.find_distance_to_null(key, known_revnos)))
409
            except errors.GhostRevisionsHaveNoRevno:
410
                ghosts.append(key)
411
        for key in ghosts:
412
            known_revnos.append((key, -1))
4332.3.4 by Robert Collins
Add a graph API for getting multiple distances to NULL at once.
413
        return dict(known_revnos)
414
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
415
    def find_unique_ancestors(self, unique_revision, common_revisions):
416
        """Find the unique ancestors for a revision versus others.
417
418
        This returns the ancestry of unique_revision, excluding all revisions
419
        in the ancestry of common_revisions. If unique_revision is in the
420
        ancestry, then the empty set will be returned.
421
422
        :param unique_revision: The revision_id whose ancestry we are
423
            interested in.
5891.1.2 by Andrew Bennetts
Fix a bunch of docstring formatting nits, making pydoctor a bit happier.
424
            (XXX: Would this API be better if we allowed multiple revisions on
425
            to be searched here?)
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
426
        :param common_revisions: Revision_ids of ancestries to exclude.
427
        :return: A set of revisions in the ancestry of unique_revision
428
        """
429
        if unique_revision in common_revisions:
430
            return set()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
431
432
        # Algorithm description
433
        # 1) Walk backwards from the unique node and all common nodes.
434
        # 2) When a node is seen by both sides, stop searching it in the unique
435
        #    walker, include it in the common walker.
436
        # 3) Stop searching when there are no nodes left for the unique walker.
437
        #    At this point, you have a maximal set of unique nodes. Some of
438
        #    them may actually be common, and you haven't reached them yet.
439
        # 4) Start new searchers for the unique nodes, seeded with the
440
        #    information you have so far.
441
        # 5) Continue searching, stopping the common searches when the search
442
        #    tip is an ancestor of all unique nodes.
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
443
        # 6) Aggregate together unique searchers when they are searching the
444
        #    same tips. When all unique searchers are searching the same node,
445
        #    stop move it to a single 'all_unique_searcher'.
446
        # 7) The 'all_unique_searcher' represents the very 'tip' of searching.
447
        #    Most of the time this produces very little important information.
448
        #    So don't step it as quickly as the other searchers.
449
        # 8) Search is done when all common searchers have completed.
450
451
        unique_searcher, common_searcher = self._find_initial_unique_nodes(
452
            [unique_revision], common_revisions)
453
454
        unique_nodes = unique_searcher.seen.difference(common_searcher.seen)
455
        if not unique_nodes:
456
            return unique_nodes
457
458
        (all_unique_searcher,
459
         unique_tip_searchers) = self._make_unique_searchers(unique_nodes,
460
                                    unique_searcher, common_searcher)
461
462
        self._refine_unique_nodes(unique_searcher, all_unique_searcher,
463
                                  unique_tip_searchers, common_searcher)
464
        true_unique_nodes = unique_nodes.difference(common_searcher.seen)
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
465
        if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
466
            trace.mutter('Found %d truly unique nodes out of %d',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
467
                         len(true_unique_nodes), len(unique_nodes))
468
        return true_unique_nodes
469
470
    def _find_initial_unique_nodes(self, unique_revisions, common_revisions):
471
        """Steps 1-3 of find_unique_ancestors.
472
473
        Find the maximal set of unique nodes. Some of these might actually
474
        still be common, but we are sure that there are no other unique nodes.
475
476
        :return: (unique_searcher, common_searcher)
477
        """
478
479
        unique_searcher = self._make_breadth_first_searcher(unique_revisions)
480
        # we know that unique_revisions aren't in common_revisions, so skip
481
        # past them.
3377.3.27 by John Arbash Meinel
some simple updates
482
        unique_searcher.next()
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
483
        common_searcher = self._make_breadth_first_searcher(common_revisions)
484
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
485
        # As long as we are still finding unique nodes, keep searching
3377.3.27 by John Arbash Meinel
some simple updates
486
        while unique_searcher._next_query:
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
487
            next_unique_nodes = set(unique_searcher.step())
488
            next_common_nodes = set(common_searcher.step())
489
490
            # Check if either searcher encounters new nodes seen by the other
491
            # side.
492
            unique_are_common_nodes = next_unique_nodes.intersection(
493
                common_searcher.seen)
494
            unique_are_common_nodes.update(
495
                next_common_nodes.intersection(unique_searcher.seen))
496
            if unique_are_common_nodes:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
497
                ancestors = unique_searcher.find_seen_ancestors(
498
                                unique_are_common_nodes)
3377.4.5 by John Arbash Meinel
Several updates. A bit more debug logging, only step the all_unique searcher 1/10th of the time.
499
                # TODO: This is a bit overboard, we only really care about
500
                #       the ancestors of the tips because the rest we
501
                #       already know. This is *correct* but causes us to
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
502
                #       search too much ancestry.
3377.3.29 by John Arbash Meinel
Revert the _find_any_seen change.
503
                ancestors.update(common_searcher.find_seen_ancestors(ancestors))
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
504
                unique_searcher.stop_searching_any(ancestors)
505
                common_searcher.start_searching(ancestors)
506
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
507
        return unique_searcher, common_searcher
508
509
    def _make_unique_searchers(self, unique_nodes, unique_searcher,
510
                               common_searcher):
511
        """Create a searcher for all the unique search tips (step 4).
512
513
        As a side effect, the common_searcher will stop searching any nodes
514
        that are ancestors of the unique searcher tips.
515
516
        :return: (all_unique_searcher, unique_tip_searchers)
517
        """
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
518
        unique_tips = self._remove_simple_descendants(unique_nodes,
519
                        self.get_parent_map(unique_nodes))
520
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
521
        if len(unique_tips) == 1:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
522
            unique_tip_searchers = []
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
523
            ancestor_all_unique = unique_searcher.find_seen_ancestors(unique_tips)
524
        else:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
525
            unique_tip_searchers = []
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
526
            for tip in unique_tips:
527
                revs_to_search = unique_searcher.find_seen_ancestors([tip])
528
                revs_to_search.update(
529
                    common_searcher.find_seen_ancestors(revs_to_search))
530
                searcher = self._make_breadth_first_searcher(revs_to_search)
531
                # We don't care about the starting nodes.
532
                searcher._label = tip
533
                searcher.step()
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
534
                unique_tip_searchers.append(searcher)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
535
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
536
            ancestor_all_unique = None
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
537
            for searcher in unique_tip_searchers:
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
538
                if ancestor_all_unique is None:
539
                    ancestor_all_unique = set(searcher.seen)
540
                else:
541
                    ancestor_all_unique = ancestor_all_unique.intersection(
542
                                                searcher.seen)
3377.3.33 by John Arbash Meinel
Add some logging with -Dgraph
543
        # Collapse all the common nodes into a single searcher
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
544
        all_unique_searcher = self._make_breadth_first_searcher(
545
                                ancestor_all_unique)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
546
        if ancestor_all_unique:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
547
            # We've seen these nodes in all the searchers, so we'll just go to
548
            # the next
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
549
            all_unique_searcher.step()
550
551
            # Stop any search tips that are already known as ancestors of the
552
            # unique nodes
553
            stopped_common = common_searcher.stop_searching_any(
554
                common_searcher.find_seen_ancestors(ancestor_all_unique))
555
556
            total_stopped = 0
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
557
            for searcher in unique_tip_searchers:
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
558
                total_stopped += len(searcher.stop_searching_any(
559
                    searcher.find_seen_ancestors(ancestor_all_unique)))
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
560
        if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
561
            trace.mutter('For %d unique nodes, created %d + 1 unique searchers'
562
                         ' (%d stopped search tips, %d common ancestors'
563
                         ' (%d stopped common)',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
564
                         len(unique_nodes), len(unique_tip_searchers),
565
                         total_stopped, len(ancestor_all_unique),
566
                         len(stopped_common))
567
        return all_unique_searcher, unique_tip_searchers
568
569
    def _step_unique_and_common_searchers(self, common_searcher,
570
                                          unique_tip_searchers,
571
                                          unique_searcher):
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
572
        """Step all the searchers"""
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
573
        newly_seen_common = set(common_searcher.step())
574
        newly_seen_unique = set()
575
        for searcher in unique_tip_searchers:
576
            next = set(searcher.step())
577
            next.update(unique_searcher.find_seen_ancestors(next))
578
            next.update(common_searcher.find_seen_ancestors(next))
579
            for alt_searcher in unique_tip_searchers:
580
                if alt_searcher is searcher:
581
                    continue
582
                next.update(alt_searcher.find_seen_ancestors(next))
583
            searcher.start_searching(next)
584
            newly_seen_unique.update(next)
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
585
        return newly_seen_common, newly_seen_unique
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
586
587
    def _find_nodes_common_to_all_unique(self, unique_tip_searchers,
588
                                         all_unique_searcher,
589
                                         newly_seen_unique, step_all_unique):
590
        """Find nodes that are common to all unique_tip_searchers.
591
592
        If it is time, step the all_unique_searcher, and add its nodes to the
593
        result.
594
        """
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
595
        common_to_all_unique_nodes = newly_seen_unique.copy()
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
596
        for searcher in unique_tip_searchers:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
597
            common_to_all_unique_nodes.intersection_update(searcher.seen)
598
        common_to_all_unique_nodes.intersection_update(
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
599
                                    all_unique_searcher.seen)
600
        # Step all-unique less frequently than the other searchers.
601
        # In the common case, we don't need to spider out far here, so
602
        # avoid doing extra work.
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
603
        if step_all_unique:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
604
            tstart = time.clock()
605
            nodes = all_unique_searcher.step()
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
606
            common_to_all_unique_nodes.update(nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
607
            if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
608
                tdelta = time.clock() - tstart
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
609
                trace.mutter('all_unique_searcher step() took %.3fs'
610
                             'for %d nodes (%d total), iteration: %s',
611
                             tdelta, len(nodes), len(all_unique_searcher.seen),
612
                             all_unique_searcher._iterations)
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
613
        return common_to_all_unique_nodes
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
614
615
    def _collapse_unique_searchers(self, unique_tip_searchers,
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
616
                                   common_to_all_unique_nodes):
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
617
        """Combine searchers that are searching the same tips.
618
619
        When two searchers are searching the same tips, we can stop one of the
620
        searchers. We also know that the maximal set of common ancestors is the
621
        intersection of the two original searchers.
622
623
        :return: A list of searchers that are searching unique nodes.
624
        """
625
        # Filter out searchers that don't actually search different
626
        # nodes. We already have the ancestry intersection for them
627
        unique_search_tips = {}
628
        for searcher in unique_tip_searchers:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
629
            stopped = searcher.stop_searching_any(common_to_all_unique_nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
630
            will_search_set = frozenset(searcher._next_query)
631
            if not will_search_set:
632
                if 'graph' in debug.debug_flags:
633
                    trace.mutter('Unique searcher %s was stopped.'
634
                                 ' (%s iterations) %d nodes stopped',
635
                                 searcher._label,
636
                                 searcher._iterations,
637
                                 len(stopped))
638
            elif will_search_set not in unique_search_tips:
639
                # This searcher is searching a unique set of nodes, let it
640
                unique_search_tips[will_search_set] = [searcher]
641
            else:
642
                unique_search_tips[will_search_set].append(searcher)
643
        # TODO: it might be possible to collapse searchers faster when they
644
        #       only have *some* search tips in common.
645
        next_unique_searchers = []
646
        for searchers in unique_search_tips.itervalues():
647
            if len(searchers) == 1:
648
                # Searching unique tips, go for it
649
                next_unique_searchers.append(searchers[0])
650
            else:
651
                # These searchers have started searching the same tips, we
652
                # don't need them to cover the same ground. The
653
                # intersection of their ancestry won't change, so create a
654
                # new searcher, combining their histories.
655
                next_searcher = searchers[0]
656
                for searcher in searchers[1:]:
657
                    next_searcher.seen.intersection_update(searcher.seen)
658
                if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
659
                    trace.mutter('Combining %d searchers into a single'
660
                                 ' searcher searching %d nodes with'
661
                                 ' %d ancestry',
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
662
                                 len(searchers),
663
                                 len(next_searcher._next_query),
664
                                 len(next_searcher.seen))
665
                next_unique_searchers.append(next_searcher)
666
        return next_unique_searchers
667
668
    def _refine_unique_nodes(self, unique_searcher, all_unique_searcher,
669
                             unique_tip_searchers, common_searcher):
670
        """Steps 5-8 of find_unique_ancestors.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
671
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
672
        This function returns when common_searcher has stopped searching for
673
        more nodes.
674
        """
675
        # We step the ancestor_all_unique searcher only every
676
        # STEP_UNIQUE_SEARCHER_EVERY steps.
677
        step_all_unique_counter = 0
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
678
        # While we still have common nodes to search
679
        while common_searcher._next_query:
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
680
            (newly_seen_common,
681
             newly_seen_unique) = self._step_unique_and_common_searchers(
682
                common_searcher, unique_tip_searchers, unique_searcher)
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
683
            # These nodes are common ancestors of all unique nodes
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
684
            common_to_all_unique_nodes = self._find_nodes_common_to_all_unique(
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
685
                unique_tip_searchers, all_unique_searcher, newly_seen_unique,
686
                step_all_unique_counter==0)
687
            step_all_unique_counter = ((step_all_unique_counter + 1)
688
                                       % STEP_UNIQUE_SEARCHER_EVERY)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
689
690
            if newly_seen_common:
691
                # If a 'common' node is an ancestor of all unique searchers, we
692
                # can stop searching it.
693
                common_searcher.stop_searching_any(
694
                    all_unique_searcher.seen.intersection(newly_seen_common))
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
695
            if common_to_all_unique_nodes:
696
                common_to_all_unique_nodes.update(
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
697
                    common_searcher.find_seen_ancestors(
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
698
                        common_to_all_unique_nodes))
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
699
                # The all_unique searcher can start searching the common nodes
700
                # but everyone else can stop.
3377.4.7 by John Arbash Meinel
Small documentation and code wrapping cleanup
701
                # This is the sort of thing where we would like to not have it
702
                # start_searching all of the nodes, but only mark all of them
703
                # as seen, and have it search only the actual tips. Otherwise
704
                # it is another get_parent_map() traversal for it to figure out
705
                # what we already should know.
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
706
                all_unique_searcher.start_searching(common_to_all_unique_nodes)
707
                common_searcher.stop_searching_any(common_to_all_unique_nodes)
3377.4.4 by John Arbash Meinel
Restore the previous code, but bring in a couple changes. Including an update to have lsprof show where the time is spent.
708
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
709
            next_unique_searchers = self._collapse_unique_searchers(
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
710
                unique_tip_searchers, common_to_all_unique_nodes)
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
711
            if len(unique_tip_searchers) != len(next_unique_searchers):
712
                if 'graph' in debug.debug_flags:
3377.4.8 by John Arbash Meinel
Final tweaks from Ian
713
                    trace.mutter('Collapsed %d unique searchers => %d'
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
714
                                 ' at %s iterations',
715
                                 len(unique_tip_searchers),
716
                                 len(next_unique_searchers),
717
                                 all_unique_searcher._iterations)
718
            unique_tip_searchers = next_unique_searchers
3377.3.21 by John Arbash Meinel
Simple brute-force implementation of find_unique_ancestors
719
3172.1.2 by Robert Collins
Parent Providers should now implement ``get_parent_map`` returning a
720
    def get_parent_map(self, revisions):
721
        """Get a map of key:parent_list for revisions.
722
723
        This implementation delegates to get_parents, for old parent_providers
724
        that do not supply get_parent_map.
725
        """
726
        result = {}
727
        for rev, parents in self.get_parents(revisions):
728
            if parents is not None:
729
                result[rev] = parents
730
        return result
731
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
732
    def _make_breadth_first_searcher(self, revisions):
733
        return _BreadthFirstSearcher(revisions, self)
734
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
735
    def _find_border_ancestors(self, revisions):
2490.2.12 by Aaron Bentley
Improve documentation
736
        """Find common ancestors with at least one uncommon descendant.
737
738
        Border ancestors are identified using a breadth-first
739
        search starting at the bottom of the graph.  Searches are stopped
740
        whenever a node or one of its descendants is determined to be common.
741
742
        This will scale with the number of uncommon ancestors.
2490.2.25 by Aaron Bentley
Update from review
743
744
        As well as the border ancestors, a set of seen common ancestors and a
745
        list of sets of seen ancestors for each input revision is returned.
746
        This allows calculation of graph difference from the results of this
747
        operation.
2490.2.12 by Aaron Bentley
Improve documentation
748
        """
2490.2.28 by Aaron Bentley
Fix handling of null revision
749
        if None in revisions:
750
            raise errors.InvalidRevisionId(None, self)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
751
        common_ancestors = set()
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
752
        searchers = [self._make_breadth_first_searcher([r])
753
                     for r in revisions]
754
        active_searchers = searchers[:]
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
755
        border_ancestors = set()
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
756
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
757
        while True:
758
            newly_seen = set()
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
759
            for searcher in searchers:
760
                new_ancestors = searcher.step()
761
                if new_ancestors:
762
                    newly_seen.update(new_ancestors)
763
            new_common = set()
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
764
            for revision in newly_seen:
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
765
                if revision in common_ancestors:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
766
                    # Not a border ancestor because it was seen as common
767
                    # already
768
                    new_common.add(revision)
2490.2.19 by Aaron Bentley
Implement common-ancestor-based culling
769
                    continue
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
770
                for searcher in searchers:
771
                    if revision not in searcher.seen:
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
772
                        break
773
                else:
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
774
                    # This is a border because it is a first common that we see
775
                    # after walking for a while.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
776
                    border_ancestors.add(revision)
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
777
                    new_common.add(revision)
778
            if new_common:
779
                for searcher in searchers:
780
                    new_common.update(searcher.find_seen_ancestors(new_common))
781
                for searcher in searchers:
782
                    searcher.start_searching(new_common)
783
                common_ancestors.update(new_common)
784
785
            # Figure out what the searchers will be searching next, and if
786
            # there is only 1 set being searched, then we are done searching,
787
            # since all searchers would have to be searching the same data,
788
            # thus it *must* be in common.
789
            unique_search_sets = set()
790
            for searcher in searchers:
791
                will_search_set = frozenset(searcher._next_query)
792
                if will_search_set not in unique_search_sets:
793
                    # This searcher is searching a unique set of nodes, let it
794
                    unique_search_sets.add(will_search_set)
795
796
            if len(unique_search_sets) == 1:
797
                nodes = unique_search_sets.pop()
798
                uncommon_nodes = nodes.difference(common_ancestors)
3376.2.14 by Martin Pool
Remove recently-introduced assert statements
799
                if uncommon_nodes:
800
                    raise AssertionError("Somehow we ended up converging"
801
                                         " without actually marking them as"
802
                                         " in common."
803
                                         "\nStart_nodes: %s"
804
                                         "\nuncommon_nodes: %s"
805
                                         % (revisions, uncommon_nodes))
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
806
                break
807
        return border_ancestors, common_ancestors, searchers
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
808
2776.3.1 by Robert Collins
* Deprecated method ``find_previous_heads`` on
809
    def heads(self, keys):
810
        """Return the heads from amongst keys.
811
812
        This is done by searching the ancestries of each key.  Any key that is
813
        reachable from another key is not returned; all the others are.
814
815
        This operation scales with the relative depth between any two keys. If
816
        any two keys are completely disconnected all ancestry of both sides
817
        will be retrieved.
818
819
        :param keys: An iterable of keys.
2776.1.4 by Robert Collins
Trivial review feedback changes.
820
        :return: A set of the heads. Note that as a set there is no ordering
821
            information. Callers will need to filter their input to create
822
            order if they need it.
2490.2.12 by Aaron Bentley
Improve documentation
823
        """
2776.1.4 by Robert Collins
Trivial review feedback changes.
824
        candidate_heads = set(keys)
3052.5.5 by John Arbash Meinel
Special case Graph.heads() for NULL_REVISION rather than is_ancestor.
825
        if revision.NULL_REVISION in candidate_heads:
826
            # NULL_REVISION is only a head if it is the only entry
827
            candidate_heads.remove(revision.NULL_REVISION)
828
            if not candidate_heads:
829
                return set([revision.NULL_REVISION])
2850.2.1 by Robert Collins
(robertc) Special case the zero-or-no-heads case for Graph.heads(). (Robert Collins)
830
        if len(candidate_heads) < 2:
831
            return candidate_heads
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
832
        searchers = dict((c, self._make_breadth_first_searcher([c]))
2776.1.4 by Robert Collins
Trivial review feedback changes.
833
                          for c in candidate_heads)
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
834
        active_searchers = dict(searchers)
835
        # skip over the actual candidate for each searcher
836
        for searcher in active_searchers.itervalues():
1551.15.81 by Aaron Bentley
Remove testing code
837
            searcher.next()
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
838
        # The common walker finds nodes that are common to two or more of the
839
        # input keys, so that we don't access all history when a currently
840
        # uncommon search point actually meets up with something behind a
841
        # common search point. Common search points do not keep searches
842
        # active; they just allow us to make searches inactive without
843
        # accessing all history.
844
        common_walker = self._make_breadth_first_searcher([])
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
845
        while len(active_searchers) > 0:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
846
            ancestors = set()
847
            # advance searches
848
            try:
849
                common_walker.next()
850
            except StopIteration:
2921.3.4 by Robert Collins
Review feedback.
851
                # No common points being searched at this time.
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
852
                pass
1551.15.78 by Aaron Bentley
Fix KeyError in filter_candidate_lca
853
            for candidate in active_searchers.keys():
854
                try:
855
                    searcher = active_searchers[candidate]
856
                except KeyError:
857
                    # rare case: we deleted candidate in a previous iteration
858
                    # through this for loop, because it was determined to be
859
                    # a descendant of another candidate.
860
                    continue
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
861
                try:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
862
                    ancestors.update(searcher.next())
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
863
                except StopIteration:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
864
                    del active_searchers[candidate]
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
865
                    continue
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
866
            # process found nodes
867
            new_common = set()
868
            for ancestor in ancestors:
869
                if ancestor in candidate_heads:
870
                    candidate_heads.remove(ancestor)
871
                    del searchers[ancestor]
872
                    if ancestor in active_searchers:
873
                        del active_searchers[ancestor]
874
                # it may meet up with a known common node
2921.3.4 by Robert Collins
Review feedback.
875
                if ancestor in common_walker.seen:
876
                    # some searcher has encountered our known common nodes:
877
                    # just stop it
878
                    ancestor_set = set([ancestor])
879
                    for searcher in searchers.itervalues():
880
                        searcher.stop_searching_any(ancestor_set)
881
                else:
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
882
                    # or it may have been just reached by all the searchers:
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
883
                    for searcher in searchers.itervalues():
884
                        if ancestor not in searcher.seen:
2490.2.9 by Aaron Bentley
Fix minimal common ancestor algorithm for non-minimal perhipheral ancestors
885
                            break
886
                    else:
2921.3.4 by Robert Collins
Review feedback.
887
                        # The final active searcher has just reached this node,
888
                        # making it be known as a descendant of all candidates,
889
                        # so we can stop searching it, and any seen ancestors
890
                        new_common.add(ancestor)
891
                        for searcher in searchers.itervalues():
892
                            seen_ancestors =\
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
893
                                searcher.find_seen_ancestors([ancestor])
2921.3.4 by Robert Collins
Review feedback.
894
                            searcher.stop_searching_any(seen_ancestors)
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
895
            common_walker.start_searching(new_common)
2776.1.4 by Robert Collins
Trivial review feedback changes.
896
        return candidate_heads
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
897
3514.2.8 by John Arbash Meinel
The insertion ordering into the weave has an impact on conflicts.
898
    def find_merge_order(self, tip_revision_id, lca_revision_ids):
899
        """Find the order that each revision was merged into tip.
900
901
        This basically just walks backwards with a stack, and walks left-first
902
        until it finds a node to stop.
903
        """
904
        if len(lca_revision_ids) == 1:
905
            return list(lca_revision_ids)
906
        looking_for = set(lca_revision_ids)
907
        # TODO: Is there a way we could do this "faster" by batching up the
908
        # get_parent_map requests?
909
        # TODO: Should we also be culling the ancestry search right away? We
910
        # could add looking_for to the "stop" list, and walk their
911
        # ancestry in batched mode. The flip side is it might mean we walk a
912
        # lot of "stop" nodes, rather than only the minimum.
913
        # Then again, without it we may trace back into ancestry we could have
914
        # stopped early.
915
        stack = [tip_revision_id]
916
        found = []
917
        stop = set()
918
        while stack and looking_for:
919
            next = stack.pop()
920
            stop.add(next)
921
            if next in looking_for:
922
                found.append(next)
923
                looking_for.remove(next)
924
                if len(looking_for) == 1:
925
                    found.append(looking_for.pop())
926
                    break
927
                continue
928
            parent_ids = self.get_parent_map([next]).get(next, None)
929
            if not parent_ids: # Ghost, nothing to search here
930
                continue
931
            for parent_id in reversed(parent_ids):
932
                # TODO: (performance) We see the parent at this point, but we
933
                #       wait to mark it until later to make sure we get left
934
                #       parents before right parents. However, instead of
935
                #       waiting until we have traversed enough parents, we
936
                #       could instead note that we've found it, and once all
937
                #       parents are in the stack, just reverse iterate the
938
                #       stack for them.
939
                if parent_id not in stop:
940
                    # this will need to be searched
941
                    stack.append(parent_id)
942
                stop.add(parent_id)
943
        return found
944
5365.6.3 by Aaron Bentley
Implement find_lefthand_merger.
945
    def find_lefthand_merger(self, merged_key, tip_key):
946
        """Find the first lefthand ancestor of tip_key that merged merged_key.
947
948
        We do this by first finding the descendants of merged_key, then
949
        walking through the lefthand ancestry of tip_key until we find a key
950
        that doesn't descend from merged_key.  Its child is the key that
951
        merged merged_key.
952
953
        :return: The first lefthand ancestor of tip_key to merge merged_key.
954
            merged_key if it is a lefthand ancestor of tip_key.
955
            None if no ancestor of tip_key merged merged_key.
956
        """
957
        descendants = self.find_descendants(merged_key, tip_key)
958
        candidate_iterator = self.iter_lefthand_ancestry(tip_key)
959
        last_candidate = None
960
        for candidate in candidate_iterator:
961
            if candidate not in descendants:
962
                return last_candidate
963
            last_candidate = candidate
964
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
965
    def find_unique_lca(self, left_revision, right_revision,
966
                        count_steps=False):
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
967
        """Find a unique LCA.
968
969
        Find lowest common ancestors.  If there is no unique  common
970
        ancestor, find the lowest common ancestors of those ancestors.
971
972
        Iteration stops when a unique lowest common ancestor is found.
973
        The graph origin is necessarily a unique lowest common ancestor.
2490.2.5 by Aaron Bentley
Use GraphWalker.unique_ancestor to determine merge base
974
975
        Note that None is not an acceptable substitute for NULL_REVISION.
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
976
        in the input for this method.
1551.19.12 by Aaron Bentley
Add documentation for the count_steps parameter of Graph.find_unique_lca
977
978
        :param count_steps: If True, the return value will be a tuple of
979
            (unique_lca, steps) where steps is the number of times that
980
            find_lca was run.  If False, only unique_lca is returned.
2490.2.3 by Aaron Bentley
Implement new merge base picker
981
        """
982
        revisions = [left_revision, right_revision]
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
983
        steps = 0
2490.2.3 by Aaron Bentley
Implement new merge base picker
984
        while True:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
985
            steps += 1
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
986
            lca = self.find_lca(*revisions)
987
            if len(lca) == 1:
1551.19.10 by Aaron Bentley
Merge now warns when it encounters a criss-cross
988
                result = lca.pop()
989
                if count_steps:
990
                    return result, steps
991
                else:
992
                    return result
2520.4.104 by Aaron Bentley
Avoid infinite loop when there is no unique lca
993
            if len(lca) == 0:
994
                raise errors.NoCommonAncestor(left_revision, right_revision)
2490.2.13 by Aaron Bentley
Update distinct -> lowest, refactor, add ParentsProvider concept
995
            revisions = lca
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
996
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
997
    def iter_ancestry(self, revision_ids):
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
998
        """Iterate the ancestry of this revision.
999
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
1000
        :param revision_ids: Nodes to start the search
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
1001
        :return: Yield tuples mapping a revision_id to its parents for the
1002
            ancestry of revision_id.
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
1003
            Ghosts will be returned with None as their parents, and nodes
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
1004
            with no parents will have NULL_REVISION as their only parent. (As
1005
            defined by get_parent_map.)
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
1006
            There will also be a node for (NULL_REVISION, ())
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
1007
        """
3228.4.4 by John Arbash Meinel
Change iter_ancestry to take a group instead of a single node,
1008
        pending = set(revision_ids)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
1009
        processed = set()
1010
        while pending:
1011
            processed.update(pending)
1012
            next_map = self.get_parent_map(pending)
1013
            next_pending = set()
1014
            for item in next_map.iteritems():
1015
                yield item
1016
                next_pending.update(p for p in item[1] if p not in processed)
1017
            ghosts = pending.difference(next_map)
1018
            for ghost in ghosts:
3228.4.10 by John Arbash Meinel
Respond to abentley's review comments.
1019
                yield (ghost, None)
3228.4.2 by John Arbash Meinel
Add a Graph.iter_ancestry()
1020
            pending = next_pending
1021
5365.6.2 by Aaron Bentley
Extract iter_lefthand_ancestry from Repository.iter_ancestry.
1022
    def iter_lefthand_ancestry(self, start_key, stop_keys=None):
1023
        if stop_keys is None:
1024
            stop_keys = ()
1025
        next_key = start_key
1026
        def get_parents(key):
1027
            try:
1028
                return self._parents_provider.get_parent_map([key])[key]
1029
            except KeyError:
1030
                raise errors.RevisionNotPresent(next_key, self)
1031
        while True:
1032
            if next_key in stop_keys:
1033
                return
1034
            parents = get_parents(next_key)
1035
            yield next_key
1036
            if len(parents) == 0:
1037
                return
1038
            else:
1039
                next_key = parents[0]
1040
2490.2.31 by Aaron Bentley
Fix iter_topo_order to permit un-included parents
1041
    def iter_topo_order(self, revisions):
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
1042
        """Iterate through the input revisions in topological order.
1043
1044
        This sorting only ensures that parents come before their children.
1045
        An ancestor may sort after a descendant if the relationship is not
1046
        visible in the supplied list of revisions.
1047
        """
4593.5.30 by John Arbash Meinel
Move the topo_sort implementation into KnownGraph, rather than calling back to it.
1048
        from bzrlib import tsort
3099.3.3 by John Arbash Meinel
Deprecate get_parents() in favor of get_parent_map()
1049
        sorter = tsort.TopoSorter(self.get_parent_map(revisions))
2490.2.34 by Aaron Bentley
Update NEWS and change implementation to return an iterator
1050
        return sorter.iter_topo_order()
2490.2.30 by Aaron Bentley
Add functionality for tsorting graphs
1051
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
1052
    def is_ancestor(self, candidate_ancestor, candidate_descendant):
2653.2.5 by Aaron Bentley
Update to clarify algorithm
1053
        """Determine whether a revision is an ancestor of another.
1054
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
1055
        We answer this using heads() as heads() has the logic to perform the
3078.2.6 by Ian Clatworthy
fix efficiency of local commit detection as recommended by jameinel's review
1056
        smallest number of parent lookups to determine the ancestral
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
1057
        relationship between N revisions.
2653.2.5 by Aaron Bentley
Update to clarify algorithm
1058
        """
2921.3.1 by Robert Collins
* Graph ``heads()`` queries have been bugfixed to no longer access all
1059
        return set([candidate_descendant]) == self.heads(
1060
            [candidate_ancestor, candidate_descendant])
2653.2.1 by Aaron Bentley
Implement Graph.is_ancestor
1061
3921.3.5 by Marius Kruger
extract graph.is_between from builtins.cmd_tags.run, and test it
1062
    def is_between(self, revid, lower_bound_revid, upper_bound_revid):
1063
        """Determine whether a revision is between two others.
1064
1065
        returns true if and only if:
1066
        lower_bound_revid <= revid <= upper_bound_revid
1067
        """
1068
        return ((upper_bound_revid is None or
1069
                    self.is_ancestor(revid, upper_bound_revid)) and
1070
               (lower_bound_revid is None or
1071
                    self.is_ancestor(lower_bound_revid, revid)))
1072
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1073
    def _search_for_extra_common(self, common, searchers):
1074
        """Make sure that unique nodes are genuinely unique.
1075
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1076
        After _find_border_ancestors, all nodes marked "common" are indeed
1077
        common. Some of the nodes considered unique are not, due to history
1078
        shortcuts stopping the searches early.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1079
1080
        We know that we have searched enough when all common search tips are
1081
        descended from all unique (uncommon) nodes because we know that a node
1082
        cannot be an ancestor of its own ancestor.
1083
1084
        :param common: A set of common nodes
1085
        :param searchers: The searchers returned from _find_border_ancestors
1086
        :return: None
1087
        """
1088
        # Basic algorithm...
1089
        #   A) The passed in searchers should all be on the same tips, thus
1090
        #      they should be considered the "common" searchers.
1091
        #   B) We find the difference between the searchers, these are the
1092
        #      "unique" nodes for each side.
1093
        #   C) We do a quick culling so that we only start searching from the
1094
        #      more interesting unique nodes. (A unique ancestor is more
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1095
        #      interesting than any of its children.)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1096
        #   D) We start searching for ancestors common to all unique nodes.
1097
        #   E) We have the common searchers stop searching any ancestors of
1098
        #      nodes found by (D)
1099
        #   F) When there are no more common search tips, we stop
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1100
1101
        # TODO: We need a way to remove unique_searchers when they overlap with
1102
        #       other unique searchers.
3376.2.14 by Martin Pool
Remove recently-introduced assert statements
1103
        if len(searchers) != 2:
1104
            raise NotImplementedError(
1105
                "Algorithm not yet implemented for > 2 searchers")
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1106
        common_searchers = searchers
1107
        left_searcher = searchers[0]
1108
        right_searcher = searchers[1]
3377.3.15 by John Arbash Meinel
minor update
1109
        unique = left_searcher.seen.symmetric_difference(right_searcher.seen)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1110
        if not unique: # No unique nodes, nothing to do
1111
            return
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1112
        total_unique = len(unique)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1113
        unique = self._remove_simple_descendants(unique,
1114
                    self.get_parent_map(unique))
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1115
        simple_unique = len(unique)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
1116
1117
        unique_searchers = []
1118
        for revision_id in unique:
3377.3.15 by John Arbash Meinel
minor update
1119
            if revision_id in left_searcher.seen:
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
1120
                parent_searcher = left_searcher
1121
            else:
1122
                parent_searcher = right_searcher
1123
            revs_to_search = parent_searcher.find_seen_ancestors([revision_id])
1124
            if not revs_to_search: # XXX: This shouldn't be possible
1125
                revs_to_search = [revision_id]
3377.3.15 by John Arbash Meinel
minor update
1126
            searcher = self._make_breadth_first_searcher(revs_to_search)
1127
            # We don't care about the starting nodes.
1128
            searcher.step()
1129
            unique_searchers.append(searcher)
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
1130
3377.3.16 by John Arbash Meinel
small cleanups
1131
        # possible todo: aggregate the common searchers into a single common
1132
        #   searcher, just make sure that we include the nodes into the .seen
1133
        #   properties of the original searchers
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1134
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1135
        ancestor_all_unique = None
1136
        for searcher in unique_searchers:
1137
            if ancestor_all_unique is None:
1138
                ancestor_all_unique = set(searcher.seen)
1139
            else:
1140
                ancestor_all_unique = ancestor_all_unique.intersection(
1141
                                            searcher.seen)
1142
3377.3.23 by John Arbash Meinel
Implement find_unique_ancestors using more explicit graph searching.
1143
        trace.mutter('Started %s unique searchers for %s unique revisions',
1144
                     simple_unique, total_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
1145
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1146
        while True: # If we have no more nodes we have nothing to do
1147
            newly_seen_common = set()
1148
            for searcher in common_searchers:
1149
                newly_seen_common.update(searcher.step())
1150
            newly_seen_unique = set()
1151
            for searcher in unique_searchers:
1152
                newly_seen_unique.update(searcher.step())
1153
            new_common_unique = set()
1154
            for revision in newly_seen_unique:
1155
                for searcher in unique_searchers:
1156
                    if revision not in searcher.seen:
1157
                        break
1158
                else:
1159
                    # This is a border because it is a first common that we see
1160
                    # after walking for a while.
1161
                    new_common_unique.add(revision)
1162
            if newly_seen_common:
1163
                # These are nodes descended from one of the 'common' searchers.
1164
                # Make sure all searchers are on the same page
1165
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
1166
                    newly_seen_common.update(
1167
                        searcher.find_seen_ancestors(newly_seen_common))
3377.3.14 by John Arbash Meinel
Take another tack on _search_for_extra
1168
                # We start searching the whole ancestry. It is a bit wasteful,
1169
                # though. We really just want to mark all of these nodes as
1170
                # 'seen' and then start just the tips. However, it requires a
1171
                # get_parent_map() call to figure out the tips anyway, and all
1172
                # redundant requests should be fairly fast.
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1173
                for searcher in common_searchers:
1174
                    searcher.start_searching(newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
1175
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1176
                # If a 'common' node is an ancestor of all unique searchers, we
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
1177
                # can stop searching it.
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1178
                stop_searching_common = ancestor_all_unique.intersection(
1179
                                            newly_seen_common)
3377.3.13 by John Arbash Meinel
Change _search_for_extra_common slightly.
1180
                if stop_searching_common:
1181
                    for searcher in common_searchers:
1182
                        searcher.stop_searching_any(stop_searching_common)
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1183
            if new_common_unique:
3377.3.20 by John Arbash Meinel
comment cleanups.
1184
                # We found some ancestors that are common
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1185
                for searcher in unique_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
1186
                    new_common_unique.update(
1187
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1188
                # Since these are common, we can grab another set of ancestors
1189
                # that we have seen
1190
                for searcher in common_searchers:
3377.3.16 by John Arbash Meinel
small cleanups
1191
                    new_common_unique.update(
1192
                        searcher.find_seen_ancestors(new_common_unique))
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1193
1194
                # We can tell all of the unique searchers to start at these
1195
                # nodes, and tell all of the common searchers to *stop*
1196
                # searching these nodes
1197
                for searcher in unique_searchers:
1198
                    searcher.start_searching(new_common_unique)
1199
                for searcher in common_searchers:
1200
                    searcher.stop_searching_any(new_common_unique)
3377.3.17 by John Arbash Meinel
Keep track of the intersection of unique ancestry,
1201
                ancestor_all_unique.update(new_common_unique)
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
1202
3377.3.20 by John Arbash Meinel
comment cleanups.
1203
                # Filter out searchers that don't actually search different
1204
                # nodes. We already have the ancestry intersection for them
3377.3.19 by John Arbash Meinel
Start culling unique searchers once they converge.
1205
                next_unique_searchers = []
1206
                unique_search_sets = set()
1207
                for searcher in unique_searchers:
1208
                    will_search_set = frozenset(searcher._next_query)
1209
                    if will_search_set not in unique_search_sets:
1210
                        # This searcher is searching a unique set of nodes, let it
1211
                        unique_search_sets.add(will_search_set)
1212
                        next_unique_searchers.append(searcher)
1213
                unique_searchers = next_unique_searchers
3377.3.2 by John Arbash Meinel
find_difference is fixed by updating _find_border_ancestors.... is that reasonable?
1214
            for searcher in common_searchers:
1215
                if searcher._next_query:
1216
                    break
1217
            else:
1218
                # All common searcher have stopped searching
3377.3.16 by John Arbash Meinel
small cleanups
1219
                return
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1220
1221
    def _remove_simple_descendants(self, revisions, parent_map):
1222
        """remove revisions which are children of other ones in the set
1223
1224
        This doesn't do any graph searching, it just checks the immediate
1225
        parent_map to find if there are any children which can be removed.
1226
1227
        :param revisions: A set of revision_ids
1228
        :return: A set of revision_ids with the children removed
1229
        """
1230
        simple_ancestors = revisions.copy()
1231
        # TODO: jam 20071214 we *could* restrict it to searching only the
1232
        #       parent_map of revisions already present in 'revisions', but
1233
        #       considering the general use case, I think this is actually
1234
        #       better.
1235
1236
        # This is the same as the following loop. I don't know that it is any
1237
        # faster.
1238
        ## simple_ancestors.difference_update(r for r, p_ids in parent_map.iteritems()
1239
        ##     if p_ids is not None and revisions.intersection(p_ids))
1240
        ## return simple_ancestors
1241
1242
        # Yet Another Way, invert the parent map (which can be cached)
1243
        ## descendants = {}
1244
        ## for revision_id, parent_ids in parent_map.iteritems():
1245
        ##   for p_id in parent_ids:
1246
        ##       descendants.setdefault(p_id, []).append(revision_id)
1247
        ## for revision in revisions.intersection(descendants):
1248
        ##   simple_ancestors.difference_update(descendants[revision])
1249
        ## return simple_ancestors
1250
        for revision, parent_ids in parent_map.iteritems():
1251
            if parent_ids is None:
1252
                continue
1253
            for parent_id in parent_ids:
1254
                if parent_id in revisions:
1255
                    # This node has a parent present in the set, so we can
1256
                    # remove it
1257
                    simple_ancestors.discard(revision)
1258
                    break
1259
        return simple_ancestors
1260
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1261
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1262
class HeadsCache(object):
1263
    """A cache of results for graph heads calls."""
1264
1265
    def __init__(self, graph):
1266
        self.graph = graph
1267
        self._heads = {}
1268
1269
    def heads(self, keys):
1270
        """Return the heads of keys.
1271
2911.4.3 by Robert Collins
Make the contract of HeadsCache.heads() more clear.
1272
        This matches the API of Graph.heads(), specifically the return value is
1273
        a set which can be mutated, and ordering of the input is not preserved
1274
        in the output.
1275
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1276
        :see also: Graph.heads.
1277
        :param keys: The keys to calculate heads for.
1278
        :return: A set containing the heads, which may be mutated without
1279
            affecting future lookups.
1280
        """
2911.4.2 by Robert Collins
Make HeadsCache actually work.
1281
        keys = frozenset(keys)
2911.4.1 by Robert Collins
Factor out the Graph.heads() cache from _RevisionTextVersionCache for reuse, and use it in commit.
1282
        try:
1283
            return set(self._heads[keys])
1284
        except KeyError:
1285
            heads = self.graph.heads(keys)
1286
            self._heads[keys] = heads
1287
            return set(heads)
1288
1289
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1290
class FrozenHeadsCache(object):
1291
    """Cache heads() calls, assuming the caller won't modify them."""
1292
1293
    def __init__(self, graph):
1294
        self.graph = graph
1295
        self._heads = {}
1296
1297
    def heads(self, keys):
1298
        """Return the heads of keys.
1299
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
1300
        Similar to Graph.heads(). The main difference is that the return value
1301
        is a frozen set which cannot be mutated.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1302
1303
        :see also: Graph.heads.
1304
        :param keys: The keys to calculate heads for.
3224.1.24 by John Arbash Meinel
Fix up docstring since FrozenHeadsCache doesn't let you mutate the result.
1305
        :return: A frozenset containing the heads.
3224.1.20 by John Arbash Meinel
Reduce the number of cache misses by caching known heads answers
1306
        """
1307
        keys = frozenset(keys)
1308
        try:
1309
            return self._heads[keys]
1310
        except KeyError:
1311
            heads = frozenset(self.graph.heads(keys))
1312
            self._heads[keys] = heads
1313
            return heads
1314
1315
    def cache(self, keys, heads):
1316
        """Store a known value."""
1317
        self._heads[frozenset(keys)] = frozenset(heads)
1318
1319
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
1320
class _BreadthFirstSearcher(object):
2921.3.4 by Robert Collins
Review feedback.
1321
    """Parallel search breadth-first the ancestry of revisions.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1322
1323
    This class implements the iterator protocol, but additionally
1324
    1. provides a set of seen ancestors, and
1325
    2. allows some ancestries to be unsearched, via stop_searching_any
1326
    """
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1327
2490.2.22 by Aaron Bentley
Rename GraphWalker -> Graph, _AncestryWalker -> _BreadthFirstSearcher
1328
    def __init__(self, revisions, parents_provider):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1329
        self._iterations = 0
1330
        self._next_query = set(revisions)
1331
        self.seen = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1332
        self._started_keys = set(self._next_query)
1333
        self._stopped_keys = set()
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
1334
        self._parents_provider = parents_provider
3177.3.3 by Robert Collins
Review feedback.
1335
        self._returning = 'next_with_ghosts'
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1336
        self._current_present = set()
1337
        self._current_ghosts = set()
1338
        self._current_parents = {}
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1339
1340
    def __repr__(self):
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1341
        if self._iterations:
1342
            prefix = "searching"
3099.3.1 by John Arbash Meinel
Implement get_parent_map for ParentProviders
1343
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1344
            prefix = "starting"
1345
        search = '%s=%r' % (prefix, list(self._next_query))
1346
        return ('_BreadthFirstSearcher(iterations=%d, %s,'
1347
                ' seen=%r)' % (self._iterations, search, list(self.seen)))
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1348
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1349
    def get_result(self):
1350
        """Get a SearchResult for the current state of this searcher.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1351
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1352
        :return: A SearchResult for this search so far. The SearchResult is
1353
            static - the search can be advanced and the search result will not
1354
            be invalidated or altered.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1355
        """
1356
        if self._returning == 'next':
1357
            # We have to know the current nodes children to be able to list the
1358
            # exclude keys for them. However, while we could have a second
1359
            # look-ahead result buffer and shuffle things around, this method
1360
            # is typically only called once per search - when memoising the
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1361
            # results of the search.
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1362
            found, ghosts, next, parents = self._do_query(self._next_query)
1363
            # pretend we didn't query: perhaps we should tweak _do_query to be
1364
            # entirely stateless?
1365
            self.seen.difference_update(next)
3184.1.3 by Robert Collins
Automatically exclude ghosts.
1366
            next_query = next.union(ghosts)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1367
        else:
1368
            next_query = self._next_query
3184.1.5 by Robert Collins
Record the number of found revisions for cross checking.
1369
        excludes = self._stopped_keys.union(next_query)
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1370
        included_keys = self.seen.difference(excludes)
1371
        return SearchResult(self._started_keys, excludes, len(included_keys),
1372
            included_keys)
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1373
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1374
    def step(self):
1375
        try:
1376
            return self.next()
1377
        except StopIteration:
1378
            return ()
1379
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1380
    def next(self):
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1381
        """Return the next ancestors of this revision.
1382
2490.2.12 by Aaron Bentley
Improve documentation
1383
        Ancestors are returned in the order they are seen in a breadth-first
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1384
        traversal.  No ancestor will be returned more than once. Ancestors are
1385
        returned before their parentage is queried, so ghosts and missing
1386
        revisions (including the start revisions) are included in the result.
1387
        This can save a round trip in LCA style calculation by allowing
1388
        convergence to be detected without reading the data for the revision
1389
        the convergence occurs on.
1390
1391
        :return: A set of revision_ids.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1392
        """
3177.3.3 by Robert Collins
Review feedback.
1393
        if self._returning != 'next':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1394
            # switch to returning the query, not the results.
3177.3.3 by Robert Collins
Review feedback.
1395
            self._returning = 'next'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1396
            self._iterations += 1
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1397
        else:
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1398
            self._advance()
1399
        if len(self._next_query) == 0:
1400
            raise StopIteration()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1401
        # We have seen what we're querying at this point as we are returning
1402
        # the query, not the results.
1403
        self.seen.update(self._next_query)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1404
        return self._next_query
1405
1406
    def next_with_ghosts(self):
1407
        """Return the next found ancestors, with ghosts split out.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1408
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1409
        Ancestors are returned in the order they are seen in a breadth-first
1410
        traversal.  No ancestor will be returned more than once. Ancestors are
3177.3.3 by Robert Collins
Review feedback.
1411
        returned only after asking for their parents, which allows us to detect
1412
        which revisions are ghosts and which are not.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1413
1414
        :return: A tuple with (present ancestors, ghost ancestors) sets.
1415
        """
3177.3.3 by Robert Collins
Review feedback.
1416
        if self._returning != 'next_with_ghosts':
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1417
            # switch to returning the results, not the current query.
3177.3.3 by Robert Collins
Review feedback.
1418
            self._returning = 'next_with_ghosts'
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1419
            self._advance()
1420
        if len(self._next_query) == 0:
1421
            raise StopIteration()
1422
        self._advance()
1423
        return self._current_present, self._current_ghosts
1424
1425
    def _advance(self):
1426
        """Advance the search.
1427
1428
        Updates self.seen, self._next_query, self._current_present,
3177.3.3 by Robert Collins
Review feedback.
1429
        self._current_ghosts, self._current_parents and self._iterations.
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1430
        """
1431
        self._iterations += 1
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1432
        found, ghosts, next, parents = self._do_query(self._next_query)
1433
        self._current_present = found
1434
        self._current_ghosts = ghosts
1435
        self._next_query = next
1436
        self._current_parents = parents
3184.1.3 by Robert Collins
Automatically exclude ghosts.
1437
        # ghosts are implicit stop points, otherwise the search cannot be
1438
        # repeated when ghosts are filled.
1439
        self._stopped_keys.update(ghosts)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1440
1441
    def _do_query(self, revisions):
1442
        """Query for revisions.
1443
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1444
        Adds revisions to the seen set.
1445
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1446
        :param revisions: Revisions to query.
1447
        :return: A tuple: (set(found_revisions), set(ghost_revisions),
1448
           set(parents_of_found_revisions), dict(found_revisions:parents)).
1449
        """
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1450
        found_revisions = set()
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1451
        parents_of_found = set()
3184.1.1 by Robert Collins
Add basic get_recipe to the graph breadth first searcher.
1452
        # revisions may contain nodes that point to other nodes in revisions:
1453
        # we want to filter them out.
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
1454
        seen = self.seen
1455
        seen.update(revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1456
        parent_map = self._parents_provider.get_parent_map(revisions)
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1457
        found_revisions.update(parent_map)
3177.3.1 by Robert Collins
* New method ``next_with_ghosts`` on the Graph breadth-first-search objects
1458
        for rev_id, parents in parent_map.iteritems():
3517.4.2 by Martin Pool
Make simple-annotation and graph code more tolerant of knits with no graph
1459
            if parents is None:
1460
                continue
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
1461
            new_found_parents = [p for p in parents if p not in seen]
3377.3.9 by John Arbash Meinel
Small tweaks to _do_query
1462
            if new_found_parents:
1463
                # Calling set.update() with an empty generator is actually
1464
                # rather expensive.
1465
                parents_of_found.update(new_found_parents)
1466
        ghost_revisions = revisions - found_revisions
1467
        return found_revisions, ghost_revisions, parents_of_found, parent_map
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1468
2490.2.8 by Aaron Bentley
fix iteration stuff
1469
    def __iter__(self):
1470
        return self
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1471
3377.3.1 by John Arbash Meinel
Bring in some of the changes from graph_update and graph_optimization
1472
    def find_seen_ancestors(self, revisions):
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
1473
        """Find ancestors of these revisions that have already been seen.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1474
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
1475
        This function generally makes the assumption that querying for the
1476
        parents of a node that has already been queried is reasonably cheap.
1477
        (eg, not a round trip to a remote host).
1478
        """
1479
        # TODO: Often we might ask one searcher for its seen ancestors, and
1480
        #       then ask another searcher the same question. This can result in
1481
        #       searching the same revisions repeatedly if the two searchers
1482
        #       have a lot of overlap.
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1483
        all_seen = self.seen
1484
        pending = set(revisions).intersection(all_seen)
1485
        seen_ancestors = set(pending)
1486
1487
        if self._returning == 'next':
1488
            # self.seen contains what nodes have been returned, not what nodes
1489
            # have been queried. We don't want to probe for nodes that haven't
1490
            # been searched yet.
1491
            not_searched_yet = self._next_query
1492
        else:
1493
            not_searched_yet = ()
3377.3.11 by John Arbash Meinel
Committing a debug thunk that was very helpful
1494
        pending.difference_update(not_searched_yet)
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1495
        get_parent_map = self._parents_provider.get_parent_map
3377.3.12 by John Arbash Meinel
Remove the helpful but ugly thunk
1496
        while pending:
1497
            parent_map = get_parent_map(pending)
1498
            all_parents = []
1499
            # We don't care if it is a ghost, since it can't be seen if it is
1500
            # a ghost
1501
            for parent_ids in parent_map.itervalues():
1502
                all_parents.extend(parent_ids)
1503
            next_pending = all_seen.intersection(all_parents).difference(seen_ancestors)
1504
            seen_ancestors.update(next_pending)
1505
            next_pending.difference_update(not_searched_yet)
1506
            pending = next_pending
3377.3.10 by John Arbash Meinel
Tweak _BreadthFirstSearcher.find_seen_ancestors()
1507
2490.2.7 by Aaron Bentley
Start implementing mca that scales with number of uncommon ancestors
1508
        return seen_ancestors
1509
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1510
    def stop_searching_any(self, revisions):
1511
        """
1512
        Remove any of the specified revisions from the search list.
1513
1514
        None of the specified revisions are required to be present in the
3808.1.4 by John Arbash Meinel
make _walk_to_common responsible for stopping ancestors
1515
        search list.
3808.1.1 by Andrew Bennetts
Possible fix for bug in new _walk_to_common_revisions.
1516
3808.1.4 by John Arbash Meinel
make _walk_to_common responsible for stopping ancestors
1517
        It is okay to call stop_searching_any() for revisions which were seen
1518
        in previous iterations. It is the callers responsibility to call
1519
        find_seen_ancestors() to make sure that current search tips that are
1520
        ancestors of those revisions are also stopped.  All explicitly stopped
1521
        revisions will be excluded from the search result's get_keys(), though.
2490.2.10 by Aaron Bentley
Clarify text, remove unused _get_ancestry method
1522
        """
3377.4.6 by John Arbash Meinel
Lots of refactoring for find_unique_ancestors.
1523
        # TODO: does this help performance?
1524
        # if not revisions:
1525
        #     return set()
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1526
        revisions = frozenset(revisions)
3177.3.3 by Robert Collins
Review feedback.
1527
        if self._returning == 'next':
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1528
            stopped = self._next_query.intersection(revisions)
1529
            self._next_query = self._next_query.difference(revisions)
1530
        else:
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1531
            stopped_present = self._current_present.intersection(revisions)
1532
            stopped = stopped_present.union(
1533
                self._current_ghosts.intersection(revisions))
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1534
            self._current_present.difference_update(stopped)
1535
            self._current_ghosts.difference_update(stopped)
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1536
            # stopping 'x' should stop returning parents of 'x', but
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1537
            # not if 'y' always references those same parents
1538
            stop_rev_references = {}
3184.2.1 by Robert Collins
Handle stopping ghosts in searches properly.
1539
            for rev in stopped_present:
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1540
                for parent_id in self._current_parents[rev]:
1541
                    if parent_id not in stop_rev_references:
1542
                        stop_rev_references[parent_id] = 0
1543
                    stop_rev_references[parent_id] += 1
1544
            # if only the stopped revisions reference it, the ref count will be
1545
            # 0 after this loop
3177.3.3 by Robert Collins
Review feedback.
1546
            for parents in self._current_parents.itervalues():
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1547
                for parent_id in parents:
1548
                    try:
1549
                        stop_rev_references[parent_id] -= 1
1550
                    except KeyError:
1551
                        pass
1552
            stop_parents = set()
1553
            for rev_id, refs in stop_rev_references.iteritems():
1554
                if refs == 0:
1555
                    stop_parents.add(rev_id)
1556
            self._next_query.difference_update(stop_parents)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1557
        self._stopped_keys.update(stopped)
4053.2.2 by Andrew Bennetts
Better fix, with test.
1558
        self._stopped_keys.update(revisions)
2490.2.25 by Aaron Bentley
Update from review
1559
        return stopped
2490.2.17 by Aaron Bentley
Add start_searching, tweak stop_searching_any
1560
1561
    def start_searching(self, revisions):
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1562
        """Add revisions to the search.
1563
1564
        The parents of revisions will be returned from the next call to next()
1565
        or next_with_ghosts(). If next_with_ghosts was the most recently used
1566
        next* call then the return value is the result of looking up the
1567
        ghost/not ghost status of revisions. (A tuple (present, ghosted)).
1568
        """
1569
        revisions = frozenset(revisions)
3184.1.2 by Robert Collins
Add tests for starting and stopping searches in combination with get_recipe.
1570
        self._started_keys.update(revisions)
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1571
        new_revisions = revisions.difference(self.seen)
3177.3.3 by Robert Collins
Review feedback.
1572
        if self._returning == 'next':
3184.1.4 by Robert Collins
Correctly exclude ghosts when ghosts are started on an existing search.
1573
            self._next_query.update(new_revisions)
3377.3.30 by John Arbash Meinel
Can we avoid the extra _do_query in start_searching?
1574
            self.seen.update(new_revisions)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1575
        else:
1576
            # perform a query on revisions
3377.3.30 by John Arbash Meinel
Can we avoid the extra _do_query in start_searching?
1577
            revs, ghosts, query, parents = self._do_query(revisions)
1578
            self._stopped_keys.update(ghosts)
3177.3.2 by Robert Collins
Update graph searchers stop_searching_any and start_searching for next_with_ghosts.
1579
            self._current_present.update(revs)
1580
            self._current_ghosts.update(ghosts)
1581
            self._next_query.update(query)
1582
            self._current_parents.update(parents)
1583
            return revs, ghosts
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1584
1585
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1586
class AbstractSearchResult(object):
5535.3.50 by Andrew Bennetts
Add docstrings to AbstractSearch and AbstractSearchResult.
1587
    """The result of a search, describing a set of keys.
1588
    
1589
    Search results are typically used as the 'fetch_spec' parameter when
1590
    fetching revisions.
1591
1592
    :seealso: AbstractSearch
1593
    """
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1594
1595
    def get_recipe(self):
1596
        """Return a recipe that can be used to replay this search.
1597
1598
        The recipe allows reconstruction of the same results at a later date.
1599
5891.1.2 by Andrew Bennetts
Fix a bunch of docstring formatting nits, making pydoctor a bit happier.
1600
        :return: A tuple of `(search_kind_str, *details)`.  The details vary by
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1601
            kind of search result.
1602
        """
1603
        raise NotImplementedError(self.get_recipe)
1604
1605
    def get_network_struct(self):
1606
        """Return a tuple that can be transmitted via the HPSS protocol."""
1607
        raise NotImplementedError(self.get_network_struct)
1608
1609
    def get_keys(self):
1610
        """Return the keys found in this search.
1611
1612
        :return: A set of keys.
1613
        """
1614
        raise NotImplementedError(self.get_keys)
1615
1616
    def is_empty(self):
1617
        """Return false if the search lists 1 or more revisions."""
1618
        raise NotImplementedError(self.is_empty)
1619
1620
    def refine(self, seen, referenced):
1621
        """Create a new search by refining this search.
1622
1623
        :param seen: Revisions that have been satisfied.
1624
        :param referenced: Revision references observed while satisfying some
1625
            of this search.
1626
        :return: A search result.
1627
        """
1628
        raise NotImplementedError(self.refine)
1629
1630
1631
class AbstractSearch(object):
5535.3.50 by Andrew Bennetts
Add docstrings to AbstractSearch and AbstractSearchResult.
1632
    """A search that can be executed, producing a search result.
1633
1634
    :seealso: AbstractSearchResult
1635
    """
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1636
5536.3.1 by Andrew Bennetts
Rename get_search_result to execute, require a SearchResult (and not a search) be passed to fetch.
1637
    def execute(self):
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1638
        """Construct a network-ready search result from this search description.
1639
1640
        This may take some time to search repositories, etc.
1641
5536.3.1 by Andrew Bennetts
Rename get_search_result to execute, require a SearchResult (and not a search) be passed to fetch.
1642
        :return: A search result (an object that implements
1643
            AbstractSearchResult's API).
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1644
        """
5536.3.1 by Andrew Bennetts
Rename get_search_result to execute, require a SearchResult (and not a search) be passed to fetch.
1645
        raise NotImplementedError(self.execute)
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1646
1647
1648
class SearchResult(AbstractSearchResult):
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1649
    """The result of a breadth first search.
1650
1651
    A SearchResult provides the ability to reconstruct the search or access a
1652
    set of the keys the search found.
1653
    """
1654
1655
    def __init__(self, start_keys, exclude_keys, key_count, keys):
1656
        """Create a SearchResult.
1657
1658
        :param start_keys: The keys the search started at.
1659
        :param exclude_keys: The keys the search excludes.
1660
        :param key_count: The total number of keys (from start to but not
1661
            including exclude).
1662
        :param keys: The keys the search found. Note that in future we may get
1663
            a SearchResult from a smart server, in which case the keys list is
1664
            not necessarily immediately available.
1665
        """
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1666
        self._recipe = ('search', start_keys, exclude_keys, key_count)
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1667
        self._keys = frozenset(keys)
1668
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1669
    def __repr__(self):
1670
        kind, start_keys, exclude_keys, key_count = self._recipe
1671
        if len(start_keys) > 5:
1672
            start_keys_repr = repr(list(start_keys)[:5])[:-1] + ', ...]'
1673
        else:
1674
            start_keys_repr = repr(start_keys)
1675
        if len(exclude_keys) > 5:
1676
            exclude_keys_repr = repr(list(exclude_keys)[:5])[:-1] + ', ...]'
1677
        else:
1678
            exclude_keys_repr = repr(exclude_keys)
1679
        return '<%s %s:(%s, %s, %d)>' % (self.__class__.__name__,
1680
            kind, start_keys_repr, exclude_keys_repr, key_count)
1681
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1682
    def get_recipe(self):
1683
        """Return a recipe that can be used to replay this search.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1684
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1685
        The recipe allows reconstruction of the same results at a later date
1686
        without knowing all the found keys. The essential elements are a list
4031.3.1 by Frank Aspell
Fixing various typos
1687
        of keys to start and to stop at. In order to give reproducible
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1688
        results when ghosts are encountered by a search they are automatically
1689
        added to the exclude list (or else ghost filling may alter the
1690
        results).
1691
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1692
        :return: A tuple ('search', start_keys_set, exclude_keys_set,
1693
            revision_count). To recreate the results of this search, create a
1694
            breadth first searcher on the same graph starting at start_keys.
1695
            Then call next() (or next_with_ghosts()) repeatedly, and on every
1696
            result, call stop_searching_any on any keys from the exclude_keys
1697
            set. The revision_count value acts as a trivial cross-check - the
1698
            found revisions of the new search should have as many elements as
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1699
            revision_count. If it does not, then additional revisions have been
1700
            ghosted since the search was executed the first time and the second
1701
            time.
1702
        """
1703
        return self._recipe
1704
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1705
    def get_network_struct(self):
1706
        start_keys = ' '.join(self._recipe[1])
1707
        stop_keys = ' '.join(self._recipe[2])
1708
        count = str(self._recipe[3])
1709
        return (self._recipe[0], '\n'.join((start_keys, stop_keys, count)))
1710
3184.1.6 by Robert Collins
Create a SearchResult object which can be used as a replacement for sets.
1711
    def get_keys(self):
1712
        """Return the keys found in this search.
1713
1714
        :return: A set of keys.
1715
        """
1716
        return self._keys
1717
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1718
    def is_empty(self):
4204.2.2 by Matt Nordhoff
Fix docstrings on graph.py's is_empty methods that said they returned true when they were *not* empty.
1719
        """Return false if the search lists 1 or more revisions."""
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1720
        return self._recipe[3] == 0
1721
1722
    def refine(self, seen, referenced):
1723
        """Create a new search by refining this search.
1724
1725
        :param seen: Revisions that have been satisfied.
1726
        :param referenced: Revision references observed while satisfying some
1727
            of this search.
1728
        """
1729
        start = self._recipe[1]
1730
        exclude = self._recipe[2]
1731
        count = self._recipe[3]
1732
        keys = self.get_keys()
1733
        # New heads = referenced + old heads - seen things - exclude
1734
        pending_refs = set(referenced)
1735
        pending_refs.update(start)
1736
        pending_refs.difference_update(seen)
1737
        pending_refs.difference_update(exclude)
1738
        # New exclude = old exclude + satisfied heads
1739
        seen_heads = start.intersection(seen)
1740
        exclude.update(seen_heads)
1741
        # keys gets seen removed
1742
        keys = keys - seen
1743
        # length is reduced by len(seen)
1744
        count -= len(seen)
1745
        return SearchResult(pending_refs, exclude, count, keys)
1746
3514.2.14 by John Arbash Meinel
Bring in the code to collapse linear portions of the graph.
1747
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1748
class PendingAncestryResult(AbstractSearchResult):
4070.9.14 by Andrew Bennetts
Tweaks requested by Robert's review.
1749
    """A search result that will reconstruct the ancestry for some graph heads.
4086.1.4 by Andrew Bennetts
Fix whitespace nit.
1750
4070.9.14 by Andrew Bennetts
Tweaks requested by Robert's review.
1751
    Unlike SearchResult, this doesn't hold the complete search result in
1752
    memory, it just holds a description of how to generate it.
1753
    """
1754
1755
    def __init__(self, heads, repo):
1756
        """Constructor.
1757
1758
        :param heads: an iterable of graph heads.
1759
        :param repo: a repository to use to generate the ancestry for the given
1760
            heads.
1761
        """
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1762
        self.heads = frozenset(heads)
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1763
        self.repo = repo
1764
5539.2.20 by Andrew Bennetts
Add PendingAncestryResult.__repr__.
1765
    def __repr__(self):
1766
        if len(self.heads) > 5:
5535.4.21 by Andrew Bennetts
Slightly more informative __repr__.
1767
            heads_repr = repr(list(self.heads)[:5])[:-1]
1768
            heads_repr += ', <%d more>...]' % (len(self.heads) - 5,)
5539.2.20 by Andrew Bennetts
Add PendingAncestryResult.__repr__.
1769
        else:
1770
            heads_repr = repr(self.heads)
1771
        return '<%s heads:%s repo:%r>' % (
1772
            self.__class__.__name__, heads_repr, self.repo)
1773
4070.9.5 by Andrew Bennetts
Better wire protocol: don't shoehorn MiniSearchResult serialisation into previous serialisation format.
1774
    def get_recipe(self):
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1775
        """Return a recipe that can be used to replay this search.
1776
1777
        The recipe allows reconstruction of the same results at a later date.
1778
1779
        :seealso SearchResult.get_recipe:
1780
1781
        :return: A tuple ('proxy-search', start_keys_set, set(), -1)
1782
            To recreate this result, create a PendingAncestryResult with the
1783
            start_keys_set.
1784
        """
1785
        return ('proxy-search', self.heads, set(), -1)
4070.9.5 by Andrew Bennetts
Better wire protocol: don't shoehorn MiniSearchResult serialisation into previous serialisation format.
1786
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1787
    def get_network_struct(self):
1788
        parts = ['ancestry-of']
1789
        parts.extend(self.heads)
1790
        return parts
1791
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1792
    def get_keys(self):
4098.1.1 by Andrew Bennetts
Fix a bug with how PendingAncestryResult.get_keys handles NULL_REVISION.
1793
        """See SearchResult.get_keys.
4098.1.2 by Andrew Bennetts
Fix 'trailing' whitespace (actually just a blank line in an indented docstring).
1794
4098.1.1 by Andrew Bennetts
Fix a bug with how PendingAncestryResult.get_keys handles NULL_REVISION.
1795
        Returns all the keys for the ancestry of the heads, excluding
1796
        NULL_REVISION.
1797
        """
1798
        return self._get_keys(self.repo.get_graph())
4098.1.3 by Andrew Bennetts
Fix 'trailing' whitespace (actually just a blank line between methods).
1799
4098.1.1 by Andrew Bennetts
Fix a bug with how PendingAncestryResult.get_keys handles NULL_REVISION.
1800
    def _get_keys(self, graph):
1801
        NULL_REVISION = revision.NULL_REVISION
1802
        keys = [key for (key, parents) in graph.iter_ancestry(self.heads)
4343.3.11 by John Arbash Meinel
Change PendingAncestryResult to strip ghosts from .get_keys()
1803
                if key != NULL_REVISION and parents is not None]
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1804
        return keys
1805
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1806
    def is_empty(self):
4204.2.2 by Matt Nordhoff
Fix docstrings on graph.py's is_empty methods that said they returned true when they were *not* empty.
1807
        """Return false if the search lists 1 or more revisions."""
4152.1.2 by Robert Collins
Add streaming from a stacked branch when the sort order is compatible with doing so.
1808
        if revision.NULL_REVISION in self.heads:
1809
            return len(self.heads) == 1
1810
        else:
1811
            return len(self.heads) == 0
1812
1813
    def refine(self, seen, referenced):
1814
        """Create a new search by refining this search.
1815
1816
        :param seen: Revisions that have been satisfied.
1817
        :param referenced: Revision references observed while satisfying some
1818
            of this search.
1819
        """
1820
        referenced = self.heads.union(referenced)
1821
        return PendingAncestryResult(referenced - seen, self.repo)
1822
4070.9.2 by Andrew Bennetts
Rough prototype of allowing a SearchResult to be passed to fetch, and using that to improve network conversations.
1823
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1824
class EmptySearchResult(AbstractSearchResult):
5539.2.15 by Andrew Bennetts
Add a docstring.
1825
    """An empty search result."""
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1826
1827
    def is_empty(self):
1828
        return True
1829
    
1830
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1831
class EverythingResult(AbstractSearchResult):
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1832
    """A search result that simply requests everything in the repository."""
1833
1834
    def __init__(self, repo):
1835
        self._repo = repo
1836
5535.3.33 by Andrew Bennetts
Fix a bug.
1837
    def __repr__(self):
1838
        return '%s(%r)' % (self.__class__.__name__, self._repo)
1839
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1840
    def get_recipe(self):
1841
        raise NotImplementedError(self.get_recipe)
1842
1843
    def get_network_struct(self):
1844
        return ('everything',)
1845
1846
    def get_keys(self):
1847
        if 'evil' in debug.debug_flags:
1848
            from bzrlib import remote
1849
            if isinstance(self._repo, remote.RemoteRepository):
1850
                # warn developers (not users) not to do this
1851
                trace.mutter_callsite(
1852
                    2, "EverythingResult(RemoteRepository).get_keys() is slow.")
5539.2.4 by Andrew Bennetts
Add some basic tests for the new verb, fix some shallow bugs.
1853
        return self._repo.all_revision_ids()
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1854
1855
    def is_empty(self):
1856
        # It's ok for this to wrongly return False: the worst that can happen
1857
        # is that RemoteStreamSource will initiate a get_stream on an empty
1858
        # repository.  And almost all repositories are non-empty.
1859
        return False
1860
1861
    def refine(self, seen, referenced):
5539.3.3 by Andrew Bennetts
Implement EverythingResult.refine.
1862
        heads = set(self._repo.all_revision_ids())
1863
        heads.difference_update(seen)
1864
        heads.update(referenced)
1865
        return PendingAncestryResult(heads, self._repo)
5539.2.1 by Andrew Bennetts
Start defining an 'everything' fetch spec.
1866
1867
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1868
class EverythingNotInOther(AbstractSearch):
1869
    """Find all revisions in that are in one repo but not the other."""
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1870
1871
    def __init__(self, to_repo, from_repo, find_ghosts=False):
1872
        self.to_repo = to_repo
1873
        self.from_repo = from_repo
1874
        self.find_ghosts = find_ghosts
1875
5536.3.1 by Andrew Bennetts
Rename get_search_result to execute, require a SearchResult (and not a search) be passed to fetch.
1876
    def execute(self):
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1877
        return self.to_repo.search_missing_revision_ids(
1878
            self.from_repo, find_ghosts=self.find_ghosts)
1879
1880
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1881
class NotInOtherForRevs(AbstractSearch):
1882
    """Find all revisions missing in one repo for a some specific heads."""
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1883
5535.3.31 by Andrew Bennetts
Cope with tags that reference missing revisions.
1884
    def __init__(self, to_repo, from_repo, required_ids, if_present_ids=None,
5852.1.5 by Andrew Bennetts, Jelmer Vernooij
Support limit= for fetching between Bazaar branches.
1885
            find_ghosts=False, limit=None):
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1886
        """Constructor.
1887
1888
        :param required_ids: revision IDs of heads that must be found, or else
1889
            the search will fail with NoSuchRevision.  All revisions in their
1890
            ancestry not already in the other repository will be included in
1891
            the search result.
1892
        :param if_present_ids: revision IDs of heads that may be absent in the
1893
            source repository.  If present, then their ancestry not already
1894
            found in other will be included in the search result.
5852.1.5 by Andrew Bennetts, Jelmer Vernooij
Support limit= for fetching between Bazaar branches.
1895
        :param limit: maximum number of revisions to fetch
5539.2.19 by Andrew Bennetts
Define SearchResult/Search interfaces with explicit abstract base classes, add some docstrings and change a method name.
1896
        """
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1897
        self.to_repo = to_repo
1898
        self.from_repo = from_repo
1899
        self.find_ghosts = find_ghosts
5535.3.31 by Andrew Bennetts
Cope with tags that reference missing revisions.
1900
        self.required_ids = required_ids
1901
        self.if_present_ids = if_present_ids
5852.1.5 by Andrew Bennetts, Jelmer Vernooij
Support limit= for fetching between Bazaar branches.
1902
        self.limit = limit
5535.3.31 by Andrew Bennetts
Cope with tags that reference missing revisions.
1903
1904
    def __repr__(self):
1905
        if len(self.required_ids) > 5:
1906
            reqd_revs_repr = repr(list(self.required_ids)[:5])[:-1] + ', ...]'
1907
        else:
1908
            reqd_revs_repr = repr(self.required_ids)
1909
        if self.if_present_ids and len(self.if_present_ids) > 5:
1910
            ifp_revs_repr = repr(list(self.if_present_ids)[:5])[:-1] + ', ...]'
1911
        else:
1912
            ifp_revs_repr = repr(self.if_present_ids)
1913
5852.1.5 by Andrew Bennetts, Jelmer Vernooij
Support limit= for fetching between Bazaar branches.
1914
        return ("<%s from:%r to:%r find_ghosts:%r req'd:%r if-present:%r"
1915
                "limit:%r>") % (
1916
                self.__class__.__name__, self.from_repo, self.to_repo,
1917
                self.find_ghosts, reqd_revs_repr, ifp_revs_repr,
1918
                self.limit)
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1919
5536.3.1 by Andrew Bennetts
Rename get_search_result to execute, require a SearchResult (and not a search) be passed to fetch.
1920
    def execute(self):
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1921
        return self.to_repo.search_missing_revision_ids(
5535.3.31 by Andrew Bennetts
Cope with tags that reference missing revisions.
1922
            self.from_repo, revision_ids=self.required_ids,
5852.1.5 by Andrew Bennetts, Jelmer Vernooij
Support limit= for fetching between Bazaar branches.
1923
            if_present_ids=self.if_present_ids, find_ghosts=self.find_ghosts,
1924
            limit=self.limit)
5539.2.8 by Andrew Bennetts
Refactor call to search_missing_revision_ids out from RepoFetcher._revids_to_fetch into the 'fetch spec'.
1925
1926
6015.23.4 by John Arbash Meinel
Prototype the walk-backwards-and-then-forwards code.
1927
def invert_parent_map(parent_map):
1928
    """Given a map from child => parents, create a map of parent=>children"""
1929
    child_map = {}
1930
    for child, parents in parent_map.iteritems():
1931
        for p in parents:
1932
            # Any given parent is likely to have only a small handful
1933
            # of children, many will have only one. So we avoid mem overhead of
1934
            # a list, in exchange for extra copying of tuples
1935
            if p not in child_map:
1936
                child_map[p] = (child,)
1937
            else:
1938
                child_map[p] = child_map[p] + (child,)
1939
    return child_map
1940
1941
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
1942
def _find_possible_heads(parent_map, tip_keys, depth):
6015.23.10 by John Arbash Meinel
Small tweaks to search performance, though still at depth=100 the primary time
1943
    """Walk backwards (towards children) through the parent_map.
1944
1945
    This finds 'heads' that will hopefully succinctly describe our search
1946
    graph.
1947
    """
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
1948
    child_map = invert_parent_map(parent_map)
1949
    heads = set()
6015.23.10 by John Arbash Meinel
Small tweaks to search performance, though still at depth=100 the primary time
1950
    current_roots = tip_keys
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
1951
    walked = set(current_roots)
6015.23.10 by John Arbash Meinel
Small tweaks to search performance, though still at depth=100 the primary time
1952
    while current_roots and depth > 0:
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
1953
        depth -= 1
1954
        children = set()
6015.23.10 by John Arbash Meinel
Small tweaks to search performance, though still at depth=100 the primary time
1955
        children_update = children.update
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
1956
        for p in current_roots:
1957
            # Is it better to pre- or post- filter the children?
1958
            try:
6015.23.10 by John Arbash Meinel
Small tweaks to search performance, though still at depth=100 the primary time
1959
                children_update(child_map[p])
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
1960
            except KeyError:
1961
                heads.add(p)
6015.23.10 by John Arbash Meinel
Small tweaks to search performance, though still at depth=100 the primary time
1962
        # If we've seen a key before, we don't want to walk it again. Note that
1963
        # 'children' stays relatively small while 'walked' grows large. So
1964
        # don't use 'difference_update' here which has to walk all of 'walked'.
1965
        # '.difference' is smart enough to walk only children and compare it to
1966
        # walked.
1967
        children = children.difference(walked)
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
1968
        walked.update(children)
1969
        current_roots = children
1970
    if current_roots:
6015.23.10 by John Arbash Meinel
Small tweaks to search performance, though still at depth=100 the primary time
1971
        # We walked to the end of depth, so these are the new tips.
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
1972
        heads.update(current_roots)
1973
    return heads
1974
1975
1976
def _run_search(parent_map, heads, exclude_keys):
6015.23.15 by John Arbash Meinel
Clean out more of the cruft that got left by accident.
1977
    """Given a parent map, run a _BreadthFirstSearcher on it.
1978
1979
    Start at heads, walk until you hit exclude_keys. As a further improvement,
1980
    watch for any heads that you encounter while walking, which means they were
1981
    not heads of the search.
1982
1983
    This is mostly used to generate a succinct recipe for how to walk through
1984
    most of parent_map.
1985
1986
    :return: (_BreadthFirstSearcher, set(heads_encountered_by_walking))
1987
    """
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
1988
    g = Graph(DictParentsProvider(parent_map))
1989
    s = g._make_breadth_first_searcher(heads)
1990
    found_heads = set()
1991
    while True:
1992
        try:
1993
            next_revs = s.next()
1994
        except StopIteration:
1995
            break
1996
        for parents in s._current_parents.itervalues():
1997
            f_heads = heads.intersection(parents)
1998
            if f_heads:
1999
                found_heads.update(f_heads)
2000
        stop_keys = exclude_keys.intersection(next_revs)
2001
        if stop_keys:
2002
            s.stop_searching_any(stop_keys)
2003
    for parents in s._current_parents.itervalues():
2004
        f_heads = heads.intersection(parents)
2005
        if f_heads:
2006
            found_heads.update(f_heads)
2007
    return s, found_heads
2008
2009
6015.23.5 by John Arbash Meinel
Implement something that seems to work for limited search recipies.
2010
def limited_search_result_from_parent_map(parent_map, missing_keys, tip_keys,
2011
                                          depth):
6015.23.4 by John Arbash Meinel
Prototype the walk-backwards-and-then-forwards code.
2012
    """Transform a parent_map that is searching 'tip_keys' into an
2013
    approximate SearchResult.
2014
2015
    We should be able to generate a SearchResult from a given set of starting
2016
    keys, that covers a subset of parent_map that has the last step pointing at
2017
    tip_keys. This is to handle the case that really-long-searches shouldn't be
2018
    started from scratch on each get_parent_map request, but we *do* want to
2019
    filter out some of the keys that we've already seen, so we don't get
2020
    information that we already know about on every request.
2021
2022
    The server will validate the search (that starting at start_keys and
2023
    stopping at stop_keys yields the exact key_count), so we have to be careful
2024
    to give an exact recipe.
2025
2026
    Basic algorithm is:
2027
        1) Invert parent_map to get child_map (todo: have it cached and pass it
2028
           in)
2029
        2) Starting at tip_keys, walk towards children for 'depth' steps.
2030
        3) At that point, we have the 'start' keys.
2031
        4) Start walking parent_map from 'start' keys, counting how many keys
2032
           are seen, and generating stop_keys for anything that would walk
2033
           outside of the parent_map.
2034
2035
    :param parent_map: A map from {child_id: (parent_ids,)}
2036
    :param missing_keys: parent_ids that we know are unavailable
2037
    :param tip_keys: the revision_ids that we are searching
2038
    :param depth: How far back to walk.
2039
    """
6015.23.17 by John Arbash Meinel
Code was relying on an empty parent map to yield an empty search.
2040
    if not parent_map:
2041
        # No search to send, because we haven't done any searching yet.
2042
        return [], [], 0
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
2043
    heads = _find_possible_heads(parent_map, tip_keys, depth)
6015.23.13 by John Arbash Meinel
Don't walk the graph a second time.
2044
    s, found_heads = _run_search(parent_map, heads, set(tip_keys))
2045
    _, start_keys, exclude_keys, key_count = s.get_result().get_recipe()
6015.23.7 by John Arbash Meinel
Run the search a 3rd time if we encounter heads.
2046
    if found_heads:
6015.23.13 by John Arbash Meinel
Don't walk the graph a second time.
2047
        # Anything in found_heads are redundant start_keys, we hit them while
2048
        # walking, so we can exclude them from the start list.
2049
        start_keys = set(start_keys).difference(found_heads)
2050
    return start_keys, exclude_keys, key_count
6015.23.4 by John Arbash Meinel
Prototype the walk-backwards-and-then-forwards code.
2051
2052
6015.23.3 by John Arbash Meinel
Start refactoring code into graph.py code for easier testing.
2053
def search_result_from_parent_map(parent_map, missing_keys):
2054
    """Transform a parent_map into SearchResult information."""
2055
    if not parent_map:
2056
        # parent_map is empty or None, simple search result
2057
        return [], [], 0
2058
    # start_set is all the keys in the cache
2059
    start_set = set(parent_map)
2060
    # result set is all the references to keys in the cache
2061
    result_parents = set()
2062
    for parents in parent_map.itervalues():
2063
        result_parents.update(parents)
2064
    stop_keys = result_parents.difference(start_set)
2065
    # We don't need to send ghosts back to the server as a position to
2066
    # stop either.
2067
    stop_keys.difference_update(missing_keys)
2068
    key_count = len(parent_map)
2069
    if (revision.NULL_REVISION in result_parents
2070
        and revision.NULL_REVISION in missing_keys):
2071
        # If we pruned NULL_REVISION from the stop_keys because it's also
2072
        # in our cache of "missing" keys we need to increment our key count
2073
        # by 1, because the reconsitituted SearchResult on the server will
2074
        # still consider NULL_REVISION to be an included key.
2075
        key_count += 1
2076
    included_keys = start_set.intersection(result_parents)
2077
    start_set.difference_update(included_keys)
2078
    return start_set, stop_keys, key_count
2079
2080
3514.2.14 by John Arbash Meinel
Bring in the code to collapse linear portions of the graph.
2081
def collapse_linear_regions(parent_map):
2082
    """Collapse regions of the graph that are 'linear'.
2083
2084
    For example::
2085
2086
      A:[B], B:[C]
2087
2088
    can be collapsed by removing B and getting::
2089
2090
      A:[C]
2091
2092
    :param parent_map: A dictionary mapping children to their parents
2093
    :return: Another dictionary with 'linear' chains collapsed
2094
    """
2095
    # Note: this isn't a strictly minimal collapse. For example:
2096
    #   A
2097
    #  / \
2098
    # B   C
2099
    #  \ /
2100
    #   D
2101
    #   |
2102
    #   E
2103
    # Will not have 'D' removed, even though 'E' could fit. Also:
2104
    #   A
2105
    #   |    A
2106
    #   B => |
2107
    #   |    C
2108
    #   C
2109
    # A and C are both kept because they are edges of the graph. We *could* get
2110
    # rid of A if we wanted.
2111
    #   A
2112
    #  / \
2113
    # B   C
2114
    # |   |
2115
    # D   E
2116
    #  \ /
2117
    #   F
2118
    # Will not have any nodes removed, even though you do have an
2119
    # 'uninteresting' linear D->B and E->C
2120
    children = {}
2121
    for child, parents in parent_map.iteritems():
2122
        children.setdefault(child, [])
2123
        for p in parents:
2124
            children.setdefault(p, []).append(child)
2125
2126
    orig_children = dict(children)
2127
    removed = set()
2128
    result = dict(parent_map)
2129
    for node in parent_map:
2130
        parents = result[node]
2131
        if len(parents) == 1:
2132
            parent_children = children[parents[0]]
2133
            if len(parent_children) != 1:
2134
                # This is not the only child
2135
                continue
2136
            node_children = children[node]
2137
            if len(node_children) != 1:
2138
                continue
2139
            child_parents = result.get(node_children[0], None)
2140
            if len(child_parents) != 1:
2141
                # This is not its only parent
2142
                continue
2143
            # The child of this node only points at it, and the parent only has
2144
            # this as a child. remove this node, and join the others together
2145
            result[node_children[0]] = parents
2146
            children[parents[0]] = node_children
2147
            del result[node]
2148
            del children[node]
2149
            removed.add(node)
2150
2151
    return result
4371.3.18 by John Arbash Meinel
Change VF.annotate to use the new KnownGraph code.
2152
2153
4819.2.3 by John Arbash Meinel
Add a GraphThunkIdsToKeys as a tested class.
2154
class GraphThunkIdsToKeys(object):
2155
    """Forwards calls about 'ids' to be about keys internally."""
2156
2157
    def __init__(self, graph):
2158
        self._graph = graph
2159
4913.4.4 by Jelmer Vernooij
Add test for Repository.get_known_graph_ancestry().
2160
    def topo_sort(self):
2161
        return [r for (r,) in self._graph.topo_sort()]
2162
4819.2.3 by John Arbash Meinel
Add a GraphThunkIdsToKeys as a tested class.
2163
    def heads(self, ids):
2164
        """See Graph.heads()"""
2165
        as_keys = [(i,) for i in ids]
2166
        head_keys = self._graph.heads(as_keys)
2167
        return set([h[0] for h in head_keys])
2168
4913.4.2 by Jelmer Vernooij
Add Repository.get_known_graph_ancestry.
2169
    def merge_sort(self, tip_revision):
5988.1.1 by Jelmer Vernooij
Fix GraphThunkIdsToKeys.merge_sort
2170
        nodes = self._graph.merge_sort((tip_revision,))
2171
        for node in nodes:
2172
            node.key = node.key[0]
2173
        return nodes
4913.4.2 by Jelmer Vernooij
Add Repository.get_known_graph_ancestry.
2174
5559.3.1 by Jelmer Vernooij
Add GraphThunkIdsToKeys.add_node.
2175
    def add_node(self, revision, parents):
2176
        self._graph.add_node((revision,), [(p,) for p in parents])
2177
4819.2.3 by John Arbash Meinel
Add a GraphThunkIdsToKeys as a tested class.
2178
4371.3.38 by John Arbash Meinel
Add a failing test for handling nodes that are in the same linear chain.
2179
_counters = [0,0,0,0,0,0,0]
4371.3.18 by John Arbash Meinel
Change VF.annotate to use the new KnownGraph code.
2180
try:
2181
    from bzrlib._known_graph_pyx import KnownGraph
4574.3.6 by Martin Pool
More warnings when failing to load extensions
2182
except ImportError, e:
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
2183
    osutils.failed_to_load_extension(e)
4371.3.18 by John Arbash Meinel
Change VF.annotate to use the new KnownGraph code.
2184
    from bzrlib._known_graph_py import KnownGraph