~bzr-pqm/bzr/bzr.dev

3640.2.4 by John Arbash Meinel
Copyright updates
1
# Copyright (C) 2007, 2008 Canonical Ltd
2474.1.1 by John Arbash Meinel
Create a Pyrex extension for reading the dirstate file.
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
2474.1.1 by John Arbash Meinel
Create a Pyrex extension for reading the dirstate file.
16
17
"""Helper functions for DirState.
18
19
This is the python implementation for DirState functions.
20
"""
21
3696.4.3 by Robert Collins
Some tuning of update_entry.
22
import binascii
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
23
import bisect
24
import errno
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
25
import os
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
26
import stat
3763.1.1 by Benjamin Peterson
fix two small oversights
27
import sys
3696.4.3 by Robert Collins
Some tuning of update_entry.
28
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
29
from bzrlib import cache_utf8, errors, osutils
3696.4.17 by Robert Collins
Review feedback.
30
from bzrlib.dirstate import DirState
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
31
from bzrlib.osutils import parent_directories, pathjoin, splitpath
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
32
33
34
# This is the Windows equivalent of ENOTDIR
35
# It is defined in pywin32.winerror, but we don't want a strong dependency for
36
# just an error code.
37
# XXX: Perhaps we could get it from a windows header ?
38
cdef int ERROR_PATH_NOT_FOUND
39
ERROR_PATH_NOT_FOUND = 3
40
cdef int ERROR_DIRECTORY
41
ERROR_DIRECTORY = 267
2474.1.1 by John Arbash Meinel
Create a Pyrex extension for reading the dirstate file.
42
3737.1.3 by John Arbash Meinel
Move more compatibility code into python-compat.h
43
#python2.4 support, and other platform-dependent includes
3731.1.1 by Robert Collins
* The C extensions now build on python 2.4 (Robert Collins, #271939)
44
cdef extern from "python-compat.h":
3737.1.3 by John Arbash Meinel
Move more compatibility code into python-compat.h
45
    unsigned long htonl(unsigned long)
3731.1.1 by Robert Collins
* The C extensions now build on python 2.4 (Robert Collins, #271939)
46
2474.1.72 by John Arbash Meinel
Document a bit more what is going on in _dirstate_helpers_c.pyx, from Martin's comments
47
# Give Pyrex some function definitions for it to understand.
48
# All of these are just hints to Pyrex, so that it can try to convert python
49
# objects into similar C objects. (such as PyInt => int).
50
# In anything defined 'cdef extern from XXX' the real C header will be
51
# imported, and the real definition will be used from there. So these are just
52
# hints, and do not need to match exactly to the C definitions.
53
2474.1.13 by John Arbash Meinel
Now that we have bisect_dirblock working again, bring back cmp_dirblock_strings.
54
cdef extern from *:
2474.1.72 by John Arbash Meinel
Document a bit more what is going on in _dirstate_helpers_c.pyx, from Martin's comments
55
    ctypedef unsigned long size_t
2474.1.69 by John Arbash Meinel
Thanks to Jan 'RedBully' Seiffert, some review cleanups
56
4459.2.1 by Vincent Ladeuil
Use a consistent scheme for naming pyrex source files.
57
cdef extern from "_dirstate_helpers_pyx.h":
2474.1.69 by John Arbash Meinel
Thanks to Jan 'RedBully' Seiffert, some review cleanups
58
    ctypedef int intptr_t
2474.1.13 by John Arbash Meinel
Now that we have bisect_dirblock working again, bring back cmp_dirblock_strings.
59
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
60
3696.4.3 by Robert Collins
Some tuning of update_entry.
61
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
62
cdef extern from "stdlib.h":
63
    unsigned long int strtoul(char *nptr, char **endptr, int base)
64
3696.4.3 by Robert Collins
Some tuning of update_entry.
65
66
cdef extern from 'sys/stat.h':
67
    int S_ISDIR(int mode)
68
    int S_ISREG(int mode)
3737.1.3 by John Arbash Meinel
Move more compatibility code into python-compat.h
69
    # On win32, this actually comes from "python-compat.h"
3696.4.3 by Robert Collins
Some tuning of update_entry.
70
    int S_ISLNK(int mode)
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
71
    int S_IXUSR
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
72
2474.1.72 by John Arbash Meinel
Document a bit more what is going on in _dirstate_helpers_c.pyx, from Martin's comments
73
# These functions allow us access to a bit of the 'bare metal' of python
74
# objects, rather than going through the object abstraction. (For example,
75
# PyList_Append, rather than getting the 'append' attribute of the object, and
76
# creating a tuple, and then using PyCallObject).
77
# Functions that return (or take) a void* are meant to grab a C PyObject*. This
78
# differs from the Pyrex 'object'. If you declare a variable as 'object' Pyrex
79
# will automatically Py_INCREF and Py_DECREF when appropriate. But for some
80
# inner loops, we don't need to do that at all, as the reference only lasts for
81
# a very short time.
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
82
# Note that the C API GetItem calls borrow references, so pyrex does the wrong
83
# thing if you declare e.g. object PyList_GetItem(object lst, int index) - you
84
# need to manually Py_INCREF yourself.
2474.1.4 by John Arbash Meinel
Add benchmarks for dirstate.bisect_dirblocks, and implement bisect_dirblocks in pyrex.
85
cdef extern from "Python.h":
3640.2.6 by John Arbash Meinel
PQM's pyrex needs a Py_ssize_t typedef.
86
    ctypedef int Py_ssize_t
3696.4.3 by Robert Collins
Some tuning of update_entry.
87
    ctypedef struct PyObject:
88
        pass
2474.1.23 by John Arbash Meinel
A C implementation of _fields_to_entry_0_parents drops the time from 400ms to 330ms for a 21k-entry tree
89
    int PyList_Append(object lst, object item) except -1
2474.1.12 by John Arbash Meinel
Clean up bisect_dirstate to not use temporary variables.
90
    void *PyList_GetItem_object_void "PyList_GET_ITEM" (object lst, int index)
3696.4.3 by Robert Collins
Some tuning of update_entry.
91
    void *PyList_GetItem_void_void "PyList_GET_ITEM" (void * lst, int index)
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
92
    object PyList_GET_ITEM(object lst, Py_ssize_t index)
2474.1.10 by John Arbash Meinel
Explicitly calling Py_INCREF makes things happier again.
93
    int PyList_CheckExact(object)
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
94
    Py_ssize_t PyList_GET_SIZE (object p)
2474.1.23 by John Arbash Meinel
A C implementation of _fields_to_entry_0_parents drops the time from 400ms to 330ms for a 21k-entry tree
95
96
    void *PyTuple_GetItem_void_void "PyTuple_GET_ITEM" (void* tpl, int index)
3696.4.3 by Robert Collins
Some tuning of update_entry.
97
    object PyTuple_GetItem_void_object "PyTuple_GET_ITEM" (void* tpl, int index)
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
98
    object PyTuple_GET_ITEM(object tpl, Py_ssize_t index)
99
2474.1.10 by John Arbash Meinel
Explicitly calling Py_INCREF makes things happier again.
100
2474.1.13 by John Arbash Meinel
Now that we have bisect_dirblock working again, bring back cmp_dirblock_strings.
101
    char *PyString_AsString(object p)
3696.4.3 by Robert Collins
Some tuning of update_entry.
102
    char *PyString_AsString_obj "PyString_AsString" (PyObject *string)
2474.1.16 by John Arbash Meinel
Shave off maybe 10% by using the PyString_* macros instead of functions.
103
    char *PyString_AS_STRING_void "PyString_AS_STRING" (void *p)
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
104
    int PyString_AsStringAndSize(object str, char **buffer, Py_ssize_t *length) except -1
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
105
    object PyString_FromString(char *)
3640.2.1 by John Arbash Meinel
More safety checks around PyString_FromStringAndSize,
106
    object PyString_FromStringAndSize(char *, Py_ssize_t)
2474.1.13 by John Arbash Meinel
Now that we have bisect_dirblock working again, bring back cmp_dirblock_strings.
107
    int PyString_Size(object p)
2474.1.16 by John Arbash Meinel
Shave off maybe 10% by using the PyString_* macros instead of functions.
108
    int PyString_GET_SIZE_void "PyString_GET_SIZE" (void *p)
2474.1.14 by John Arbash Meinel
Switching bisect_dirblocks remove the extra .split('/')
109
    int PyString_CheckExact(object p)
3696.4.3 by Robert Collins
Some tuning of update_entry.
110
    void Py_INCREF(object o)
111
    void Py_DECREF(object o)
2474.1.13 by John Arbash Meinel
Now that we have bisect_dirblock working again, bring back cmp_dirblock_strings.
112
2474.1.23 by John Arbash Meinel
A C implementation of _fields_to_entry_0_parents drops the time from 400ms to 330ms for a 21k-entry tree
113
2474.1.13 by John Arbash Meinel
Now that we have bisect_dirblock working again, bring back cmp_dirblock_strings.
114
cdef extern from "string.h":
2474.1.25 by John Arbash Meinel
Refactor into a helper function to make implementation clearer
115
    int strncmp(char *s1, char *s2, int len)
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
116
    void *memchr(void *s, int c, size_t len)
117
    int memcmp(void *b1, void *b2, size_t len)
118
    # ??? memrchr is a GNU extension :(
119
    # void *memrchr(void *s, int c, size_t len)
120
121
122
cdef void* _my_memrchr(void *s, int c, size_t n):
123
    # memrchr seems to be a GNU extension, so we have to implement it ourselves
2474.1.69 by John Arbash Meinel
Thanks to Jan 'RedBully' Seiffert, some review cleanups
124
    cdef char *pos
125
    cdef char *start
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
126
2474.1.69 by John Arbash Meinel
Thanks to Jan 'RedBully' Seiffert, some review cleanups
127
    start = <char*>s
128
    pos = start + n - 1
129
    while pos >= start:
130
        if pos[0] == c:
131
            return <void*>pos
132
        pos = pos - 1
133
    return NULL
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
134
135
136
def _py_memrchr(s, c):
137
    """Just to expose _my_memrchr for testing.
138
139
    :param s: The Python string to search
140
    :param c: The character to search for
141
    :return: The offset to the last instance of 'c' in s
142
    """
143
    cdef void *_s
144
    cdef void *found
145
    cdef int length
146
    cdef char *_c
147
148
    _s = PyString_AsString(s)
149
    length = PyString_Size(s)
150
151
    _c = PyString_AsString(c)
152
    assert PyString_Size(c) == 1,\
153
        'Must be a single character string, not %s' % (c,)
154
    found = _my_memrchr(_s, _c[0], length)
155
    if found == NULL:
156
        return None
2668.1.1 by John Arbash Meinel
(Lukáš Lalinský) Add a special header for intptr_t for MSVC which doesn't have it in the standard place
157
    return <char*>found - <char*>_s
2474.1.4 by John Arbash Meinel
Add benchmarks for dirstate.bisect_dirblocks, and implement bisect_dirblocks in pyrex.
158
3640.2.1 by John Arbash Meinel
More safety checks around PyString_FromStringAndSize,
159
cdef object safe_string_from_size(char *s, Py_ssize_t size):
160
    if size < 0:
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
161
        # XXX: On 64-bit machines the <int> cast causes a C compiler warning.
3640.2.1 by John Arbash Meinel
More safety checks around PyString_FromStringAndSize,
162
        raise AssertionError(
163
            'tried to create a string with an invalid size: %d @0x%x'
164
            % (size, <int>s))
165
    return PyString_FromStringAndSize(s, size)
166
2474.1.4 by John Arbash Meinel
Add benchmarks for dirstate.bisect_dirblocks, and implement bisect_dirblocks in pyrex.
167
2474.1.69 by John Arbash Meinel
Thanks to Jan 'RedBully' Seiffert, some review cleanups
168
cdef int _is_aligned(void *ptr):
169
    """Is this pointer aligned to an integer size offset?
170
171
    :return: 1 if this pointer is aligned, 0 otherwise.
172
    """
173
    return ((<intptr_t>ptr) & ((sizeof(int))-1)) == 0
174
175
2474.1.41 by John Arbash Meinel
Change the name of cmp_dirblock_strings to cmp_by_dirs
176
cdef int _cmp_by_dirs(char *path1, int size1, char *path2, int size2):
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
177
    cdef unsigned char *cur1
178
    cdef unsigned char *cur2
179
    cdef unsigned char *end1
180
    cdef unsigned char *end2
2474.1.18 by John Arbash Meinel
Add an integer-size comparison loop at the begining, and
181
    cdef int *cur_int1
182
    cdef int *cur_int2
183
    cdef int *end_int1
184
    cdef int *end_int2
2474.1.17 by John Arbash Meinel
Using a custom loop seems to be the same speed, but is probably
185
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
186
    if path1 == path2 and size1 == size2:
2474.1.54 by John Arbash Meinel
Optimize the simple case that the strings are the same object.
187
        return 0
188
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
189
    end1 = <unsigned char*>path1+size1
190
    end2 = <unsigned char*>path2+size2
2474.1.17 by John Arbash Meinel
Using a custom loop seems to be the same speed, but is probably
191
2474.1.18 by John Arbash Meinel
Add an integer-size comparison loop at the begining, and
192
    # Use 32-bit comparisons for the matching portion of the string.
193
    # Almost all CPU's are faster at loading and comparing 32-bit integers,
194
    # than they are at 8-bit integers.
2474.1.69 by John Arbash Meinel
Thanks to Jan 'RedBully' Seiffert, some review cleanups
195
    # 99% of the time, these will be aligned, but in case they aren't just skip
196
    # this loop
197
    if _is_aligned(path1) and _is_aligned(path2):
198
        cur_int1 = <int*>path1
199
        cur_int2 = <int*>path2
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
200
        end_int1 = <int*>(path1 + size1 - (size1 % sizeof(int)))
201
        end_int2 = <int*>(path2 + size2 - (size2 % sizeof(int)))
2474.1.69 by John Arbash Meinel
Thanks to Jan 'RedBully' Seiffert, some review cleanups
202
203
        while cur_int1 < end_int1 and cur_int2 < end_int2:
204
            if cur_int1[0] != cur_int2[0]:
205
                break
206
            cur_int1 = cur_int1 + 1
207
            cur_int2 = cur_int2 + 1
208
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
209
        cur1 = <unsigned char*>cur_int1
210
        cur2 = <unsigned char*>cur_int2
2474.1.69 by John Arbash Meinel
Thanks to Jan 'RedBully' Seiffert, some review cleanups
211
    else:
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
212
        cur1 = <unsigned char*>path1
213
        cur2 = <unsigned char*>path2
2474.1.18 by John Arbash Meinel
Add an integer-size comparison loop at the begining, and
214
2474.1.17 by John Arbash Meinel
Using a custom loop seems to be the same speed, but is probably
215
    while cur1 < end1 and cur2 < end2:
216
        if cur1[0] == cur2[0]:
217
            # This character matches, just go to the next one
218
            cur1 = cur1 + 1
219
            cur2 = cur2 + 1
220
            continue
221
        # The current characters do not match
222
        if cur1[0] == c'/':
2474.1.19 by John Arbash Meinel
Clean up _cmp_dirblock_strings_alt to make it the default.
223
            return -1 # Reached the end of path1 segment first
2474.1.17 by John Arbash Meinel
Using a custom loop seems to be the same speed, but is probably
224
        elif cur2[0] == c'/':
2474.1.19 by John Arbash Meinel
Clean up _cmp_dirblock_strings_alt to make it the default.
225
            return 1 # Reached the end of path2 segment first
2474.1.17 by John Arbash Meinel
Using a custom loop seems to be the same speed, but is probably
226
        elif cur1[0] < cur2[0]:
227
            return -1
228
        else:
229
            return 1
2474.1.19 by John Arbash Meinel
Clean up _cmp_dirblock_strings_alt to make it the default.
230
231
    # We reached the end of at least one of the strings
2474.1.17 by John Arbash Meinel
Using a custom loop seems to be the same speed, but is probably
232
    if cur1 < end1:
2474.1.19 by John Arbash Meinel
Clean up _cmp_dirblock_strings_alt to make it the default.
233
        return 1 # Not at the end of cur1, must be at the end of cur2
2474.1.18 by John Arbash Meinel
Add an integer-size comparison loop at the begining, and
234
    if cur2 < end2:
2474.1.19 by John Arbash Meinel
Clean up _cmp_dirblock_strings_alt to make it the default.
235
        return -1 # At the end of cur1, but not at cur2
2474.1.17 by John Arbash Meinel
Using a custom loop seems to be the same speed, but is probably
236
    # We reached the end of both strings
237
    return 0
238
239
4459.2.2 by Vincent Ladeuil
Use the same method or function names for _dirstate_helpers in pyrex and
240
def cmp_by_dirs(path1, path2):
2474.1.41 by John Arbash Meinel
Change the name of cmp_dirblock_strings to cmp_by_dirs
241
    """Compare two paths directory by directory.
242
243
    This is equivalent to doing::
244
245
       cmp(path1.split('/'), path2.split('/'))
246
247
    The idea is that you should compare path components separately. This
248
    differs from plain ``cmp(path1, path2)`` for paths like ``'a-b'`` and
249
    ``a/b``. "a-b" comes after "a" but would come before "a/b" lexically.
250
251
    :param path1: first path
252
    :param path2: second path
2872.4.10 by Martin Pool
docstrings for cmp_ functions seem to be backwards
253
    :return: negative number if ``path1`` comes first,
2474.1.41 by John Arbash Meinel
Change the name of cmp_dirblock_strings to cmp_by_dirs
254
        0 if paths are equal,
2872.4.10 by Martin Pool
docstrings for cmp_ functions seem to be backwards
255
        and positive number if ``path2`` sorts first
2474.1.41 by John Arbash Meinel
Change the name of cmp_dirblock_strings to cmp_by_dirs
256
    """
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
257
    if not PyString_CheckExact(path1):
258
        raise TypeError("'path1' must be a plain string, not %s: %r"
259
                        % (type(path1), path1))
260
    if not PyString_CheckExact(path2):
261
        raise TypeError("'path2' must be a plain string, not %s: %r"
262
                        % (type(path2), path2))
2474.1.41 by John Arbash Meinel
Change the name of cmp_dirblock_strings to cmp_by_dirs
263
    return _cmp_by_dirs(PyString_AsString(path1),
264
                        PyString_Size(path1),
265
                        PyString_AsString(path2),
266
                        PyString_Size(path2))
2474.1.13 by John Arbash Meinel
Now that we have bisect_dirblock working again, bring back cmp_dirblock_strings.
267
268
4459.2.2 by Vincent Ladeuil
Use the same method or function names for _dirstate_helpers in pyrex and
269
def _cmp_path_by_dirblock(path1, path2):
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
270
    """Compare two paths based on what directory they are in.
271
272
    This generates a sort order, such that all children of a directory are
273
    sorted together, and grandchildren are in the same order as the
274
    children appear. But all grandchildren come after all children.
275
2474.1.66 by John Arbash Meinel
Some restructuring.
276
    In other words, all entries in a directory are sorted together, and
277
    directorys are sorted in cmp_by_dirs order.
278
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
279
    :param path1: first path
280
    :param path2: the second path
2872.4.10 by Martin Pool
docstrings for cmp_ functions seem to be backwards
281
    :return: negative number if ``path1`` comes first,
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
282
        0 if paths are equal
2872.4.10 by Martin Pool
docstrings for cmp_ functions seem to be backwards
283
        and a positive number if ``path2`` sorts first
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
284
    """
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
285
    if not PyString_CheckExact(path1):
286
        raise TypeError("'path1' must be a plain string, not %s: %r"
287
                        % (type(path1), path1))
288
    if not PyString_CheckExact(path2):
289
        raise TypeError("'path2' must be a plain string, not %s: %r"
290
                        % (type(path2), path2))
4459.2.4 by Vincent Ladeuil
Fixed as per John and Martin reviews.
291
    return _cmp_path_by_dirblock_intern(PyString_AsString(path1),
292
                                        PyString_Size(path1),
293
                                        PyString_AsString(path2),
294
                                        PyString_Size(path2))
295
296
297
cdef int _cmp_path_by_dirblock_intern(char *path1, int path1_len,
298
                                      char *path2, int path2_len):
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
299
    """Compare two paths by what directory they are in.
300
4459.2.2 by Vincent Ladeuil
Use the same method or function names for _dirstate_helpers in pyrex and
301
    see ``_cmp_path_by_dirblock`` for details.
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
302
    """
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
303
    cdef char *dirname1
304
    cdef int dirname1_len
305
    cdef char *dirname2
306
    cdef int dirname2_len
307
    cdef char *basename1
308
    cdef int basename1_len
309
    cdef char *basename2
310
    cdef int basename2_len
311
    cdef int cur_len
312
    cdef int cmp_val
313
314
    if path1_len == 0 and path2_len == 0:
315
        return 0
316
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
317
    if path1 == path2 and path1_len == path2_len:
318
        return 0
319
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
320
    if path1_len == 0:
321
        return -1
322
323
    if path2_len == 0:
324
        return 1
325
2474.1.59 by John Arbash Meinel
Make sure to set basename_len. With that patch, the tests pass.
326
    basename1 = <char*>_my_memrchr(path1, c'/', path1_len)
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
327
328
    if basename1 == NULL:
2474.1.59 by John Arbash Meinel
Make sure to set basename_len. With that patch, the tests pass.
329
        basename1 = path1
330
        basename1_len = path1_len
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
331
        dirname1 = ''
332
        dirname1_len = 0
333
    else:
2474.1.59 by John Arbash Meinel
Make sure to set basename_len. With that patch, the tests pass.
334
        dirname1 = path1
335
        dirname1_len = basename1 - path1
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
336
        basename1 = basename1 + 1
2474.1.59 by John Arbash Meinel
Make sure to set basename_len. With that patch, the tests pass.
337
        basename1_len = path1_len - dirname1_len - 1
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
338
2474.1.59 by John Arbash Meinel
Make sure to set basename_len. With that patch, the tests pass.
339
    basename2 = <char*>_my_memrchr(path2, c'/', path2_len)
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
340
341
    if basename2 == NULL:
2474.1.59 by John Arbash Meinel
Make sure to set basename_len. With that patch, the tests pass.
342
        basename2 = path2
343
        basename2_len = path2_len
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
344
        dirname2 = ''
345
        dirname2_len = 0
346
    else:
2474.1.59 by John Arbash Meinel
Make sure to set basename_len. With that patch, the tests pass.
347
        dirname2 = path2
348
        dirname2_len = basename2 - path2
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
349
        basename2 = basename2 + 1
2474.1.59 by John Arbash Meinel
Make sure to set basename_len. With that patch, the tests pass.
350
        basename2_len = path2_len - dirname2_len - 1
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
351
352
    cmp_val = _cmp_by_dirs(dirname1, dirname1_len,
353
                           dirname2, dirname2_len)
354
    if cmp_val != 0:
355
        return cmp_val
356
357
    cur_len = basename1_len
358
    if basename2_len < basename1_len:
359
        cur_len = basename2_len
360
361
    cmp_val = memcmp(basename1, basename2, cur_len)
362
    if cmp_val != 0:
363
        return cmp_val
364
    if basename1_len == basename2_len:
365
        return 0
366
    if basename1_len < basename2_len:
367
        return -1
368
    return 1
369
370
4459.2.2 by Vincent Ladeuil
Use the same method or function names for _dirstate_helpers in pyrex and
371
def _bisect_path_left(paths, path):
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
372
    """Return the index where to insert path into paths.
373
374
    This uses a path-wise comparison so we get::
375
        a
376
        a-b
377
        a=b
378
        a/b
379
    Rather than::
380
        a
381
        a-b
382
        a/b
383
        a=b
384
    :param paths: A list of paths to search through
385
    :param path: A single path to insert
386
    :return: An offset where 'path' can be inserted.
387
    :seealso: bisect.bisect_left
388
    """
389
    cdef int _lo
390
    cdef int _hi
391
    cdef int _mid
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
392
    cdef char *path_cstr
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
393
    cdef int path_size
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
394
    cdef char *cur_cstr
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
395
    cdef int cur_size
396
    cdef void *cur
397
398
    if not PyList_CheckExact(paths):
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
399
        raise TypeError("you must pass a python list for 'paths' not: %s %r"
400
                        % (type(paths), paths))
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
401
    if not PyString_CheckExact(path):
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
402
        raise TypeError("you must pass a string for 'path' not: %s %r"
403
                        % (type(path), path))
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
404
405
    _hi = len(paths)
406
    _lo = 0
407
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
408
    path_cstr = PyString_AsString(path)
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
409
    path_size = PyString_Size(path)
410
411
    while _lo < _hi:
412
        _mid = (_lo + _hi) / 2
413
        cur = PyList_GetItem_object_void(paths, _mid)
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
414
        cur_cstr = PyString_AS_STRING_void(cur)
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
415
        cur_size = PyString_GET_SIZE_void(cur)
4459.2.4 by Vincent Ladeuil
Fixed as per John and Martin reviews.
416
        if _cmp_path_by_dirblock_intern(cur_cstr, cur_size,
417
                                        path_cstr, path_size) < 0:
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
418
            _lo = _mid + 1
419
        else:
420
            _hi = _mid
421
    return _lo
422
423
4459.2.2 by Vincent Ladeuil
Use the same method or function names for _dirstate_helpers in pyrex and
424
def _bisect_path_right(paths, path):
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
425
    """Return the index where to insert path into paths.
426
427
    This uses a path-wise comparison so we get::
428
        a
429
        a-b
430
        a=b
431
        a/b
432
    Rather than::
433
        a
434
        a-b
435
        a/b
436
        a=b
437
    :param paths: A list of paths to search through
438
    :param path: A single path to insert
439
    :return: An offset where 'path' can be inserted.
440
    :seealso: bisect.bisect_right
441
    """
442
    cdef int _lo
443
    cdef int _hi
444
    cdef int _mid
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
445
    cdef char *path_cstr
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
446
    cdef int path_size
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
447
    cdef char *cur_cstr
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
448
    cdef int cur_size
449
    cdef void *cur
450
451
    if not PyList_CheckExact(paths):
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
452
        raise TypeError("you must pass a python list for 'paths' not: %s %r"
453
                        % (type(paths), paths))
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
454
    if not PyString_CheckExact(path):
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
455
        raise TypeError("you must pass a string for 'path' not: %s %r"
456
                        % (type(path), path))
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
457
458
    _hi = len(paths)
459
    _lo = 0
460
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
461
    path_cstr = PyString_AsString(path)
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
462
    path_size = PyString_Size(path)
463
464
    while _lo < _hi:
465
        _mid = (_lo + _hi) / 2
466
        cur = PyList_GetItem_object_void(paths, _mid)
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
467
        cur_cstr = PyString_AS_STRING_void(cur)
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
468
        cur_size = PyString_GET_SIZE_void(cur)
4459.2.4 by Vincent Ladeuil
Fixed as per John and Martin reviews.
469
        if _cmp_path_by_dirblock_intern(path_cstr, path_size,
470
                                        cur_cstr, cur_size) < 0:
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
471
            _hi = _mid
472
        else:
473
            _lo = _mid + 1
474
    return _lo
475
476
4459.2.2 by Vincent Ladeuil
Use the same method or function names for _dirstate_helpers in pyrex and
477
def bisect_dirblock(dirblocks, dirname, lo=0, hi=None, cache=None):
2474.1.4 by John Arbash Meinel
Add benchmarks for dirstate.bisect_dirblocks, and implement bisect_dirblocks in pyrex.
478
    """Return the index where to insert dirname into the dirblocks.
479
480
    The return value idx is such that all directories blocks in dirblock[:idx]
481
    have names < dirname, and all blocks in dirblock[idx:] have names >=
482
    dirname.
483
484
    Optional args lo (default 0) and hi (default len(dirblocks)) bound the
485
    slice of a to be searched.
486
    """
487
    cdef int _lo
488
    cdef int _hi
489
    cdef int _mid
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
490
    cdef char *dirname_cstr
2474.1.14 by John Arbash Meinel
Switching bisect_dirblocks remove the extra .split('/')
491
    cdef int dirname_size
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
492
    cdef char *cur_cstr
2474.1.14 by John Arbash Meinel
Switching bisect_dirblocks remove the extra .split('/')
493
    cdef int cur_size
2474.1.12 by John Arbash Meinel
Clean up bisect_dirstate to not use temporary variables.
494
    cdef void *cur
2474.1.4 by John Arbash Meinel
Add benchmarks for dirstate.bisect_dirblocks, and implement bisect_dirblocks in pyrex.
495
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
496
    if not PyList_CheckExact(dirblocks):
497
        raise TypeError("you must pass a python list for 'dirblocks' not: %s %r"
498
                        % (type(dirblocks), dirblocks))
499
    if not PyString_CheckExact(dirname):
500
        raise TypeError("you must pass a string for dirname not: %s %r"
501
                        % (type(dirname), dirname))
2474.1.4 by John Arbash Meinel
Add benchmarks for dirstate.bisect_dirblocks, and implement bisect_dirblocks in pyrex.
502
    if hi is None:
503
        _hi = len(dirblocks)
504
    else:
505
        _hi = hi
506
507
    _lo = lo
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
508
    dirname_cstr = PyString_AsString(dirname)
2474.1.14 by John Arbash Meinel
Switching bisect_dirblocks remove the extra .split('/')
509
    dirname_size = PyString_Size(dirname)
510
2474.1.4 by John Arbash Meinel
Add benchmarks for dirstate.bisect_dirblocks, and implement bisect_dirblocks in pyrex.
511
    while _lo < _hi:
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
512
        _mid = (_lo + _hi) / 2
2474.1.4 by John Arbash Meinel
Add benchmarks for dirstate.bisect_dirblocks, and implement bisect_dirblocks in pyrex.
513
        # Grab the dirname for the current dirblock
2474.1.12 by John Arbash Meinel
Clean up bisect_dirstate to not use temporary variables.
514
        # cur = dirblocks[_mid][0]
515
        cur = PyTuple_GetItem_void_void(
516
                PyList_GetItem_object_void(dirblocks, _mid), 0)
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
517
        cur_cstr = PyString_AS_STRING_void(cur)
2474.1.16 by John Arbash Meinel
Shave off maybe 10% by using the PyString_* macros instead of functions.
518
        cur_size = PyString_GET_SIZE_void(cur)
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
519
        if _cmp_by_dirs(cur_cstr, cur_size, dirname_cstr, dirname_size) < 0:
2474.1.58 by John Arbash Meinel
(broken) Try to properly implement DirState._bisect*
520
            _lo = _mid + 1
2474.1.14 by John Arbash Meinel
Switching bisect_dirblocks remove the extra .split('/')
521
        else:
522
            _hi = _mid
2474.1.4 by John Arbash Meinel
Add benchmarks for dirstate.bisect_dirblocks, and implement bisect_dirblocks in pyrex.
523
    return _lo
524
525
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
526
cdef class Reader:
527
    """Maintain the current location, and return fields as you parse them."""
528
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
529
    cdef object state # The DirState object
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
530
    cdef object text # The overall string object
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
531
    cdef char *text_cstr # Pointer to the beginning of text
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
532
    cdef int text_size # Length of text
533
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
534
    cdef char *end_cstr # End of text
535
    cdef char *cur_cstr # Pointer to the current record
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
536
    cdef char *next # Pointer to the end of this record
537
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
538
    def __init__(self, text, state):
539
        self.state = state
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
540
        self.text = text
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
541
        self.text_cstr = PyString_AsString(text)
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
542
        self.text_size = PyString_Size(text)
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
543
        self.end_cstr = self.text_cstr + self.text_size
544
        self.cur_cstr = self.text_cstr
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
545
3640.2.1 by John Arbash Meinel
More safety checks around PyString_FromStringAndSize,
546
    cdef char *get_next(self, int *size) except NULL:
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
547
        """Return a pointer to the start of the next field."""
548
        cdef char *next
3640.2.1 by John Arbash Meinel
More safety checks around PyString_FromStringAndSize,
549
        cdef Py_ssize_t extra_len
550
551
        if self.cur_cstr == NULL:
552
            raise AssertionError('get_next() called when cur_str is NULL')
553
        elif self.cur_cstr >= self.end_cstr:
554
            raise AssertionError('get_next() called when there are no chars'
555
                                 ' left')
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
556
        next = self.cur_cstr
3640.2.1 by John Arbash Meinel
More safety checks around PyString_FromStringAndSize,
557
        self.cur_cstr = <char*>memchr(next, c'\0', self.end_cstr - next)
558
        if self.cur_cstr == NULL:
559
            extra_len = self.end_cstr - next
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
560
            raise errors.DirstateCorrupt(self.state,
561
                'failed to find trailing NULL (\\0).'
3640.2.1 by John Arbash Meinel
More safety checks around PyString_FromStringAndSize,
562
                ' Trailing garbage: %r'
563
                % safe_string_from_size(next, extra_len))
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
564
        size[0] = self.cur_cstr - next
565
        self.cur_cstr = self.cur_cstr + 1
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
566
        return next
567
2474.1.53 by John Arbash Meinel
Changing Reader.get_next_str (which returns a Python String)
568
    cdef object get_next_str(self):
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
569
        """Get the next field as a Python string."""
2474.1.37 by John Arbash Meinel
get_next() returns the length of the string,
570
        cdef int size
571
        cdef char *next
572
        next = self.get_next(&size)
3640.2.1 by John Arbash Meinel
More safety checks around PyString_FromStringAndSize,
573
        return safe_string_from_size(next, size)
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
574
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
575
    cdef int _init(self) except -1:
576
        """Get the pointer ready.
577
578
        This assumes that the dirstate header has already been read, and we
579
        already have the dirblock string loaded into memory.
580
        This just initializes our memory pointers, etc for parsing of the
581
        dirblock string.
582
        """
2474.1.32 by John Arbash Meinel
Skip past the first entry while reading,
583
        cdef char *first
2474.1.37 by John Arbash Meinel
get_next() returns the length of the string,
584
        cdef int size
2474.1.32 by John Arbash Meinel
Skip past the first entry while reading,
585
        # The first field should be an empty string left over from the Header
2474.1.37 by John Arbash Meinel
get_next() returns the length of the string,
586
        first = self.get_next(&size)
587
        if first[0] != c'\0' and size == 0:
2474.1.32 by John Arbash Meinel
Skip past the first entry while reading,
588
            raise AssertionError('First character should be null not: %s'
589
                                 % (first,))
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
590
        return 0
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
591
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
592
    cdef object _get_entry(self, int num_trees, void **p_current_dirname,
593
                           int *new_block):
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
594
        """Extract the next entry.
595
596
        This parses the next entry based on the current location in
597
        ``self.cur_cstr``.
598
        Each entry can be considered a "row" in the total table. And each row
599
        has a fixed number of columns. It is generally broken up into "key"
600
        columns, then "current" columns, and then "parent" columns.
601
602
        :param num_trees: How many parent trees need to be parsed
603
        :param p_current_dirname: A pointer to the current PyString
604
            representing the directory name.
605
            We pass this in as a void * so that pyrex doesn't have to
606
            increment/decrement the PyObject reference counter for each
607
            _get_entry call.
608
            We use a pointer so that _get_entry can update it with the new
609
            value.
610
        :param new_block: This is to let the caller know that it needs to
611
            create a new directory block to store the next entry.
612
        """
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
613
        cdef object path_name_file_id_key
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
614
        cdef char *entry_size_cstr
2474.1.38 by John Arbash Meinel
Finally, faster than text.split() (156ms)
615
        cdef unsigned long int entry_size
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
616
        cdef char* executable_cstr
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
617
        cdef int is_executable
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
618
        cdef char* dirname_cstr
2474.1.38 by John Arbash Meinel
Finally, faster than text.split() (156ms)
619
        cdef char* trailing
620
        cdef int cur_size
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
621
        cdef int i
2474.1.38 by John Arbash Meinel
Finally, faster than text.split() (156ms)
622
        cdef object minikind
623
        cdef object fingerprint
624
        cdef object info
625
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
626
        # Read the 'key' information (dirname, name, file_id)
627
        dirname_cstr = self.get_next(&cur_size)
628
        # Check to see if we have started a new directory block.
629
        # If so, then we need to create a new dirname PyString, so that it can
630
        # be used in all of the tuples. This saves time and memory, by re-using
631
        # the same object repeatedly.
632
633
        # Do the cheap 'length of string' check first. If the string is a
634
        # different length, then we *have* to be a different directory.
635
        if (cur_size != PyString_GET_SIZE_void(p_current_dirname[0])
636
            or strncmp(dirname_cstr,
637
                       # Extract the char* from our current dirname string.  We
638
                       # know it is a PyString, so we can use
639
                       # PyString_AS_STRING, we use the _void version because
640
                       # we are tricking Pyrex by using a void* rather than an
641
                       # <object>
642
                       PyString_AS_STRING_void(p_current_dirname[0]),
643
                       cur_size+1) != 0):
3640.2.1 by John Arbash Meinel
More safety checks around PyString_FromStringAndSize,
644
            dirname = safe_string_from_size(dirname_cstr, cur_size)
2474.1.38 by John Arbash Meinel
Finally, faster than text.split() (156ms)
645
            p_current_dirname[0] = <void*>dirname
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
646
            new_block[0] = 1
647
        else:
648
            new_block[0] = 0
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
649
650
        # Build up the key that will be used.
651
        # By using <object>(void *) Pyrex will automatically handle the
652
        # Py_INCREF that we need.
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
653
        path_name_file_id_key = (<object>p_current_dirname[0],
2474.1.38 by John Arbash Meinel
Finally, faster than text.split() (156ms)
654
                                 self.get_next_str(),
655
                                 self.get_next_str(),
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
656
                                )
657
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
658
        # Parse all of the per-tree information. current has the information in
659
        # the same location as parent trees. The only difference is that 'info'
660
        # is a 'packed_stat' for current, while it is a 'revision_id' for
661
        # parent trees.
662
        # minikind, fingerprint, and info will be returned as regular python
663
        # strings
664
        # entry_size and is_executable will be parsed into a python Long and
665
        # python Boolean, respectively.
666
        # TODO: jam 20070718 Consider changin the entry_size conversion to
667
        #       prefer python Int when possible. They are generally faster to
668
        #       work with, and it will be rare that we have a file >2GB.
669
        #       Especially since this code is pretty much fixed at a max of
670
        #       4GB.
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
671
        trees = []
672
        for i from 0 <= i < num_trees:
673
            minikind = self.get_next_str()
674
            fingerprint = self.get_next_str()
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
675
            entry_size_cstr = self.get_next(&cur_size)
676
            entry_size = strtoul(entry_size_cstr, NULL, 10)
677
            executable_cstr = self.get_next(&cur_size)
678
            is_executable = (executable_cstr[0] == c'y')
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
679
            info = self.get_next_str()
680
            PyList_Append(trees, (
2474.1.38 by John Arbash Meinel
Finally, faster than text.split() (156ms)
681
                minikind,     # minikind
682
                fingerprint,  # fingerprint
683
                entry_size,   # size
684
                is_executable,# executable
685
                info,         # packed_stat or revision_id
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
686
            ))
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
687
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
688
        # The returned tuple is (key, [trees])
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
689
        ret = (path_name_file_id_key, trees)
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
690
        # Ignore the trailing newline, but assert that it does exist, this
691
        # ensures that we always finish parsing a line on an end-of-entry
692
        # marker.
2474.1.38 by John Arbash Meinel
Finally, faster than text.split() (156ms)
693
        trailing = self.get_next(&cur_size)
694
        if cur_size != 1 or trailing[0] != c'\n':
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
695
            raise errors.DirstateCorrupt(self.state,
2474.1.38 by John Arbash Meinel
Finally, faster than text.split() (156ms)
696
                'Bad parse, we expected to end on \\n, not: %d %s: %s'
3640.2.1 by John Arbash Meinel
More safety checks around PyString_FromStringAndSize,
697
                % (cur_size, safe_string_from_size(trailing, cur_size),
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
698
                   ret))
2474.1.38 by John Arbash Meinel
Finally, faster than text.split() (156ms)
699
        return ret
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
700
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
701
    def _parse_dirblocks(self):
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
702
        """Parse all dirblocks in the state file."""
703
        cdef int num_trees
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
704
        cdef object current_block
705
        cdef object entry
706
        cdef void * current_dirname
707
        cdef int new_block
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
708
        cdef int expected_entry_count
709
        cdef int entry_count
710
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
711
        num_trees = self.state._num_present_parents() + 1
712
        expected_entry_count = self.state._num_entries
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
713
714
        # Ignore the first record
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
715
        self._init()
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
716
2474.1.37 by John Arbash Meinel
get_next() returns the length of the string,
717
        current_block = []
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
718
        dirblocks = [('', current_block), ('', [])]
719
        self.state._dirblocks = dirblocks
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
720
        obj = ''
2474.1.37 by John Arbash Meinel
get_next() returns the length of the string,
721
        current_dirname = <void*>obj
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
722
        new_block = 0
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
723
        entry_count = 0
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
724
2474.1.54 by John Arbash Meinel
Optimize the simple case that the strings are the same object.
725
        # TODO: jam 2007-05-07 Consider pre-allocating some space for the
726
        #       members, and then growing and shrinking from there. If most
727
        #       directories have close to 10 entries in them, it would save a
728
        #       few mallocs if we default our list size to something
729
        #       reasonable. Or we could malloc it to something large (100 or
730
        #       so), and then truncate. That would give us a malloc + realloc,
731
        #       rather than lots of reallocs.
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
732
        while self.cur_cstr < self.end_cstr:
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
733
            entry = self._get_entry(num_trees, &current_dirname, &new_block)
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
734
            if new_block:
735
                # new block - different dirname
736
                current_block = []
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
737
                PyList_Append(dirblocks,
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
738
                              (<object>current_dirname, current_block))
739
            PyList_Append(current_block, entry)
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
740
            entry_count = entry_count + 1
741
        if entry_count != expected_entry_count:
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
742
            raise errors.DirstateCorrupt(self.state,
743
                    'We read the wrong number of entries.'
2474.1.46 by John Arbash Meinel
Finish implementing _c_read_dirblocks for any number of parents.
744
                    ' We expected to read %s, but read %s'
745
                    % (expected_entry_count, entry_count))
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
746
        self.state._split_root_dirblock_into_contents()
2474.1.36 by John Arbash Meinel
Move functions into member functions on reader() class.
747
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
748
4459.2.2 by Vincent Ladeuil
Use the same method or function names for _dirstate_helpers in pyrex and
749
def _read_dirblocks(state):
2474.1.1 by John Arbash Meinel
Create a Pyrex extension for reading the dirstate file.
750
    """Read in the dirblocks for the given DirState object.
751
752
    This is tightly bound to the DirState internal representation. It should be
753
    thought of as a member function, which is only separated out so that we can
754
    re-write it in pyrex.
755
756
    :param state: A DirState object.
757
    :return: None
2474.1.70 by John Arbash Meinel
Lot's of fixes from Martin's comments.
758
    :postcondition: The dirblocks will be loaded into the appropriate fields in
759
        the DirState object.
2474.1.1 by John Arbash Meinel
Create a Pyrex extension for reading the dirstate file.
760
    """
761
    state._state_file.seek(state._end_of_header)
762
    text = state._state_file.read()
763
    # TODO: check the crc checksums. crc_measured = zlib.crc32(text)
764
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
765
    reader = Reader(text, state)
2474.1.30 by John Arbash Meinel
Start working towards a parser which uses a Reader (producer)
766
3640.2.5 by John Arbash Meinel
Change from using AssertionError to using DirstateCorrupt in a few places
767
    reader._parse_dirblocks()
2474.1.1 by John Arbash Meinel
Create a Pyrex extension for reading the dirstate file.
768
    state._dirblock_state = DirState.IN_MEMORY_UNMODIFIED
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
769
770
3696.4.3 by Robert Collins
Some tuning of update_entry.
771
cdef int minikind_from_mode(int mode):
772
    # in order of frequency:
773
    if S_ISREG(mode):
774
        return c"f"
775
    if S_ISDIR(mode):
776
        return c"d"
777
    if S_ISLNK(mode):
778
        return c"l"
779
    return 0
780
781
782
_encode = binascii.b2a_base64
783
3696.4.17 by Robert Collins
Review feedback.
784
3696.4.3 by Robert Collins
Some tuning of update_entry.
785
from struct import pack
786
cdef _pack_stat(stat_value):
787
    """return a string representing the stat value's key fields.
788
789
    :param stat_value: A stat oject with st_size, st_mtime, st_ctime, st_dev,
790
        st_ino and st_mode fields.
791
    """
792
    cdef char result[6*4] # 6 long ints
793
    cdef int *aliased
794
    aliased = <int *>result
795
    aliased[0] = htonl(stat_value.st_size)
796
    aliased[1] = htonl(int(stat_value.st_mtime))
797
    aliased[2] = htonl(int(stat_value.st_ctime))
798
    aliased[3] = htonl(stat_value.st_dev)
799
    aliased[4] = htonl(stat_value.st_ino & 0xFFFFFFFF)
800
    aliased[5] = htonl(stat_value.st_mode)
801
    packed = PyString_FromStringAndSize(result, 6*4)
802
    return _encode(packed)[:-1]
803
804
805
def update_entry(self, entry, abspath, stat_value):
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
806
    """Update the entry based on what is actually on disk.
807
3696.5.2 by Robert Collins
Integrate less aggressive sha logic with C iter-changes.
808
    This function only calculates the sha if it needs to - if the entry is
809
    uncachable, or clearly different to the first parent's entry, no sha
810
    is calculated, and None is returned.
811
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
812
    :param entry: This is the dirblock entry for the file in question.
813
    :param abspath: The path on disk for this file.
814
    :param stat_value: (optional) if we already have done a stat on the
815
        file, re-use it.
3696.5.2 by Robert Collins
Integrate less aggressive sha logic with C iter-changes.
816
    :return: None, or The sha1 hexdigest of the file (40 bytes) or link
817
        target of a symlink.
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
818
    """
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
819
    return _update_entry(self, entry, abspath, stat_value)
820
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
821
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
822
cdef _update_entry(self, entry, abspath, stat_value):
823
    """Update the entry based on what is actually on disk.
824
3696.5.2 by Robert Collins
Integrate less aggressive sha logic with C iter-changes.
825
    This function only calculates the sha if it needs to - if the entry is
826
    uncachable, or clearly different to the first parent's entry, no sha
827
    is calculated, and None is returned.
828
3696.4.17 by Robert Collins
Review feedback.
829
    :param self: The dirstate object this is operating on.
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
830
    :param entry: This is the dirblock entry for the file in question.
831
    :param abspath: The path on disk for this file.
3696.4.17 by Robert Collins
Review feedback.
832
    :param stat_value: The stat value done on the path.
3696.5.2 by Robert Collins
Integrate less aggressive sha logic with C iter-changes.
833
    :return: None, or The sha1 hexdigest of the file (40 bytes) or link
834
        target of a symlink.
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
835
    """
3696.4.17 by Robert Collins
Review feedback.
836
    # TODO - require pyrex 0.9.8, then use a pyd file to define access to the
837
    # _st mode of the compiled stat objects.
3696.4.3 by Robert Collins
Some tuning of update_entry.
838
    cdef int minikind, saved_minikind
839
    cdef void * details
840
    minikind = minikind_from_mode(stat_value.st_mode)
841
    if 0 == minikind:
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
842
        return None
843
    packed_stat = _pack_stat(stat_value)
3696.4.3 by Robert Collins
Some tuning of update_entry.
844
    details = PyList_GetItem_void_void(PyTuple_GetItem_void_void(<void *>entry, 1), 0)
845
    saved_minikind = PyString_AsString_obj(<PyObject *>PyTuple_GetItem_void_void(details, 0))[0]
4100.2.5 by Aaron Bentley
Stop resetting minikind to 'd' when comparing.
846
    if minikind == c'd' and saved_minikind == c't':
847
        minikind = c't'
3696.4.3 by Robert Collins
Some tuning of update_entry.
848
    saved_link_or_sha1 = PyTuple_GetItem_void_object(details, 1)
849
    saved_file_size = PyTuple_GetItem_void_object(details, 2)
850
    saved_executable = PyTuple_GetItem_void_object(details, 3)
851
    saved_packed_stat = PyTuple_GetItem_void_object(details, 4)
852
    # Deal with pyrex decrefing the objects
853
    Py_INCREF(saved_link_or_sha1)
854
    Py_INCREF(saved_file_size)
855
    Py_INCREF(saved_executable)
856
    Py_INCREF(saved_packed_stat)
857
    #(saved_minikind, saved_link_or_sha1, saved_file_size,
858
    # saved_executable, saved_packed_stat) = entry[1][0]
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
859
860
    if (minikind == saved_minikind
861
        and packed_stat == saved_packed_stat):
862
        # The stat hasn't changed since we saved, so we can re-use the
863
        # saved sha hash.
3696.4.3 by Robert Collins
Some tuning of update_entry.
864
        if minikind == c'd':
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
865
            return None
866
867
        # size should also be in packed_stat
868
        if saved_file_size == stat_value.st_size:
869
            return saved_link_or_sha1
870
871
    # If we have gotten this far, that means that we need to actually
872
    # process this entry.
873
    link_or_sha1 = None
3696.4.3 by Robert Collins
Some tuning of update_entry.
874
    if minikind == c'f':
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
875
        executable = self._is_executable(stat_value.st_mode,
876
                                         saved_executable)
877
        if self._cutoff_time is None:
878
            self._sha_cutoff_time()
879
        if (stat_value.st_mtime < self._cutoff_time
3696.5.2 by Robert Collins
Integrate less aggressive sha logic with C iter-changes.
880
            and stat_value.st_ctime < self._cutoff_time
881
            and len(entry[1]) > 1
882
            and entry[1][1][0] != 'a'):
883
                # Could check for size changes for further optimised
884
                # avoidance of sha1's. However the most prominent case of
885
                # over-shaing is during initial add, which this catches.
886
            link_or_sha1 = self._sha1_file(abspath)
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
887
            entry[1][0] = ('f', link_or_sha1, stat_value.st_size,
888
                           executable, packed_stat)
889
        else:
890
            entry[1][0] = ('f', '', stat_value.st_size,
891
                           executable, DirState.NULLSTAT)
3696.4.3 by Robert Collins
Some tuning of update_entry.
892
    elif minikind == c'd':
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
893
        link_or_sha1 = None
894
        entry[1][0] = ('d', '', 0, False, packed_stat)
3696.4.3 by Robert Collins
Some tuning of update_entry.
895
        if saved_minikind != c'd':
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
896
            # This changed from something into a directory. Make sure we
897
            # have a directory block for it. This doesn't happen very
898
            # often, so this doesn't have to be super fast.
899
            block_index, entry_index, dir_present, file_present = \
900
                self._get_block_entry_index(entry[0][0], entry[0][1], 0)
901
            self._ensure_block(block_index, entry_index,
3696.4.17 by Robert Collins
Review feedback.
902
                               pathjoin(entry[0][0], entry[0][1]))
3696.4.3 by Robert Collins
Some tuning of update_entry.
903
    elif minikind == c'l':
3696.4.1 by Robert Collins
Refactor to allow a pluggable dirstate update_entry with interface tests.
904
        link_or_sha1 = self._read_link(abspath, saved_link_or_sha1)
905
        if self._cutoff_time is None:
906
            self._sha_cutoff_time()
907
        if (stat_value.st_mtime < self._cutoff_time
908
            and stat_value.st_ctime < self._cutoff_time):
909
            entry[1][0] = ('l', link_or_sha1, stat_value.st_size,
910
                           False, packed_stat)
911
        else:
912
            entry[1][0] = ('l', '', stat_value.st_size,
913
                           False, DirState.NULLSTAT)
914
    self._dirblock_state = DirState.IN_MEMORY_MODIFIED
915
    return link_or_sha1
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
916
917
3696.4.7 by Robert Collins
More tuning.
918
cdef char _minikind_from_string(object string):
919
    """Convert a python string to a char."""
920
    return PyString_AsString(string)[0]
921
922
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
923
cdef object _kind_absent
924
cdef object _kind_file
925
cdef object _kind_directory
926
cdef object _kind_symlink
927
cdef object _kind_relocated
928
cdef object _kind_tree_reference
929
_kind_absent = "absent"
930
_kind_file = "file"
931
_kind_directory = "directory"
932
_kind_symlink = "symlink"
933
_kind_relocated = "relocated"
934
_kind_tree_reference = "tree-reference"
935
936
3696.4.7 by Robert Collins
More tuning.
937
cdef object _minikind_to_kind(char minikind):
938
    """Create a string kind for minikind."""
939
    cdef char _minikind[1]
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
940
    if minikind == c'f':
941
        return _kind_file
942
    elif minikind == c'd':
943
        return _kind_directory
944
    elif minikind == c'a':
945
        return _kind_absent
946
    elif minikind == c'r':
947
        return _kind_relocated
948
    elif minikind == c'l':
949
        return _kind_symlink
950
    elif minikind == c't':
951
        return _kind_tree_reference
3696.4.7 by Robert Collins
More tuning.
952
    _minikind[0] = minikind
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
953
    raise KeyError(PyString_FromStringAndSize(_minikind, 1))
3696.4.7 by Robert Collins
More tuning.
954
955
956
cdef int _versioned_minikind(char minikind):
957
    """Return non-zero if minikind is in fltd"""
958
    return (minikind == c'f' or
959
            minikind == c'd' or
960
            minikind == c'l' or
961
            minikind == c't')
962
963
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
964
cdef class ProcessEntryC:
965
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
966
    cdef int doing_consistency_expansion
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
967
    cdef object old_dirname_to_file_id # dict
968
    cdef object new_dirname_to_file_id # dict
969
    cdef object last_source_parent
970
    cdef object last_target_parent
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
971
    cdef int include_unchanged
972
    cdef int partial
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
973
    cdef object use_filesystem_for_exec
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
974
    cdef object utf8_decode
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
975
    cdef readonly object searched_specific_files
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
976
    cdef readonly object searched_exact_paths
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
977
    cdef object search_specific_files
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
978
    # The parents up to the root of the paths we are searching.
979
    # After all normal paths are returned, these specific items are returned.
980
    cdef object search_specific_file_parents
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
981
    cdef object state
982
    # Current iteration variables:
983
    cdef object current_root
984
    cdef object current_root_unicode
985
    cdef object root_entries
986
    cdef int root_entries_pos, root_entries_len
987
    cdef object root_abspath
988
    cdef int source_index, target_index
989
    cdef int want_unversioned
990
    cdef object tree
991
    cdef object dir_iterator
992
    cdef int block_index
993
    cdef object current_block
994
    cdef int current_block_pos
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
995
    cdef object current_block_list
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
996
    cdef object current_dir_info
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
997
    cdef object current_dir_list
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
998
    cdef object _pending_consistent_entries # list
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
999
    cdef int path_index
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1000
    cdef object root_dir_info
1001
    cdef object bisect_left
3696.4.17 by Robert Collins
Review feedback.
1002
    cdef object pathjoin
3696.5.2 by Robert Collins
Integrate less aggressive sha logic with C iter-changes.
1003
    cdef object fstat
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1004
    # A set of the ids we've output when doing partial output.
1005
    cdef object seen_ids
3696.5.2 by Robert Collins
Integrate less aggressive sha logic with C iter-changes.
1006
    cdef object sha_file
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1007
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1008
    def __init__(self, include_unchanged, use_filesystem_for_exec,
1009
        search_specific_files, state, source_index, target_index,
1010
        want_unversioned, tree):
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1011
        self.doing_consistency_expansion = 0
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1012
        self.old_dirname_to_file_id = {}
1013
        self.new_dirname_to_file_id = {}
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1014
        # Are we doing a partial iter_changes?
1015
        self.partial = set(['']).__ne__(search_specific_files)
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1016
        # Using a list so that we can access the values and change them in
1017
        # nested scope. Each one is [path, file_id, entry]
1018
        self.last_source_parent = [None, None]
1019
        self.last_target_parent = [None, None]
4570.2.4 by Robert Collins
Older pyrex compatibility.
1020
        if include_unchanged is None:
1021
            self.include_unchanged = False
1022
        else:
4570.2.9 by Robert Collins
Give a clearer error on bad input to dirstate iter_changes objects.
1023
            self.include_unchanged = int(include_unchanged)
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1024
        self.use_filesystem_for_exec = use_filesystem_for_exec
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1025
        self.utf8_decode = cache_utf8._utf8_decode
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1026
        # for all search_indexs in each path at or under each element of
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1027
        # search_specific_files, if the detail is relocated: add the id, and
1028
        # add the relocated path as one to search if its not searched already.
1029
        # If the detail is not relocated, add the id.
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1030
        self.searched_specific_files = set()
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1031
        # When we search exact paths without expanding downwards, we record
1032
        # that here.
1033
        self.searched_exact_paths = set()
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1034
        self.search_specific_files = search_specific_files
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1035
        # The parents up to the root of the paths we are searching.
1036
        # After all normal paths are returned, these specific items are returned.
1037
        self.search_specific_file_parents = set()
1038
        # The ids we've sent out in the delta.
1039
        self.seen_ids = set()
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1040
        self.state = state
1041
        self.current_root = None
1042
        self.current_root_unicode = None
1043
        self.root_entries = None
1044
        self.root_entries_pos = 0
1045
        self.root_entries_len = 0
1046
        self.root_abspath = None
1047
        if source_index is None:
1048
            self.source_index = -1
1049
        else:
1050
            self.source_index = source_index
1051
        self.target_index = target_index
1052
        self.want_unversioned = want_unversioned
1053
        self.tree = tree
1054
        self.dir_iterator = None
1055
        self.block_index = -1
1056
        self.current_block = None
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1057
        self.current_block_list = None
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1058
        self.current_block_pos = -1
1059
        self.current_dir_info = None
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1060
        self.current_dir_list = None
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1061
        self._pending_consistent_entries = []
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1062
        self.path_index = 0
1063
        self.root_dir_info = None
1064
        self.bisect_left = bisect.bisect_left
3696.4.17 by Robert Collins
Review feedback.
1065
        self.pathjoin = osutils.pathjoin
3696.5.2 by Robert Collins
Integrate less aggressive sha logic with C iter-changes.
1066
        self.fstat = os.fstat
1067
        self.sha_file = osutils.sha_file
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1068
        if target_index != 0:
1069
            # A lot of code in here depends on target_index == 0
1070
            raise errors.BzrError('unsupported target index')
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1071
3696.4.11 by Robert Collins
Some Cification of iter_changes, and making the python one actually work.
1072
    cdef _process_entry(self, entry, path_info):
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1073
        """Compare an entry and real disk to generate delta information.
1074
1075
        :param path_info: top_relpath, basename, kind, lstat, abspath for
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1076
            the path of entry. If None, then the path is considered absent in 
1077
            the target (Perhaps we should pass in a concrete entry for this ?)
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1078
            Basename is returned as a utf8 string because we expect this
1079
            tuple will be ignored, and don't want to take the time to
1080
            decode.
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1081
        :return: (iter_changes_result, changed). If the entry has not been
1082
            handled then changed is None. Otherwise it is False if no content
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1083
            or metadata changes have occured, and True if any content or
1084
            metadata change has occurred. If self.include_unchanged is True then
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1085
            if changed is not None, iter_changes_result will always be a result
1086
            tuple. Otherwise, iter_changes_result is None unless changed is
1087
            True.
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1088
        """
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1089
        cdef char target_minikind
3696.4.7 by Robert Collins
More tuning.
1090
        cdef char source_minikind
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1091
        cdef object file_id
1092
        cdef int content_change
3696.4.9 by Robert Collins
Reduce lookups in process_entry.
1093
        cdef object details_list
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1094
        file_id = None
3696.4.9 by Robert Collins
Reduce lookups in process_entry.
1095
        details_list = entry[1]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1096
        if -1 == self.source_index:
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1097
            source_details = DirState.NULL_PARENT_DETAILS
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1098
        else:
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1099
            source_details = details_list[self.source_index]
1100
        target_details = details_list[self.target_index]
3696.4.7 by Robert Collins
More tuning.
1101
        target_minikind = _minikind_from_string(target_details[0])
1102
        if path_info is not None and _versioned_minikind(target_minikind):
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1103
            if self.target_index != 0:
3763.1.1 by Benjamin Peterson
fix two small oversights
1104
                raise AssertionError("Unsupported target index %d" %
1105
                                     self.target_index)
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1106
            link_or_sha1 = _update_entry(self.state, entry, path_info[4], path_info[3])
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1107
            # The entry may have been modified by update_entry
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1108
            target_details = details_list[self.target_index]
3696.4.7 by Robert Collins
More tuning.
1109
            target_minikind = _minikind_from_string(target_details[0])
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1110
        else:
1111
            link_or_sha1 = None
3696.4.7 by Robert Collins
More tuning.
1112
        # the rest of this function is 0.3 seconds on 50K paths, or
1113
        # 0.000006 seconds per call.
1114
        source_minikind = _minikind_from_string(source_details[0])
1115
        if ((_versioned_minikind(source_minikind) or source_minikind == c'r')
1116
            and _versioned_minikind(target_minikind)):
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1117
            # claimed content in both: diff
1118
            #   r    | fdlt   |      | add source to search, add id path move and perform
1119
            #        |        |      | diff check on source-target
1120
            #   r    | fdlt   |  a   | dangling file that was present in the basis.
1121
            #        |        |      | ???
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1122
            if source_minikind != c'r':
1123
                old_dirname = entry[0][0]
1124
                old_basename = entry[0][1]
1125
                old_path = path = None
1126
            else:
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1127
                # add the source to the search path to find any children it
1128
                # has.  TODO ? : only add if it is a container ?
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1129
                if (not self.doing_consistency_expansion and 
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1130
                    not osutils.is_inside_any(self.searched_specific_files,
1131
                                             source_details[1])):
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1132
                    self.search_specific_files.add(source_details[1])
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1133
                    # expanding from a user requested path, parent expansion
1134
                    # for delta consistency happens later.
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1135
                # generate the old path; this is needed for stating later
1136
                # as well.
1137
                old_path = source_details[1]
1138
                old_dirname, old_basename = os.path.split(old_path)
3696.4.17 by Robert Collins
Review feedback.
1139
                path = self.pathjoin(entry[0][0], entry[0][1])
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1140
                old_entry = self.state._get_entry(self.source_index,
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1141
                                             path_utf8=old_path)
1142
                # update the source details variable to be the real
1143
                # location.
1144
                if old_entry == (None, None):
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1145
                    raise errors.CorruptDirstate(self.state._filename,
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1146
                        "entry '%s/%s' is considered renamed from %r"
1147
                        " but source does not exist\n"
1148
                        "entry: %s" % (entry[0][0], entry[0][1], old_path, entry))
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1149
                source_details = old_entry[1][self.source_index]
3696.4.7 by Robert Collins
More tuning.
1150
                source_minikind = _minikind_from_string(source_details[0])
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1151
            if path_info is None:
1152
                # the file is missing on disk, show as removed.
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1153
                content_change = 1
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1154
                target_kind = None
1155
                target_exec = False
1156
            else:
1157
                # source and target are both versioned and disk file is present.
1158
                target_kind = path_info[2]
1159
                if target_kind == 'directory':
1160
                    if path is None:
3696.4.17 by Robert Collins
Review feedback.
1161
                        old_path = path = self.pathjoin(old_dirname, old_basename)
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1162
                    file_id = entry[0][2]
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1163
                    self.new_dirname_to_file_id[path] = file_id
3696.4.7 by Robert Collins
More tuning.
1164
                    if source_minikind != c'd':
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1165
                        content_change = 1
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1166
                    else:
1167
                        # directories have no fingerprint
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1168
                        content_change = 0
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1169
                    target_exec = False
1170
                elif target_kind == 'file':
3696.4.7 by Robert Collins
More tuning.
1171
                    if source_minikind != c'f':
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1172
                        content_change = 1
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1173
                    else:
4393.3.2 by Ian Clatworthy
fix pyrex version of _process_entry as well
1174
                        # Check the sha. We can't just rely on the size as
1175
                        # content filtering may mean differ sizes actually
1176
                        # map to the same content
1177
                        if link_or_sha1 is None:
1178
                            # Stat cache miss:
1179
                            statvalue, link_or_sha1 = \
1180
                                self.state._sha1_provider.stat_and_sha1(
1181
                                path_info[4])
1182
                            self.state._observed_sha1(entry, link_or_sha1,
1183
                                statvalue)
1184
                        content_change = (link_or_sha1 != source_details[1])
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1185
                    # Target details is updated at update_entry time
1186
                    if self.use_filesystem_for_exec:
1187
                        # We don't need S_ISREG here, because we are sure
1188
                        # we are dealing with a file.
1189
                        target_exec = bool(S_IXUSR & path_info[3].st_mode)
1190
                    else:
1191
                        target_exec = target_details[3]
1192
                elif target_kind == 'symlink':
3696.4.7 by Robert Collins
More tuning.
1193
                    if source_minikind != c'l':
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1194
                        content_change = 1
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1195
                    else:
1196
                        content_change = (link_or_sha1 != source_details[1])
1197
                    target_exec = False
1198
                elif target_kind == 'tree-reference':
3696.4.7 by Robert Collins
More tuning.
1199
                    if source_minikind != c't':
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1200
                        content_change = 1
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1201
                    else:
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1202
                        content_change = 0
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1203
                    target_exec = False
1204
                else:
4634.58.1 by Robert Collins
Show a sensible error when a previously versionable path becomes a FIFO or other unversionable file.
1205
                    if path is None:
1206
                        path = self.pathjoin(old_dirname, old_basename)
1207
                    raise errors.BadFileKindError(path, path_info[2])
3696.4.7 by Robert Collins
More tuning.
1208
            if source_minikind == c'd':
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1209
                if path is None:
3696.4.17 by Robert Collins
Review feedback.
1210
                    old_path = path = self.pathjoin(old_dirname, old_basename)
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1211
                if file_id is None:
1212
                    file_id = entry[0][2]
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1213
                self.old_dirname_to_file_id[old_path] = file_id
1214
            # parent id is the entry for the path in the target tree
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1215
            if old_basename and old_dirname == self.last_source_parent[0]:
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1216
                # use a cached hit for non-root source entries.
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1217
                source_parent_id = self.last_source_parent[1]
1218
            else:
1219
                try:
1220
                    source_parent_id = self.old_dirname_to_file_id[old_dirname]
1221
                except KeyError:
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1222
                    source_parent_entry = self.state._get_entry(self.source_index,
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1223
                                                           path_utf8=old_dirname)
1224
                    source_parent_id = source_parent_entry[0][2]
1225
                if source_parent_id == entry[0][2]:
1226
                    # This is the root, so the parent is None
1227
                    source_parent_id = None
1228
                else:
1229
                    self.last_source_parent[0] = old_dirname
1230
                    self.last_source_parent[1] = source_parent_id
1231
            new_dirname = entry[0][0]
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1232
            if entry[0][1] and new_dirname == self.last_target_parent[0]:
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1233
                # use a cached hit for non-root target entries.
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1234
                target_parent_id = self.last_target_parent[1]
1235
            else:
1236
                try:
1237
                    target_parent_id = self.new_dirname_to_file_id[new_dirname]
1238
                except KeyError:
1239
                    # TODO: We don't always need to do the lookup, because the
1240
                    #       parent entry will be the same as the source entry.
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1241
                    target_parent_entry = self.state._get_entry(self.target_index,
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1242
                                                           path_utf8=new_dirname)
1243
                    if target_parent_entry == (None, None):
1244
                        raise AssertionError(
1245
                            "Could not find target parent in wt: %s\nparent of: %s"
1246
                            % (new_dirname, entry))
1247
                    target_parent_id = target_parent_entry[0][2]
1248
                if target_parent_id == entry[0][2]:
1249
                    # This is the root, so the parent is None
1250
                    target_parent_id = None
1251
                else:
1252
                    self.last_target_parent[0] = new_dirname
1253
                    self.last_target_parent[1] = target_parent_id
1254
1255
            source_exec = source_details[3]
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1256
            changed = (content_change
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1257
                or source_parent_id != target_parent_id
1258
                or old_basename != entry[0][1]
1259
                or source_exec != target_exec
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1260
                )
1261
            if not changed and not self.include_unchanged:
1262
                return None, False
1263
            else:
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1264
                if old_path is None:
3696.4.17 by Robert Collins
Review feedback.
1265
                    path = self.pathjoin(old_dirname, old_basename)
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1266
                    old_path = path
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1267
                    old_path_u = self.utf8_decode(old_path)[0]
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1268
                    path_u = old_path_u
1269
                else:
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1270
                    old_path_u = self.utf8_decode(old_path)[0]
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1271
                    if old_path == path:
1272
                        path_u = old_path_u
1273
                    else:
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1274
                        path_u = self.utf8_decode(path)[0]
3696.4.7 by Robert Collins
More tuning.
1275
                source_kind = _minikind_to_kind(source_minikind)
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1276
                return (entry[0][2],
1277
                       (old_path_u, path_u),
1278
                       content_change,
1279
                       (True, True),
1280
                       (source_parent_id, target_parent_id),
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1281
                       (self.utf8_decode(old_basename)[0], self.utf8_decode(entry[0][1])[0]),
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1282
                       (source_kind, target_kind),
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1283
                       (source_exec, target_exec)), changed
3696.4.7 by Robert Collins
More tuning.
1284
        elif source_minikind == c'a' and _versioned_minikind(target_minikind):
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1285
            # looks like a new file
3696.4.17 by Robert Collins
Review feedback.
1286
            path = self.pathjoin(entry[0][0], entry[0][1])
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1287
            # parent id is the entry for the path in the target tree
1288
            # TODO: these are the same for an entire directory: cache em.
4005.1.1 by John Arbash Meinel
Related to bug #328674, give a better error for a corrupt dirstate.
1289
            parent_entry = self.state._get_entry(self.target_index,
1290
                                                 path_utf8=entry[0][0])
1291
            if parent_entry is None:
1292
                raise errors.DirstateCorrupt(self.state,
1293
                    "We could not find the parent entry in index %d"
1294
                    " for the entry: %s"
1295
                    % (self.target_index, entry[0]))
1296
            parent_id = parent_entry[0][2]
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1297
            if parent_id == entry[0][2]:
1298
                parent_id = None
1299
            if path_info is not None:
1300
                # Present on disk:
1301
                if self.use_filesystem_for_exec:
1302
                    # We need S_ISREG here, because we aren't sure if this
1303
                    # is a file or not.
1304
                    target_exec = bool(
1305
                        S_ISREG(path_info[3].st_mode)
1306
                        and S_IXUSR & path_info[3].st_mode)
1307
                else:
1308
                    target_exec = target_details[3]
1309
                return (entry[0][2],
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1310
                       (None, self.utf8_decode(path)[0]),
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1311
                       True,
1312
                       (False, True),
1313
                       (None, parent_id),
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1314
                       (None, self.utf8_decode(entry[0][1])[0]),
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1315
                       (None, path_info[2]),
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1316
                       (None, target_exec)), True
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1317
            else:
1318
                # Its a missing file, report it as such.
1319
                return (entry[0][2],
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1320
                       (None, self.utf8_decode(path)[0]),
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1321
                       False,
1322
                       (False, True),
1323
                       (None, parent_id),
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1324
                       (None, self.utf8_decode(entry[0][1])[0]),
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1325
                       (None, None),
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1326
                       (None, False)), True
3696.4.7 by Robert Collins
More tuning.
1327
        elif _versioned_minikind(source_minikind) and target_minikind == c'a':
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1328
            # unversioned, possibly, or possibly not deleted: we dont care.
1329
            # if its still on disk, *and* theres no other entry at this
1330
            # path [we dont know this in this routine at the moment -
1331
            # perhaps we should change this - then it would be an unknown.
3696.4.17 by Robert Collins
Review feedback.
1332
            old_path = self.pathjoin(entry[0][0], entry[0][1])
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1333
            # parent id is the entry for the path in the target tree
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1334
            parent_id = self.state._get_entry(self.source_index, path_utf8=entry[0][0])[0][2]
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1335
            if parent_id == entry[0][2]:
1336
                parent_id = None
1337
            return (entry[0][2],
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1338
                   (self.utf8_decode(old_path)[0], None),
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1339
                   True,
1340
                   (True, False),
1341
                   (parent_id, None),
3696.4.6 by Robert Collins
Fixes for the relocated code, and use _update_entry within the C accelerated code, another 8 percent saving.
1342
                   (self.utf8_decode(entry[0][1])[0], None),
3696.4.7 by Robert Collins
More tuning.
1343
                   (_minikind_to_kind(source_minikind), None),
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1344
                   (source_details[3], None)), True
3696.4.7 by Robert Collins
More tuning.
1345
        elif _versioned_minikind(source_minikind) and target_minikind == c'r':
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1346
            # a rename; could be a true rename, or a rename inherited from
1347
            # a renamed parent. TODO: handle this efficiently. Its not
1348
            # common case to rename dirs though, so a correct but slow
1349
            # implementation will do.
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1350
            if (not self.doing_consistency_expansion and 
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1351
                not osutils.is_inside_any(self.searched_specific_files,
1352
                    target_details[1])):
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
1353
                self.search_specific_files.add(target_details[1])
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1354
                # We don't expand the specific files parents list here as
1355
                # the path is absent in target and won't create a delta with
1356
                # missing parent.
3696.4.7 by Robert Collins
More tuning.
1357
        elif ((source_minikind == c'r' or source_minikind == c'a') and
1358
              (target_minikind == c'r' or target_minikind == c'a')):
3696.4.13 by Robert Collins
More Cification, removing redundant string comparisons
1359
            # neither of the selected trees contain this path,
3696.4.5 by Robert Collins
Simple 'compiled with pyrex' ProcessEntry class. faster.
1360
            # so skip over it. This is not currently directly tested, but
1361
            # is indirectly via test_too_much.TestCommands.test_conflicts.
1362
            pass
1363
        else:
1364
            raise AssertionError("don't know how to compare "
1365
                "source_minikind=%r, target_minikind=%r"
1366
                % (source_minikind, target_minikind))
1367
            ## import pdb;pdb.set_trace()
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1368
        return None, None
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1369
1370
    def __iter__(self):
1371
        return self
1372
1373
    def iter_changes(self):
1374
        return self
1375
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1376
    cdef void _gather_result_for_consistency(self, result):
1377
        """Check a result we will yield to make sure we are consistent later.
1378
        
1379
        This gathers result's parents into a set to output later.
1380
1381
        :param result: A result tuple.
1382
        """
1383
        if not self.partial or not result[0]:
1384
            return
1385
        self.seen_ids.add(result[0])
1386
        new_path = result[1][1]
1387
        if new_path:
1388
            # Not the root and not a delete: queue up the parents of the path.
1389
            self.search_specific_file_parents.update(
1390
                osutils.parent_directories(new_path.encode('utf8')))
1391
            # Add the root directory which parent_directories does not
1392
            # provide.
1393
            self.search_specific_file_parents.add('')
1394
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1395
    cdef void _update_current_block(self):
1396
        if (self.block_index < len(self.state._dirblocks) and
1397
            osutils.is_inside(self.current_root, self.state._dirblocks[self.block_index][0])):
1398
            self.current_block = self.state._dirblocks[self.block_index]
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1399
            self.current_block_list = self.current_block[1]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1400
            self.current_block_pos = 0
1401
        else:
1402
            self.current_block = None
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1403
            self.current_block_list = None
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1404
1405
    def __next__(self):
1406
        # Simple thunk to allow tail recursion without pyrex confusion
1407
        return self._iter_next()
1408
1409
    cdef _iter_next(self):
1410
        """Iterate over the changes."""
1411
        # This function single steps through an iterator. As such while loops
1412
        # are often exited by 'return' - the code is structured so that the
1413
        # next call into the function will return to the same while loop. Note
1414
        # that all flow control needed to re-reach that step is reexecuted,
1415
        # which can be a performance problem. It has not yet been tuned to
1416
        # minimise this; a state machine is probably the simplest restructuring
1417
        # to both minimise this overhead and make the code considerably more
1418
        # understandable.
1419
1420
        # sketch: 
1421
        # compare source_index and target_index at or under each element of search_specific_files.
1422
        # follow the following comparison table. Note that we only want to do diff operations when
1423
        # the target is fdl because thats when the walkdirs logic will have exposed the pathinfo 
1424
        # for the target.
1425
        # cases:
1426
        # 
1427
        # Source | Target | disk | action
1428
        #   r    | fdlt   |      | add source to search, add id path move and perform
1429
        #        |        |      | diff check on source-target
1430
        #   r    | fdlt   |  a   | dangling file that was present in the basis. 
1431
        #        |        |      | ???
1432
        #   r    |  a     |      | add source to search
1433
        #   r    |  a     |  a   | 
1434
        #   r    |  r     |      | this path is present in a non-examined tree, skip.
1435
        #   r    |  r     |  a   | this path is present in a non-examined tree, skip.
1436
        #   a    | fdlt   |      | add new id
1437
        #   a    | fdlt   |  a   | dangling locally added file, skip
1438
        #   a    |  a     |      | not present in either tree, skip
1439
        #   a    |  a     |  a   | not present in any tree, skip
1440
        #   a    |  r     |      | not present in either tree at this path, skip as it
1441
        #        |        |      | may not be selected by the users list of paths.
1442
        #   a    |  r     |  a   | not present in either tree at this path, skip as it
1443
        #        |        |      | may not be selected by the users list of paths.
1444
        #  fdlt  | fdlt   |      | content in both: diff them
1445
        #  fdlt  | fdlt   |  a   | deleted locally, but not unversioned - show as deleted ?
1446
        #  fdlt  |  a     |      | unversioned: output deleted id for now
1447
        #  fdlt  |  a     |  a   | unversioned and deleted: output deleted id
1448
        #  fdlt  |  r     |      | relocated in this tree, so add target to search.
1449
        #        |        |      | Dont diff, we will see an r,fd; pair when we reach
1450
        #        |        |      | this id at the other path.
1451
        #  fdlt  |  r     |  a   | relocated in this tree, so add target to search.
1452
        #        |        |      | Dont diff, we will see an r,fd; pair when we reach
1453
        #        |        |      | this id at the other path.
1454
1455
        # TODO: jam 20070516 - Avoid the _get_entry lookup overhead by
1456
        #       keeping a cache of directories that we have seen.
1457
        cdef object current_dirname, current_blockname
1458
        cdef char * current_dirname_c, * current_blockname_c
3696.4.11 by Robert Collins
Some Cification of iter_changes, and making the python one actually work.
1459
        cdef int advance_entry, advance_path
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1460
        cdef int path_handled
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1461
        searched_specific_files = self.searched_specific_files
1462
        # Are we walking a root?
1463
        while self.root_entries_pos < self.root_entries_len:
1464
            entry = self.root_entries[self.root_entries_pos]
1465
            self.root_entries_pos = self.root_entries_pos + 1
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1466
            result, changed = self._process_entry(entry, self.root_dir_info)
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1467
            if changed is not None:
1468
                if changed:
1469
                    self._gather_result_for_consistency(result)
1470
                if changed or self.include_unchanged:
1471
                    return result
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1472
        # Have we finished the prior root, or never started one ?
1473
        if self.current_root is None:
1474
            # TODO: the pending list should be lexically sorted?  the
1475
            # interface doesn't require it.
1476
            try:
1477
                self.current_root = self.search_specific_files.pop()
1478
            except KeyError:
1479
                raise StopIteration()
1480
            self.searched_specific_files.add(self.current_root)
1481
            # process the entries for this containing directory: the rest will be
1482
            # found by their parents recursively.
1483
            self.root_entries = self.state._entries_for_path(self.current_root)
1484
            self.root_entries_len = len(self.root_entries)
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1485
            self.current_root_unicode = self.current_root.decode('utf8')
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1486
            self.root_abspath = self.tree.abspath(self.current_root_unicode)
1487
            try:
1488
                root_stat = os.lstat(self.root_abspath)
1489
            except OSError, e:
1490
                if e.errno == errno.ENOENT:
1491
                    # the path does not exist: let _process_entry know that.
1492
                    self.root_dir_info = None
1493
                else:
1494
                    # some other random error: hand it up.
1495
                    raise
1496
            else:
1497
                self.root_dir_info = ('', self.current_root,
1498
                    osutils.file_kind_from_stat_mode(root_stat.st_mode), root_stat,
1499
                    self.root_abspath)
1500
                if self.root_dir_info[2] == 'directory':
1501
                    if self.tree._directory_is_tree_reference(
1502
                        self.current_root_unicode):
1503
                        self.root_dir_info = self.root_dir_info[:2] + \
1504
                            ('tree-reference',) + self.root_dir_info[3:]
1505
            if not self.root_entries and not self.root_dir_info:
1506
                # this specified path is not present at all, skip it.
1507
                # (tail recursion, can do a loop once the full structure is
1508
                # known).
1509
                return self._iter_next()
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1510
            path_handled = 0
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1511
            self.root_entries_pos = 0
1512
            # XXX Clarity: This loop is duplicated a out the self.current_root
1513
            # is None guard above: if we return from it, it completes there
1514
            # (and the following if block cannot trigger because
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1515
            # path_handled must be true, so the if block is not # duplicated.
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1516
            while self.root_entries_pos < self.root_entries_len:
1517
                entry = self.root_entries[self.root_entries_pos]
1518
                self.root_entries_pos = self.root_entries_pos + 1
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1519
                result, changed = self._process_entry(entry, self.root_dir_info)
1520
                if changed is not None:
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1521
                    path_handled = -1
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1522
                    if changed:
1523
                        self._gather_result_for_consistency(result)
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1524
                    if changed or self.include_unchanged:
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1525
                        return result
1526
            # handle unversioned specified paths:
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1527
            if self.want_unversioned and not path_handled and self.root_dir_info:
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1528
                new_executable = bool(
1529
                    stat.S_ISREG(self.root_dir_info[3].st_mode)
1530
                    and stat.S_IEXEC & self.root_dir_info[3].st_mode)
1531
                return (None,
1532
                       (None, self.current_root_unicode),
1533
                       True,
1534
                       (False, False),
1535
                       (None, None),
1536
                       (None, splitpath(self.current_root_unicode)[-1]),
1537
                       (None, self.root_dir_info[2]),
1538
                       (None, new_executable)
1539
                      )
1540
            # If we reach here, the outer flow continues, which enters into the
1541
            # per-root setup logic.
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1542
        if (self.current_dir_info is None and self.current_block is None and not
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1543
            self.doing_consistency_expansion):
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1544
            # setup iteration of this root:
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1545
            self.current_dir_list = None
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1546
            if self.root_dir_info and self.root_dir_info[2] == 'tree-reference':
1547
                self.current_dir_info = None
1548
            else:
1549
                self.dir_iterator = osutils._walkdirs_utf8(self.root_abspath,
1550
                    prefix=self.current_root)
1551
                self.path_index = 0
1552
                try:
1553
                    self.current_dir_info = self.dir_iterator.next()
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1554
                    self.current_dir_list = self.current_dir_info[1]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1555
                except OSError, e:
1556
                    # there may be directories in the inventory even though
1557
                    # this path is not a file on disk: so mark it as end of
1558
                    # iterator
1559
                    if e.errno in (errno.ENOENT, errno.ENOTDIR, errno.EINVAL):
1560
                        self.current_dir_info = None
1561
                    elif sys.platform == 'win32':
1562
                        # on win32, python2.4 has e.errno == ERROR_DIRECTORY, but
1563
                        # python 2.5 has e.errno == EINVAL,
1564
                        #            and e.winerror == ERROR_DIRECTORY
1565
                        try:
1566
                            e_winerror = e.winerror
1567
                        except AttributeError:
1568
                            e_winerror = None
1569
                        win_errors = (ERROR_DIRECTORY, ERROR_PATH_NOT_FOUND)
1570
                        if (e.errno in win_errors or e_winerror in win_errors):
1571
                            self.current_dir_info = None
1572
                        else:
1573
                            # Will this really raise the right exception ?
1574
                            raise
1575
                    else:
1576
                        raise
1577
                else:
1578
                    if self.current_dir_info[0][0] == '':
1579
                        # remove .bzr from iteration
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1580
                        bzr_index = self.bisect_left(self.current_dir_list, ('.bzr',))
1581
                        if self.current_dir_list[bzr_index][0] != '.bzr':
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1582
                            raise AssertionError()
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1583
                        del self.current_dir_list[bzr_index]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1584
            initial_key = (self.current_root, '', '')
1585
            self.block_index, _ = self.state._find_block_index_from_key(initial_key)
1586
            if self.block_index == 0:
1587
                # we have processed the total root already, but because the
1588
                # initial key matched it we should skip it here.
1589
                self.block_index = self.block_index + 1
1590
            self._update_current_block()
1591
        # walk until both the directory listing and the versioned metadata
1592
        # are exhausted. 
1593
        while (self.current_dir_info is not None
1594
            or self.current_block is not None):
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1595
            # Uncommon case - a missing directory or an unversioned directory:
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1596
            if (self.current_dir_info and self.current_block
1597
                and self.current_dir_info[0][0] != self.current_block[0]):
1598
                # Work around pyrex broken heuristic - current_dirname has
1599
                # the same scope as current_dirname_c
1600
                current_dirname = self.current_dir_info[0][0]
1601
                current_dirname_c = PyString_AS_STRING_void(
1602
                    <void *>current_dirname)
1603
                current_blockname = self.current_block[0]
1604
                current_blockname_c = PyString_AS_STRING_void(
1605
                    <void *>current_blockname)
1606
                # In the python generator we evaluate this if block once per
1607
                # dir+block; because we reenter in the pyrex version its being
1608
                # evaluated once per path: we could cache the result before
1609
                # doing the while loop and probably save time.
1610
                if _cmp_by_dirs(current_dirname_c,
1611
                    PyString_Size(current_dirname),
1612
                    current_blockname_c,
1613
                    PyString_Size(current_blockname)) < 0:
1614
                    # filesystem data refers to paths not covered by the
1615
                    # dirblock.  this has two possibilities:
1616
                    # A) it is versioned but empty, so there is no block for it
1617
                    # B) it is not versioned.
1618
1619
                    # if (A) then we need to recurse into it to check for
1620
                    # new unknown files or directories.
1621
                    # if (B) then we should ignore it, because we don't
1622
                    # recurse into unknown directories.
1623
                    # We are doing a loop
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1624
                    while self.path_index < len(self.current_dir_list):
1625
                        current_path_info = self.current_dir_list[self.path_index]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1626
                        # dont descend into this unversioned path if it is
1627
                        # a dir
1628
                        if current_path_info[2] in ('directory',
1629
                                                    'tree-reference'):
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1630
                            del self.current_dir_list[self.path_index]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1631
                            self.path_index = self.path_index - 1
1632
                        self.path_index = self.path_index + 1
1633
                        if self.want_unversioned:
1634
                            if current_path_info[2] == 'directory':
1635
                                if self.tree._directory_is_tree_reference(
1636
                                    self.utf8_decode(current_path_info[0])[0]):
1637
                                    current_path_info = current_path_info[:2] + \
1638
                                        ('tree-reference',) + current_path_info[3:]
1639
                            new_executable = bool(
1640
                                stat.S_ISREG(current_path_info[3].st_mode)
1641
                                and stat.S_IEXEC & current_path_info[3].st_mode)
1642
                            return (None,
1643
                                (None, self.utf8_decode(current_path_info[0])[0]),
1644
                                True,
1645
                                (False, False),
1646
                                (None, None),
1647
                                (None, self.utf8_decode(current_path_info[1])[0]),
1648
                                (None, current_path_info[2]),
1649
                                (None, new_executable))
1650
                    # This dir info has been handled, go to the next
1651
                    self.path_index = 0
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1652
                    self.current_dir_list = None
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1653
                    try:
1654
                        self.current_dir_info = self.dir_iterator.next()
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1655
                        self.current_dir_list = self.current_dir_info[1]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1656
                    except StopIteration:
1657
                        self.current_dir_info = None
1658
                else: #(dircmp > 0)
1659
                    # We have a dirblock entry for this location, but there
1660
                    # is no filesystem path for this. This is most likely
1661
                    # because a directory was removed from the disk.
1662
                    # We don't have to report the missing directory,
1663
                    # because that should have already been handled, but we
1664
                    # need to handle all of the files that are contained
1665
                    # within.
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1666
                    while self.current_block_pos < len(self.current_block_list):
1667
                        current_entry = self.current_block_list[self.current_block_pos]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1668
                        self.current_block_pos = self.current_block_pos + 1
1669
                        # entry referring to file not present on disk.
1670
                        # advance the entry only, after processing.
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1671
                        result, changed = self._process_entry(current_entry, None)
1672
                        if changed is not None:
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1673
                            if changed:
1674
                                self._gather_result_for_consistency(result)
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1675
                            if changed or self.include_unchanged:
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1676
                                return result
1677
                    self.block_index = self.block_index + 1
1678
                    self._update_current_block()
1679
                continue # next loop-on-block/dir
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1680
            result = self._loop_one_block()
1681
            if result is not None:
1682
                return result
1683
        if len(self.search_specific_files):
1684
            # More supplied paths to process
1685
            self.current_root = None
1686
            return self._iter_next()
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1687
        # Start expanding more conservatively, adding paths the user may not
1688
        # have intended but required for consistent deltas.
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1689
        self.doing_consistency_expansion = 1
1690
        if not self._pending_consistent_entries:
1691
            self._pending_consistent_entries = self._next_consistent_entries()
1692
        while self._pending_consistent_entries:
1693
            result, changed = self._pending_consistent_entries.pop()
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1694
            if changed is not None:
1695
                return result
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1696
        raise StopIteration()
1697
1698
    cdef object _maybe_tree_ref(self, current_path_info):
1699
        if self.tree._directory_is_tree_reference(
1700
            self.utf8_decode(current_path_info[0])[0]):
1701
            return current_path_info[:2] + \
1702
                ('tree-reference',) + current_path_info[3:]
1703
        else:
1704
            return current_path_info
1705
1706
    cdef object _loop_one_block(self):
3696.4.11 by Robert Collins
Some Cification of iter_changes, and making the python one actually work.
1707
            # current_dir_info and current_block refer to the same directory -
1708
            # this is the common case code.
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1709
            # Assign local variables for current path and entry:
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1710
            cdef object current_entry
1711
            cdef object current_path_info
1712
            cdef int path_handled
3696.4.13 by Robert Collins
More Cification, removing redundant string comparisons
1713
            cdef char minikind
1714
            cdef int cmp_result
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1715
            # cdef char * temp_str
1716
            # cdef Py_ssize_t temp_str_length
1717
            # PyString_AsStringAndSize(disk_kind, &temp_str, &temp_str_length)
1718
            # if not strncmp(temp_str, "directory", temp_str_length):
1719
            if (self.current_block is not None and
1720
                self.current_block_pos < PyList_GET_SIZE(self.current_block_list)):
1721
                current_entry = PyList_GET_ITEM(self.current_block_list,
1722
                    self.current_block_pos)
1723
                # accomodate pyrex
1724
                Py_INCREF(current_entry)
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1725
            else:
1726
                current_entry = None
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1727
            if (self.current_dir_info is not None and
1728
                self.path_index < PyList_GET_SIZE(self.current_dir_list)):
1729
                current_path_info = PyList_GET_ITEM(self.current_dir_list,
1730
                    self.path_index)
1731
                # accomodate pyrex
1732
                Py_INCREF(current_path_info)
1733
                disk_kind = PyTuple_GET_ITEM(current_path_info, 2)
1734
                # accomodate pyrex
1735
                Py_INCREF(disk_kind)
1736
                if disk_kind == "directory":
1737
                    current_path_info = self._maybe_tree_ref(current_path_info)
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1738
            else:
1739
                current_path_info = None
1740
            while (current_entry is not None or current_path_info is not None):
3696.4.11 by Robert Collins
Some Cification of iter_changes, and making the python one actually work.
1741
                advance_entry = -1
1742
                advance_path = -1
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1743
                result = None
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1744
                path_handled = 0
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1745
                if current_entry is None:
1746
                    # unversioned -  the check for path_handled when the path
1747
                    # is advanced will yield this path if needed.
1748
                    pass
1749
                elif current_path_info is None:
1750
                    # no path is fine: the per entry code will handle it.
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1751
                    result, changed = self._process_entry(current_entry,
1752
                        current_path_info)
3696.4.13 by Robert Collins
More Cification, removing redundant string comparisons
1753
                else:
1754
                    minikind = _minikind_from_string(
1755
                        current_entry[1][self.target_index][0])
1756
                    cmp_result = cmp(current_path_info[1], current_entry[0][1])
1757
                    if (cmp_result or minikind == c'a' or minikind == c'r'):
1758
                        # The current path on disk doesn't match the dirblock
1759
                        # record. Either the dirblock record is marked as
1760
                        # absent/renamed, or the file on disk is not present at all
1761
                        # in the dirblock. Either way, report about the dirblock
1762
                        # entry, and let other code handle the filesystem one.
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1763
3696.4.13 by Robert Collins
More Cification, removing redundant string comparisons
1764
                        # Compare the basename for these files to determine
1765
                        # which comes first
1766
                        if cmp_result < 0:
1767
                            # extra file on disk: pass for now, but only
1768
                            # increment the path, not the entry
1769
                            advance_entry = 0
1770
                        else:
1771
                            # entry referring to file not present on disk.
1772
                            # advance the entry only, after processing.
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1773
                            result, changed = self._process_entry(current_entry,
1774
                                None)
3696.4.13 by Robert Collins
More Cification, removing redundant string comparisons
1775
                            advance_path = 0
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1776
                    else:
3696.4.13 by Robert Collins
More Cification, removing redundant string comparisons
1777
                        # paths are the same,and the dirstate entry is not
1778
                        # absent or renamed.
4570.2.1 by Robert Collins
Refactoring of dirstate iter_changes to make it possible to tell changed vs unchanged results internally.
1779
                        result, changed = self._process_entry(current_entry,
1780
                            current_path_info)
1781
                        if changed is not None:
3696.4.13 by Robert Collins
More Cification, removing redundant string comparisons
1782
                            path_handled = -1
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1783
                            if not changed and not self.include_unchanged:
1784
                                changed = None
3696.4.11 by Robert Collins
Some Cification of iter_changes, and making the python one actually work.
1785
                # >- loop control starts here:
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1786
                # >- entry
1787
                if advance_entry and current_entry is not None:
1788
                    self.current_block_pos = self.current_block_pos + 1
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1789
                    if self.current_block_pos < PyList_GET_SIZE(self.current_block_list):
1790
                        current_entry = self.current_block_list[self.current_block_pos]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1791
                    else:
1792
                        current_entry = None
1793
                # >- path
1794
                if advance_path and current_path_info is not None:
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1795
                    if not path_handled:
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1796
                        # unversioned in all regards
1797
                        if self.want_unversioned:
1798
                            new_executable = bool(
1799
                                stat.S_ISREG(current_path_info[3].st_mode)
1800
                                and stat.S_IEXEC & current_path_info[3].st_mode)
1801
                            try:
1802
                                relpath_unicode = self.utf8_decode(current_path_info[0])[0]
1803
                            except UnicodeDecodeError:
1804
                                raise errors.BadFilenameEncoding(
1805
                                    current_path_info[0], osutils._fs_enc)
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1806
                            if changed is not None:
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1807
                                raise AssertionError(
1808
                                    "result is not None: %r" % result)
1809
                            result = (None,
1810
                                (None, relpath_unicode),
1811
                                True,
1812
                                (False, False),
1813
                                (None, None),
1814
                                (None, self.utf8_decode(current_path_info[1])[0]),
1815
                                (None, current_path_info[2]),
1816
                                (None, new_executable))
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1817
                            changed = True
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1818
                        # dont descend into this unversioned path if it is
1819
                        # a dir
1820
                        if current_path_info[2] in ('directory'):
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1821
                            del self.current_dir_list[self.path_index]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1822
                            self.path_index = self.path_index - 1
1823
                    # dont descend the disk iterator into any tree 
1824
                    # paths.
1825
                    if current_path_info[2] == 'tree-reference':
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1826
                        del self.current_dir_list[self.path_index]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1827
                        self.path_index = self.path_index - 1
1828
                    self.path_index = self.path_index + 1
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1829
                    if self.path_index < len(self.current_dir_list):
1830
                        current_path_info = self.current_dir_list[self.path_index]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1831
                        if current_path_info[2] == 'directory':
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1832
                            current_path_info = self._maybe_tree_ref(
1833
                                current_path_info)
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1834
                    else:
1835
                        current_path_info = None
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1836
                if changed is not None:
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1837
                    # Found a result on this pass, yield it
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1838
                    if changed:
1839
                        self._gather_result_for_consistency(result)
1840
                    if changed or self.include_unchanged:
1841
                        return result
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1842
            if self.current_block is not None:
1843
                self.block_index = self.block_index + 1
1844
                self._update_current_block()
1845
            if self.current_dir_info is not None:
1846
                self.path_index = 0
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1847
                self.current_dir_list = None
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1848
                try:
1849
                    self.current_dir_info = self.dir_iterator.next()
3696.4.12 by Robert Collins
More Cification of iter_changes, smaller functions, explicit type usage.
1850
                    self.current_dir_list = self.current_dir_info[1]
3696.4.10 by Robert Collins
Basic first cut of full-pyrex iter_changes.
1851
                except StopIteration:
1852
                    self.current_dir_info = None
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1853
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1854
    cdef object _next_consistent_entries(self):
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1855
        """Grabs the next specific file parent case to consider.
1856
        
1857
        :return: A list of the results, each of which is as for _process_entry.
1858
        """
1859
        results = []
1860
        while self.search_specific_file_parents:
1861
            # Process the parent directories for the paths we were iterating.
1862
            # Even in extremely large trees this should be modest, so currently
1863
            # no attempt is made to optimise.
1864
            path_utf8 = self.search_specific_file_parents.pop()
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1865
            if path_utf8 in self.searched_exact_paths:
1866
                # We've examined this path.
1867
                continue
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1868
            if osutils.is_inside_any(self.searched_specific_files, path_utf8):
1869
                # We've examined this path.
1870
                continue
1871
            path_entries = self.state._entries_for_path(path_utf8)
1872
            # We need either one or two entries. If the path in
1873
            # self.target_index has moved (so the entry in source_index is in
1874
            # 'ar') then we need to also look for the entry for this path in
1875
            # self.source_index, to output the appropriate delete-or-rename.
1876
            selected_entries = []
1877
            found_item = False
1878
            for candidate_entry in path_entries:
1879
                # Find entries present in target at this path:
1880
                if candidate_entry[1][self.target_index][0] not in 'ar':
1881
                    found_item = True
1882
                    selected_entries.append(candidate_entry)
1883
                # Find entries present in source at this path:
1884
                elif (self.source_index is not None and
1885
                    candidate_entry[1][self.source_index][0] not in 'ar'):
1886
                    found_item = True
1887
                    if candidate_entry[1][self.target_index][0] == 'a':
1888
                        # Deleted, emit it here.
1889
                        selected_entries.append(candidate_entry)
1890
                    else:
1891
                        # renamed, emit it when we process the directory it
1892
                        # ended up at.
1893
                        self.search_specific_file_parents.add(
1894
                            candidate_entry[1][self.target_index][1])
1895
            if not found_item:
1896
                raise AssertionError(
1897
                    "Missing entry for specific path parent %r, %r" % (
1898
                    path_utf8, path_entries))
1899
            path_info = self._path_info(path_utf8, path_utf8.decode('utf8'))
1900
            for entry in selected_entries:
1901
                if entry[0][2] in self.seen_ids:
1902
                    continue
1903
                result, changed = self._process_entry(entry, path_info)
1904
                if changed is None:
1905
                    raise AssertionError(
1906
                        "Got entry<->path mismatch for specific path "
1907
                        "%r entry %r path_info %r " % (
1908
                        path_utf8, entry, path_info))
1909
                # Only include changes - we're outside the users requested
1910
                # expansion.
1911
                if changed:
1912
                    self._gather_result_for_consistency(result)
1913
                    if (result[6][0] == 'directory' and
1914
                        result[6][1] != 'directory'):
1915
                        # This stopped being a directory, the old children have
1916
                        # to be included.
1917
                        if entry[1][self.source_index][0] == 'r':
1918
                            # renamed, take the source path
1919
                            entry_path_utf8 = entry[1][self.source_index][1]
1920
                        else:
1921
                            entry_path_utf8 = path_utf8
1922
                        initial_key = (entry_path_utf8, '', '')
1923
                        block_index, _ = self.state._find_block_index_from_key(
1924
                            initial_key)
1925
                        if block_index == 0:
1926
                            # The children of the root are in block index 1.
4570.2.4 by Robert Collins
Older pyrex compatibility.
1927
                            block_index = block_index + 1
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1928
                        current_block = None
1929
                        if block_index < len(self.state._dirblocks):
1930
                            current_block = self.state._dirblocks[block_index]
1931
                            if not osutils.is_inside(
1932
                                entry_path_utf8, current_block[0]):
1933
                                # No entries for this directory at all.
1934
                                current_block = None
1935
                        if current_block is not None:
1936
                            for entry in current_block[1]:
1937
                                if entry[1][self.source_index][0] in 'ar':
1938
                                    # Not in the source tree, so doesn't have to be
1939
                                    # included.
1940
                                    continue
1941
                                # Path of the entry itself.
1942
                                self.search_specific_file_parents.add(
4570.2.5 by Robert Collins
Review feedback, including finding a bug with changes at the root.
1943
                                    self.pathjoin(*entry[0][:2]))
4570.2.3 by Robert Collins
Change the way iter_changes treats specific files to prevent InconsistentDeltas.
1944
                if changed or self.include_unchanged:
1945
                    results.append((result, changed))
1946
            self.searched_exact_paths.add(path_utf8)
1947
        return results
1948
1949
    cdef object _path_info(self, utf8_path, unicode_path):
1950
        """Generate path_info for unicode_path.
1951
1952
        :return: None if unicode_path does not exist, or a path_info tuple.
1953
        """
1954
        abspath = self.tree.abspath(unicode_path)
1955
        try:
1956
            stat = os.lstat(abspath)
1957
        except OSError, e:
1958
            if e.errno == errno.ENOENT:
1959
                # the path does not exist.
1960
                return None
1961
            else:
1962
                raise
1963
        utf8_basename = utf8_path.rsplit('/', 1)[-1]
1964
        dir_info = (utf8_path, utf8_basename,
1965
            osutils.file_kind_from_stat_mode(stat.st_mode), stat,
1966
            abspath)
1967
        if dir_info[2] == 'directory':
1968
            if self.tree._directory_is_tree_reference(
1969
                unicode_path):
1970
                self.root_dir_info = self.root_dir_info[:2] + \
1971
                    ('tree-reference',) + self.root_dir_info[3:]
1972
        return dir_info