~bzr-pqm/bzr/bzr.dev

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
# Copyright (C) 2007 Canonical Ltd
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

"""Pyrex extensions to knit parsing."""

import sys

from bzrlib import errors


cdef extern from "stdlib.h":
    ctypedef unsigned size_t
    long int strtol(char *nptr, char **endptr, int base)


cdef extern from "Python.h":
    int PyDict_CheckExact(object)
    void *PyDict_GetItem_void "PyDict_GetItem" (object p, object key)
    int PyDict_SetItem(object p, object key, object val) except -1

    int PyList_Append(object lst, object item) except -1
    object PyList_GET_ITEM(object lst, int index)
    int PyList_CheckExact(object)

    void *PyTuple_GetItem_void_void "PyTuple_GET_ITEM" (void* tpl, int index)

    char *PyString_AsString(object p)
    object PyString_FromStringAndSize(char *, int)
    int PyString_Size(object p)

    void Py_INCREF(object)


cdef extern from "string.h":
    void *memchr(void *s, int c, size_t n)


cdef int string_to_int_safe(char *s, char *end, int *out) except -1:
    """Convert a base10 string to an integer.

    This makes sure the whole string is consumed, or it raises ValueError.
    This is similar to how int(s) works, except you don't need a Python
    String object.

    :param s: The string to convert
    :param end: The character after the integer. So if the string is '12\0',
        this should be pointing at the '\0'. If the string was '12 ' then this
        should point at the ' '.
    :param out: This is the integer that will be returned
    :return: -1 if an exception is raised. 0 otherwise
    """
    cdef char *integer_end

    # We can't just return the integer because of how pyrex determines when
    # there is an exception.
    out[0] = <int>strtol(s, &integer_end, 10)
    if integer_end != end:
        py_s = PyString_FromStringAndSize(s, end-s)
        raise ValueError('%r is not a valid integer' % (py_s,))
    return 0


cdef class KnitIndexReader:

    cdef object kndx
    cdef object fp

    cdef object cache
    cdef object history

    cdef char * cur_str
    cdef char * end_str

    cdef int history_len

    def __init__(self, kndx, fp):
        self.kndx = kndx
        self.fp = fp

        self.cache = kndx._cache
        self.history = kndx._history

        self.cur_str = NULL
        self.end_str = NULL
        self.history_len = 0

    cdef void validate(self):
        if not PyDict_CheckExact(self.cache):
            raise TypeError('kndx._cache must be a python dict')
        if not PyList_CheckExact(self.history):
            raise TypeError('kndx._history must be a python list')

    cdef object process_options(self, char *option_str, char *end):
        """Process the options string into a list."""
        cdef char *next

        # This is alternative code which creates a python string and splits it.
        # It is "correct" and more obvious, but slower than the following code.
        # It can be uncommented to switch in case the other code is seen as
        # suspect.
        # options = PyString_FromStringAndSize(option_str,
        #                                      end - option_str)
        # return options.split(',')

        final_options = []

        while option_str < end:
            next = <char*>memchr(option_str, c',', end - option_str)
            if next == NULL:
                next = end
            next_option = PyString_FromStringAndSize(option_str,
                                                     next - option_str)
            PyList_Append(final_options, next_option)

            # Move past the ','
            option_str = next+1

        return final_options

    cdef object process_parents(self, char *parent_str, char *end):
        cdef char *next
        cdef int int_parent
        cdef char *parent_end

        # Alternative, correct but slower code.
        #
        # parents = PyString_FromStringAndSize(parent_str,
        #                                      end - parent_str)
        # real_parents = []
        # for parent in parents.split():
        #     if parent[0].startswith('.'):
        #         real_parents.append(parent[1:])
        #     else:
        #         real_parents.append(self.history[int(parent)])
        # return real_parents

        parents = []
        while parent_str <= end:
            next = <char*>memchr(parent_str, c' ', end - parent_str)
            if next == NULL or next >= end or next == parent_str:
                break

            if parent_str[0] == c'.':
                # This is an explicit revision id
                parent_str = parent_str + 1
                parent = PyString_FromStringAndSize(parent_str,
                                                    next - parent_str)
            else:
                # This in an integer mapping to original
                string_to_int_safe(parent_str, next, &int_parent)

                if int_parent >= self.history_len:
                    raise IndexError('Parent index refers to a revision which'
                        ' does not exist yet.'
                        ' %d > %d' % (int_parent, self.history_len))
                parent = PyList_GET_ITEM(self.history, int_parent)
                # PyList_GET_ITEM steals a reference
                Py_INCREF(parent)
            PyList_Append(parents, parent)
            parent_str = next + 1
        return parents

    cdef int process_one_record(self, char *start, char *end) except -1:
        """Take a simple string and split it into an index record."""
        cdef char *version_id_str
        cdef int version_id_size
        cdef char *option_str
        cdef char *option_end
        cdef char *pos_str
        cdef int pos
        cdef char *size_str
        cdef int size
        cdef char *parent_str
        cdef int parent_size
        cdef void *cache_entry

        version_id_str = start
        option_str = <char*>memchr(version_id_str, c' ', end - version_id_str)
        if option_str == NULL or option_str >= end:
            # Short entry
            return 0
        version_id_size = <int>(option_str - version_id_str)
        # Move past the space character
        option_str = option_str + 1

        pos_str = <char*>memchr(option_str, c' ', end - option_str)
        if pos_str == NULL or pos_str >= end:
            # Short entry
            return 0
        option_end = pos_str
        pos_str = pos_str + 1

        size_str = <char*>memchr(pos_str, c' ', end - pos_str)
        if size_str == NULL or size_str >= end:
            # Short entry
            return 0
        size_str = size_str + 1

        parent_str = <char*>memchr(size_str, c' ', end - size_str)
        if parent_str == NULL or parent_str >= end:
            # Missing parents
            return 0
        parent_str = parent_str + 1

        version_id = PyString_FromStringAndSize(version_id_str,
                                                version_id_size)
        options = self.process_options(option_str, option_end)

        try:
            string_to_int_safe(pos_str, size_str - 1, &pos)
            string_to_int_safe(size_str, parent_str - 1, &size)
            parents = self.process_parents(parent_str, end)
        except (ValueError, IndexError), e:
            py_line = PyString_FromStringAndSize(start, end - start)
            raise errors.KnitCorrupt(self.kndx._filename,
                                     "line %r: %s" % (py_line, e))

        cache_entry = PyDict_GetItem_void(self.cache, version_id)
        if cache_entry == NULL:
            PyList_Append(self.history, version_id)
            index = self.history_len
            self.history_len = self.history_len + 1
        else:
            # PyTuple_GetItem_void_void does *not* increment the reference
            # counter, but casting to <object> does.
            index = <object>PyTuple_GetItem_void_void(cache_entry, 5)

        PyDict_SetItem(self.cache, version_id,
                       (version_id,
                        options,
                        pos,
                        size,
                        parents,
                        index,
                       ))
        return 1

    cdef int process_next_record(self) except -1:
        """Process the next record in the file."""
        cdef char *last
        cdef char *start

        start = self.cur_str
        # Find the next newline
        last = <char*>memchr(start, c'\n', self.end_str - start)
        if last == NULL:
            # Process until the end of the file
            last = self.end_str - 1
            self.cur_str = self.end_str
        else:
            # The last character is right before the '\n'
            # And the next string is right after it
            self.cur_str = last + 1
            last = last - 1

        if last <= start or last[0] != c':':
            # Incomplete record
            return 0

        return self.process_one_record(start, last)

    def read(self):
        cdef int text_size

        self.validate()

        self.kndx.check_header(self.fp)

        # We read the whole thing at once
        # TODO: jam 2007-05-09 Consider reading incrementally rather than
        #       having to have the whole thing read up front.
        #       we already know that calling f.readlines() versus lots of
        #       f.readline() calls is faster.
        #       The other possibility is to avoid a Python String here
        #       completely. However self.fp may be a 'file-like' object
        #       it is not guaranteed to be a real file.
        text = self.fp.read()
        text_size = PyString_Size(text)
        self.cur_str = PyString_AsString(text)
        # This points to the last character in the string
        self.end_str = self.cur_str + text_size

        while self.cur_str < self.end_str:
            self.process_next_record()


def _load_data_c(kndx, fp):
    """Load the knit index file into memory."""
    reader = KnitIndexReader(kndx, fp)
    reader.read()