~bzr-pqm/bzr/bzr.dev

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
# Copyright (C) 2008 Canonical Ltd
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#

"""Pyrex extensions for converting chunks to lines."""

#python2.4 support
cdef extern from "python-compat.h":
    pass

cdef extern from "stdlib.h":
    ctypedef unsigned size_t

cdef extern from "Python.h":
    ctypedef int Py_ssize_t # Required for older pyrex versions
    ctypedef struct PyObject:
        pass
    int PyList_Append(object lst, object item) except -1

    int PyString_CheckExact(object p)
    char *PyString_AS_STRING(object p)
    Py_ssize_t PyString_GET_SIZE(object p)
    object PyString_FromStringAndSize(char *c_str, Py_ssize_t len)

cdef extern from "string.h":
    void *memchr(void *s, int c, size_t n)


def chunks_to_lines(chunks):
    """Re-split chunks into simple lines.

    Each entry in the result should contain a single newline at the end. Except
    for the last entry which may not have a final newline. If chunks is already
    a simple list of lines, we return it directly.

    :param chunks: An list/tuple of strings. If chunks is already a list of
        lines, then we will return it as-is.
    :return: A list of strings.
    """
    cdef char *c_str
    cdef char *newline
    cdef char *c_last
    cdef Py_ssize_t the_len
    cdef int last_no_newline

    # Check to see if the chunks are already lines
    last_no_newline = 0
    for chunk in chunks:
        if last_no_newline:
            # We have a chunk which followed a chunk without a newline, so this
            # is not a simple list of lines.
            break
        # Switching from PyString_AsStringAndSize to PyString_CheckExact and
        # then the macros GET_SIZE and AS_STRING saved us 40us / 470us.
        # It seems PyString_AsStringAndSize can actually trigger a conversion,
        # which we don't want anyway.
        if not PyString_CheckExact(chunk):
            raise TypeError('chunk is not a string')
        the_len = PyString_GET_SIZE(chunk)
        if the_len == 0:
            # An empty string is never a valid line
            break
        c_str = PyString_AS_STRING(chunk)
        c_last = c_str + the_len - 1
        newline = <char *>memchr(c_str, c'\n', the_len)
        if newline != c_last:
            if newline == NULL:
                # Missing a newline. Only valid as the last line
                last_no_newline = 1
            else:
                # There is a newline in the middle, we must resplit
                break
    else:
        # Everything was already a list of lines
        return chunks

    # We know we need to create a new list of lines
    lines = []
    tail = None # Any remainder from the previous chunk
    for chunk in chunks:
        if tail is not None:
            chunk = tail + chunk
            tail = None
        if not PyString_CheckExact(chunk):
            raise TypeError('chunk is not a string')
        the_len = PyString_GET_SIZE(chunk)
        if the_len == 0:
            # An empty string is never a valid line, and we don't need to
            # append anything
            continue
        c_str = PyString_AS_STRING(chunk)
        c_last = c_str + the_len - 1
        newline = <char *>memchr(c_str, c'\n', the_len)
        if newline == c_last:
            # A simple line
            PyList_Append(lines, chunk)
        elif newline == NULL:
            # A chunk without a newline, if this is the last entry, then we
            # allow it
            tail = chunk
        else:
            # We have a newline in the middle, loop until we've consumed all
            # lines
            while newline != NULL:
                line = PyString_FromStringAndSize(c_str, newline - c_str + 1)
                PyList_Append(lines, line)
                c_str = newline + 1
                if c_str > c_last: # We are done
                    break
                the_len = c_last - c_str + 1
                newline = <char *>memchr(c_str, c'\n', the_len)
                if newline == NULL:
                    tail = PyString_FromStringAndSize(c_str, the_len)
                    break
    if tail is not None:
        PyList_Append(lines, tail)
    return lines