« back to all changes in this revision
Viewing changes to bzrlib/_walkdirs_win32.pyx
- browse files at revision 3557
- compare with another revision
- compare with revision 3641.2.2
- download diff
- download tarball
- view history from revision 3557
- Committer: Canonical.com Patch Queue Manager
- Date: 2008-07-17 13:27:12 UTC
- mfrom: (3504.4.15 win32_find_files)
- Revision ID: pqm@pqm.ubuntu.com-20080717132712-1zbt1asfsuslh1v9
(jam) Implement _walkdirs_win32,
going directly to Win32 apis improves status performance 2-6x
going directly to Win32 apis improves status performance 2-6x
added
removed
bzrlib/_walkdirs_win32.pyx
1
# Copyright (C) 2008 Canonical Ltd
2
#
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
#
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
12
#
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
17
"""Helper functions for Walkdirs on win32."""
18
19
20
cdef extern from "_walkdirs_win32.h":
21
cdef struct _HANDLE:
22
pass
23
ctypedef _HANDLE *HANDLE
24
ctypedef unsigned long DWORD
25
ctypedef long long __int64
26
ctypedef unsigned short WCHAR
27
cdef struct _FILETIME:
28
DWORD dwHighDateTime
29
DWORD dwLowDateTime
30
ctypedef _FILETIME FILETIME
31
32
cdef struct _WIN32_FIND_DATAW:
33
DWORD dwFileAttributes
34
FILETIME ftCreationTime
35
FILETIME ftLastAccessTime
36
FILETIME ftLastWriteTime
37
DWORD nFileSizeHigh
38
DWORD nFileSizeLow
39
# Some reserved stuff here
40
WCHAR cFileName[260] # MAX_PATH
41
WCHAR cAlternateFilename[14]
42
43
# We have to use the typedef trick, otherwise pyrex uses:
44
# struct WIN32_FIND_DATAW
45
# which fails due to 'incomplete type'
46
ctypedef _WIN32_FIND_DATAW WIN32_FIND_DATAW
47
48
cdef HANDLE INVALID_HANDLE_VALUE
49
cdef HANDLE FindFirstFileW(WCHAR *path, WIN32_FIND_DATAW *data)
50
cdef int FindNextFileW(HANDLE search, WIN32_FIND_DATAW *data)
51
cdef int FindClose(HANDLE search)
52
53
cdef DWORD FILE_ATTRIBUTE_READONLY
54
cdef DWORD FILE_ATTRIBUTE_DIRECTORY
55
cdef int ERROR_NO_MORE_FILES
56
57
cdef int GetLastError()
58
59
# Wide character functions
60
DWORD wcslen(WCHAR *)
61
62
63
cdef extern from "Python.h":
64
WCHAR *PyUnicode_AS_UNICODE(object)
65
Py_ssize_t PyUnicode_GET_SIZE(object)
66
object PyUnicode_FromUnicode(WCHAR *, Py_ssize_t)
67
int PyList_Append(object, object) except -1
68
object PyUnicode_AsUTF8String(object)
69
70
71
import operator
72
import stat
73
74
from bzrlib import osutils
75
76
77
cdef class _Win32Stat:
78
"""Represent a 'stat' result generated from WIN32_FIND_DATA"""
79
80
cdef readonly int st_mode
81
cdef readonly double st_ctime
82
cdef readonly double st_mtime
83
cdef readonly double st_atime
84
cdef readonly __int64 st_size
85
86
# os.stat always returns 0, so we hard code it here
87
cdef readonly int st_dev
88
cdef readonly int st_ino
89
90
def __repr__(self):
91
"""Repr is the same as a Stat object.
92
93
(mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)
94
"""
95
return repr((self.st_mode, 0, 0, 0, 0, 0, self.st_size, self.st_atime,
96
self.st_mtime, self.st_ctime))
97
98
99
cdef object _get_name(WIN32_FIND_DATAW *data):
100
"""Extract the Unicode name for this file/dir."""
101
return PyUnicode_FromUnicode(data.cFileName,
102
wcslen(data.cFileName))
103
104
105
cdef int _get_mode_bits(WIN32_FIND_DATAW *data):
106
cdef int mode_bits
107
108
mode_bits = 0100666 # writeable file, the most common
109
if data.dwFileAttributes & FILE_ATTRIBUTE_READONLY == FILE_ATTRIBUTE_READONLY:
110
mode_bits ^= 0222 # remove the write bits
111
if data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY == FILE_ATTRIBUTE_DIRECTORY:
112
# Remove the FILE bit, set the DIR bit, and set the EXEC bits
113
mode_bits ^= 0140111
114
return mode_bits
115
116
117
cdef __int64 _get_size(WIN32_FIND_DATAW *data):
118
# Pyrex casts a DWORD into a PyLong anyway, so it is safe to do << 32
119
# on a DWORD
120
return ((<__int64>data.nFileSizeHigh) << 32) + data.nFileSizeLow
121
122
123
cdef double _ftime_to_timestamp(FILETIME *ft):
124
"""Convert from a FILETIME struct into a floating point timestamp.
125
126
The fields of a FILETIME structure are the hi and lo part
127
of a 64-bit value expressed in 100 nanosecond units.
128
1e7 is one second in such units; 1e-7 the inverse.
129
429.4967296 is 2**32 / 1e7 or 2**32 * 1e-7.
130
It also uses the epoch 1601-01-01 rather than 1970-01-01
131
(taken from posixmodule.c)
132
"""
133
cdef __int64 val
134
# NB: This gives slightly different results versus casting to a 64-bit
135
# integer and doing integer math before casting into a floating
136
# point number. But the difference is in the sub millisecond range,
137
# which doesn't seem critical here.
138
# secs between epochs: 11,644,473,600
139
val = ((<__int64>ft.dwHighDateTime) << 32) + ft.dwLowDateTime
140
return (val * 1.0e-7) - 11644473600.0
141
142
143
cdef int _should_skip(WIN32_FIND_DATAW *data):
144
"""Is this '.' or '..' so we should skip it?"""
145
if (data.cFileName[0] != c'.'):
146
return 0
147
if data.cFileName[1] == c'\0':
148
return 1
149
if data.cFileName[1] == c'.' and data.cFileName[2] == c'\0':
150
return 1
151
return 0
152
153
154
cdef class Win32Finder:
155
"""A class which encapsulates the search of files in a given directory"""
156
157
cdef object _top
158
cdef object _prefix
159
160
cdef object _directory_kind
161
cdef object _file_kind
162
163
cdef object _pending
164
cdef object _last_dirblock
165
166
def __init__(self, top, prefix=""):
167
self._top = top
168
self._prefix = prefix
169
170
self._directory_kind = osutils._directory_kind
171
self._file_kind = osutils._formats[stat.S_IFREG]
172
173
self._pending = [(osutils.safe_utf8(prefix), osutils.safe_unicode(top))]
174
self._last_dirblock = None
175
176
def __iter__(self):
177
return self
178
179
cdef object _get_kind(self, WIN32_FIND_DATAW *data):
180
if data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY:
181
return self._directory_kind
182
return self._file_kind
183
184
cdef _Win32Stat _get_stat_value(self, WIN32_FIND_DATAW *data):
185
"""Get the filename and the stat information."""
186
cdef _Win32Stat statvalue
187
188
statvalue = _Win32Stat()
189
statvalue.st_mode = _get_mode_bits(data)
190
statvalue.st_ctime = _ftime_to_timestamp(&data.ftCreationTime)
191
statvalue.st_mtime = _ftime_to_timestamp(&data.ftLastWriteTime)
192
statvalue.st_atime = _ftime_to_timestamp(&data.ftLastAccessTime)
193
statvalue.st_size = _get_size(data)
194
statvalue.st_ino = 0
195
statvalue.st_dev = 0
196
return statvalue
197
198
def _get_files_in(self, directory, relprefix):
199
cdef WIN32_FIND_DATAW search_data
200
cdef HANDLE hFindFile
201
cdef int last_err
202
cdef WCHAR *query
203
cdef int result
204
205
top_star = directory + '*'
206
207
dirblock = []
208
209
query = PyUnicode_AS_UNICODE(top_star)
210
hFindFile = FindFirstFileW(query, &search_data)
211
if hFindFile == INVALID_HANDLE_VALUE:
212
# Raise an exception? This path doesn't seem to exist
213
raise WindowsError(GetLastError(), top_star)
214
215
try:
216
result = 1
217
while result:
218
# Skip '.' and '..'
219
if _should_skip(&search_data):
220
result = FindNextFileW(hFindFile, &search_data)
221
continue
222
name_unicode = _get_name(&search_data)
223
name_utf8 = PyUnicode_AsUTF8String(name_unicode)
224
relpath = relprefix + name_utf8
225
abspath = directory + name_unicode
226
PyList_Append(dirblock,
227
(relpath, name_utf8,
228
self._get_kind(&search_data),
229
self._get_stat_value(&search_data),
230
abspath))
231
232
result = FindNextFileW(hFindFile, &search_data)
233
# FindNextFileW sets GetLastError() == ERROR_NO_MORE_FILES when it
234
# actually finishes. If we have anything else, then we have a
235
# genuine problem
236
last_err = GetLastError()
237
if last_err != ERROR_NO_MORE_FILES:
238
raise WindowsError(last_err)
239
finally:
240
result = FindClose(hFindFile)
241
if result == 0:
242
last_err = GetLastError()
243
pass
244
return dirblock
245
246
cdef _update_pending(self):
247
"""If we had a result before, add the subdirs to pending."""
248
if self._last_dirblock is not None:
249
# push the entries left in the dirblock onto the pending queue
250
# we do this here, because we allow the user to modified the
251
# queue before the next iteration
252
for d in reversed(self._last_dirblock):
253
if d[2] == self._directory_kind:
254
self._pending.append((d[0], d[-1]))
255
self._last_dirblock = None
256
257
def __next__(self):
258
self._update_pending()
259
if not self._pending:
260
raise StopIteration()
261
relroot, top = self._pending.pop()
262
# NB: At the moment Pyrex doesn't support Unicode literals, which means
263
# that all of these string literals are going to be upcasted to Unicode
264
# at runtime... :(
265
# Maybe we could use unicode(x) during __init__?
266
if relroot:
267
relprefix = relroot + '/'
268
else:
269
relprefix = ''
270
top_slash = top + '/'
271
272
dirblock = self._get_files_in(top_slash, relprefix)
273
dirblock.sort(key=operator.itemgetter(1))
274
self._last_dirblock = dirblock
275
return (relroot, top), dirblock
276
277
278
def _walkdirs_utf8_win32_find_file(top, prefix=""):
279
"""Implement a version of walkdirs_utf8 for win32.
280
281
This uses the find files api to both list the files and to stat them.
282
"""
283
return Win32Finder(top, prefix=prefix)
Loggerhead is a web-based interface for Breezy
Version: 2.0.1