« back to all changes in this revision
Viewing changes to bzrlib/osutils.py
- browse files at revision 3731
- compare with another revision
- compare with revision 334
- download diff
- download tarball
- view history from revision 3731
- Committer: Canonical.com Patch Queue Manager
- Date: 2008-09-24 01:43:25 UTC
- mfrom: (3696.3.12 integration)
- Revision ID: pqm@pqm.ubuntu.com-20080924014325-ucivgbdmsbuthnqw
(robertc) Accelerate _walkdirs_utf8 on unix platforms. (Robert
Collins)
Collins)
added
removed
bzrlib/osutils.py
123
123
124
124
_directory_kind = 'directory'
125
125
126
_formats = {
127
stat.S_IFDIR:_directory_kind,
128
stat.S_IFCHR:'chardev',
129
stat.S_IFBLK:'block',
130
stat.S_IFREG:'file',
131
stat.S_IFIFO:'fifo',
132
stat.S_IFLNK:'symlink',
133
stat.S_IFSOCK:'socket',
134
}
135
136
137
def file_kind_from_stat_mode(stat_mode, _formats=_formats, _unknown='unknown'):
138
"""Generate a file kind from a stat mode. This is used in walkdirs.
139
140
Its performance is critical: Do not mutate without careful benchmarking.
141
"""
142
try:
143
return _formats[stat_mode & 0170000]
144
except KeyError:
145
return _unknown
146
147
148
def file_kind(f, _lstat=os.lstat, _mapper=file_kind_from_stat_mode):
149
try:
150
return _mapper(_lstat(f).st_mode)
151
except OSError, e:
152
if getattr(e, 'errno', None) in (errno.ENOENT, errno.ENOTDIR):
153
raise errors.NoSuchFile(f)
154
raise
155
156
157
126
def get_umask():
158
127
"""Return the current umask"""
159
128
# Assume that people aren't messing with the umask while running
1201
1170
_lstat = os.lstat
1202
1171
_directory = _directory_kind
1203
1172
_listdir = os.listdir
1204
_kind_from_mode = _formats.get
1173
_kind_from_mode = file_kind_from_stat_mode
1205
1174
pending = [(safe_unicode(prefix), "", _directory, None, safe_unicode(top))]
1206
1175
while pending:
1207
1176
# 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
1223
1192
for name in names:
1224
1193
abspath = top_slash + name
1225
1194
statvalue = _lstat(abspath)
1226
kind = _kind_from_mode(statvalue.st_mode & 0170000, 'unknown')
1195
kind = _kind_from_mode(statvalue.st_mode)
1227
1196
append((relprefix + name, name, kind, statvalue, abspath))
1228
1197
yield (relroot, top), dirblock
1229
1198
1231
1200
pending.extend(d for d in reversed(dirblock) if d[2] == _directory)
1232
1201
1233
1202
1234
_real_walkdirs_utf8 = None
1203
class DirReader(object):
1204
"""An interface for reading directories."""
1205
1206
def top_prefix_to_starting_dir(self, top, prefix=""):
1207
"""Converts top and prefix to a starting dir entry
1208
1209
:param top: A utf8 path
1210
:param prefix: An optional utf8 path to prefix output relative paths
1211
with.
1212
:return: A tuple starting with prefix, and ending with the native
1213
encoding of top.
1214
"""
1215
raise NotImplementedError(self.top_prefix_to_starting_dir)
1216
1217
def read_dir(self, prefix, top):
1218
"""Read a specific dir.
1219
1220
:param prefix: A utf8 prefix to be preprended to the path basenames.
1221
:param top: A natively encoded path to read.
1222
:return: A list of the directories contents. Each item contains:
1223
(utf8_relpath, utf8_name, kind, lstatvalue, native_abspath)
1224
"""
1225
raise NotImplementedError(self.read_dir)
1226
1227
1228
_selected_dir_reader = None
1229
1235
1230
1236
1231
def _walkdirs_utf8(top, prefix=""):
1237
1232
"""Yield data about all the directories in a tree.
1247
1242
path-from-top might be unicode or utf8, but it is the correct path to
1248
1243
pass to os functions to affect the file in question. (such as os.lstat)
1249
1244
"""
1250
global _real_walkdirs_utf8
1251
if _real_walkdirs_utf8 is None:
1245
global _selected_dir_reader
1246
if _selected_dir_reader is None:
1252
1247
fs_encoding = _fs_enc.upper()
1253
1248
if win32utils.winver == 'Windows NT':
1254
1249
# Win98 doesn't have unicode apis like FindFirstFileW
1257
1252
# but that gets a bit tricky, and requires custom compiling
1258
1253
# for win98 anyway.
1259
1254
try:
1260
from bzrlib._walkdirs_win32 import _walkdirs_utf8_win32_find_file
1255
from bzrlib._walkdirs_win32 import Win32ReadDir
1261
1256
except ImportError:
1262
_real_walkdirs_utf8 = _walkdirs_unicode_to_utf8
1257
_selected_dir_reader = UnicodeDirReader()
1263
1258
else:
1264
_real_walkdirs_utf8 = _walkdirs_utf8_win32_find_file
1259
_selected_dir_reader = Win32ReadDir()
1265
1260
elif fs_encoding not in ('UTF-8', 'US-ASCII', 'ANSI_X3.4-1968'):
1266
1261
# ANSI_X3.4-1968 is a form of ASCII
1267
_real_walkdirs_utf8 = _walkdirs_unicode_to_utf8
1262
_selected_dir_reader = UnicodeDirReader()
1268
1263
else:
1269
_real_walkdirs_utf8 = _walkdirs_fs_utf8
1270
return _real_walkdirs_utf8(top, prefix=prefix)
1271
1272
1273
def _walkdirs_fs_utf8(top, prefix=""):
1274
"""See _walkdirs_utf8.
1275
1276
This sub-function is called when we know the filesystem is already in utf8
1277
encoding. So we don't need to transcode filenames.
1278
"""
1279
_lstat = os.lstat
1280
_directory = _directory_kind
1281
# Use C accelerated directory listing.
1282
_listdir = _read_dir
1283
_kind_from_mode = _formats.get
1284
1264
try:
1265
from bzrlib._readdir_pyx import UTF8DirReader
1266
except ImportError:
1267
# No optimised code path
1268
_selected_dir_reader = UnicodeDirReader()
1269
else:
1270
_selected_dir_reader = UTF8DirReader()
1285
1271
# 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
1286
1272
# But we don't actually uses 1-3 in pending, so set them to None
1287
pending = [(safe_utf8(prefix), None, None, None, safe_utf8(top))]
1273
pending = [[_selected_dir_reader.top_prefix_to_starting_dir(top, prefix)]]
1274
read_dir = _selected_dir_reader.read_dir
1275
_directory = _directory_kind
1288
1276
while pending:
1289
relroot, _, _, _, top = pending.pop()
1290
if relroot:
1291
relprefix = relroot + '/'
1292
else:
1293
relprefix = ''
1294
top_slash = top + '/'
1295
1296
dirblock = []
1297
append = dirblock.append
1298
# read_dir supplies in should-stat order.
1299
for _, name in sorted(_listdir(top)):
1300
abspath = top_slash + name
1301
statvalue = _lstat(abspath)
1302
kind = _kind_from_mode(statvalue.st_mode & 0170000, 'unknown')
1303
append((relprefix + name, name, kind, statvalue, abspath))
1304
dirblock.sort()
1277
relroot, _, _, _, top = pending[-1].pop()
1278
if not pending[-1]:
1279
pending.pop()
1280
dirblock = sorted(read_dir(relroot, top))
1305
1281
yield (relroot, top), dirblock
1306
1307
1282
# push the user specified dirs from dirblock
1308
pending.extend(d for d in reversed(dirblock) if d[2] == _directory)
1309
1310
1311
def _walkdirs_unicode_to_utf8(top, prefix=""):
1312
"""See _walkdirs_utf8
1313
1314
Because Win32 has a Unicode api, all of the 'path-from-top' entries will be
1315
Unicode paths.
1316
This is currently the fallback code path when the filesystem encoding is
1317
not UTF-8. It may be better to implement an alternative so that we can
1318
safely handle paths that are not properly decodable in the current
1319
encoding.
1320
"""
1321
_utf8_encode = codecs.getencoder('utf8')
1322
_lstat = os.lstat
1323
_directory = _directory_kind
1324
_listdir = os.listdir
1325
_kind_from_mode = _formats.get
1326
1327
pending = [(safe_utf8(prefix), None, None, None, safe_unicode(top))]
1328
while pending:
1329
relroot, _, _, _, top = pending.pop()
1330
if relroot:
1331
relprefix = relroot + '/'
1283
next = [d for d in reversed(dirblock) if d[2] == _directory]
1284
if next:
1285
pending.append(next)
1286
1287
1288
class UnicodeDirReader(DirReader):
1289
"""A dir reader for non-utf8 file systems, which transcodes."""
1290
1291
__slots__ = ['_utf8_encode']
1292
1293
def __init__(self):
1294
self._utf8_encode = codecs.getencoder('utf8')
1295
1296
def top_prefix_to_starting_dir(self, top, prefix=""):
1297
"""See DirReader.top_prefix_to_starting_dir."""
1298
return (safe_utf8(prefix), None, None, None, safe_unicode(top))
1299
1300
def read_dir(self, prefix, top):
1301
"""Read a single directory from a non-utf8 file system.
1302
1303
top, and the abspath element in the output are unicode, all other paths
1304
are utf8. Local disk IO is done via unicode calls to listdir etc.
1305
1306
This is currently the fallback code path when the filesystem encoding is
1307
not UTF-8. It may be better to implement an alternative so that we can
1308
safely handle paths that are not properly decodable in the current
1309
encoding.
1310
1311
See DirReader.read_dir for details.
1312
"""
1313
_utf8_encode = self._utf8_encode
1314
_lstat = os.lstat
1315
_listdir = os.listdir
1316
_kind_from_mode = file_kind_from_stat_mode
1317
1318
if prefix:
1319
relprefix = prefix + '/'
1332
1320
else:
1333
1321
relprefix = ''
1334
1322
top_slash = top + u'/'
1336
1324
dirblock = []
1337
1325
append = dirblock.append
1338
1326
for name in sorted(_listdir(top)):
1339
name_utf8 = _utf8_encode(name)[0]
1327
try:
1328
name_utf8 = _utf8_encode(name)[0]
1329
except UnicodeDecodeError:
1330
raise errors.BadFilenameEncoding(
1331
_utf8_encode(relprefix)[0] + name, _fs_enc)
1340
1332
abspath = top_slash + name
1341
1333
statvalue = _lstat(abspath)
1342
kind = _kind_from_mode(statvalue.st_mode & 0170000, 'unknown')
1334
kind = _kind_from_mode(statvalue.st_mode)
1343
1335
append((relprefix + name_utf8, name_utf8, kind, statvalue, abspath))
1344
yield (relroot, top), dirblock
1345
1346
# push the user specified dirs from dirblock
1347
pending.extend(d for d in reversed(dirblock) if d[2] == _directory)
1336
return dirblock
1348
1337
1349
1338
1350
1339
def copy_tree(from_path, to_path, handlers={}):
1572
1561
return open(filename, 'rU').read()
1573
1562
1574
1563
1575
try:
1576
from bzrlib._readdir_pyx import read_dir as _read_dir
1577
except ImportError:
1578
from bzrlib._readdir_py import read_dir as _read_dir
1564
def file_kind_from_stat_mode_thunk(mode):
1565
global file_kind_from_stat_mode
1566
if file_kind_from_stat_mode is file_kind_from_stat_mode_thunk:
1567
try:
1568
from bzrlib._readdir_pyx import UTF8DirReader
1569
file_kind_from_stat_mode = UTF8DirReader().kind_from_mode
1570
except ImportError:
1571
from bzrlib._readdir_py import (
1572
_kind_from_mode as file_kind_from_stat_mode
1573
)
1574
return file_kind_from_stat_mode(mode)
1575
file_kind_from_stat_mode = file_kind_from_stat_mode_thunk
1576
1577
1578
def file_kind(f, _lstat=os.lstat):
1579
try:
1580
return file_kind_from_stat_mode(_lstat(f).st_mode)
1581
except OSError, e:
1582
if getattr(e, 'errno', None) in (errno.ENOENT, errno.ENOTDIR):
1583
raise errors.NoSuchFile(f)
1584
raise
1585
1586
Loggerhead is a web-based interface for Breezy
Version: 2.0.1