13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
from cStringIO import StringIO
21
from stat import (S_ISREG, S_ISDIR, S_ISLNK, ST_MODE, ST_SIZE,
22
S_ISCHR, S_ISBLK, S_ISFIFO, S_ISSOCK)
26
25
from bzrlib.lazy_import import lazy_import
27
26
lazy_import(globals(), """
29
27
from datetime import datetime
31
from ntpath import (abspath as _nt_abspath,
33
normpath as _nt_normpath,
34
realpath as _nt_realpath,
35
splitdrive as _nt_splitdrive,
31
# We need to import both shutil and rmtree as we export the later on posix
32
# and need the former on windows
34
from shutil import rmtree
37
# We need to import both tempfile and mkdtemp as we export the later on posix
38
# and need the former on windows
44
from tempfile import (
40
from tempfile import mkdtemp
49
43
from bzrlib import (
51
from bzrlib.symbol_versioning import (
58
63
from bzrlib import symbol_versioning
59
from bzrlib.symbol_versioning import (
62
from bzrlib.trace import mutter
66
# Cross platform wall-clock time functionality with decent resolution.
67
# On Linux ``time.clock`` returns only CPU time. On Windows, ``time.time()``
68
# only has a resolution of ~15ms. Note that ``time.clock()`` is not
69
# synchronized with ``time.time()``, this is only meant to be used to find
70
# delta times by subtracting from another call to this function.
71
timer_func = time.time
72
if sys.platform == 'win32':
73
timer_func = time.clock
65
75
# On win32, O_BINARY is used to indicate the file should
66
76
# be opened in binary mode, rather than text mode.
67
77
# On other platforms, O_BINARY doesn't exist, because
68
78
# they always open in binary mode, so it is okay to
69
# OR with 0 on those platforms
79
# OR with 0 on those platforms.
80
# O_NOINHERIT and O_TEXT exists only on win32 too.
70
81
O_BINARY = getattr(os, 'O_BINARY', 0)
82
O_TEXT = getattr(os, 'O_TEXT', 0)
83
O_NOINHERIT = getattr(os, 'O_NOINHERIT', 0)
86
def get_unicode_argv():
88
user_encoding = get_user_encoding()
89
return [a.decode(user_encoding) for a in sys.argv[1:]]
90
except UnicodeDecodeError:
91
raise errors.BzrError("Parameter %r encoding is unsupported by %s "
92
"application locale." % (a, user_encoding))
73
95
def make_readonly(filename):
409
432
def rmtree(path, ignore_errors=False, onerror=_win32_delete_readonly):
410
433
"""Replacer for shutil.rmtree: could remove readonly dirs/files"""
411
434
return shutil.rmtree(path, ignore_errors, onerror)
436
f = win32utils.get_unicode_argv # special function or None
412
440
elif sys.platform == 'darwin':
413
441
getcwd = _mac_getcwd
416
def get_terminal_encoding():
444
def get_terminal_encoding(trace=False):
417
445
"""Find the best encoding for printing to the screen.
419
447
This attempts to check both sys.stdout and sys.stdin to see
420
448
what encoding they are in, and if that fails it falls back to
421
bzrlib.user_encoding.
449
osutils.get_user_encoding().
422
450
The problem is that on Windows, locale.getpreferredencoding()
423
451
is not the same encoding as that used by the console:
424
452
http://mail.python.org/pipermail/python-list/2003-May/162357.html
426
454
On my standard US Windows XP, the preferred encoding is
427
455
cp1252, but the console is cp437
457
:param trace: If True trace the selected encoding via mutter().
459
from bzrlib.trace import mutter
429
460
output_encoding = getattr(sys.stdout, 'encoding', None)
430
461
if not output_encoding:
431
462
input_encoding = getattr(sys.stdin, 'encoding', None)
432
463
if not input_encoding:
433
output_encoding = bzrlib.user_encoding
434
mutter('encoding stdout as bzrlib.user_encoding %r', output_encoding)
464
output_encoding = get_user_encoding()
466
mutter('encoding stdout as osutils.get_user_encoding() %r',
436
469
output_encoding = input_encoding
437
mutter('encoding stdout as sys.stdin encoding %r', output_encoding)
471
mutter('encoding stdout as sys.stdin encoding %r',
439
mutter('encoding stdout as sys.stdout encoding %r', output_encoding)
475
mutter('encoding stdout as sys.stdout encoding %r', output_encoding)
440
476
if output_encoding == 'cp0':
441
477
# invalid encoding (cp0 means 'no codepage' on Windows)
442
output_encoding = bzrlib.user_encoding
443
mutter('cp0 is invalid encoding.'
444
' encoding stdout as bzrlib.user_encoding %r', output_encoding)
478
output_encoding = get_user_encoding()
480
mutter('cp0 is invalid encoding.'
481
' encoding stdout as osutils.get_user_encoding() %r',
447
485
codecs.lookup(output_encoding)
671
738
:param timezone: How to display the time: 'utc', 'original' for the
672
739
timezone specified by offset, or 'local' for the process's current
674
:param show_offset: Whether to append the timezone.
675
:param date_fmt: strftime format.
741
:param date_fmt: strftime format.
742
:param show_offset: Whether to append the timezone.
744
(date_fmt, tt, offset_str) = \
745
_format_date(t, offset, timezone, date_fmt, show_offset)
746
date_fmt = date_fmt.replace('%a', weekdays[tt[6]])
747
date_str = time.strftime(date_fmt, tt)
748
return date_str + offset_str
751
# Cache of formatted offset strings
755
def format_date_with_offset_in_original_timezone(t, offset=0,
756
_cache=_offset_cache):
757
"""Return a formatted date string in the original timezone.
759
This routine may be faster then format_date.
761
:param t: Seconds since the epoch.
762
:param offset: Timezone offset in seconds east of utc.
766
tt = time.gmtime(t + offset)
767
date_fmt = _default_format_by_weekday_num[tt[6]]
768
date_str = time.strftime(date_fmt, tt)
769
offset_str = _cache.get(offset, None)
770
if offset_str is None:
771
offset_str = ' %+03d%02d' % (offset / 3600, (offset / 60) % 60)
772
_cache[offset] = offset_str
773
return date_str + offset_str
776
def format_local_date(t, offset=0, timezone='original', date_fmt=None,
778
"""Return an unicode date string formatted according to the current locale.
780
:param t: Seconds since the epoch.
781
:param offset: Timezone offset in seconds east of utc.
782
:param timezone: How to display the time: 'utc', 'original' for the
783
timezone specified by offset, or 'local' for the process's current
785
:param date_fmt: strftime format.
786
:param show_offset: Whether to append the timezone.
788
(date_fmt, tt, offset_str) = \
789
_format_date(t, offset, timezone, date_fmt, show_offset)
790
date_str = time.strftime(date_fmt, tt)
791
if not isinstance(date_str, unicode):
792
date_str = date_str.decode(get_user_encoding(), 'replace')
793
return date_str + offset_str
796
def _format_date(t, offset, timezone, date_fmt, show_offset):
677
797
if timezone == 'utc':
678
798
tt = time.gmtime(t)
817
936
return pathjoin(*p)
939
def parent_directories(filename):
940
"""Return the list of parent directories, deepest first.
942
For example, parent_directories("a/b/c") -> ["a/b", "a"].
945
parts = splitpath(dirname(filename))
947
parents.append(joinpath(parts))
952
_extension_load_failures = []
955
def failed_to_load_extension(exception):
956
"""Handle failing to load a binary extension.
958
This should be called from the ImportError block guarding the attempt to
959
import the native extension. If this function returns, the pure-Python
960
implementation should be loaded instead::
963
>>> import bzrlib._fictional_extension_pyx
964
>>> except ImportError, e:
965
>>> bzrlib.osutils.failed_to_load_extension(e)
966
>>> import bzrlib._fictional_extension_py
968
# NB: This docstring is just an example, not a doctest, because doctest
969
# currently can't cope with the use of lazy imports in this namespace --
972
# This currently doesn't report the failure at the time it occurs, because
973
# they tend to happen very early in startup when we can't check config
974
# files etc, and also we want to report all failures but not spam the user
976
exception_str = str(exception)
977
if exception_str not in _extension_load_failures:
978
trace.mutter("failed to load compiled extension: %s" % exception_str)
979
_extension_load_failures.append(exception_str)
982
def report_extension_load_failures():
983
if not _extension_load_failures:
985
from bzrlib.config import GlobalConfig
986
if GlobalConfig().get_user_option_as_bool('ignore_missing_extensions'):
988
# the warnings framework should by default show this only once
989
from bzrlib.trace import warning
991
"bzr: warning: some compiled extensions could not be loaded; "
992
"see <https://answers.launchpad.net/bzr/+faq/703>")
993
# we no longer show the specific missing extensions here, because it makes
994
# the message too long and scary - see
995
# https://bugs.launchpad.net/bzr/+bug/430529
999
from bzrlib._chunks_to_lines_pyx import chunks_to_lines
1000
except ImportError, e:
1001
failed_to_load_extension(e)
1002
from bzrlib._chunks_to_lines_py import chunks_to_lines
820
1005
def split_lines(s):
821
1006
"""Split s into lines, but without removing the newline characters."""
1007
# Trivially convert a fulltext into a 'chunked' representation, and let
1008
# chunks_to_lines do the heavy lifting.
1009
if isinstance(s, str):
1010
# chunks_to_lines only supports 8-bit strings
1011
return chunks_to_lines([s])
1013
return _split_lines(s)
1016
def _split_lines(s):
1017
"""Split s into lines, but without removing the newline characters.
1019
This supports Unicode or plain string objects.
822
1021
lines = s.split('\n')
823
1022
result = [line + '\n' for line in lines[:-1]]
927
while len(head) >= len(base):
1162
if len(head) <= len(base) and head != base:
1163
raise errors.PathNotChild(rp, base)
928
1164
if head == base:
930
head, tail = os.path.split(head)
1166
head, tail = split(head)
934
raise errors.PathNotChild(rp, base)
1171
return pathjoin(*reversed(s))
1176
def _cicp_canonical_relpath(base, path):
1177
"""Return the canonical path relative to base.
1179
Like relpath, but on case-insensitive-case-preserving file-systems, this
1180
will return the relpath as stored on the file-system rather than in the
1181
case specified in the input string, for all existing portions of the path.
1183
This will cause O(N) behaviour if called for every path in a tree; if you
1184
have a number of paths to convert, you should use canonical_relpaths().
1186
# TODO: it should be possible to optimize this for Windows by using the
1187
# win32 API FindFiles function to look for the specified name - but using
1188
# os.listdir() still gives us the correct, platform agnostic semantics in
1191
rel = relpath(base, path)
1192
# '.' will have been turned into ''
1196
abs_base = abspath(base)
1198
_listdir = os.listdir
1200
# use an explicit iterator so we can easily consume the rest on early exit.
1201
bit_iter = iter(rel.split('/'))
1202
for bit in bit_iter:
1205
next_entries = _listdir(current)
1206
except OSError: # enoent, eperm, etc
1207
# We can't find this in the filesystem, so just append the
1209
current = pathjoin(current, bit, *list(bit_iter))
1211
for look in next_entries:
1212
if lbit == look.lower():
1213
current = pathjoin(current, look)
1216
# got to the end, nothing matched, so we just return the
1217
# non-existing bits as they were specified (the filename may be
1218
# the target of a move, for example).
1219
current = pathjoin(current, bit, *list(bit_iter))
1221
return current[len(abs_base):].lstrip('/')
1223
# XXX - TODO - we need better detection/integration of case-insensitive
1224
# file-systems; Linux often sees FAT32 devices (or NFS-mounted OSX
1225
# filesystems), for example, so could probably benefit from the same basic
1226
# support there. For now though, only Windows and OSX get that support, and
1227
# they get it for *all* file-systems!
1228
if sys.platform in ('win32', 'darwin'):
1229
canonical_relpath = _cicp_canonical_relpath
1231
canonical_relpath = relpath
1233
def canonical_relpaths(base, paths):
1234
"""Create an iterable to canonicalize a sequence of relative paths.
1236
The intent is for this implementation to use a cache, vastly speeding
1237
up multiple transformations in the same directory.
1239
# but for now, we haven't optimized...
1240
return [canonical_relpath(base, p) for p in paths]
1243
def decode_filename(filename):
1244
"""Decode the filename using the filesystem encoding
1246
If it is unicode, it is returned.
1247
Otherwise it is decoded from the the filesystem's encoding. If decoding
1248
fails, a errors.BadFilenameEncoding exception is raised.
1250
if type(filename) is unicode:
1253
return filename.decode(_fs_enc)
1254
except UnicodeDecodeError:
1255
raise errors.BadFilenameEncoding(filename, _fs_enc)
942
1258
def safe_unicode(unicode_or_utf8_string):
943
1259
"""Coerce unicode_or_utf8_string into unicode.
945
1261
If it is unicode, it is returned.
946
Otherwise it is decoded from utf-8. If a decoding error
947
occurs, it is wrapped as a If the decoding fails, the exception is wrapped
948
as a BzrBadParameter exception.
1262
Otherwise it is decoded from utf-8. If decoding fails, the exception is
1263
wrapped in a BzrBadParameterNotUnicode exception.
950
1265
if isinstance(unicode_or_utf8_string, unicode):
951
1266
return unicode_or_utf8_string
1064
1379
normalized_filename = _inaccessible_normalized_filename
1382
def set_signal_handler(signum, handler, restart_syscall=True):
1383
"""A wrapper for signal.signal that also calls siginterrupt(signum, False)
1384
on platforms that support that.
1386
:param restart_syscall: if set, allow syscalls interrupted by a signal to
1387
automatically restart (by calling `signal.siginterrupt(signum,
1388
False)`). May be ignored if the feature is not available on this
1389
platform or Python version.
1393
siginterrupt = signal.siginterrupt
1395
# This python implementation doesn't provide signal support, hence no
1398
except AttributeError:
1399
# siginterrupt doesn't exist on this platform, or for this version
1401
siginterrupt = lambda signum, flag: None
1403
def sig_handler(*args):
1404
# Python resets the siginterrupt flag when a signal is
1405
# received. <http://bugs.python.org/issue8354>
1406
# As a workaround for some cases, set it back the way we want it.
1407
siginterrupt(signum, False)
1408
# Now run the handler function passed to set_signal_handler.
1411
sig_handler = handler
1412
old_handler = signal.signal(signum, sig_handler)
1414
siginterrupt(signum, False)
1418
default_terminal_width = 80
1419
"""The default terminal width for ttys.
1421
This is defined so that higher levels can share a common fallback value when
1422
terminal_width() returns None.
1425
# Keep some state so that terminal_width can detect if _terminal_size has
1426
# returned a different size since the process started. See docstring and
1427
# comments of terminal_width for details.
1428
# _terminal_size_state has 3 possible values: no_data, unchanged, and changed.
1429
_terminal_size_state = 'no_data'
1430
_first_terminal_size = None
1067
1432
def terminal_width():
1068
"""Return estimated terminal width."""
1069
if sys.platform == 'win32':
1070
return win32utils.get_console_size()[0]
1433
"""Return terminal width.
1435
None is returned if the width can't established precisely.
1438
- if BZR_COLUMNS is set, returns its value
1439
- if there is no controlling terminal, returns None
1440
- query the OS, if the queried size has changed since the last query,
1442
- if COLUMNS is set, returns its value,
1443
- if the OS has a value (even though it's never changed), return its value.
1445
From there, we need to query the OS to get the size of the controlling
1448
On Unices we query the OS by:
1449
- get termios.TIOCGWINSZ
1450
- if an error occurs or a negative value is obtained, returns None
1452
On Windows we query the OS by:
1453
- win32utils.get_console_size() decides,
1454
- returns None on error (provided default value)
1456
# Note to implementors: if changing the rules for determining the width,
1457
# make sure you've considered the behaviour in these cases:
1458
# - M-x shell in emacs, where $COLUMNS is set and TIOCGWINSZ returns 0,0.
1459
# - bzr log | less, in bash, where $COLUMNS not set and TIOCGWINSZ returns
1461
# - (add more interesting cases here, if you find any)
1462
# Some programs implement "Use $COLUMNS (if set) until SIGWINCH occurs",
1463
# but we don't want to register a signal handler because it is impossible
1464
# to do so without risking EINTR errors in Python <= 2.6.5 (see
1465
# <http://bugs.python.org/issue8354>). Instead we check TIOCGWINSZ every
1466
# time so we can notice if the reported size has changed, which should have
1469
# If BZR_COLUMNS is set, take it, user is always right
1470
# Except if they specified 0 in which case, impose no limit here
1472
width = int(os.environ['BZR_COLUMNS'])
1473
except (KeyError, ValueError):
1475
if width is not None:
1481
isatty = getattr(sys.stdout, 'isatty', None)
1482
if isatty is None or not isatty():
1483
# Don't guess, setting BZR_COLUMNS is the recommended way to override.
1487
width, height = os_size = _terminal_size(None, None)
1488
global _first_terminal_size, _terminal_size_state
1489
if _terminal_size_state == 'no_data':
1490
_first_terminal_size = os_size
1491
_terminal_size_state = 'unchanged'
1492
elif (_terminal_size_state == 'unchanged' and
1493
_first_terminal_size != os_size):
1494
_terminal_size_state = 'changed'
1496
# If the OS claims to know how wide the terminal is, and this value has
1497
# ever changed, use that.
1498
if _terminal_size_state == 'changed':
1499
if width is not None and width > 0:
1502
# If COLUMNS is set, use it.
1504
return int(os.environ['COLUMNS'])
1505
except (KeyError, ValueError):
1508
# Finally, use an unchanged size from the OS, if we have one.
1509
if _terminal_size_state == 'unchanged':
1510
if width is not None and width > 0:
1513
# The width could not be determined.
1517
def _win32_terminal_size(width, height):
1518
width, height = win32utils.get_console_size(defaultx=width, defaulty=height)
1519
return width, height
1522
def _ioctl_terminal_size(width, height):
1073
1524
import struct, fcntl, termios
1074
1525
s = struct.pack('HHHH', 0, 0, 0, 0)
1075
1526
x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
1076
width = struct.unpack('HHHH', x)[1]
1527
height, width = struct.unpack('HHHH', x)[0:2]
1528
except (IOError, AttributeError):
1081
width = int(os.environ['COLUMNS'])
1530
return width, height
1532
_terminal_size = None
1533
"""Returns the terminal size as (width, height).
1535
:param width: Default value for width.
1536
:param height: Default value for height.
1538
This is defined specifically for each OS and query the size of the controlling
1539
terminal. If any error occurs, the provided default values should be returned.
1541
if sys.platform == 'win32':
1542
_terminal_size = _win32_terminal_size
1544
_terminal_size = _ioctl_terminal_size
1090
1547
def supports_executable():
1247
1730
path-from-top might be unicode or utf8, but it is the correct path to
1248
1731
pass to os functions to affect the file in question. (such as os.lstat)
1250
global _real_walkdirs_utf8
1251
if _real_walkdirs_utf8 is None:
1733
global _selected_dir_reader
1734
if _selected_dir_reader is None:
1252
1735
fs_encoding = _fs_enc.upper()
1253
if win32utils.winver == 'Windows NT':
1736
if sys.platform == "win32" and win32utils.winver == 'Windows NT':
1254
1737
# Win98 doesn't have unicode apis like FindFirstFileW
1255
1738
# TODO: We possibly could support Win98 by falling back to the
1256
1739
# original FindFirstFile, and using TCHAR instead of WCHAR,
1257
1740
# but that gets a bit tricky, and requires custom compiling
1258
1741
# for win98 anyway.
1260
from bzrlib._walkdirs_win32 import _walkdirs_utf8_win32_find_file
1743
from bzrlib._walkdirs_win32 import Win32ReadDir
1744
_selected_dir_reader = Win32ReadDir()
1261
1745
except ImportError:
1262
_real_walkdirs_utf8 = _walkdirs_unicode_to_utf8
1264
_real_walkdirs_utf8 = _walkdirs_utf8_win32_find_file
1265
elif fs_encoding not in ('UTF-8', 'US-ASCII', 'ANSI_X3.4-1968'):
1747
elif fs_encoding in ('UTF-8', 'US-ASCII', 'ANSI_X3.4-1968'):
1266
1748
# ANSI_X3.4-1968 is a form of ASCII
1267
_real_walkdirs_utf8 = _walkdirs_unicode_to_utf8
1269
_real_walkdirs_utf8 = _walkdirs_fs_utf8
1270
return _real_walkdirs_utf8(top, prefix=prefix)
1273
def _walkdirs_fs_utf8(top, prefix=""):
1274
"""See _walkdirs_utf8.
1276
This sub-function is called when we know the filesystem is already in utf8
1277
encoding. So we don't need to transcode filenames.
1280
_directory = _directory_kind
1281
# Use C accelerated directory listing.
1282
_listdir = _read_dir
1283
_kind_from_mode = _formats.get
1750
from bzrlib._readdir_pyx import UTF8DirReader
1751
_selected_dir_reader = UTF8DirReader()
1752
except ImportError, e:
1753
failed_to_load_extension(e)
1756
if _selected_dir_reader is None:
1757
# Fallback to the python version
1758
_selected_dir_reader = UnicodeDirReader()
1285
1760
# 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
1286
1761
# But we don't actually uses 1-3 in pending, so set them to None
1287
pending = [(safe_utf8(prefix), None, None, None, safe_utf8(top))]
1762
pending = [[_selected_dir_reader.top_prefix_to_starting_dir(top, prefix)]]
1763
read_dir = _selected_dir_reader.read_dir
1764
_directory = _directory_kind
1289
relroot, _, _, _, top = pending.pop()
1291
relprefix = relroot + '/'
1294
top_slash = top + '/'
1297
append = dirblock.append
1298
# read_dir supplies in should-stat order.
1299
for _, name in sorted(_listdir(top)):
1300
abspath = top_slash + name
1301
statvalue = _lstat(abspath)
1302
kind = _kind_from_mode(statvalue.st_mode & 0170000, 'unknown')
1303
append((relprefix + name, name, kind, statvalue, abspath))
1766
relroot, _, _, _, top = pending[-1].pop()
1769
dirblock = sorted(read_dir(relroot, top))
1305
1770
yield (relroot, top), dirblock
1307
1771
# push the user specified dirs from dirblock
1308
pending.extend(d for d in reversed(dirblock) if d[2] == _directory)
1311
def _walkdirs_unicode_to_utf8(top, prefix=""):
1312
"""See _walkdirs_utf8
1314
Because Win32 has a Unicode api, all of the 'path-from-top' entries will be
1316
This is currently the fallback code path when the filesystem encoding is
1317
not UTF-8. It may be better to implement an alternative so that we can
1318
safely handle paths that are not properly decodable in the current
1321
_utf8_encode = codecs.getencoder('utf8')
1323
_directory = _directory_kind
1324
_listdir = os.listdir
1325
_kind_from_mode = _formats.get
1327
pending = [(safe_utf8(prefix), None, None, None, safe_unicode(top))]
1329
relroot, _, _, _, top = pending.pop()
1331
relprefix = relroot + '/'
1772
next = [d for d in reversed(dirblock) if d[2] == _directory]
1774
pending.append(next)
1777
class UnicodeDirReader(DirReader):
1778
"""A dir reader for non-utf8 file systems, which transcodes."""
1780
__slots__ = ['_utf8_encode']
1783
self._utf8_encode = codecs.getencoder('utf8')
1785
def top_prefix_to_starting_dir(self, top, prefix=""):
1786
"""See DirReader.top_prefix_to_starting_dir."""
1787
return (safe_utf8(prefix), None, None, None, safe_unicode(top))
1789
def read_dir(self, prefix, top):
1790
"""Read a single directory from a non-utf8 file system.
1792
top, and the abspath element in the output are unicode, all other paths
1793
are utf8. Local disk IO is done via unicode calls to listdir etc.
1795
This is currently the fallback code path when the filesystem encoding is
1796
not UTF-8. It may be better to implement an alternative so that we can
1797
safely handle paths that are not properly decodable in the current
1800
See DirReader.read_dir for details.
1802
_utf8_encode = self._utf8_encode
1804
_listdir = os.listdir
1805
_kind_from_mode = file_kind_from_stat_mode
1808
relprefix = prefix + '/'
1334
1811
top_slash = top + u'/'
1484
2001
return socket.gethostname().decode(get_user_encoding())
1487
def recv_all(socket, bytes):
2004
# We must not read/write any more than 64k at a time from/to a socket so we
2005
# don't risk "no buffer space available" errors on some platforms. Windows in
2006
# particular is likely to throw WSAECONNABORTED or WSAENOBUFS if given too much
2008
MAX_SOCKET_CHUNK = 64 * 1024
2010
_end_of_stream_errors = [errno.ECONNRESET]
2011
for _eno in ['WSAECONNRESET', 'WSAECONNABORTED']:
2012
_eno = getattr(errno, _eno, None)
2013
if _eno is not None:
2014
_end_of_stream_errors.append(_eno)
2018
def read_bytes_from_socket(sock, report_activity=None,
2019
max_read_size=MAX_SOCKET_CHUNK):
2020
"""Read up to max_read_size of bytes from sock and notify of progress.
2022
Translates "Connection reset by peer" into file-like EOF (return an
2023
empty string rather than raise an error), and repeats the recv if
2024
interrupted by a signal.
2028
bytes = sock.recv(max_read_size)
2029
except socket.error, e:
2031
if eno in _end_of_stream_errors:
2032
# The connection was closed by the other side. Callers expect
2033
# an empty string to signal end-of-stream.
2035
elif eno == errno.EINTR:
2036
# Retry the interrupted recv.
2040
if report_activity is not None:
2041
report_activity(len(bytes), 'read')
2045
def recv_all(socket, count):
1488
2046
"""Receive an exact number of bytes.
1490
2048
Regular Socket.recv() may return less than the requested number of bytes,
1491
dependning on what's in the OS buffer. MSG_WAITALL is not available
2049
depending on what's in the OS buffer. MSG_WAITALL is not available
1492
2050
on all platforms, but this should work everywhere. This will return
1493
2051
less than the requested amount if the remote end closes.
1495
2053
This isn't optimized and is intended mostly for use in testing.
1498
while len(b) < bytes:
1499
new = socket.recv(bytes - len(b))
2056
while len(b) < count:
2057
new = read_bytes_from_socket(socket, None, count - len(b))
1506
def send_all(socket, bytes):
2064
def send_all(sock, bytes, report_activity=None):
1507
2065
"""Send all bytes on a socket.
1509
Regular socket.sendall() can give socket error 10053 on Windows. This
1510
implementation sends no more than 64k at a time, which avoids this problem.
2067
Breaks large blocks in smaller chunks to avoid buffering limitations on
2068
some platforms, and catches EINTR which may be thrown if the send is
2069
interrupted by a signal.
2071
This is preferred to socket.sendall(), because it avoids portability bugs
2072
and provides activity reporting.
2074
:param report_activity: Call this as bytes are read, see
2075
Transport._report_activity
1513
for pos in xrange(0, len(bytes), chunk_size):
1514
socket.sendall(bytes[pos:pos+chunk_size])
2078
byte_count = len(bytes)
2079
while sent_total < byte_count:
2081
sent = sock.send(buffer(bytes, sent_total, MAX_SOCKET_CHUNK))
2082
except socket.error, e:
2083
if e.args[0] != errno.EINTR:
2087
report_activity(sent, 'write')
2090
def connect_socket(address):
2091
# Slight variation of the socket.create_connection() function (provided by
2092
# python-2.6) that can fail if getaddrinfo returns an empty list. We also
2093
# provide it for previous python versions. Also, we don't use the timeout
2094
# parameter (provided by the python implementation) so we don't implement
2096
err = socket.error('getaddrinfo returns an empty list')
2097
host, port = address
2098
for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
2099
af, socktype, proto, canonname, sa = res
2102
sock = socket.socket(af, socktype, proto)
2106
except socket.error, err:
2107
# 'err' is now the most recent error
2108
if sock is not None:
1517
2113
def dereference_path(path):
1558
2154
base = dirname(bzrlib.__file__)
1559
2155
if getattr(sys, 'frozen', None): # bzr.exe
1560
2156
base = abspath(pathjoin(base, '..', '..'))
1561
filename = pathjoin(base, resource_relpath)
1562
return open(filename, 'rU').read()
1566
from bzrlib._readdir_pyx import read_dir as _read_dir
1568
from bzrlib._readdir_py import read_dir as _read_dir
2157
f = file(pathjoin(base, resource_relpath), "rU")
2163
def file_kind_from_stat_mode_thunk(mode):
2164
global file_kind_from_stat_mode
2165
if file_kind_from_stat_mode is file_kind_from_stat_mode_thunk:
2167
from bzrlib._readdir_pyx import UTF8DirReader
2168
file_kind_from_stat_mode = UTF8DirReader().kind_from_mode
2169
except ImportError, e:
2170
# This is one time where we won't warn that an extension failed to
2171
# load. The extension is never available on Windows anyway.
2172
from bzrlib._readdir_py import (
2173
_kind_from_mode as file_kind_from_stat_mode
2175
return file_kind_from_stat_mode(mode)
2176
file_kind_from_stat_mode = file_kind_from_stat_mode_thunk
2179
def file_kind(f, _lstat=os.lstat):
2181
return file_kind_from_stat_mode(_lstat(f).st_mode)
2183
if getattr(e, 'errno', None) in (errno.ENOENT, errno.ENOTDIR):
2184
raise errors.NoSuchFile(f)
2188
def until_no_eintr(f, *a, **kw):
2189
"""Run f(*a, **kw), retrying if an EINTR error occurs.
2191
WARNING: you must be certain that it is safe to retry the call repeatedly
2192
if EINTR does occur. This is typically only true for low-level operations
2193
like os.read. If in any doubt, don't use this.
2195
Keep in mind that this is not a complete solution to EINTR. There is
2196
probably code in the Python standard library and other dependencies that
2197
may encounter EINTR if a signal arrives (and there is signal handler for
2198
that signal). So this function can reduce the impact for IO that bzrlib
2199
directly controls, but it is not a complete solution.
2201
# Borrowed from Twisted's twisted.python.util.untilConcludes function.
2205
except (IOError, OSError), e:
2206
if e.errno == errno.EINTR:
2211
@deprecated_function(deprecated_in((2, 2, 0)))
2212
def re_compile_checked(re_string, flags=0, where=""):
2213
"""Return a compiled re, or raise a sensible error.
2215
This should only be used when compiling user-supplied REs.
2217
:param re_string: Text form of regular expression.
2218
:param flags: eg re.IGNORECASE
2219
:param where: Message explaining to the user the context where
2220
it occurred, eg 'log search filter'.
2222
# from https://bugs.launchpad.net/bzr/+bug/251352
2224
re_obj = re.compile(re_string, flags)
2227
except errors.InvalidPattern, e:
2229
where = ' in ' + where
2230
# despite the name 'error' is a type
2231
raise errors.BzrCommandError('Invalid regular expression%s: %s'
2235
if sys.platform == "win32":
2238
return msvcrt.getch()
2243
fd = sys.stdin.fileno()
2244
settings = termios.tcgetattr(fd)
2247
ch = sys.stdin.read(1)
2249
termios.tcsetattr(fd, termios.TCSADRAIN, settings)
2252
if sys.platform == 'linux2':
2253
def _local_concurrency():
2255
return os.sysconf('SC_NPROCESSORS_ONLN')
2256
except (ValueError, OSError, AttributeError):
2258
elif sys.platform == 'darwin':
2259
def _local_concurrency():
2260
return subprocess.Popen(['sysctl', '-n', 'hw.availcpu'],
2261
stdout=subprocess.PIPE).communicate()[0]
2262
elif "bsd" in sys.platform:
2263
def _local_concurrency():
2264
return subprocess.Popen(['sysctl', '-n', 'hw.ncpu'],
2265
stdout=subprocess.PIPE).communicate()[0]
2266
elif sys.platform == 'sunos5':
2267
def _local_concurrency():
2268
return subprocess.Popen(['psrinfo', '-p',],
2269
stdout=subprocess.PIPE).communicate()[0]
2270
elif sys.platform == "win32":
2271
def _local_concurrency():
2272
# This appears to return the number of cores.
2273
return os.environ.get('NUMBER_OF_PROCESSORS')
2275
def _local_concurrency():
2280
_cached_local_concurrency = None
2282
def local_concurrency(use_cache=True):
2283
"""Return how many processes can be run concurrently.
2285
Rely on platform specific implementations and default to 1 (one) if
2286
anything goes wrong.
2288
global _cached_local_concurrency
2290
if _cached_local_concurrency is not None and use_cache:
2291
return _cached_local_concurrency
2293
concurrency = os.environ.get('BZR_CONCURRENCY', None)
2294
if concurrency is None:
2296
import multiprocessing
2298
# multiprocessing is only available on Python >= 2.6
2300
concurrency = _local_concurrency()
2301
except (OSError, IOError):
2304
concurrency = multiprocessing.cpu_count()
2306
concurrency = int(concurrency)
2307
except (TypeError, ValueError):
2310
_cached_concurrency = concurrency
2314
class UnicodeOrBytesToBytesWriter(codecs.StreamWriter):
2315
"""A stream writer that doesn't decode str arguments."""
2317
def __init__(self, encode, stream, errors='strict'):
2318
codecs.StreamWriter.__init__(self, stream, errors)
2319
self.encode = encode
2321
def write(self, object):
2322
if type(object) is str:
2323
self.stream.write(object)
2325
data, _ = self.encode(object, self.errors)
2326
self.stream.write(data)
2328
if sys.platform == 'win32':
2329
def open_file(filename, mode='r', bufsize=-1):
2330
"""This function is used to override the ``open`` builtin.
2332
But it uses O_NOINHERIT flag so the file handle is not inherited by
2333
child processes. Deleting or renaming a closed file opened with this
2334
function is not blocking child processes.
2336
writing = 'w' in mode
2337
appending = 'a' in mode
2338
updating = '+' in mode
2339
binary = 'b' in mode
2342
# see http://msdn.microsoft.com/en-us/library/yeby3zcb%28VS.71%29.aspx
2343
# for flags for each modes.
2353
flags |= os.O_WRONLY
2354
flags |= os.O_CREAT | os.O_TRUNC
2359
flags |= os.O_WRONLY
2360
flags |= os.O_CREAT | os.O_APPEND
2365
flags |= os.O_RDONLY
2367
return os.fdopen(os.open(filename, flags), mode, bufsize)
2372
def getuser_unicode():
2373
"""Return the username as unicode.
2376
user_encoding = get_user_encoding()
2377
username = getpass.getuser().decode(user_encoding)
2378
except UnicodeDecodeError:
2379
raise errors.BzrError("Can't decode username as %s." % \
2384
def available_backup_name(base, exists):
2385
"""Find a non-existing backup file name.
2387
This will *not* create anything, this only return a 'free' entry. This
2388
should be used for checking names in a directory below a locked
2389
tree/branch/repo to avoid race conditions. This is LBYL (Look Before You
2390
Leap) and generally discouraged.
2392
:param base: The base name.
2394
:param exists: A callable returning True if the path parameter exists.
2397
name = "%s.~%d~" % (base, counter)
2400
name = "%s.~%d~" % (base, counter)
2404
def set_fd_cloexec(fd):
2405
"""Set a Unix file descriptor's FD_CLOEXEC flag. Do nothing if platform
2406
support for this is not available.
2410
old = fcntl.fcntl(fd, fcntl.F_GETFD)
2411
fcntl.fcntl(fd, fcntl.F_SETFD, old | fcntl.FD_CLOEXEC)
2412
except (ImportError, AttributeError):
2413
# Either the fcntl module or specific constants are not present
2417
def find_executable_on_path(name):
2418
"""Finds an executable on the PATH.
2420
On Windows, this will try to append each extension in the PATHEXT
2421
environment variable to the name, if it cannot be found with the name
2424
:param name: The base name of the executable.
2425
:return: The path to the executable found or None.
2427
path = os.environ.get('PATH')
2430
path = path.split(os.pathsep)
2431
if sys.platform == 'win32':
2432
exts = os.environ.get('PATHEXT', '').split(os.pathsep)
2433
exts = [ext.lower() for ext in exts]
2434
base, ext = os.path.splitext(name)
2436
if ext.lower() not in exts:
2444
f = os.path.join(d, name) + ext
2445
if os.access(f, os.X_OK):
added
removed
bzrlib/osutils.py
