69
72
from bzrlib import symbol_versioning
75
# Cross platform wall-clock time functionality with decent resolution.
76
# On Linux ``time.clock`` returns only CPU time. On Windows, ``time.time()``
77
# only has a resolution of ~15ms. Note that ``time.clock()`` is not
78
# synchronized with ``time.time()``, this is only meant to be used to find
79
# delta times by subtracting from another call to this function.
80
timer_func = time.time
81
if sys.platform == 'win32':
82
timer_func = time.clock
72
84
# On win32, O_BINARY is used to indicate the file should
73
85
# be opened in binary mode, rather than text mode.
74
86
# On other platforms, O_BINARY doesn't exist, because
98
119
:param paths: A container (and hence not None) of paths.
99
120
:return: A set of paths sufficient to include everything in paths via
100
is_inside_any, drawn from the paths parameter.
121
is_inside, drawn from the paths parameter.
105
other_paths = paths.difference([path])
106
if not is_inside_any(other_paths, path):
107
# this is a top level path, we must check it.
108
search_paths.add(path)
127
return path.split('/')
128
sorted_paths = sorted(list(paths), key=sort_key)
130
search_paths = [sorted_paths[0]]
131
for path in sorted_paths[1:]:
132
if not is_inside(search_paths[-1], path):
133
# This path is unique, add it
134
search_paths.append(path)
136
return set(search_paths)
172
199
def fancy_rename(old, new, rename_func, unlink_func):
173
200
"""A fancy rename, when you don't have atomic rename.
175
202
:param old: The old path, to rename from
176
203
:param new: The new path, to rename to
177
204
:param rename_func: The potentially non-atomic rename function
178
:param unlink_func: A way to delete the target file if the full rename succeeds
205
:param unlink_func: A way to delete the target file if the full rename
181
208
# sftp rename doesn't allow overwriting, so play tricks:
182
209
base = os.path.basename(new)
183
210
dirname = os.path.dirname(new)
184
tmp_name = u'tmp.%s.%.9f.%d.%s' % (base, time.time(), os.getpid(), rand_chars(10))
211
# callers use different encodings for the paths so the following MUST
212
# respect that. We rely on python upcasting to unicode if new is unicode
213
# and keeping a str if not.
214
tmp_name = 'tmp.%s.%.9f.%d.%s' % (base, time.time(),
215
os.getpid(), rand_chars(10))
185
216
tmp_name = pathjoin(dirname, tmp_name)
187
218
# Rename the file out of the way, but keep track if it didn't exist
218
250
# source and target may be aliases of each other (e.g. on a
219
251
# case-insensitive filesystem), so we may have accidentally renamed
220
252
# source by when we tried to rename target
221
if not (file_existed and e.errno in (None, errno.ENOENT)):
253
failure_exc = sys.exc_info()
254
if (file_existed and e.errno in (None, errno.ENOENT)
255
and old.lower() == new.lower()):
256
# source and target are the same file on a case-insensitive
257
# filesystem, so we don't generate an exception
225
261
# If the file used to exist, rename it back into place
317
355
"""We expect to be able to atomically replace 'new' with old.
319
357
On win32, if new exists, it must be moved out of the way first,
323
361
fancy_rename(old, new, rename_func=os.rename, unlink_func=os.unlink)
324
362
except OSError, e:
325
363
if e.errno in (errno.EPERM, errno.EACCES, errno.EBUSY, errno.EINVAL):
326
# If we try to rename a non-existant file onto cwd, we get
327
# EPERM or EACCES instead of ENOENT, this will raise ENOENT
364
# If we try to rename a non-existant file onto cwd, we get
365
# EPERM or EACCES instead of ENOENT, this will raise ENOENT
328
366
# if the old path doesn't exist, sometimes we get EACCES
329
367
# On Linux, we seem to get EBUSY, on Mac we get EINVAL
667
730
date_str = time.strftime(date_fmt, tt)
668
731
return date_str + offset_str
734
# Cache of formatted offset strings
738
def format_date_with_offset_in_original_timezone(t, offset=0,
739
_cache=_offset_cache):
740
"""Return a formatted date string in the original timezone.
742
This routine may be faster then format_date.
744
:param t: Seconds since the epoch.
745
:param offset: Timezone offset in seconds east of utc.
749
tt = time.gmtime(t + offset)
750
date_fmt = _default_format_by_weekday_num[tt[6]]
751
date_str = time.strftime(date_fmt, tt)
752
offset_str = _cache.get(offset, None)
753
if offset_str is None:
754
offset_str = ' %+03d%02d' % (offset / 3600, (offset / 60) % 60)
755
_cache[offset] = offset_str
756
return date_str + offset_str
670
759
def format_local_date(t, offset=0, timezone='original', date_fmt=None,
671
760
show_offset=True):
672
761
"""Return an unicode date string formatted according to the current locale.
683
772
_format_date(t, offset, timezone, date_fmt, show_offset)
684
773
date_str = time.strftime(date_fmt, tt)
685
774
if not isinstance(date_str, unicode):
686
date_str = date_str.decode(bzrlib.user_encoding, 'replace')
775
date_str = date_str.decode(get_user_encoding(), 'replace')
687
776
return date_str + offset_str
689
779
def _format_date(t, offset, timezone, date_fmt, show_offset):
690
780
if timezone == 'utc':
691
781
tt = time.gmtime(t)
829
919
return pathjoin(*p)
922
def parent_directories(filename):
923
"""Return the list of parent directories, deepest first.
925
For example, parent_directories("a/b/c") -> ["a/b", "a"].
928
parts = splitpath(dirname(filename))
930
parents.append(joinpath(parts))
935
_extension_load_failures = []
938
def failed_to_load_extension(exception):
939
"""Handle failing to load a binary extension.
941
This should be called from the ImportError block guarding the attempt to
942
import the native extension. If this function returns, the pure-Python
943
implementation should be loaded instead::
946
>>> import bzrlib._fictional_extension_pyx
947
>>> except ImportError, e:
948
>>> bzrlib.osutils.failed_to_load_extension(e)
949
>>> import bzrlib._fictional_extension_py
951
# NB: This docstring is just an example, not a doctest, because doctest
952
# currently can't cope with the use of lazy imports in this namespace --
955
# This currently doesn't report the failure at the time it occurs, because
956
# they tend to happen very early in startup when we can't check config
957
# files etc, and also we want to report all failures but not spam the user
959
from bzrlib import trace
960
exception_str = str(exception)
961
if exception_str not in _extension_load_failures:
962
trace.mutter("failed to load compiled extension: %s" % exception_str)
963
_extension_load_failures.append(exception_str)
966
def report_extension_load_failures():
967
if not _extension_load_failures:
969
from bzrlib.config import GlobalConfig
970
if GlobalConfig().get_user_option_as_bool('ignore_missing_extensions'):
972
# the warnings framework should by default show this only once
973
from bzrlib.trace import warning
975
"bzr: warning: some compiled extensions could not be loaded; "
976
"see <https://answers.launchpad.net/bzr/+faq/703>")
977
# we no longer show the specific missing extensions here, because it makes
978
# the message too long and scary - see
979
# https://bugs.launchpad.net/bzr/+bug/430529
833
983
from bzrlib._chunks_to_lines_pyx import chunks_to_lines
984
except ImportError, e:
985
failed_to_load_extension(e)
835
986
from bzrlib._chunks_to_lines_py import chunks_to_lines
875
1026
shutil.copyfile(src, dest)
878
# Look Before You Leap (LBYL) is appropriate here instead of Easier to Ask for
879
# Forgiveness than Permission (EAFP) because:
880
# - root can damage a solaris file system by using unlink,
881
# - unlink raises different exceptions on different OSes (linux: EISDIR, win32:
882
# EACCES, OSX: EPERM) when invoked on a directory.
883
1029
def delete_any(path):
884
"""Delete a file or directory."""
1030
"""Delete a file, symlink or directory.
1032
Will delete even if readonly.
1035
_delete_file_or_dir(path)
1036
except (OSError, IOError), e:
1037
if e.errno in (errno.EPERM, errno.EACCES):
1038
# make writable and try again
1041
except (OSError, IOError):
1043
_delete_file_or_dir(path)
1048
def _delete_file_or_dir(path):
1049
# Look Before You Leap (LBYL) is appropriate here instead of Easier to Ask for
1050
# Forgiveness than Permission (EAFP) because:
1051
# - root can damage a solaris file system by using unlink,
1052
# - unlink raises different exceptions on different OSes (linux: EISDIR, win32:
1053
# EACCES, OSX: EPERM) when invoked on a directory.
885
1054
if isdir(path): # Takes care of symlinks
1009
1199
# the target of a move, for example).
1010
1200
current = pathjoin(current, bit, *list(bit_iter))
1012
return current[len(abs_base)+1:]
1202
return current[len(abs_base):].lstrip('/')
1014
1204
# XXX - TODO - we need better detection/integration of case-insensitive
1015
# file-systems; Linux often sees FAT32 devices, for example, so could
1016
# probably benefit from the same basic support there. For now though, only
1017
# Windows gets that support, and it gets it for *all* file-systems!
1018
if sys.platform == "win32":
1205
# file-systems; Linux often sees FAT32 devices (or NFS-mounted OSX
1206
# filesystems), for example, so could probably benefit from the same basic
1207
# support there. For now though, only Windows and OSX get that support, and
1208
# they get it for *all* file-systems!
1209
if sys.platform in ('win32', 'darwin'):
1019
1210
canonical_relpath = _cicp_canonical_relpath
1021
1212
canonical_relpath = relpath
1129
1319
On platforms where the system normalizes filenames (Mac OSX),
1130
1320
you can access a file by any path which will normalize correctly.
1131
On platforms where the system does not normalize filenames
1321
On platforms where the system does not normalize filenames
1132
1322
(Windows, Linux), you have to access a file by its exact path.
1134
Internally, bzr only supports NFC normalization, since that is
1324
Internally, bzr only supports NFC normalization, since that is
1135
1325
the standard for XML documents.
1137
1327
So return the normalized path, and a flag indicating if the file
1154
1344
normalized_filename = _inaccessible_normalized_filename
1347
default_terminal_width = 80
1348
"""The default terminal width for ttys.
1350
This is defined so that higher levels can share a common fallback value when
1351
terminal_width() returns None.
1157
1355
def terminal_width():
1158
"""Return estimated terminal width."""
1159
if sys.platform == 'win32':
1160
return win32utils.get_console_size()[0]
1356
"""Return terminal width.
1358
None is returned if the width can't established precisely.
1361
- if BZR_COLUMNS is set, returns its value
1362
- if there is no controlling terminal, returns None
1363
- if COLUMNS is set, returns its value,
1365
From there, we need to query the OS to get the size of the controlling
1369
- get termios.TIOCGWINSZ
1370
- if an error occurs or a negative value is obtained, returns None
1374
- win32utils.get_console_size() decides,
1375
- returns None on error (provided default value)
1378
# If BZR_COLUMNS is set, take it, user is always right
1380
return int(os.environ['BZR_COLUMNS'])
1381
except (KeyError, ValueError):
1384
isatty = getattr(sys.stdout, 'isatty', None)
1385
if isatty is None or not isatty():
1386
# Don't guess, setting BZR_COLUMNS is the recommended way to override.
1389
# If COLUMNS is set, take it, the terminal knows better (even inside a
1390
# given terminal, the application can decide to set COLUMNS to a lower
1391
# value (splitted screen) or a bigger value (scroll bars))
1393
return int(os.environ['COLUMNS'])
1394
except (KeyError, ValueError):
1397
width, height = _terminal_size(None, None)
1399
# Consider invalid values as meaning no width
1405
def _win32_terminal_size(width, height):
1406
width, height = win32utils.get_console_size(defaultx=width, defaulty=height)
1407
return width, height
1410
def _ioctl_terminal_size(width, height):
1163
1412
import struct, fcntl, termios
1164
1413
s = struct.pack('HHHH', 0, 0, 0, 0)
1165
1414
x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
1166
width = struct.unpack('HHHH', x)[1]
1415
height, width = struct.unpack('HHHH', x)[0:2]
1416
except (IOError, AttributeError):
1171
width = int(os.environ['COLUMNS'])
1418
return width, height
1420
_terminal_size = None
1421
"""Returns the terminal size as (width, height).
1423
:param width: Default value for width.
1424
:param height: Default value for height.
1426
This is defined specifically for each OS and query the size of the controlling
1427
terminal. If any error occurs, the provided default values should be returned.
1429
if sys.platform == 'win32':
1430
_terminal_size = _win32_terminal_size
1432
_terminal_size = _ioctl_terminal_size
1435
def _terminal_size_changed(signum, frame):
1436
"""Set COLUMNS upon receiving a SIGnal for WINdow size CHange."""
1437
width, height = _terminal_size(None, None)
1438
if width is not None:
1439
os.environ['COLUMNS'] = str(width)
1441
if sys.platform == 'win32':
1442
# Martin (gz) mentioned WINDOW_BUFFER_SIZE_RECORD from ReadConsoleInput but
1443
# I've no idea how to plug that in the current design -- vila 20091216
1446
signal.signal(signal.SIGWINCH, _terminal_size_changed)
1180
1449
def supports_executable():
1258
1527
def walkdirs(top, prefix=""):
1259
1528
"""Yield data about all the directories in a tree.
1261
1530
This yields all the data about the contents of a directory at a time.
1262
1531
After each directory has been yielded, if the caller has mutated the list
1263
1532
to exclude some directories, they are then not descended into.
1265
1534
The data yielded is of the form:
1266
1535
((directory-relpath, directory-path-from-top),
1267
1536
[(relpath, basename, kind, lstat, path-from-top), ...]),
1268
1537
- directory-relpath is the relative path of the directory being returned
1269
1538
with respect to top. prefix is prepended to this.
1270
- directory-path-from-root is the path including top for this directory.
1539
- directory-path-from-root is the path including top for this directory.
1271
1540
It is suitable for use with os functions.
1272
1541
- relpath is the relative path within the subtree being walked.
1273
1542
- basename is the basename of the path
1275
1544
present within the tree - but it may be recorded as versioned. See
1276
1545
versioned_kind.
1277
1546
- lstat is the stat data *if* the file was statted.
1278
- planned, not implemented:
1547
- planned, not implemented:
1279
1548
path_from_tree_root is the path from the root of the tree.
1281
:param prefix: Prefix the relpaths that are yielded with 'prefix'. This
1550
:param prefix: Prefix the relpaths that are yielded with 'prefix'. This
1282
1551
allows one to walk a subtree but get paths that are relative to a tree
1283
1552
rooted higher up.
1284
1553
:return: an iterator over the dirs.
1286
1555
#TODO there is a bit of a smell where the results of the directory-
1287
# summary in this, and the path from the root, may not agree
1556
# summary in this, and the path from the root, may not agree
1288
1557
# depending on top and prefix - i.e. ./foo and foo as a pair leads to
1289
1558
# potentially confusing output. We should make this more robust - but
1290
1559
# not at a speed cost. RBC 20060731
1374
1643
# for win98 anyway.
1376
1645
from bzrlib._walkdirs_win32 import Win32ReadDir
1378
_selected_dir_reader = UnicodeDirReader()
1380
1646
_selected_dir_reader = Win32ReadDir()
1381
elif fs_encoding not in ('UTF-8', 'US-ASCII', 'ANSI_X3.4-1968'):
1649
elif fs_encoding in ('UTF-8', 'US-ASCII', 'ANSI_X3.4-1968'):
1382
1650
# ANSI_X3.4-1968 is a form of ASCII
1383
_selected_dir_reader = UnicodeDirReader()
1386
1652
from bzrlib._readdir_pyx import UTF8DirReader
1388
# No optimised code path
1389
_selected_dir_reader = UnicodeDirReader()
1391
1653
_selected_dir_reader = UTF8DirReader()
1654
except ImportError, e:
1655
failed_to_load_extension(e)
1658
if _selected_dir_reader is None:
1659
# Fallback to the python version
1660
_selected_dir_reader = UnicodeDirReader()
1392
1662
# 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
1393
1663
# But we don't actually uses 1-3 in pending, so set them to None
1394
1664
pending = [[_selected_dir_reader.top_prefix_to_starting_dir(top, prefix)]]
1626
def send_all(socket, bytes):
1896
def send_all(socket, bytes, report_activity=None):
1627
1897
"""Send all bytes on a socket.
1629
1899
Regular socket.sendall() can give socket error 10053 on Windows. This
1630
1900
implementation sends no more than 64k at a time, which avoids this problem.
1902
:param report_activity: Call this as bytes are read, see
1903
Transport._report_activity
1632
1905
chunk_size = 2**16
1633
1906
for pos in xrange(0, len(bytes), chunk_size):
1634
until_no_eintr(socket.sendall, bytes[pos:pos+chunk_size])
1907
block = bytes[pos:pos+chunk_size]
1908
if report_activity is not None:
1909
report_activity(len(block), 'write')
1910
until_no_eintr(socket.sendall, block)
1637
1913
def dereference_path(path):
1997
def re_compile_checked(re_string, flags=0, where=""):
1998
"""Return a compiled re, or raise a sensible error.
2000
This should only be used when compiling user-supplied REs.
2002
:param re_string: Text form of regular expression.
2003
:param flags: eg re.IGNORECASE
2004
:param where: Message explaining to the user the context where
2005
it occurred, eg 'log search filter'.
2007
# from https://bugs.launchpad.net/bzr/+bug/251352
2009
re_obj = re.compile(re_string, flags)
2014
where = ' in ' + where
2015
# despite the name 'error' is a type
2016
raise errors.BzrCommandError('Invalid regular expression%s: %r: %s'
2017
% (where, re_string, e))
1720
2020
if sys.platform == "win32":
1734
2034
termios.tcsetattr(fd, termios.TCSADRAIN, settings)
2038
if sys.platform == 'linux2':
2039
def _local_concurrency():
2041
prefix = 'processor'
2042
for line in file('/proc/cpuinfo', 'rb'):
2043
if line.startswith(prefix):
2044
concurrency = int(line[line.find(':')+1:]) + 1
2046
elif sys.platform == 'darwin':
2047
def _local_concurrency():
2048
return subprocess.Popen(['sysctl', '-n', 'hw.availcpu'],
2049
stdout=subprocess.PIPE).communicate()[0]
2050
elif sys.platform[0:7] == 'freebsd':
2051
def _local_concurrency():
2052
return subprocess.Popen(['sysctl', '-n', 'hw.ncpu'],
2053
stdout=subprocess.PIPE).communicate()[0]
2054
elif sys.platform == 'sunos5':
2055
def _local_concurrency():
2056
return subprocess.Popen(['psrinfo', '-p',],
2057
stdout=subprocess.PIPE).communicate()[0]
2058
elif sys.platform == "win32":
2059
def _local_concurrency():
2060
# This appears to return the number of cores.
2061
return os.environ.get('NUMBER_OF_PROCESSORS')
2063
def _local_concurrency():
2068
_cached_local_concurrency = None
2070
def local_concurrency(use_cache=True):
2071
"""Return how many processes can be run concurrently.
2073
Rely on platform specific implementations and default to 1 (one) if
2074
anything goes wrong.
2076
global _cached_local_concurrency
2078
if _cached_local_concurrency is not None and use_cache:
2079
return _cached_local_concurrency
2081
concurrency = os.environ.get('BZR_CONCURRENCY', None)
2082
if concurrency is None:
2084
concurrency = _local_concurrency()
2085
except (OSError, IOError):
2088
concurrency = int(concurrency)
2089
except (TypeError, ValueError):
2092
_cached_concurrency = concurrency
2096
class UnicodeOrBytesToBytesWriter(codecs.StreamWriter):
2097
"""A stream writer that doesn't decode str arguments."""
2099
def __init__(self, encode, stream, errors='strict'):
2100
codecs.StreamWriter.__init__(self, stream, errors)
2101
self.encode = encode
2103
def write(self, object):
2104
if type(object) is str:
2105
self.stream.write(object)
2107
data, _ = self.encode(object, self.errors)
2108
self.stream.write(data)
added
removed
bzrlib/osutils.py
