1
1
# Bazaar-NG -- distributed version control
3
3
# Copyright (C) 2005 by Canonical Ltd
5
5
# This program is free software; you can redistribute it and/or modify
6
6
# it under the terms of the GNU General Public License as published by
7
7
# the Free Software Foundation; either version 2 of the License, or
8
8
# (at your option) any later version.
10
10
# This program is distributed in the hope that it will be useful,
11
11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
13
# GNU General Public License for more details.
15
15
# You should have received a copy of the GNU General Public License
16
16
# along with this program; if not, write to the Free Software
17
17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
from cStringIO import StringIO
21
from ntpath import (abspath as _nt_abspath,
23
normpath as _nt_normpath,
24
realpath as _nt_realpath,
27
from os import listdir
32
from shutil import copyfile
34
from stat import (S_ISREG, S_ISDIR, S_ISLNK, ST_MODE, ST_SIZE,
35
S_ISCHR, S_ISBLK, S_ISFIFO, S_ISSOCK)
44
from bzrlib.errors import (BzrError,
45
BzrBadParameterNotUnicode,
50
from bzrlib.symbol_versioning import (deprecated_function,
52
from bzrlib.trace import mutter
19
import os, types, re, time, types
20
from stat import S_ISREG, S_ISDIR, ST_MODE, ST_SIZE
22
from errors import bailout
55
24
def make_readonly(filename):
56
25
"""Make a filename read-only."""
26
# TODO: probably needs to be fixed for windows
57
27
mod = os.stat(filename).st_mode
58
28
mod = mod & 0777555
59
29
os.chmod(filename, mod)
65
35
os.chmod(filename, mod)
38
_QUOTE_RE = re.compile(r'([^a-zA-Z0-9.,:/_~-])')
72
"""Return a quoted filename filename
74
This previously used backslash quoting, but that works poorly on
76
# TODO: I'm not really sure this is the best format either.x
79
_QUOTE_RE = re.compile(r'([^a-zA-Z0-9.,:/\\_~-])')
81
if _QUOTE_RE.search(f):
87
_directory_kind = 'directory'
90
stat.S_IFDIR:_directory_kind,
91
stat.S_IFCHR:'chardev',
95
stat.S_IFLNK:'symlink',
96
stat.S_IFSOCK:'socket',
100
def file_kind_from_stat_mode(stat_mode, _formats=_formats, _unknown='unknown'):
101
"""Generate a file kind from a stat mode. This is used in walkdirs.
103
Its performance is critical: Do not mutate without careful benchmarking.
106
return _formats[stat_mode & 0170000]
111
def file_kind(f, _lstat=os.lstat, _mapper=file_kind_from_stat_mode):
113
return _mapper(_lstat(f).st_mode)
115
if getattr(e, 'errno', None) == errno.ENOENT:
116
raise bzrlib.errors.NoSuchFile(f)
120
def kind_marker(kind):
123
elif kind == _directory_kind:
125
elif kind == 'symlink':
128
raise BzrError('invalid file kind %r' % kind)
130
lexists = getattr(os.path, 'lexists', None)
134
if hasattr(os, 'lstat'):
140
if e.errno == errno.ENOENT:
143
raise BzrError("lstat/stat of (%r): %r" % (f, e))
146
def fancy_rename(old, new, rename_func, unlink_func):
147
"""A fancy rename, when you don't have atomic rename.
149
:param old: The old path, to rename from
150
:param new: The new path, to rename to
151
:param rename_func: The potentially non-atomic rename function
152
:param unlink_func: A way to delete the target file if the full rename succeeds
155
# sftp rename doesn't allow overwriting, so play tricks:
157
base = os.path.basename(new)
158
dirname = os.path.dirname(new)
159
tmp_name = u'tmp.%s.%.9f.%d.%s' % (base, time.time(), os.getpid(), rand_chars(10))
160
tmp_name = pathjoin(dirname, tmp_name)
162
# Rename the file out of the way, but keep track if it didn't exist
163
# We don't want to grab just any exception
164
# something like EACCES should prevent us from continuing
165
# The downside is that the rename_func has to throw an exception
166
# with an errno = ENOENT, or NoSuchFile
169
rename_func(new, tmp_name)
170
except (NoSuchFile,), e:
173
# RBC 20060103 abstraction leakage: the paramiko SFTP clients rename
174
# function raises an IOError with errno == None when a rename fails.
175
# This then gets caught here.
176
if e.errno not in (None, errno.ENOENT, errno.ENOTDIR):
179
if (not hasattr(e, 'errno')
180
or e.errno not in (errno.ENOENT, errno.ENOTDIR)):
187
# This may throw an exception, in which case success will
189
rename_func(old, new)
193
# If the file used to exist, rename it back into place
194
# otherwise just delete it from the tmp location
196
unlink_func(tmp_name)
198
rename_func(tmp_name, new)
201
# In Python 2.4.2 and older, os.path.abspath and os.path.realpath
202
# choke on a Unicode string containing a relative path if
203
# os.getcwd() returns a non-sys.getdefaultencoding()-encoded
205
_fs_enc = sys.getfilesystemencoding()
206
def _posix_abspath(path):
207
# jam 20060426 rather than encoding to fsencoding
208
# copy posixpath.abspath, but use os.getcwdu instead
209
if not posixpath.isabs(path):
210
path = posixpath.join(getcwd(), path)
211
return posixpath.normpath(path)
214
def _posix_realpath(path):
215
return posixpath.realpath(path.encode(_fs_enc)).decode(_fs_enc)
218
def _win32_abspath(path):
219
# Real _nt_abspath doesn't have a problem with a unicode cwd
220
return _nt_abspath(unicode(path)).replace('\\', '/')
223
def _win32_realpath(path):
224
# Real _nt_realpath doesn't have a problem with a unicode cwd
225
return _nt_realpath(unicode(path)).replace('\\', '/')
228
def _win32_pathjoin(*args):
229
return _nt_join(*args).replace('\\', '/')
232
def _win32_normpath(path):
233
return _nt_normpath(unicode(path)).replace('\\', '/')
237
return os.getcwdu().replace('\\', '/')
240
def _win32_mkdtemp(*args, **kwargs):
241
return tempfile.mkdtemp(*args, **kwargs).replace('\\', '/')
244
def _win32_rename(old, new):
245
fancy_rename(old, new, rename_func=os.rename, unlink_func=os.unlink)
248
# Default is to just use the python builtins, but these can be rebound on
249
# particular platforms.
250
abspath = _posix_abspath
251
realpath = _posix_realpath
252
pathjoin = os.path.join
253
normpath = os.path.normpath
255
mkdtemp = tempfile.mkdtemp
257
dirname = os.path.dirname
258
basename = os.path.basename
259
rmtree = shutil.rmtree
261
MIN_ABS_PATHLENGTH = 1
264
if sys.platform == 'win32':
265
abspath = _win32_abspath
266
realpath = _win32_realpath
267
pathjoin = _win32_pathjoin
268
normpath = _win32_normpath
269
getcwd = _win32_getcwd
270
mkdtemp = _win32_mkdtemp
271
rename = _win32_rename
273
MIN_ABS_PATHLENGTH = 3
275
def _win32_delete_readonly(function, path, excinfo):
276
"""Error handler for shutil.rmtree function [for win32]
277
Helps to remove files and dirs marked as read-only.
279
type_, value = excinfo[:2]
280
if function in (os.remove, os.rmdir) \
281
and type_ == OSError \
282
and value.errno == errno.EACCES:
283
bzrlib.osutils.make_writable(path)
288
def rmtree(path, ignore_errors=False, onerror=_win32_delete_readonly):
289
"""Replacer for shutil.rmtree: could remove readonly dirs/files"""
290
return shutil.rmtree(path, ignore_errors, onerror)
293
def get_terminal_encoding():
294
"""Find the best encoding for printing to the screen.
296
This attempts to check both sys.stdout and sys.stdin to see
297
what encoding they are in, and if that fails it falls back to
298
bzrlib.user_encoding.
299
The problem is that on Windows, locale.getpreferredencoding()
300
is not the same encoding as that used by the console:
301
http://mail.python.org/pipermail/python-list/2003-May/162357.html
303
On my standard US Windows XP, the preferred encoding is
304
cp1252, but the console is cp437
306
output_encoding = getattr(sys.stdout, 'encoding', None)
307
if not output_encoding:
308
input_encoding = getattr(sys.stdin, 'encoding', None)
309
if not input_encoding:
310
output_encoding = bzrlib.user_encoding
311
mutter('encoding stdout as bzrlib.user_encoding %r', output_encoding)
313
output_encoding = input_encoding
314
mutter('encoding stdout as sys.stdin encoding %r', output_encoding)
316
mutter('encoding stdout as sys.stdout encoding %r', output_encoding)
317
return output_encoding
320
def normalizepath(f):
321
if hasattr(os.path, 'realpath'):
325
[p,e] = os.path.split(f)
326
if e == "" or e == "." or e == "..":
329
return pathjoin(F(p), e)
333
"""Copy a file to a backup.
335
Backups are named in GNU-style, with a ~ suffix.
337
If the file is already a backup, it's not copied.
343
if has_symlinks() and os.path.islink(fn):
344
target = os.readlink(fn)
345
os.symlink(target, bfn)
353
outf = file(bfn, 'wb')
40
"""Return shell-quoted filename"""
41
## We could be a bit more terse by using double-quotes etc
42
f = _QUOTE_RE.sub(r'\\\1', f)
49
mode = os.lstat(f)[ST_MODE]
55
bailout("can't handle file kind of %r" % fp)
376
"""True if f is a symlink."""
378
return S_ISLNK(os.lstat(f)[ST_MODE])
382
def is_inside(dir, fname):
383
"""True if fname is inside dir.
385
The parameters should typically be passed to osutils.normpath first, so
386
that . and .. and repeated slashes are eliminated, and the separators
387
are canonical for the platform.
389
The empty string as a dir name is taken as top-of-tree and matches
392
>>> is_inside('src', pathjoin('src', 'foo.c'))
394
>>> is_inside('src', 'srccontrol')
396
>>> is_inside('src', pathjoin('src', 'a', 'a', 'a', 'foo.c'))
398
>>> is_inside('foo.c', 'foo.c')
400
>>> is_inside('foo.c', '')
402
>>> is_inside('', 'foo.c')
405
# XXX: Most callers of this can actually do something smarter by
406
# looking at the inventory
416
return fname.startswith(dir)
419
def is_inside_any(dir_list, fname):
420
"""True if fname is inside any of given dirs."""
421
for dirname in dir_list:
422
if is_inside(dirname, fname):
428
def is_inside_or_parent_of_any(dir_list, fname):
429
"""True if fname is a child or a parent of any of the given files."""
430
for dirname in dir_list:
431
if is_inside(dirname, fname) or is_inside(fname, dirname):
437
76
def pumpfile(fromfile, tofile):
438
77
"""Copy contents of one file to another."""
441
b = fromfile.read(BUFSIZE)
447
def file_iterator(input_file, readsize=32768):
449
b = input_file.read(readsize)
78
tofile.write(fromfile.read())
82
"""Return a new UUID"""
84
## XXX: Could alternatively read /proc/sys/kernel/random/uuid on
85
## Linux, but we need something portable for other systems;
86
## preferably an implementation in Python.
87
bailout('uuids not allowed!')
88
return chomp(os.popen('uuidgen').readline())
91
if s and (s[-1] == '\n'):
99
## TODO: Maybe read in chunks to handle big files
456
100
if hasattr(f, 'tell'):
457
101
assert f.tell() == 0
469
def sha_strings(strings):
470
"""Return the sha-1 of concatenation of strings"""
472
map(s.update, strings)
473
104
return s.hexdigest()
476
107
def sha_string(f):
479
111
return s.hexdigest()
482
def fingerprint_file(f):
487
return {'size': size,
488
'sha1': s.hexdigest()}
116
"""Return email-style username.
118
Something similar to 'Martin Pool <mbp@sourcefrog.net>'
120
:todo: Check it's reasonably well-formed.
122
:todo: Allow taking it from a dotfile to help people on windows
123
who can't easily set variables.
125
:todo: Cope without pwd module, which is only on unix.
127
e = os.environ.get('BZREMAIL') or os.environ.get('EMAIL')
135
w = pwd.getpwuid(uid)
136
realname, junk = w.pw_gecos.split(',', 1)
137
return '%s <%s@%s>' % (realname, w.pw_name, socket.getfqdn())
141
import getpass, socket
142
return '<%s@%s>' % (getpass.getuser(), socket.getfqdn())
146
"""Return just the email component of a username."""
147
e = os.environ.get('BZREMAIL') or os.environ.get('EMAIL')
150
m = re.search(r'[\w+.-]+@[\w+.-]+', e)
152
bailout('%r is not a reasonable email address' % e)
156
import getpass, socket
157
return '%s@%s' % (getpass.getuser(), socket.getfqdn())
491
162
def compare_files(a, b):
492
163
"""Returns true if equal in contents"""
503
def local_time_offset(t=None):
504
"""Return offset of local zone from GMT, either at present or at time t."""
505
# python2.3 localtime() can't take None
509
if time.localtime(t).tm_isdst and time.daylight:
512
return -time.timezone
515
def format_date(t, offset=0, timezone='original', date_fmt=None,
164
# TODO: don't read the whole thing in one go.
165
result = a.read() == b.read()
170
def format_date(t, inutc=False):
517
171
## TODO: Perhaps a global option to use either universal or local time?
518
172
## Or perhaps just let people set $TZ?
519
175
assert isinstance(t, float)
521
if timezone == 'utc':
522
178
tt = time.gmtime(t)
524
elif timezone == 'original':
527
tt = time.gmtime(t + offset)
528
elif timezone == 'local':
529
182
tt = time.localtime(t)
530
offset = local_time_offset(t)
532
raise BzrError("unsupported timezone format %r" % timezone,
533
['options are "utc", "original", "local"'])
535
date_fmt = "%a %Y-%m-%d %H:%M:%S"
537
offset_str = ' %+03d%02d' % (offset / 3600, (offset / 60) % 60)
540
return (time.strftime(date_fmt, tt) + offset_str)
184
zonename = time.tzname[1]
185
offset = - time.altzone
187
zonename = time.tzname[0]
188
offset = - time.timezone
190
return (time.strftime("%a %Y-%m-%d %H:%M:%S", tt)
191
+ ' ' + zonename + ' '
192
+ '%+03d%02d' % (offset / 3600, (offset / 60) % 60))
543
195
def compact_date(when):
603
227
>>> splitpath('a/../b')
604
228
Traceback (most recent call last):
606
BzrError: sorry, '..' not allowed in path
230
BzrError: ("sorry, '..' not allowed in path", [])
608
232
assert isinstance(p, types.StringTypes)
610
# split on either delimiter because people might use either on
612
ps = re.split(r'[\\/]', p)
233
ps = [f for f in p.split('/') if f != '.']
617
raise BzrError("sorry, %r not allowed in path" % f)
618
elif (f == '.') or (f == ''):
236
bailout("sorry, %r not allowed in path" % f)
625
240
assert isinstance(p, list)
627
if (f == '..') or (f == None) or (f == ''):
628
raise BzrError("sorry, %r not allowed in path" % f)
632
@deprecated_function(zero_nine)
242
if (f == '..') or (f is None) or (f == ''):
243
bailout("sorry, %r not allowed in path" % f)
633
247
def appendpath(p1, p2):
637
return pathjoin(p1, p2)
641
"""Split s into lines, but without removing the newline characters."""
642
lines = s.split('\n')
643
result = [line + '\n' for line in lines[:-1]]
645
result.append(lines[-1])
649
def hardlinks_good():
650
return sys.platform not in ('win32', 'cygwin', 'darwin')
653
def link_or_copy(src, dest):
654
"""Hardlink a file, or copy it if it can't be hardlinked."""
655
if not hardlinks_good():
660
except (OSError, IOError), e:
661
if e.errno != errno.EXDEV:
665
def delete_any(full_path):
666
"""Delete a file or directory."""
670
# We may be renaming a dangling inventory id
671
if e.errno not in (errno.EISDIR, errno.EACCES, errno.EPERM):
677
if hasattr(os, 'symlink'):
683
def contains_whitespace(s):
684
"""True if there are any whitespace characters in s."""
685
for ch in string.whitespace:
692
def contains_linebreaks(s):
693
"""True if there is any vertical whitespace in s."""
701
def relpath(base, path):
702
"""Return path relative to base, or raise exception.
704
The path may be either an absolute path or a path relative to the
705
current working directory.
707
os.path.commonprefix (python2.4) has a bad bug that it works just
708
on string prefixes, assuming that '/u' is a prefix of '/u2'. This
712
assert len(base) >= MIN_ABS_PATHLENGTH, ('Length of base must be equal or'
713
' exceed the platform minimum length (which is %d)' %
720
while len(head) >= len(base):
723
head, tail = os.path.split(head)
727
raise PathNotChild(rp, base)
735
def safe_unicode(unicode_or_utf8_string):
736
"""Coerce unicode_or_utf8_string into unicode.
738
If it is unicode, it is returned.
739
Otherwise it is decoded from utf-8. If a decoding error
740
occurs, it is wrapped as a If the decoding fails, the exception is wrapped
741
as a BzrBadParameter exception.
743
if isinstance(unicode_or_utf8_string, unicode):
744
return unicode_or_utf8_string
746
return unicode_or_utf8_string.decode('utf8')
747
except UnicodeDecodeError:
748
raise BzrBadParameterNotUnicode(unicode_or_utf8_string)
751
_platform_normalizes_filenames = False
752
if sys.platform == 'darwin':
753
_platform_normalizes_filenames = True
756
def normalizes_filenames():
757
"""Return True if this platform normalizes unicode filenames.
759
Mac OSX does, Windows/Linux do not.
761
return _platform_normalizes_filenames
764
if _platform_normalizes_filenames:
765
def unicode_filename(path):
766
"""Make sure 'path' is a properly normalized filename.
768
On platforms where the system normalizes filenames (Mac OSX),
769
you can access a file by any path which will normalize
771
Internally, bzr only supports NFC/NFKC normalization, since
772
that is the standard for XML documents.
773
So we return an normalized path, and indicate this has been
776
:return: (path, is_normalized) Return a path which can
777
access the file, and whether or not this path is
780
return unicodedata.normalize('NFKC', path), True
782
def unicode_filename(path):
783
"""Make sure 'path' is a properly normalized filename.
785
On platforms where the system does not normalize filenames
786
(Windows, Linux), you have to access a file by its exact path.
787
Internally, bzr only supports NFC/NFKC normalization, since
788
that is the standard for XML documents.
789
So we return the original path, and indicate if this is
792
:return: (path, is_normalized) Return a path which can
793
access the file, and whether or not this path is
796
return path, unicodedata.normalize('NFKC', path) == path
799
def terminal_width():
800
"""Return estimated terminal width."""
801
if sys.platform == 'win32':
802
import bzrlib.win32console
803
return bzrlib.win32console.get_console_size()[0]
806
import struct, fcntl, termios
807
s = struct.pack('HHHH', 0, 0, 0, 0)
808
x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
809
width = struct.unpack('HHHH', x)[1]
814
width = int(os.environ['COLUMNS'])
822
def supports_executable():
823
return sys.platform != "win32"
826
_validWin32PathRE = re.compile(r'^([A-Za-z]:[/\\])?[^:<>*"?\|]*$')
829
def check_legal_path(path):
830
"""Check whether the supplied path is legal.
831
This is only required on Windows, so we don't test on other platforms
834
if sys.platform != "win32":
836
if _validWin32PathRE.match(path) is None:
837
raise IllegalPath(path)
840
def walkdirs(top, prefix=""):
841
"""Yield data about all the directories in a tree.
843
This yields all the data about the contents of a directory at a time.
844
After each directory has been yielded, if the caller has mutated the list
845
to exclude some directories, they are then not descended into.
847
The data yielded is of the form:
848
[(relpath, basename, kind, lstat, path_from_top), ...]
850
:param prefix: Prefix the relpaths that are yielded with 'prefix'. This
851
allows one to walk a subtree but get paths that are relative to a tree
853
:return: an iterator over the dirs.
857
_directory = _directory_kind
859
pending = [(prefix, "", _directory, None, top)]
862
currentdir = pending.pop()
863
# 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
866
relroot = currentdir[0] + '/'
869
for name in sorted(_listdir(top)):
870
abspath = top + '/' + name
871
statvalue = lstat(abspath)
872
dirblock.append ((relroot + name, name, file_kind_from_stat_mode(statvalue.st_mode), statvalue, abspath))
874
# push the user specified dirs from dirblock
875
for dir in reversed(dirblock):
876
if dir[2] == _directory:
880
def path_prefix_key(path):
881
"""Generate a prefix-order path key for path.
883
This can be used to sort paths in the same way that walkdirs does.
885
return (dirname(path) , path)
888
def compare_paths_prefix_order(path_a, path_b):
889
"""Compare path_a and path_b to generate the same order walkdirs uses."""
890
key_a = path_prefix_key(path_a)
891
key_b = path_prefix_key(path_b)
892
return cmp(key_a, key_b)
254
def extern_command(cmd, ignore_errors = False):
255
mutter('external command: %s' % `cmd`)
257
if not ignore_errors:
258
bailout('command failed')
added
removed
bzrlib/osutils.py
