~bzr-pqm/bzr/bzr.dev

5594.1.1 by Vincent Ladeuil
Fix socketpair-based SSH transport leaking socket into other child processes
1
# Copyright (C) 2005-2011 Canonical Ltd
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
2
#
1 by mbp at sourcefrog
import from baz patch-364
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
7
#
1 by mbp at sourcefrog
import from baz patch-364
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
12
#
1 by mbp at sourcefrog
import from baz patch-364
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
4183.7.1 by Sabin Iacob
update FSF mailing address
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
1 by mbp at sourcefrog
import from baz patch-364
16
5225.1.1 by Andrew Bennetts
Don't lazy_import errno in osutils; it's a builtin module, and that lazy_import is implicated in a FreeBSD builder failure.
17
import errno
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
18
import os
19
import re
20
import stat
21
import sys
22
import time
5051.2.1 by Benjamin Peterson
move codecs import out of lazy section since it is used on module import
23
import codecs
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
24
25
from bzrlib.lazy_import import lazy_import
26
lazy_import(globals(), """
2215.6.1 by James Henstridge
Don't rely on time.timezone and time.altzone in local_time_offset(),
27
from datetime import datetime
5187.2.7 by Parth Malwankar
moved getuser_unicode to osutils.
28
import getpass
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
29
import ntpath
1711.4.5 by John Arbash Meinel
the _posix_* routines should use posixpath not os.path, so tests pass on win32
30
import posixpath
5273.1.11 by Vincent Ladeuil
Really fix and better explain why we need both the module and the symbol.
31
# We need to import both shutil and rmtree as we export the later on posix
32
# and need the former on windows
33
import shutil
5273.1.10 by Vincent Ladeuil
Fixed as per jam's review.
34
from shutil import rmtree
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
35
import socket
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
36
import subprocess
5273.1.11 by Vincent Ladeuil
Really fix and better explain why we need both the module and the symbol.
37
# We need to import both tempfile and mkdtemp as we export the later on posix
38
# and need the former on windows
39
import tempfile
40
from tempfile import mkdtemp
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
41
import unicodedata
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
42
43
from bzrlib import (
2249.5.16 by John Arbash Meinel
[merge] bzr.dev 2283
44
    cache_utf8,
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
45
    errors,
5011.3.16 by Andrew Bennetts
Merge lp:bzr.
46
    trace,
2245.4.6 by Alexander Belchenko
osutils.py: terminal_width() now use win32utils.get_console_size()
47
    win32utils,
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
48
    )
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
49
""")
1 by mbp at sourcefrog
import from baz patch-364
50
5011.3.3 by Martin
Reintroduce EINTR handling only for socket object functions and general cleanup
51
from bzrlib.symbol_versioning import (
52
    deprecated_function,
53
    deprecated_in,
54
    )
55
3734.2.4 by Vincent Ladeuil
Fix python2.6 deprecation warnings related to hashlib.
56
# sha and md5 modules are deprecated in python2.6 but hashlib is available as
57
# of 2.5
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
58
if sys.version_info < (2, 5):
3734.5.2 by Vincent Ladeuil
Martin's review feedback.
59
    import md5 as _mod_md5
60
    md5 = _mod_md5.new
61
    import sha as _mod_sha
62
    sha = _mod_sha.new
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
63
else:
64
    from hashlib import (
65
        md5,
66
        sha1 as sha,
67
        )
68
3504.4.1 by John Arbash Meinel
Write an alternative 'walkdirs' implementation that uses win32 apis.
69
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
70
import bzrlib
2309.4.3 by John Arbash Meinel
(broken) change safe_*_id to emit a warning.
71
from bzrlib import symbol_versioning
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
72
1 by mbp at sourcefrog
import from baz patch-364
73
4889.2.5 by John Arbash Meinel
Review feedback from Andrew.
74
# Cross platform wall-clock time functionality with decent resolution.
75
# On Linux ``time.clock`` returns only CPU time. On Windows, ``time.time()``
76
# only has a resolution of ~15ms. Note that ``time.clock()`` is not
77
# synchronized with ``time.time()``, this is only meant to be used to find
78
# delta times by subtracting from another call to this function.
4889.2.1 by John Arbash Meinel
Make -Dhpss log debug information for the server process.
79
timer_func = time.time
80
if sys.platform == 'win32':
81
    timer_func = time.clock
82
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
83
# On win32, O_BINARY is used to indicate the file should
84
# be opened in binary mode, rather than text mode.
85
# On other platforms, O_BINARY doesn't exist, because
86
# they always open in binary mode, so it is okay to
4634.140.12 by INADA Naoki
small clean up.
87
# OR with 0 on those platforms.
88
# O_NOINHERIT and O_TEXT exists only on win32 too.
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
89
O_BINARY = getattr(os, 'O_BINARY', 0)
4634.140.12 by INADA Naoki
small clean up.
90
O_TEXT = getattr(os, 'O_TEXT', 0)
4634.140.1 by INADA Naoki
Avoids child process inherits file handles on win32. by using os.fdopen and os.open with O_NOINHERIT instead of builtin open.
91
O_NOINHERIT = getattr(os, 'O_NOINHERIT', 0)
92
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
93
4355.2.2 by Alexander Belchenko
osutils.py: get_unicode_argv function (to obtain unicode command line arguments from sys.argv) moved to the beginning of module based on suggestions from review of John Meinel.
94
def get_unicode_argv():
95
    try:
96
        user_encoding = get_user_encoding()
5274.4.8 by Jason Spashett
Reverse merge -r 5280..5279
97
        return [a.decode(user_encoding) for a in sys.argv[1:]]
4355.2.2 by Alexander Belchenko
osutils.py: get_unicode_argv function (to obtain unicode command line arguments from sys.argv) moved to the beginning of module based on suggestions from review of John Meinel.
98
    except UnicodeDecodeError:
99
        raise errors.BzrError(("Parameter '%r' is unsupported by the current "
100
                                                            "encoding." % a))
101
102
1 by mbp at sourcefrog
import from baz patch-364
103
def make_readonly(filename):
104
    """Make a filename read-only."""
2949.6.1 by Alexander Belchenko
windows python has os.lstat
105
    mod = os.lstat(filename).st_mode
2568.1.1 by John Arbash Meinel
(Elliot Murphy) Use os.lstat rather than os.stat for osutils.make_readonly/make_writeable
106
    if not stat.S_ISLNK(mod):
107
        mod = mod & 0777555
108
        os.chmod(filename, mod)
1 by mbp at sourcefrog
import from baz patch-364
109
110
111
def make_writable(filename):
2949.6.1 by Alexander Belchenko
windows python has os.lstat
112
    mod = os.lstat(filename).st_mode
2568.1.1 by John Arbash Meinel
(Elliot Murphy) Use os.lstat rather than os.stat for osutils.make_readonly/make_writeable
113
    if not stat.S_ISLNK(mod):
114
        mod = mod | 0200
115
        os.chmod(filename, mod)
1 by mbp at sourcefrog
import from baz patch-364
116
117
2825.7.1 by Robert Collins
* Partial commits are now approximately 40% faster by walking over the
118
def minimum_path_selection(paths):
119
    """Return the smallset subset of paths which are outside paths.
120
2843.1.1 by Ian Clatworthy
Faster partial commits by walking less data (Robert Collins)
121
    :param paths: A container (and hence not None) of paths.
2825.7.1 by Robert Collins
* Partial commits are now approximately 40% faster by walking over the
122
    :return: A set of paths sufficient to include everything in paths via
4325.3.3 by Johan Walles
Add unit test and fix for minimum_path_selection() vs directory names with
123
        is_inside, drawn from the paths parameter.
2825.7.1 by Robert Collins
* Partial commits are now approximately 40% faster by walking over the
124
    """
4325.3.7 by Johan Walles
Style fixes for minimum_path_selection().
125
    if len(paths) < 2:
126
        return set(paths)
4325.3.3 by Johan Walles
Add unit test and fix for minimum_path_selection() vs directory names with
127
128
    def sort_key(path):
129
        return path.split('/')
130
    sorted_paths = sorted(list(paths), key=sort_key)
131
4325.3.7 by Johan Walles
Style fixes for minimum_path_selection().
132
    search_paths = [sorted_paths[0]]
133
    for path in sorted_paths[1:]:
4325.3.2 by Johan Walles
Use a linear algorithm for osutil.minimum_path_selection().
134
        if not is_inside(search_paths[-1], path):
135
            # This path is unique, add it
136
            search_paths.append(path)
4325.3.7 by Johan Walles
Style fixes for minimum_path_selection().
137
4325.3.2 by Johan Walles
Use a linear algorithm for osutil.minimum_path_selection().
138
    return set(search_paths)
2825.7.1 by Robert Collins
* Partial commits are now approximately 40% faster by walking over the
139
140
1077 by Martin Pool
- avoid compiling REs at module load time
141
_QUOTE_RE = None
969 by Martin Pool
- Add less-sucky is_within_any
142
143
1 by mbp at sourcefrog
import from baz patch-364
144
def quotefn(f):
779 by Martin Pool
- better quotefn for windows: use doublequotes for strings with
145
    """Return a quoted filename filename
146
147
    This previously used backslash quoting, but that works poorly on
148
    Windows."""
149
    # TODO: I'm not really sure this is the best format either.x
1077 by Martin Pool
- avoid compiling REs at module load time
150
    global _QUOTE_RE
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
151
    if _QUOTE_RE is None:
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
152
        _QUOTE_RE = re.compile(r'([^a-zA-Z0-9.,:/\\_~-])')
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
153
779 by Martin Pool
- better quotefn for windows: use doublequotes for strings with
154
    if _QUOTE_RE.search(f):
155
        return '"' + f + '"'
156
    else:
157
        return f
1 by mbp at sourcefrog
import from baz patch-364
158
159
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
160
_directory_kind = 'directory'
161
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
162
def get_umask():
163
    """Return the current umask"""
164
    # Assume that people aren't messing with the umask while running
165
    # XXX: This is not thread safe, but there is no way to get the
166
    #      umask without setting it
167
    umask = os.umask(0)
168
    os.umask(umask)
169
    return umask
170
171
2324.2.1 by Dmitry Vasiliev
kind_marker() optimization
172
_kind_marker_map = {
173
    "file": "",
174
    _directory_kind: "/",
175
    "symlink": "@",
1551.10.30 by Aaron Bentley
Merge from bzr.dev
176
    'tree-reference': '+',
2324.2.1 by Dmitry Vasiliev
kind_marker() optimization
177
}
1551.10.30 by Aaron Bentley
Merge from bzr.dev
178
179
488 by Martin Pool
- new helper function kind_marker()
180
def kind_marker(kind):
2324.2.1 by Dmitry Vasiliev
kind_marker() optimization
181
    try:
182
        return _kind_marker_map[kind]
183
    except KeyError:
5024.1.2 by John Arbash Meinel
Switch so that all unknown files get an empty marker, rather than failing.
184
        # Slightly faster than using .get(, '') when the common case is that
185
        # kind will be found
186
        return ''
1 by mbp at sourcefrog
import from baz patch-364
187
2324.2.1 by Dmitry Vasiliev
kind_marker() optimization
188
1732.1.2 by John Arbash Meinel
just use os.path.lexists if it exists
189
lexists = getattr(os.path, 'lexists', None)
190
if lexists is None:
191
    def lexists(f):
192
        try:
2324.2.2 by Dmitry Vasiliev
Fixed lexists() implementation
193
            stat = getattr(os, 'lstat', os.stat)
194
            stat(f)
1732.1.2 by John Arbash Meinel
just use os.path.lexists if it exists
195
            return True
2324.2.2 by Dmitry Vasiliev
Fixed lexists() implementation
196
        except OSError, e:
1732.1.2 by John Arbash Meinel
just use os.path.lexists if it exists
197
            if e.errno == errno.ENOENT:
198
                return False;
199
            else:
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
200
                raise errors.BzrError("lstat/stat of (%r): %r" % (f, e))
1732.1.2 by John Arbash Meinel
just use os.path.lexists if it exists
201
1 by mbp at sourcefrog
import from baz patch-364
202
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
203
def fancy_rename(old, new, rename_func, unlink_func):
204
    """A fancy rename, when you don't have atomic rename.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
205
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
206
    :param old: The old path, to rename from
207
    :param new: The new path, to rename to
208
    :param rename_func: The potentially non-atomic rename function
4935.1.1 by Vincent Ladeuil
Support Unicode paths for ftp transport (encoded as utf8).
209
    :param unlink_func: A way to delete the target file if the full rename
210
        succeeds
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
211
    """
212
    # sftp rename doesn't allow overwriting, so play tricks:
213
    base = os.path.basename(new)
214
    dirname = os.path.dirname(new)
4935.1.3 by Vincent Ladeuil
Better fix for fancy_rename respecting callers file encoding.
215
    # callers use different encodings for the paths so the following MUST
216
    # respect that. We rely on python upcasting to unicode if new is unicode
217
    # and keeping a str if not.
218
    tmp_name = 'tmp.%s.%.9f.%d.%s' % (base, time.time(),
219
                                      os.getpid(), rand_chars(10))
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
220
    tmp_name = pathjoin(dirname, tmp_name)
221
222
    # Rename the file out of the way, but keep track if it didn't exist
223
    # We don't want to grab just any exception
224
    # something like EACCES should prevent us from continuing
225
    # The downside is that the rename_func has to throw an exception
226
    # with an errno = ENOENT, or NoSuchFile
227
    file_existed = False
228
    try:
229
        rename_func(new, tmp_name)
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
230
    except (errors.NoSuchFile,), e:
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
231
        pass
1532 by Robert Collins
Merge in John Meinels integration branch.
232
    except IOError, e:
233
        # RBC 20060103 abstraction leakage: the paramiko SFTP clients rename
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
234
        # function raises an IOError with errno is None when a rename fails.
1532 by Robert Collins
Merge in John Meinels integration branch.
235
        # This then gets caught here.
1185.50.37 by John Arbash Meinel
Fixed exception handling for fancy_rename
236
        if e.errno not in (None, errno.ENOENT, errno.ENOTDIR):
1532 by Robert Collins
Merge in John Meinels integration branch.
237
            raise
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
238
    except Exception, e:
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
239
        if (getattr(e, 'errno', None) is None
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
240
            or e.errno not in (errno.ENOENT, errno.ENOTDIR)):
241
            raise
242
    else:
243
        file_existed = True
244
4789.17.1 by John Arbash Meinel
Change fancy_rename slightly.
245
    failure_exc = None
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
246
    success = False
247
    try:
2978.8.2 by Alexander Belchenko
teach fancy_rename to handle change case renames in possible case-insensitive filesystem
248
        try:
249
            # This may throw an exception, in which case success will
250
            # not be set.
251
            rename_func(old, new)
252
            success = True
253
        except (IOError, OSError), e:
2978.8.3 by Alexander Belchenko
Aaron's review
254
            # source and target may be aliases of each other (e.g. on a
255
            # case-insensitive filesystem), so we may have accidentally renamed
256
            # source by when we tried to rename target
4789.17.1 by John Arbash Meinel
Change fancy_rename slightly.
257
            failure_exc = sys.exc_info()
258
            if (file_existed and e.errno in (None, errno.ENOENT)
259
                and old.lower() == new.lower()):
260
                # source and target are the same file on a case-insensitive
261
                # filesystem, so we don't generate an exception
262
                failure_exc = None
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
263
    finally:
264
        if file_existed:
265
            # If the file used to exist, rename it back into place
266
            # otherwise just delete it from the tmp location
267
            if success:
1551.15.4 by Aaron Bentley
Revert now-unnecessary changes
268
                unlink_func(tmp_name)
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
269
            else:
1185.31.49 by John Arbash Meinel
Some corrections using the new osutils.rename. **ALL TESTS PASS**
270
                rename_func(tmp_name, new)
4789.17.2 by John Arbash Meinel
Also handle the case when source *and* target does not exist.
271
    if failure_exc is not None:
272
        raise failure_exc[0], failure_exc[1], failure_exc[2]
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
273
1685.1.9 by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url
274
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
275
# In Python 2.4.2 and older, os.path.abspath and os.path.realpath
276
# choke on a Unicode string containing a relative path if
277
# os.getcwd() returns a non-sys.getdefaultencoding()-encoded
278
# string.
2093.1.1 by John Arbash Meinel
(Bart Teeuwisse) if sys.getfilesystemencoding() is None, use 'utf-8'
279
_fs_enc = sys.getfilesystemencoding() or 'utf-8'
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
280
def _posix_abspath(path):
1711.4.5 by John Arbash Meinel
the _posix_* routines should use posixpath not os.path, so tests pass on win32
281
    # jam 20060426 rather than encoding to fsencoding
282
    # copy posixpath.abspath, but use os.getcwdu instead
283
    if not posixpath.isabs(path):
284
        path = posixpath.join(getcwd(), path)
285
    return posixpath.normpath(path)
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
286
287
288
def _posix_realpath(path):
1711.4.5 by John Arbash Meinel
the _posix_* routines should use posixpath not os.path, so tests pass on win32
289
    return posixpath.realpath(path.encode(_fs_enc)).decode(_fs_enc)
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
290
291
1711.5.2 by John Arbash Meinel
win32 likes to return lowercase drive letters sometimes, and uppercase at other times. normalize this
292
def _win32_fixdrive(path):
293
    """Force drive letters to be consistent.
294
295
    win32 is inconsistent whether it returns lower or upper case
296
    and even if it was consistent the user might type the other
297
    so we force it to uppercase
298
    running python.exe under cmd.exe return capital C:\\
299
    running win32 python inside a cygwin shell returns lowercase c:\\
300
    """
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
301
    drive, path = ntpath.splitdrive(path)
1711.5.2 by John Arbash Meinel
win32 likes to return lowercase drive letters sometimes, and uppercase at other times. normalize this
302
    return drive.upper() + path
303
304
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
305
def _win32_abspath(path):
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
306
    # Real ntpath.abspath doesn't have a problem with a unicode cwd
307
    return _win32_fixdrive(ntpath.abspath(unicode(path)).replace('\\', '/'))
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
308
309
2279.4.1 by Alexander Belchenko
Reimplementation of ntpath.abspath in Python for Windows98: unicode safe, UNC path safe
310
def _win98_abspath(path):
311
    """Return the absolute version of a path.
312
    Windows 98 safe implementation (python reimplementation
313
    of Win32 API function GetFullPathNameW)
314
    """
315
    # Corner cases:
316
    #   C:\path     => C:/path
317
    #   C:/path     => C:/path
318
    #   \\HOST\path => //HOST/path
319
    #   //HOST/path => //HOST/path
320
    #   path        => C:/cwd/path
321
    #   /path       => C:/path
322
    path = unicode(path)
323
    # check for absolute path
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
324
    drive = ntpath.splitdrive(path)[0]
2279.4.1 by Alexander Belchenko
Reimplementation of ntpath.abspath in Python for Windows98: unicode safe, UNC path safe
325
    if drive == '' and path[:2] not in('//','\\\\'):
326
        cwd = os.getcwdu()
327
        # we cannot simply os.path.join cwd and path
328
        # because os.path.join('C:','/path') produce '/path'
329
        # and this is incorrect
330
        if path[:1] in ('/','\\'):
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
331
            cwd = ntpath.splitdrive(cwd)[0]
2279.4.3 by Alexander Belchenko
win98_abspath: support for running in POSIX environment: cwd path has not drive letter
332
            path = path[1:]
2279.4.1 by Alexander Belchenko
Reimplementation of ntpath.abspath in Python for Windows98: unicode safe, UNC path safe
333
        path = cwd + '\\' + path
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
334
    return _win32_fixdrive(ntpath.normpath(path).replace('\\', '/'))
2279.4.1 by Alexander Belchenko
Reimplementation of ntpath.abspath in Python for Windows98: unicode safe, UNC path safe
335
336
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
337
def _win32_realpath(path):
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
338
    # Real ntpath.realpath doesn't have a problem with a unicode cwd
339
    return _win32_fixdrive(ntpath.realpath(unicode(path)).replace('\\', '/'))
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
340
341
342
def _win32_pathjoin(*args):
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
343
    return ntpath.join(*args).replace('\\', '/')
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
344
345
346
def _win32_normpath(path):
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
347
    return _win32_fixdrive(ntpath.normpath(unicode(path)).replace('\\', '/'))
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
348
349
350
def _win32_getcwd():
1711.5.2 by John Arbash Meinel
win32 likes to return lowercase drive letters sometimes, and uppercase at other times. normalize this
351
    return _win32_fixdrive(os.getcwdu().replace('\\', '/'))
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
352
353
354
def _win32_mkdtemp(*args, **kwargs):
5331.3.1 by Martin
Avoid infinite recursion with _win32_mkdtemp by using module namespaced name
355
    return _win32_fixdrive(tempfile.mkdtemp(*args, **kwargs).replace('\\', '/'))
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
356
357
358
def _win32_rename(old, new):
1711.7.6 by John Arbash Meinel
Change _win32_rename() so that it raises ENOENT *before* it tries any renaming.
359
    """We expect to be able to atomically replace 'new' with old.
360
1711.7.17 by John Arbash Meinel
Delay the extra syscall in _win32_rename until we get a failure.
361
    On win32, if new exists, it must be moved out of the way first,
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
362
    and then deleted.
1711.7.6 by John Arbash Meinel
Change _win32_rename() so that it raises ENOENT *before* it tries any renaming.
363
    """
1711.7.17 by John Arbash Meinel
Delay the extra syscall in _win32_rename until we get a failure.
364
    try:
5186.2.11 by Martin Pool
_win32_rename no longer relies on _wrapped_rename
365
        fancy_rename(old, new, rename_func=os.rename, unlink_func=os.unlink)
1711.7.17 by John Arbash Meinel
Delay the extra syscall in _win32_rename until we get a failure.
366
    except OSError, e:
1830.3.15 by John Arbash Meinel
On Mac we get EINVAL when renaming cwd
367
        if e.errno in (errno.EPERM, errno.EACCES, errno.EBUSY, errno.EINVAL):
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
368
            # If we try to rename a non-existant file onto cwd, we get
369
            # EPERM or EACCES instead of ENOENT, this will raise ENOENT
1830.3.15 by John Arbash Meinel
On Mac we get EINVAL when renaming cwd
370
            # if the old path doesn't exist, sometimes we get EACCES
371
            # On Linux, we seem to get EBUSY, on Mac we get EINVAL
1711.7.17 by John Arbash Meinel
Delay the extra syscall in _win32_rename until we get a failure.
372
            os.lstat(old)
373
        raise
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
374
375
1830.3.11 by John Arbash Meinel
Create a mac version of 'getcwd()' which normalizes the path.
376
def _mac_getcwd():
3201.1.1 by jameinel
Fix bug #185458, switch from NFKC to NFC and add tests for filenames that would be broken under NFKC
377
    return unicodedata.normalize('NFC', os.getcwdu())
1830.3.11 by John Arbash Meinel
Create a mac version of 'getcwd()' which normalizes the path.
378
379
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
380
# Default is to just use the python builtins, but these can be rebound on
381
# particular platforms.
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
382
abspath = _posix_abspath
383
realpath = _posix_realpath
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
384
pathjoin = os.path.join
385
normpath = os.path.normpath
5186.2.11 by Martin Pool
_win32_rename no longer relies on _wrapped_rename
386
getcwd = os.getcwdu
5186.2.3 by Martin Pool
Remove OS-level rename wrapper
387
rename = os.rename
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
388
dirname = os.path.dirname
389
basename = os.path.basename
2215.4.2 by Alexander Belchenko
split and splitext now the part of osutils
390
split = os.path.split
391
splitext = os.path.splitext
5273.1.10 by Vincent Ladeuil
Fixed as per jam's review.
392
# These were already lazily imported into local scope
393
# mkdtemp = tempfile.mkdtemp
394
# rmtree = shutil.rmtree
5609.29.5 by John Arbash Meinel
Fix bug #740932. Transform should update the sha cache.
395
lstat = os.lstat
396
fstat = os.fstat
397
398
def wrap_stat(st):
399
    return st
400
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
401
1551.2.53 by abentley
Strip trailing slashes in a platform-sensible way
402
MIN_ABS_PATHLENGTH = 1
403
1685.1.9 by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url
404
1185.31.47 by John Arbash Meinel
Added a fancy footwork rename to osutils, made SftpTransport use it.
405
if sys.platform == 'win32':
3224.5.35 by Andrew Bennetts
More improvements suggested by John's review.
406
    if win32utils.winver == 'Windows 98':
407
        abspath = _win98_abspath
408
    else:
409
        abspath = _win32_abspath
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
410
    realpath = _win32_realpath
411
    pathjoin = _win32_pathjoin
412
    normpath = _win32_normpath
413
    getcwd = _win32_getcwd
414
    mkdtemp = _win32_mkdtemp
415
    rename = _win32_rename
5609.29.5 by John Arbash Meinel
Fix bug #740932. Transform should update the sha cache.
416
    try:
417
        from bzrlib import _walkdirs_win32
418
    except ImportError:
419
        pass
420
    else:
421
        lstat = _walkdirs_win32.lstat
422
        fstat = _walkdirs_win32.fstat
423
        wrap_stat = _walkdirs_win32.wrap_stat
1685.1.20 by John Arbash Meinel
More changes to get 'bzr branch' and 'bzr pull' to work
424
1551.2.53 by abentley
Strip trailing slashes in a platform-sensible way
425
    MIN_ABS_PATHLENGTH = 3
1532 by Robert Collins
Merge in John Meinels integration branch.
426
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
427
    def _win32_delete_readonly(function, path, excinfo):
428
        """Error handler for shutil.rmtree function [for win32]
429
        Helps to remove files and dirs marked as read-only.
430
        """
2116.5.1 by Henri Wiechers
Fixes osutils.rmtree on Windows with Python 2.5
431
        exception = excinfo[1]
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
432
        if function in (os.remove, os.rmdir) \
2116.5.1 by Henri Wiechers
Fixes osutils.rmtree on Windows with Python 2.5
433
            and isinstance(exception, OSError) \
434
            and exception.errno == errno.EACCES:
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
435
            make_writable(path)
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
436
            function(path)
437
        else:
438
            raise
439
440
    def rmtree(path, ignore_errors=False, onerror=_win32_delete_readonly):
441
        """Replacer for shutil.rmtree: could remove readonly dirs/files"""
442
        return shutil.rmtree(path, ignore_errors, onerror)
4355.2.2 by Alexander Belchenko
osutils.py: get_unicode_argv function (to obtain unicode command line arguments from sys.argv) moved to the beginning of module based on suggestions from review of John Meinel.
443
444
    f = win32utils.get_unicode_argv     # special function or None
445
    if f is not None:
446
        get_unicode_argv = f
447
1830.3.11 by John Arbash Meinel
Create a mac version of 'getcwd()' which normalizes the path.
448
elif sys.platform == 'darwin':
449
    getcwd = _mac_getcwd
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
450
1685.1.31 by John Arbash Meinel
Adding tests for the rest of the _win32 functions.
451
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
452
def get_terminal_encoding(trace=False):
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
453
    """Find the best encoding for printing to the screen.
454
455
    This attempts to check both sys.stdout and sys.stdin to see
456
    what encoding they are in, and if that fails it falls back to
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
457
    osutils.get_user_encoding().
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
458
    The problem is that on Windows, locale.getpreferredencoding()
459
    is not the same encoding as that used by the console:
460
    http://mail.python.org/pipermail/python-list/2003-May/162357.html
461
462
    On my standard US Windows XP, the preferred encoding is
463
    cp1252, but the console is cp437
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
464
465
    :param trace: If True trace the selected encoding via mutter().
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
466
    """
5523.2.3 by Parth Malwankar
message is now shown to the user but is not too scary.
467
    from bzrlib.trace import mutter
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
468
    output_encoding = getattr(sys.stdout, 'encoding', None)
469
    if not output_encoding:
470
        input_encoding = getattr(sys.stdin, 'encoding', None)
471
        if not input_encoding:
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
472
            output_encoding = get_user_encoding()
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
473
            if trace:
474
                mutter('encoding stdout as osutils.get_user_encoding() %r',
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
475
                   output_encoding)
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
476
        else:
477
            output_encoding = input_encoding
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
478
            if trace:
479
                mutter('encoding stdout as sys.stdin encoding %r',
480
                    output_encoding)
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
481
    else:
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
482
        if trace:
483
            mutter('encoding stdout as sys.stdout encoding %r', output_encoding)
2127.4.1 by Alexander Belchenko
(jam, bialix) Workaround for cp0 console encoding on Windows
484
    if output_encoding == 'cp0':
485
        # invalid encoding (cp0 means 'no codepage' on Windows)
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
486
        output_encoding = get_user_encoding()
5320.2.4 by Robert Collins
``bzrlib.osutils.get_terminal_encoding`` will now only mutter its
487
        if trace:
488
            mutter('cp0 is invalid encoding.'
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
489
               ' encoding stdout as osutils.get_user_encoding() %r',
490
               output_encoding)
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
491
    # check encoding
492
    try:
493
        codecs.lookup(output_encoding)
494
    except LookupError:
495
        sys.stderr.write('bzr: warning:'
2192.1.9 by Alexander Belchenko
final fix suggested by John Meinel
496
                         ' unknown terminal encoding %s.\n'
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
497
                         '  Using encoding %s instead.\n'
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
498
                         % (output_encoding, get_user_encoding())
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
499
                        )
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
500
        output_encoding = get_user_encoding()
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
501
1711.4.10 by John Arbash Meinel
Pull out sys.stdout.encoding handling into a separate function so it can be tested, and used elsewhere.
502
    return output_encoding
503
504
1185.31.32 by John Arbash Meinel
Updated the bzr sourcecode to use bzrlib.osutils.pathjoin rather than os.path.join to enforce internal use of / instead of \
505
def normalizepath(f):
3287.18.2 by Matt McClure
Reverts to 3290.
506
    if getattr(os.path, 'realpath', None) is not None:
1185.31.32 by John Arbash Meinel
Updated the bzr sourcecode to use bzrlib.osutils.pathjoin rather than os.path.join to enforce internal use of / instead of \
507
        F = realpath
508
    else:
509
        F = abspath
510
    [p,e] = os.path.split(f)
511
    if e == "" or e == "." or e == "..":
512
        return F(f)
513
    else:
514
        return pathjoin(F(p), e)
515
1 by mbp at sourcefrog
import from baz patch-364
516
517
def isdir(f):
518
    """True if f is an accessible directory."""
519
    try:
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
520
        return stat.S_ISDIR(os.lstat(f)[stat.ST_MODE])
1 by mbp at sourcefrog
import from baz patch-364
521
    except OSError:
522
        return False
523
524
525
def isfile(f):
526
    """True if f is a regular file."""
527
    try:
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
528
        return stat.S_ISREG(os.lstat(f)[stat.ST_MODE])
1 by mbp at sourcefrog
import from baz patch-364
529
    except OSError:
530
        return False
531
1092.2.6 by Robert Collins
symlink support updated to work
532
def islink(f):
533
    """True if f is a symlink."""
534
    try:
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
535
        return stat.S_ISLNK(os.lstat(f)[stat.ST_MODE])
1092.2.6 by Robert Collins
symlink support updated to work
536
    except OSError:
537
        return False
1 by mbp at sourcefrog
import from baz patch-364
538
485 by Martin Pool
- move commit code into its own module
539
def is_inside(dir, fname):
540
    """True if fname is inside dir.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
541
1185.31.38 by John Arbash Meinel
Changing os.path.normpath to osutils.normpath
542
    The parameters should typically be passed to osutils.normpath first, so
969 by Martin Pool
- Add less-sucky is_within_any
543
    that . and .. and repeated slashes are eliminated, and the separators
544
    are canonical for the platform.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
545
546
    The empty string as a dir name is taken as top-of-tree and matches
974.1.26 by aaron.bentley at utoronto
merged mbp@sourcefrog.net-20050817233101-0939da1cf91f2472
547
    everything.
485 by Martin Pool
- move commit code into its own module
548
    """
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
549
    # XXX: Most callers of this can actually do something smarter by
969 by Martin Pool
- Add less-sucky is_within_any
550
    # looking at the inventory
972 by Martin Pool
- less dodgy is_inside function
551
    if dir == fname:
552
        return True
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
553
974.1.26 by aaron.bentley at utoronto
merged mbp@sourcefrog.net-20050817233101-0939da1cf91f2472
554
    if dir == '':
555
        return True
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
556
1185.31.34 by John Arbash Meinel
Removing instances of os.sep
557
    if dir[-1] != '/':
558
        dir += '/'
1185.1.41 by Robert Collins
massive patch from Alexander Belchenko - many PEP8 fixes, removes unused function uuid
559
972 by Martin Pool
- less dodgy is_inside function
560
    return fname.startswith(dir)
561
485 by Martin Pool
- move commit code into its own module
562
563
def is_inside_any(dir_list, fname):
564
    """True if fname is inside any of given dirs."""
565
    for dirname in dir_list:
566
        if is_inside(dirname, fname):
567
            return True
2324.2.3 by Dmitry Vasiliev
Fixed is_inside_* methods implementation
568
    return False
485 by Martin Pool
- move commit code into its own module
569
570
1740.3.4 by Jelmer Vernooij
Move inventory to commit builder.
571
def is_inside_or_parent_of_any(dir_list, fname):
572
    """True if fname is a child or a parent of any of the given files."""
573
    for dirname in dir_list:
574
        if is_inside(dirname, fname) or is_inside(fname, dirname):
575
            return True
2324.2.3 by Dmitry Vasiliev
Fixed is_inside_* methods implementation
576
    return False
1740.3.4 by Jelmer Vernooij
Move inventory to commit builder.
577
578
3956.2.1 by John Arbash Meinel
Add report_activity to osutils.pumpfile
579
def pumpfile(from_file, to_file, read_length=-1, buff_size=32768,
580
             report_activity=None, direction='read'):
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
581
    """Copy contents of one file to another.
3408.6.1 by Eric Holmberg
Fix for Bug #215426 in which bzr can cause a MemoryError in socket.recv while
582
583
    The read_length can either be -1 to read to end-of-file (EOF) or
584
    it can specify the maximum number of bytes to read.
585
586
    The buff_size represents the maximum size for each read operation
587
    performed on from_file.
588
3956.2.1 by John Arbash Meinel
Add report_activity to osutils.pumpfile
589
    :param report_activity: Call this as bytes are read, see
590
        Transport._report_activity
591
    :param direction: Will be passed to report_activity
592
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
593
    :return: The number of bytes copied.
594
    """
595
    length = 0
3408.6.1 by Eric Holmberg
Fix for Bug #215426 in which bzr can cause a MemoryError in socket.recv while
596
    if read_length >= 0:
597
        # read specified number of bytes
598
599
        while read_length > 0:
600
            num_bytes_to_read = min(read_length, buff_size)
601
602
            block = from_file.read(num_bytes_to_read)
603
            if not block:
604
                # EOF reached
605
                break
3956.2.1 by John Arbash Meinel
Add report_activity to osutils.pumpfile
606
            if report_activity is not None:
607
                report_activity(len(block), direction)
3408.6.1 by Eric Holmberg
Fix for Bug #215426 in which bzr can cause a MemoryError in socket.recv while
608
            to_file.write(block)
609
610
            actual_bytes_read = len(block)
611
            read_length -= actual_bytes_read
612
            length += actual_bytes_read
613
    else:
614
        # read to EOF
615
        while True:
616
            block = from_file.read(buff_size)
617
            if not block:
618
                # EOF reached
619
                break
3956.2.1 by John Arbash Meinel
Add report_activity to osutils.pumpfile
620
            if report_activity is not None:
621
                report_activity(len(block), direction)
3408.6.1 by Eric Holmberg
Fix for Bug #215426 in which bzr can cause a MemoryError in socket.recv while
622
            to_file.write(block)
623
            length += len(block)
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
624
    return length
1 by mbp at sourcefrog
import from baz patch-364
625
626
3635.1.2 by Robert Collins
Add osutils.pump_string_file helper function.
627
def pump_string_file(bytes, file_handle, segment_size=None):
628
    """Write bytes to file_handle in many smaller writes.
629
630
    :param bytes: The string to write.
631
    :param file_handle: The file to write to.
632
    """
633
    # Write data in chunks rather than all at once, because very large
634
    # writes fail on some platforms (e.g. Windows with SMB  mounted
635
    # drives).
636
    if not segment_size:
637
        segment_size = 5242880 # 5MB
638
    segments = range(len(bytes) / segment_size + 1)
639
    write = file_handle.write
640
    for segment_index in segments:
641
        segment = buffer(bytes, segment_index * segment_size, segment_size)
642
        write(segment)
643
644
1185.67.7 by Aaron Bentley
Refactored a bit
645
def file_iterator(input_file, readsize=32768):
646
    while True:
647
        b = input_file.read(readsize)
648
        if len(b) == 0:
649
            break
650
        yield b
651
652
1 by mbp at sourcefrog
import from baz patch-364
653
def sha_file(f):
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
654
    """Calculate the hexdigest of an open file.
655
656
    The file cursor should be already at the start.
657
    """
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
658
    s = sha()
320 by Martin Pool
- Compute SHA-1 of files in chunks
659
    BUFSIZE = 128<<10
660
    while True:
661
        b = f.read(BUFSIZE)
662
        if not b:
663
            break
664
        s.update(b)
1 by mbp at sourcefrog
import from baz patch-364
665
    return s.hexdigest()
666
667
3368.2.49 by Ian Clatworthy
added osutils.size_sha_file() with tests
668
def size_sha_file(f):
669
    """Calculate the size and hexdigest of an open file.
670
671
    The file cursor should be already at the start and
672
    the caller is responsible for closing the file afterwards.
673
    """
674
    size = 0
675
    s = sha()
676
    BUFSIZE = 128<<10
677
    while True:
678
        b = f.read(BUFSIZE)
679
        if not b:
680
            break
681
        size += len(b)
682
        s.update(b)
683
    return size, s.hexdigest()
684
685
2872.3.1 by Martin Pool
Add -Dhashcache option; clean up dirstate sha1 code
686
def sha_file_by_name(fname):
687
    """Calculate the SHA1 of a file by reading the full text"""
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
688
    s = sha()
4634.140.1 by INADA Naoki
Avoids child process inherits file handles on win32. by using os.fdopen and os.open with O_NOINHERIT instead of builtin open.
689
    f = os.open(fname, os.O_RDONLY | O_BINARY | O_NOINHERIT)
2872.3.1 by Martin Pool
Add -Dhashcache option; clean up dirstate sha1 code
690
    try:
2872.3.2 by Martin Pool
Do sha_file_by_name using raw os files rather than file objects; makes this routine about 12osutils.py faster
691
        while True:
692
            b = os.read(f, 1<<16)
693
            if not b:
694
                return s.hexdigest()
695
            s.update(b)
2872.3.1 by Martin Pool
Add -Dhashcache option; clean up dirstate sha1 code
696
    finally:
2872.3.2 by Martin Pool
Do sha_file_by_name using raw os files rather than file objects; makes this routine about 12osutils.py faster
697
        os.close(f)
2872.3.1 by Martin Pool
Add -Dhashcache option; clean up dirstate sha1 code
698
699
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
700
def sha_strings(strings, _factory=sha):
1235 by Martin Pool
- split sha_strings into osutils
701
    """Return the sha-1 of concatenation of strings"""
2825.2.1 by Robert Collins
Micro-tweaks to sha routines.
702
    s = _factory()
1235 by Martin Pool
- split sha_strings into osutils
703
    map(s.update, strings)
704
    return s.hexdigest()
705
706
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
707
def sha_string(f, _factory=sha):
2825.2.1 by Robert Collins
Micro-tweaks to sha routines.
708
    return _factory(f).hexdigest()
1 by mbp at sourcefrog
import from baz patch-364
709
710
124 by mbp at sourcefrog
- check file text for past revisions is correct
711
def fingerprint_file(f):
126 by mbp at sourcefrog
Use just one big read to fingerprint files
712
    b = f.read()
2825.2.1 by Robert Collins
Micro-tweaks to sha routines.
713
    return {'size': len(b),
2929.3.1 by Vincent Ladeuil
Fix python2.6 deprecation warnings (still 4 failures 5 errors in test suite).
714
            'sha1': sha(b).hexdigest()}
124 by mbp at sourcefrog
- check file text for past revisions is correct
715
716
1 by mbp at sourcefrog
import from baz patch-364
717
def compare_files(a, b):
718
    """Returns true if equal in contents"""
74 by mbp at sourcefrog
compare_files: read in one page at a time rather than
719
    BUFSIZE = 4096
720
    while True:
721
        ai = a.read(BUFSIZE)
722
        bi = b.read(BUFSIZE)
723
        if ai != bi:
724
            return False
725
        if ai == '':
726
            return True
1 by mbp at sourcefrog
import from baz patch-364
727
728
49 by mbp at sourcefrog
fix local-time-offset calculation
729
def local_time_offset(t=None):
730
    """Return offset of local zone from GMT, either at present or at time t."""
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
731
    if t is None:
73 by mbp at sourcefrog
fix time.localtime call for python 2.3
732
        t = time.time()
2215.6.1 by James Henstridge
Don't rely on time.timezone and time.altzone in local_time_offset(),
733
    offset = datetime.fromtimestamp(t) - datetime.utcfromtimestamp(t)
734
    return offset.days * 86400 + offset.seconds
8 by mbp at sourcefrog
store committer's timezone in revision and show
735
3512.3.1 by Martin von Gagern
Hand-selected minimalistic set of changes from my setlocale branch.
736
weekdays = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
4379.4.1 by Ian Clatworthy
make log --long faster
737
_default_format_by_weekday_num = [wd + " %Y-%m-%d %H:%M:%S" for wd in weekdays]
738
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
739
2425.6.2 by Martin Pool
Make timestamps use existing format_date; document that function more
740
def format_date(t, offset=0, timezone='original', date_fmt=None,
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
741
                show_offset=True):
2425.6.2 by Martin Pool
Make timestamps use existing format_date; document that function more
742
    """Return a formatted date string.
743
744
    :param t: Seconds since the epoch.
745
    :param offset: Timezone offset in seconds east of utc.
746
    :param timezone: How to display the time: 'utc', 'original' for the
747
         timezone specified by offset, or 'local' for the process's current
748
         timezone.
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
749
    :param date_fmt: strftime format.
750
    :param show_offset: Whether to append the timezone.
751
    """
752
    (date_fmt, tt, offset_str) = \
753
               _format_date(t, offset, timezone, date_fmt, show_offset)
754
    date_fmt = date_fmt.replace('%a', weekdays[tt[6]])
755
    date_str = time.strftime(date_fmt, tt)
756
    return date_str + offset_str
757
4379.4.1 by Ian Clatworthy
make log --long faster
758
759
# Cache of formatted offset strings
760
_offset_cache = {}
761
762
4379.4.2 by Ian Clatworthy
add NEWS item and tests for new date formatting API
763
def format_date_with_offset_in_original_timezone(t, offset=0,
4379.4.1 by Ian Clatworthy
make log --long faster
764
    _cache=_offset_cache):
765
    """Return a formatted date string in the original timezone.
766
767
    This routine may be faster then format_date.
768
769
    :param t: Seconds since the epoch.
770
    :param offset: Timezone offset in seconds east of utc.
771
    """
772
    if offset is None:
773
        offset = 0
774
    tt = time.gmtime(t + offset)
775
    date_fmt = _default_format_by_weekday_num[tt[6]]
776
    date_str = time.strftime(date_fmt, tt)
777
    offset_str = _cache.get(offset, None)
778
    if offset_str is None:
779
        offset_str = ' %+03d%02d' % (offset / 3600, (offset / 60) % 60)
780
        _cache[offset] = offset_str
781
    return date_str + offset_str
782
783
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
784
def format_local_date(t, offset=0, timezone='original', date_fmt=None,
785
                      show_offset=True):
786
    """Return an unicode date string formatted according to the current locale.
787
788
    :param t: Seconds since the epoch.
789
    :param offset: Timezone offset in seconds east of utc.
790
    :param timezone: How to display the time: 'utc', 'original' for the
791
         timezone specified by offset, or 'local' for the process's current
792
         timezone.
793
    :param date_fmt: strftime format.
794
    :param show_offset: Whether to append the timezone.
795
    """
796
    (date_fmt, tt, offset_str) = \
797
               _format_date(t, offset, timezone, date_fmt, show_offset)
798
    date_str = time.strftime(date_fmt, tt)
799
    if not isinstance(date_str, unicode):
4385.4.1 by Alexander Belchenko
removed all references to bzrlib.user_encoding
800
        date_str = date_str.decode(get_user_encoding(), 'replace')
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
801
    return date_str + offset_str
802
4379.4.1 by Ian Clatworthy
make log --long faster
803
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
804
def _format_date(t, offset, timezone, date_fmt, show_offset):
8 by mbp at sourcefrog
store committer's timezone in revision and show
805
    if timezone == 'utc':
1 by mbp at sourcefrog
import from baz patch-364
806
        tt = time.gmtime(t)
807
        offset = 0
8 by mbp at sourcefrog
store committer's timezone in revision and show
808
    elif timezone == 'original':
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
809
        if offset is None:
23 by mbp at sourcefrog
format_date: handle revisions with no timezone offset
810
            offset = 0
16 by mbp at sourcefrog
fix inverted calculation for original timezone -> utc
811
        tt = time.gmtime(t + offset)
12 by mbp at sourcefrog
new --timezone option for bzr log
812
    elif timezone == 'local':
1 by mbp at sourcefrog
import from baz patch-364
813
        tt = time.localtime(t)
49 by mbp at sourcefrog
fix local-time-offset calculation
814
        offset = local_time_offset(t)
12 by mbp at sourcefrog
new --timezone option for bzr log
815
    else:
3144.1.1 by Lukáš Lalinský
Fixed error reporting of unsupported timezone format.
816
        raise errors.UnsupportedTimezoneFormat(timezone)
1185.12.24 by Aaron Bentley
Made format_date more flexible
817
    if date_fmt is None:
818
        date_fmt = "%a %Y-%m-%d %H:%M:%S"
819
    if show_offset:
820
        offset_str = ' %+03d%02d' % (offset / 3600, (offset / 60) % 60)
821
    else:
822
        offset_str = ''
3526.5.4 by Martin von Gagern
Use separate function format_local_date for local weekday formats in unicode.
823
    return (date_fmt, tt, offset_str)
1 by mbp at sourcefrog
import from baz patch-364
824
825
826
def compact_date(when):
827
    return time.strftime('%Y%m%d%H%M%S', time.gmtime(when))
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
828
1 by mbp at sourcefrog
import from baz patch-364
829
1957.1.4 by John Arbash Meinel
create a helper for formatting a time delta
830
def format_delta(delta):
831
    """Get a nice looking string for a time delta.
832
833
    :param delta: The time difference in seconds, can be positive or negative.
834
        positive indicates time in the past, negative indicates time in the
835
        future. (usually time.time() - stored_time)
836
    :return: String formatted to show approximate resolution
837
    """
838
    delta = int(delta)
839
    if delta >= 0:
840
        direction = 'ago'
841
    else:
842
        direction = 'in the future'
843
        delta = -delta
844
845
    seconds = delta
846
    if seconds < 90: # print seconds up to 90 seconds
847
        if seconds == 1:
848
            return '%d second %s' % (seconds, direction,)
849
        else:
850
            return '%d seconds %s' % (seconds, direction)
851
852
    minutes = int(seconds / 60)
853
    seconds -= 60 * minutes
854
    if seconds == 1:
855
        plural_seconds = ''
856
    else:
857
        plural_seconds = 's'
858
    if minutes < 90: # print minutes, seconds up to 90 minutes
859
        if minutes == 1:
860
            return '%d minute, %d second%s %s' % (
861
                    minutes, seconds, plural_seconds, direction)
862
        else:
863
            return '%d minutes, %d second%s %s' % (
864
                    minutes, seconds, plural_seconds, direction)
865
866
    hours = int(minutes / 60)
867
    minutes -= 60 * hours
868
    if minutes == 1:
869
        plural_minutes = ''
870
    else:
871
        plural_minutes = 's'
872
873
    if hours == 1:
874
        return '%d hour, %d minute%s %s' % (hours, minutes,
875
                                            plural_minutes, direction)
876
    return '%d hours, %d minute%s %s' % (hours, minutes,
877
                                         plural_minutes, direction)
1 by mbp at sourcefrog
import from baz patch-364
878
879
def filesize(f):
880
    """Return size of given open file."""
5273.1.2 by Vincent Ladeuil
Cleanup imports in osutils.py.
881
    return os.fstat(f.fileno())[stat.ST_SIZE]
1 by mbp at sourcefrog
import from baz patch-364
882
1553.5.5 by Martin Pool
New utility routine rand_chars
883
1185.1.7 by Robert Collins
Nathaniel McCallums patch for urandom friendliness on aix.
884
# Define rand_bytes based on platform.
885
try:
886
    # Python 2.4 and later have os.urandom,
887
    # but it doesn't work on some arches
888
    os.urandom(1)
1 by mbp at sourcefrog
import from baz patch-364
889
    rand_bytes = os.urandom
1185.1.7 by Robert Collins
Nathaniel McCallums patch for urandom friendliness on aix.
890
except (NotImplementedError, AttributeError):
891
    # If python doesn't have os.urandom, or it doesn't work,
892
    # then try to first pull random data from /dev/urandom
2067.1.1 by John Arbash Meinel
Catch an exception while opening /dev/urandom rather than using os.path.exists()
893
    try:
1185.1.7 by Robert Collins
Nathaniel McCallums patch for urandom friendliness on aix.
894
        rand_bytes = file('/dev/urandom', 'rb').read
895
    # Otherwise, use this hack as a last resort
2067.1.1 by John Arbash Meinel
Catch an exception while opening /dev/urandom rather than using os.path.exists()
896
    except (IOError, OSError):
1185.1.7 by Robert Collins
Nathaniel McCallums patch for urandom friendliness on aix.
897
        # not well seeded, but better than nothing
898
        def rand_bytes(n):
899
            import random
900
            s = ''
901
            while n:
902
                s += chr(random.randint(0, 255))
903
                n -= 1
904
            return s
1 by mbp at sourcefrog
import from baz patch-364
905
1553.5.5 by Martin Pool
New utility routine rand_chars
906
907
ALNUM = '0123456789abcdefghijklmnopqrstuvwxyz'
908
def rand_chars(num):
909
    """Return a random string of num alphanumeric characters
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
910
911
    The result only contains lowercase chars because it may be used on
1553.5.5 by Martin Pool
New utility routine rand_chars
912
    case-insensitive filesystems.
913
    """
914
    s = ''
915
    for raw_byte in rand_bytes(num):
916
        s += ALNUM[ord(raw_byte) % 36]
917
    return s
918
919
1 by mbp at sourcefrog
import from baz patch-364
920
## TODO: We could later have path objects that remember their list
1759.2.2 by Jelmer Vernooij
Revert some of my spelling fixes and fix some typos after review by Aaron.
921
## decomposition (might be too tricksy though.)
1 by mbp at sourcefrog
import from baz patch-364
922
923
def splitpath(p):
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
924
    """Turn string into list of parts."""
271 by Martin Pool
- Windows path fixes
925
    # split on either delimiter because people might use either on
926
    # Windows
927
    ps = re.split(r'[\\/]', p)
928
929
    rps = []
1 by mbp at sourcefrog
import from baz patch-364
930
    for f in ps:
931
        if f == '..':
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
932
            raise errors.BzrError("sorry, %r not allowed in path" % f)
271 by Martin Pool
- Windows path fixes
933
        elif (f == '.') or (f == ''):
934
            pass
935
        else:
936
            rps.append(f)
937
    return rps
1 by mbp at sourcefrog
import from baz patch-364
938
3890.2.4 by John Arbash Meinel
Add a new function that can convert 'chunks' format to a 'lines' format.
939
1 by mbp at sourcefrog
import from baz patch-364
940
def joinpath(p):
941
    for f in p:
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
942
        if (f == '..') or (f is None) or (f == ''):
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
943
            raise errors.BzrError("sorry, %r not allowed in path" % f)
1185.31.32 by John Arbash Meinel
Updated the bzr sourcecode to use bzrlib.osutils.pathjoin rather than os.path.join to enforce internal use of / instead of \
944
    return pathjoin(*p)
1 by mbp at sourcefrog
import from baz patch-364
945
946
4370.1.1 by Ian Clatworthy
add osutils.parent_directories() API
947
def parent_directories(filename):
4371.1.1 by Ian Clatworthy
(igc) added osutils.parent_directories() (Ian Clatworthy)
948
    """Return the list of parent directories, deepest first.
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
949
4371.1.1 by Ian Clatworthy
(igc) added osutils.parent_directories() (Ian Clatworthy)
950
    For example, parent_directories("a/b/c") -> ["a/b", "a"].
951
    """
4370.1.1 by Ian Clatworthy
add osutils.parent_directories() API
952
    parents = []
953
    parts = splitpath(dirname(filename))
954
    while parts:
955
        parents.append(joinpath(parts))
956
        parts.pop()
957
    return parents
958
959
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
960
_extension_load_failures = []
961
962
963
def failed_to_load_extension(exception):
4574.3.1 by Martin Pool
Give a warning when failing to load _chunks_to_lines_pyx
964
    """Handle failing to load a binary extension.
965
966
    This should be called from the ImportError block guarding the attempt to
967
    import the native extension.  If this function returns, the pure-Python
968
    implementation should be loaded instead::
969
970
    >>> try:
971
    >>>     import bzrlib._fictional_extension_pyx
972
    >>> except ImportError, e:
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
973
    >>>     bzrlib.osutils.failed_to_load_extension(e)
4574.3.1 by Martin Pool
Give a warning when failing to load _chunks_to_lines_pyx
974
    >>>     import bzrlib._fictional_extension_py
975
    """
976
    # NB: This docstring is just an example, not a doctest, because doctest
977
    # currently can't cope with the use of lazy imports in this namespace --
978
    # mbp 20090729
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
979
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
980
    # This currently doesn't report the failure at the time it occurs, because
981
    # they tend to happen very early in startup when we can't check config
982
    # files etc, and also we want to report all failures but not spam the user
983
    # with 10 warnings.
984
    exception_str = str(exception)
985
    if exception_str not in _extension_load_failures:
5523.2.3 by Parth Malwankar
message is now shown to the user but is not too scary.
986
        trace.mutter("failed to load compiled extension: %s" % exception_str)
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
987
        _extension_load_failures.append(exception_str)
988
989
990
def report_extension_load_failures():
991
    if not _extension_load_failures:
992
        return
993
    from bzrlib.config import GlobalConfig
994
    if GlobalConfig().get_user_option_as_bool('ignore_missing_extensions'):
995
        return
996
    # the warnings framework should by default show this only once
4695.4.1 by Martin Pool
Give a shorter/cleaner message for missing extensions
997
    from bzrlib.trace import warning
998
    warning(
999
        "bzr: warning: some compiled extensions could not be loaded; "
1000
        "see <https://answers.launchpad.net/bzr/+faq/703>")
1001
    # we no longer show the specific missing extensions here, because it makes
1002
    # the message too long and scary - see
1003
    # https://bugs.launchpad.net/bzr/+bug/430529
4574.3.1 by Martin Pool
Give a warning when failing to load _chunks_to_lines_pyx
1004
1005
3890.2.7 by John Arbash Meinel
A Pyrex extension is about 5x faster than the fastest python code I could write.
1006
try:
1007
    from bzrlib._chunks_to_lines_pyx import chunks_to_lines
4574.3.1 by Martin Pool
Give a warning when failing to load _chunks_to_lines_pyx
1008
except ImportError, e:
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
1009
    failed_to_load_extension(e)
3890.2.8 by John Arbash Meinel
Move everything into properly parameterized tests.
1010
    from bzrlib._chunks_to_lines_py import chunks_to_lines
3890.2.7 by John Arbash Meinel
A Pyrex extension is about 5x faster than the fastest python code I could write.
1011
1012
1231 by Martin Pool
- more progress on fetch on top of weaves
1013
def split_lines(s):
1014
    """Split s into lines, but without removing the newline characters."""
3890.2.18 by John Arbash Meinel
Implement osutils.split_lines() in terms of chunks_to_lines if possible.
1015
    # Trivially convert a fulltext into a 'chunked' representation, and let
1016
    # chunks_to_lines do the heavy lifting.
1017
    if isinstance(s, str):
1018
        # chunks_to_lines only supports 8-bit strings
1019
        return chunks_to_lines([s])
1020
    else:
1021
        return _split_lines(s)
1022
1023
1024
def _split_lines(s):
1025
    """Split s into lines, but without removing the newline characters.
1026
1027
    This supports Unicode or plain string objects.
1028
    """
1666.1.6 by Robert Collins
Make knit the default format.
1029
    lines = s.split('\n')
1030
    result = [line + '\n' for line in lines[:-1]]
1031
    if lines[-1]:
1032
        result.append(lines[-1])
1033
    return result
1391 by Robert Collins
merge from integration
1034
1035
1185.10.4 by Aaron Bentley
Disabled hardlinks on cygwin, mac OS
1036
def hardlinks_good():
1185.10.5 by Aaron Bentley
Fixed hardlinks_good test
1037
    return sys.platform not in ('win32', 'cygwin', 'darwin')
1185.10.4 by Aaron Bentley
Disabled hardlinks on cygwin, mac OS
1038
1185.1.46 by Robert Collins
Aarons branch --basis patch
1039
1185.10.3 by Aaron Bentley
Made copy_multi_immutable create hardlinks opportunistically
1040
def link_or_copy(src, dest):
1041
    """Hardlink a file, or copy it if it can't be hardlinked."""
1185.10.4 by Aaron Bentley
Disabled hardlinks on cygwin, mac OS
1042
    if not hardlinks_good():
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
1043
        shutil.copyfile(src, dest)
1185.10.3 by Aaron Bentley
Made copy_multi_immutable create hardlinks opportunistically
1044
        return
1045
    try:
1046
        os.link(src, dest)
1047
    except (OSError, IOError), e:
1048
        if e.errno != errno.EXDEV:
1049
            raise
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
1050
        shutil.copyfile(src, dest)
1399.1.4 by Robert Collins
move diff and symlink conditionals into inventory.py from diff.py
1051
2831.5.2 by Vincent Ladeuil
Review feedback.
1052
1053
def delete_any(path):
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
1054
    """Delete a file, symlink or directory.
1055
4490.1.1 by Martin Pool
merge fix for forcing readonly deletion, and tweak
1056
    Will delete even if readonly.
1057
    """
4440.1.2 by Craig Hewetson
Fixes made after first code review.
1058
    try:
4490.1.1 by Martin Pool
merge fix for forcing readonly deletion, and tweak
1059
       _delete_file_or_dir(path)
4440.1.2 by Craig Hewetson
Fixes made after first code review.
1060
    except (OSError, IOError), e:
1061
        if e.errno in (errno.EPERM, errno.EACCES):
4490.1.1 by Martin Pool
merge fix for forcing readonly deletion, and tweak
1062
            # make writable and try again
1063
            try:
4440.1.2 by Craig Hewetson
Fixes made after first code review.
1064
                make_writable(path)
4490.1.1 by Martin Pool
merge fix for forcing readonly deletion, and tweak
1065
            except (OSError, IOError):
4440.1.2 by Craig Hewetson
Fixes made after first code review.
1066
                pass
4490.1.1 by Martin Pool
merge fix for forcing readonly deletion, and tweak
1067
            _delete_file_or_dir(path)
1068
        else:
1069
            raise
1070
1071
1072
def _delete_file_or_dir(path):
1073
    # Look Before You Leap (LBYL) is appropriate here instead of Easier to Ask for
1074
    # Forgiveness than Permission (EAFP) because:
1075
    # - root can damage a solaris file system by using unlink,
1076
    # - unlink raises different exceptions on different OSes (linux: EISDIR, win32:
1077
    #   EACCES, OSX: EPERM) when invoked on a directory.
2831.5.2 by Vincent Ladeuil
Review feedback.
1078
    if isdir(path): # Takes care of symlinks
1079
        os.rmdir(path)
1080
    else:
1081
        os.unlink(path)
1558.12.9 by Aaron Bentley
Handle resolving conflicts with directories properly
1082
1399.1.4 by Robert Collins
move diff and symlink conditionals into inventory.py from diff.py
1083
1084
def has_symlinks():
1963.2.6 by Robey Pointer
pychecker is on crack; go back to using 'is None'.
1085
    if getattr(os, 'symlink', None) is not None:
1399.1.4 by Robert Collins
move diff and symlink conditionals into inventory.py from diff.py
1086
        return True
1087
    else:
1088
        return False
2831.5.2 by Vincent Ladeuil
Review feedback.
1089
1185.16.38 by Martin Pool
- move contains_whitespace and contains_linebreaks to osutils
1090
3136.1.1 by Aaron Bentley
Add support for hardlinks to TreeTransform
1091
def has_hardlinks():
1092
    if getattr(os, 'link', None) is not None:
1093
        return True
1094
    else:
1095
        return False
1096
1097
3287.18.14 by Matt McClure
Extracted a host_os_dereferences_symlinks method.
1098
def host_os_dereferences_symlinks():
1099
    return (has_symlinks()
3287.18.19 by Matt McClure
Changed tested sys.platform value from 'windows' (mistaken) to 'win32'
1100
            and sys.platform not in ('cygwin', 'win32'))
3287.18.14 by Matt McClure
Extracted a host_os_dereferences_symlinks method.
1101
1102
4241.14.14 by Vincent Ladeuil
Test and implements osutils.readlink().
1103
def readlink(abspath):
1104
    """Return a string representing the path to which the symbolic link points.
1105
1106
    :param abspath: The link absolute unicode path.
1107
1108
    This his guaranteed to return the symbolic link in unicode in all python
1109
    versions.
1110
    """
1111
    link = abspath.encode(_fs_enc)
1112
    target = os.readlink(link)
1113
    target = target.decode(_fs_enc)
1114
    return target
1115
1116
1185.16.38 by Martin Pool
- move contains_whitespace and contains_linebreaks to osutils
1117
def contains_whitespace(s):
1118
    """True if there are any whitespace characters in s."""
2249.2.1 by John Arbash Meinel
(John Arbash Meinel) hard-code the whitespace chars to avoid problems in some locales.
1119
    # string.whitespace can include '\xa0' in certain locales, because it is
1120
    # considered "non-breaking-space" as part of ISO-8859-1. But it
1121
    # 1) Isn't a breaking whitespace
1122
    # 2) Isn't one of ' \t\r\n' which are characters we sometimes use as
1123
    #    separators
1124
    # 3) '\xa0' isn't unicode safe since it is >128.
2249.5.16 by John Arbash Meinel
[merge] bzr.dev 2283
1125
1126
    # This should *not* be a unicode set of characters in case the source
1127
    # string is not a Unicode string. We can auto-up-cast the characters since
1128
    # they are ascii, but we don't want to auto-up-cast the string in case it
1129
    # is utf-8
1130
    for ch in ' \t\n\r\v\f':
1185.16.38 by Martin Pool
- move contains_whitespace and contains_linebreaks to osutils
1131
        if ch in s:
1132
            return True
1133
    else:
1134
        return False
1135
1136
1137
def contains_linebreaks(s):
1138
    """True if there is any vertical whitespace in s."""
1139
    for ch in '\f\n\r':
1140
        if ch in s:
1141
            return True
1142
    else:
1143
        return False
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1144
1145
1146
def relpath(base, path):
5193.2.1 by Alexander Belchenko
update docstring for osutils.relpath() function.
1147
    """Return path relative to base, or raise PathNotChild exception.
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1148
1149
    The path may be either an absolute path or a path relative to the
1150
    current working directory.
1151
1152
    os.path.commonprefix (python2.4) has a bad bug that it works just
1153
    on string prefixes, assuming that '/u' is a prefix of '/u2'.  This
1636.1.1 by Robert Collins
Fix calling relpath() and abspath() on transports at their root.
1154
    avoids that problem.
5193.2.1 by Alexander Belchenko
update docstring for osutils.relpath() function.
1155
5193.2.2 by Alexander Belchenko
update wording based on spiv's review.
1156
    NOTE: `base` should not have a trailing slash otherwise you'll get
1157
    PathNotChild exceptions regardless of `path`.
1636.1.1 by Robert Collins
Fix calling relpath() and abspath() on transports at their root.
1158
    """
1685.1.12 by John Arbash Meinel
Some more work to get LocalTransport to only support URLs
1159
3376.2.4 by Martin Pool
Remove every assert statement from bzrlib!
1160
    if len(base) < MIN_ABS_PATHLENGTH:
1161
        # must have space for e.g. a drive letter
1162
        raise ValueError('%r is too short to calculate a relative path'
1163
            % (base,))
1685.1.9 by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url
1164
1685.1.12 by John Arbash Meinel
Some more work to get LocalTransport to only support URLs
1165
    rp = abspath(path)
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1166
1167
    s = []
1685.1.12 by John Arbash Meinel
Some more work to get LocalTransport to only support URLs
1168
    head = rp
4555.2.1 by John Arbash Meinel
Fix bug #394227, osutils.relpath() could get into an infinite loop.
1169
    while True:
1170
        if len(head) <= len(base) and head != base:
1171
            raise errors.PathNotChild(rp, base)
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1172
        if head == base:
1173
            break
4555.2.1 by John Arbash Meinel
Fix bug #394227, osutils.relpath() could get into an infinite loop.
1174
        head, tail = split(head)
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1175
        if tail:
4555.2.1 by John Arbash Meinel
Fix bug #394227, osutils.relpath() could get into an infinite loop.
1176
            s.append(tail)
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
1177
1185.31.35 by John Arbash Meinel
Couple small fixes, all tests pass on cygwin.
1178
    if s:
4555.2.3 by John Arbash Meinel
Fix a trivial bug that should have been caught earlier. :)
1179
        return pathjoin(*reversed(s))
1185.31.35 by John Arbash Meinel
Couple small fixes, all tests pass on cygwin.
1180
    else:
1181
        return ''
1185.33.60 by Martin Pool
Use full terminal width for verbose test output.
1182
1183
3794.5.29 by Mark Hammond
cicp_canonical_relpath -> _cicp_canonical_relpath
1184
def _cicp_canonical_relpath(base, path):
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1185
    """Return the canonical path relative to base.
1186
1187
    Like relpath, but on case-insensitive-case-preserving file-systems, this
3794.5.13 by Mark Hammond
Tweaks suggested by Martin
1188
    will return the relpath as stored on the file-system rather than in the
1189
    case specified in the input string, for all existing portions of the path.
1190
3794.5.28 by Mark Hammond
Update comments.
1191
    This will cause O(N) behaviour if called for every path in a tree; if you
1192
    have a number of paths to convert, you should use canonical_relpaths().
3794.5.31 by Mark Hammond
bulk of the simple review comments from igc.
1193
    """
1194
    # TODO: it should be possible to optimize this for Windows by using the
1195
    # win32 API FindFiles function to look for the specified name - but using
1196
    # os.listdir() still gives us the correct, platform agnostic semantics in
1197
    # the short term.
3794.5.13 by Mark Hammond
Tweaks suggested by Martin
1198
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1199
    rel = relpath(base, path)
1200
    # '.' will have been turned into ''
1201
    if not rel:
1202
        return rel
1203
1204
    abs_base = abspath(base)
1205
    current = abs_base
1206
    _listdir = os.listdir
1207
1208
    # use an explicit iterator so we can easily consume the rest on early exit.
3794.5.36 by Mark Hammond
test for, and fix problem with canonical_relpath when the tail does not exist.
1209
    bit_iter = iter(rel.split('/'))
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1210
    for bit in bit_iter:
1211
        lbit = bit.lower()
4634.70.2 by John Arbash Meinel
Fix bug #322807, teach cicp_canonical_relpath how to handle
1212
        try:
1213
            next_entries = _listdir(current)
4634.70.3 by John Arbash Meinel
Clean up some terminology, catch a double _listdir request, thanks spiv.
1214
        except OSError: # enoent, eperm, etc
1215
            # We can't find this in the filesystem, so just append the
1216
            # remaining bits.
4634.70.2 by John Arbash Meinel
Fix bug #322807, teach cicp_canonical_relpath how to handle
1217
            current = pathjoin(current, bit, *list(bit_iter))
1218
            break
4634.70.3 by John Arbash Meinel
Clean up some terminology, catch a double _listdir request, thanks spiv.
1219
        for look in next_entries:
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1220
            if lbit == look.lower():
1221
                current = pathjoin(current, look)
1222
                break
1223
        else:
1224
            # got to the end, nothing matched, so we just return the
1225
            # non-existing bits as they were specified (the filename may be
1226
            # the target of a move, for example).
1227
            current = pathjoin(current, bit, *list(bit_iter))
1228
            break
4634.70.2 by John Arbash Meinel
Fix bug #322807, teach cicp_canonical_relpath how to handle
1229
    return current[len(abs_base):].lstrip('/')
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1230
3794.5.13 by Mark Hammond
Tweaks suggested by Martin
1231
# XXX - TODO - we need better detection/integration of case-insensitive
4241.9.5 by Vincent Ladeuil
Fix unicode related OSX failures.
1232
# file-systems; Linux often sees FAT32 devices (or NFS-mounted OSX
1233
# filesystems), for example, so could probably benefit from the same basic
1234
# support there.  For now though, only Windows and OSX get that support, and
1235
# they get it for *all* file-systems!
4241.9.2 by Vincent Ladeuil
Fix most of cicp related failures on OSX.
1236
if sys.platform in ('win32', 'darwin'):
3794.5.29 by Mark Hammond
cicp_canonical_relpath -> _cicp_canonical_relpath
1237
    canonical_relpath = _cicp_canonical_relpath
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1238
else:
1239
    canonical_relpath = relpath
1240
3794.5.15 by Mark Hammond
Add canonical_relpaths() as a placeholder for a future caching implementation.
1241
def canonical_relpaths(base, paths):
1242
    """Create an iterable to canonicalize a sequence of relative paths.
1243
1244
    The intent is for this implementation to use a cache, vastly speeding
1245
    up multiple transformations in the same directory.
1246
    """
1247
    # but for now, we haven't optimized...
1248
    return [canonical_relpath(base, p) for p in paths]
3794.5.1 by Mark Hammond
Add canonical_relpath api function
1249
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
1250
1251
def decode_filename(filename):
1252
    """Decode the filename using the filesystem encoding
1253
1254
    If it is unicode, it is returned.
1255
    Otherwise it is decoded from the the filesystem's encoding. If decoding
1256
    fails, a errors.BadFilenameEncoding exception is raised.
1257
    """
5279.2.15 by Eric Moritz
used "type(filename) == unicode" per John A Meinel's suggestion. This will
1258
    if type(filename) is unicode:
5279.2.4 by Eric Moritz
Added the filename_decode
1259
        return filename
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
1260
    try:
1261
        return filename.decode(_fs_enc)
1262
    except UnicodeDecodeError:
1263
        raise errors.BadFilenameEncoding(filename, _fs_enc)
1264
5279.2.4 by Eric Moritz
Added the filename_decode
1265
1534.3.1 by Robert Collins
* bzrlib.osutils.safe_unicode now exists to provide parameter coercion
1266
def safe_unicode(unicode_or_utf8_string):
1267
    """Coerce unicode_or_utf8_string into unicode.
1268
1269
    If it is unicode, it is returned.
4204.2.1 by Matt Nordhoff
Fix a broken sentence in osutils.safe_unicode's docstring
1270
    Otherwise it is decoded from utf-8. If decoding fails, the exception is
1271
    wrapped in a BzrBadParameterNotUnicode exception.
1534.3.1 by Robert Collins
* bzrlib.osutils.safe_unicode now exists to provide parameter coercion
1272
    """
1273
    if isinstance(unicode_or_utf8_string, unicode):
1274
        return unicode_or_utf8_string
1275
    try:
1276
        return unicode_or_utf8_string.decode('utf8')
1277
    except UnicodeDecodeError:
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
1278
        raise errors.BzrBadParameterNotUnicode(unicode_or_utf8_string)
1534.3.1 by Robert Collins
* bzrlib.osutils.safe_unicode now exists to provide parameter coercion
1279
1280
2249.5.8 by John Arbash Meinel
Add osutils.safe_utf8 and safe_revision_id for the new revision_id work.
1281
def safe_utf8(unicode_or_utf8_string):
1282
    """Coerce unicode_or_utf8_string to a utf8 string.
1283
1284
    If it is a str, it is returned.
1285
    If it is Unicode, it is encoded into a utf-8 string.
1286
    """
1287
    if isinstance(unicode_or_utf8_string, str):
1288
        # TODO: jam 20070209 This is overkill, and probably has an impact on
1289
        #       performance if we are dealing with lots of apis that want a
1290
        #       utf-8 revision id
1291
        try:
1292
            # Make sure it is a valid utf-8 string
1293
            unicode_or_utf8_string.decode('utf-8')
1294
        except UnicodeDecodeError:
1295
            raise errors.BzrBadParameterNotUnicode(unicode_or_utf8_string)
1296
        return unicode_or_utf8_string
1297
    return unicode_or_utf8_string.encode('utf-8')
1298
1299
2309.4.4 by John Arbash Meinel
Change what warnings are raised, and add tests that they are used.
1300
_revision_id_warning = ('Unicode revision ids were deprecated in bzr 0.15.'
1301
                        ' Revision id generators should be creating utf8'
1302
                        ' revision ids.')
1303
1304
1305
def safe_revision_id(unicode_or_utf8_string, warn=True):
2249.5.8 by John Arbash Meinel
Add osutils.safe_utf8 and safe_revision_id for the new revision_id work.
1306
    """Revision ids should now be utf8, but at one point they were unicode.
1307
2309.4.4 by John Arbash Meinel
Change what warnings are raised, and add tests that they are used.
1308
    :param unicode_or_utf8_string: A possibly Unicode revision_id. (can also be
1309
        utf8 or None).
1310
    :param warn: Functions that are sanitizing user data can set warn=False
1311
    :return: None or a utf8 revision id.
2249.5.8 by John Arbash Meinel
Add osutils.safe_utf8 and safe_revision_id for the new revision_id work.
1312
    """
2309.4.3 by John Arbash Meinel
(broken) change safe_*_id to emit a warning.
1313
    if (unicode_or_utf8_string is None
1314
        or unicode_or_utf8_string.__class__ == str):
1315
        return unicode_or_utf8_string
2309.4.4 by John Arbash Meinel
Change what warnings are raised, and add tests that they are used.
1316
    if warn:
1317
        symbol_versioning.warn(_revision_id_warning, DeprecationWarning,
1318
                               stacklevel=2)
2309.4.3 by John Arbash Meinel
(broken) change safe_*_id to emit a warning.
1319
    return cache_utf8.encode(unicode_or_utf8_string)
1320
1321
2309.4.4 by John Arbash Meinel
Change what warnings are raised, and add tests that they are used.
1322
_file_id_warning = ('Unicode file ids were deprecated in bzr 0.15. File id'
1323
                    ' generators should be creating utf8 file ids.')
1324
1325
1326
def safe_file_id(unicode_or_utf8_string, warn=True):
2309.4.3 by John Arbash Meinel
(broken) change safe_*_id to emit a warning.
1327
    """File ids should now be utf8, but at one point they were unicode.
1328
1329
    This is the same as safe_utf8, except it uses the cached encode functions
1330
    to save a little bit of performance.
2309.4.4 by John Arbash Meinel
Change what warnings are raised, and add tests that they are used.
1331
1332
    :param unicode_or_utf8_string: A possibly Unicode file_id. (can also be
1333
        utf8 or None).
1334
    :param warn: Functions that are sanitizing user data can set warn=False
1335
    :return: None or a utf8 file id.
2309.4.3 by John Arbash Meinel
(broken) change safe_*_id to emit a warning.
1336
    """
1337
    if (unicode_or_utf8_string is None
1338
        or unicode_or_utf8_string.__class__ == str):
1339
        return unicode_or_utf8_string
2309.4.4 by John Arbash Meinel
Change what warnings are raised, and add tests that they are used.
1340
    if warn:
1341
        symbol_versioning.warn(_file_id_warning, DeprecationWarning,
1342
                               stacklevel=2)
2309.4.3 by John Arbash Meinel
(broken) change safe_*_id to emit a warning.
1343
    return cache_utf8.encode(unicode_or_utf8_string)
2294.1.4 by John Arbash Meinel
Add safe_file_id as a helper in osutils.
1344
1345
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
1346
_platform_normalizes_filenames = False
1347
if sys.platform == 'darwin':
1348
    _platform_normalizes_filenames = True
1349
1350
1351
def normalizes_filenames():
1352
    """Return True if this platform normalizes unicode filenames.
1353
5283.2.1 by Martin Pool
Additional platform name tweaks
1354
    Only Mac OSX.
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
1355
    """
1356
    return _platform_normalizes_filenames
1357
1358
1830.3.2 by John Arbash Meinel
normalized_filename is a much better name
1359
def _accessible_normalized_filename(path):
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1360
    """Get the unicode normalized path, and if you can access the file.
1361
1362
    On platforms where the system normalizes filenames (Mac OSX),
1363
    you can access a file by any path which will normalize correctly.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1364
    On platforms where the system does not normalize filenames
5283.2.1 by Martin Pool
Additional platform name tweaks
1365
    (everything else), you have to access a file by its exact path.
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1366
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1367
    Internally, bzr only supports NFC normalization, since that is
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1368
    the standard for XML documents.
1369
1370
    So return the normalized path, and a flag indicating if the file
1371
    can be accessed by that path.
1372
    """
1373
3201.1.1 by jameinel
Fix bug #185458, switch from NFKC to NFC and add tests for filenames that would be broken under NFKC
1374
    return unicodedata.normalize('NFC', unicode(path)), True
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1375
1376
1830.3.2 by John Arbash Meinel
normalized_filename is a much better name
1377
def _inaccessible_normalized_filename(path):
1378
    __doc__ = _accessible_normalized_filename.__doc__
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1379
3201.1.1 by jameinel
Fix bug #185458, switch from NFKC to NFC and add tests for filenames that would be broken under NFKC
1380
    normalized = unicodedata.normalize('NFC', unicode(path))
1830.3.1 by John Arbash Meinel
Change the return value of unicode_filename, and make it testable on all platforms
1381
    return normalized, normalized == path
1382
1383
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
1384
if _platform_normalizes_filenames:
1830.3.2 by John Arbash Meinel
normalized_filename is a much better name
1385
    normalized_filename = _accessible_normalized_filename
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
1386
else:
1830.3.2 by John Arbash Meinel
normalized_filename is a much better name
1387
    normalized_filename = _inaccessible_normalized_filename
1185.85.75 by John Arbash Meinel
Adding bzrlib.osutils.unicode_filename to handle unicode normalization for file paths.
1388
1389
4634.142.2 by Andrew Bennetts
Make calling siginterrupt in set_signal_handler conditional on a restart_syscall param (default True), and add missing import.
1390
def set_signal_handler(signum, handler, restart_syscall=True):
4634.142.1 by Andrew Bennetts
Add osutils.set_signal_handler to call signal.siginterrupt where possible, and use it in bzrlib.
1391
    """A wrapper for signal.signal that also calls siginterrupt(signum, False)
1392
    on platforms that support that.
4634.142.2 by Andrew Bennetts
Make calling siginterrupt in set_signal_handler conditional on a restart_syscall param (default True), and add missing import.
1393
1394
    :param restart_syscall: if set, allow syscalls interrupted by a signal to
1395
        automatically restart (by calling `signal.siginterrupt(signum,
1396
        False)`).  May be ignored if the feature is not available on this
1397
        platform or Python version.
4634.142.1 by Andrew Bennetts
Add osutils.set_signal_handler to call signal.siginterrupt where possible, and use it in bzrlib.
1398
    """
5141.4.1 by Andrew Bennetts
Reset siginterrupt every time we handle a signal.
1399
    try:
5169.2.2 by Vincent Ladeuil
Just rely on python to tell us what it supports.
1400
        import signal
5141.4.1 by Andrew Bennetts
Reset siginterrupt every time we handle a signal.
1401
        siginterrupt = signal.siginterrupt
5169.2.2 by Vincent Ladeuil
Just rely on python to tell us what it supports.
1402
    except ImportError:
1403
        # This python implementation doesn't provide signal support, hence no
1404
        # handler exists
1405
        return None
5141.4.1 by Andrew Bennetts
Reset siginterrupt every time we handle a signal.
1406
    except AttributeError:
1407
        # siginterrupt doesn't exist on this platform, or for this version
1408
        # of Python.
1409
        siginterrupt = lambda signum, flag: None
4634.142.2 by Andrew Bennetts
Make calling siginterrupt in set_signal_handler conditional on a restart_syscall param (default True), and add missing import.
1410
    if restart_syscall:
5141.4.1 by Andrew Bennetts
Reset siginterrupt every time we handle a signal.
1411
        def sig_handler(*args):
1412
            # Python resets the siginterrupt flag when a signal is
5141.4.3 by Andrew Bennetts
Link to Python bug in comment.
1413
            # received.  <http://bugs.python.org/issue8354>
1414
            # As a workaround for some cases, set it back the way we want it.
4634.142.2 by Andrew Bennetts
Make calling siginterrupt in set_signal_handler conditional on a restart_syscall param (default True), and add missing import.
1415
            siginterrupt(signum, False)
5141.4.1 by Andrew Bennetts
Reset siginterrupt every time we handle a signal.
1416
            # Now run the handler function passed to set_signal_handler.
1417
            handler(*args)
1418
    else:
1419
        sig_handler = handler
1420
    old_handler = signal.signal(signum, sig_handler)
1421
    if restart_syscall:
1422
        siginterrupt(signum, False)
4634.142.1 by Andrew Bennetts
Add osutils.set_signal_handler to call signal.siginterrupt where possible, and use it in bzrlib.
1423
    return old_handler
1424
1425
4747.3.6 by Vincent Ladeuil
terminal_width can now returns None.
1426
default_terminal_width = 80
1427
"""The default terminal width for ttys.
1428
1429
This is defined so that higher levels can share a common fallback value when
1430
terminal_width() returns None.
1431
"""
1432
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1433
# Keep some state so that terminal_width can detect if _terminal_size has
1434
# returned a different size since the process started.  See docstring and
1435
# comments of terminal_width for details.
1436
# _terminal_size_state has 3 possible values: no_data, unchanged, and changed.
1437
_terminal_size_state = 'no_data'
1438
_first_terminal_size = None
4747.3.6 by Vincent Ladeuil
terminal_width can now returns None.
1439
1185.33.60 by Martin Pool
Use full terminal width for verbose test output.
1440
def terminal_width():
4747.3.6 by Vincent Ladeuil
terminal_width can now returns None.
1441
    """Return terminal width.
1442
1443
    None is returned if the width can't established precisely.
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1444
1445
    The rules are:
1446
    - if BZR_COLUMNS is set, returns its value
1447
    - if there is no controlling terminal, returns None
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1448
    - query the OS, if the queried size has changed since the last query,
1449
      return its value,
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1450
    - if COLUMNS is set, returns its value,
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1451
    - if the OS has a value (even though it's never changed), return its value.
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1452
1453
    From there, we need to query the OS to get the size of the controlling
1454
    terminal.
1455
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1456
    On Unices we query the OS by:
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1457
    - get termios.TIOCGWINSZ
1458
    - if an error occurs or a negative value is obtained, returns None
1459
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1460
    On Windows we query the OS by:
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1461
    - win32utils.get_console_size() decides,
1462
    - returns None on error (provided default value)
4747.3.6 by Vincent Ladeuil
terminal_width can now returns None.
1463
    """
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1464
    # Note to implementors: if changing the rules for determining the width,
1465
    # make sure you've considered the behaviour in these cases:
1466
    #  - M-x shell in emacs, where $COLUMNS is set and TIOCGWINSZ returns 0,0.
1467
    #  - bzr log | less, in bash, where $COLUMNS not set and TIOCGWINSZ returns
1468
    #    0,0.
1469
    #  - (add more interesting cases here, if you find any)
1470
    # Some programs implement "Use $COLUMNS (if set) until SIGWINCH occurs",
1471
    # but we don't want to register a signal handler because it is impossible
1472
    # to do so without risking EINTR errors in Python <= 2.6.5 (see
1473
    # <http://bugs.python.org/issue8354>).  Instead we check TIOCGWINSZ every
1474
    # time so we can notice if the reported size has changed, which should have
1475
    # a similar effect.
4747.3.4 by Vincent Ladeuil
Add tests, introduce explicit default values, always respect COLUMNS.
1476
4747.3.7 by Vincent Ladeuil
Introduce BZR_COLUMNS since COLUMNS behaviour is too obscure.
1477
    # If BZR_COLUMNS is set, take it, user is always right
5582.7.1 by Neil Martinsen-Burrell
allow BZR_COLUMNS to be 0
1478
    # Except if they specified 0 in which case, impose no limit here
4747.3.7 by Vincent Ladeuil
Introduce BZR_COLUMNS since COLUMNS behaviour is too obscure.
1479
    try:
5582.7.1 by Neil Martinsen-Burrell
allow BZR_COLUMNS to be 0
1480
        width = int(os.environ['BZR_COLUMNS'])
4747.3.7 by Vincent Ladeuil
Introduce BZR_COLUMNS since COLUMNS behaviour is too obscure.
1481
    except (KeyError, ValueError):
5582.7.1 by Neil Martinsen-Burrell
allow BZR_COLUMNS to be 0
1482
        width = None
1483
    if width is not None:
1484
        if width > 0:
1485
            return width
1486
        else:
1487
            return None
4747.3.7 by Vincent Ladeuil
Introduce BZR_COLUMNS since COLUMNS behaviour is too obscure.
1488
4747.3.3 by Vincent Ladeuil
More complete fix (previous one changed the focus).
1489
    isatty = getattr(sys.stdout, 'isatty', None)
4797.54.1 by Andrew Bennetts
Remove the SIGWINCH signal handler.
1490
    if isatty is None or not isatty():
4747.3.7 by Vincent Ladeuil
Introduce BZR_COLUMNS since COLUMNS behaviour is too obscure.
1491
        # Don't guess, setting BZR_COLUMNS is the recommended way to override.
4747.3.6 by Vincent Ladeuil
terminal_width can now returns None.
1492
        return None
4747.3.1 by Joke de Buhr
Prevent linebreaks in output if it's not connected to a tty.
1493
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1494
    # Query the OS
1495
    width, height = os_size = _terminal_size(None, None)
1496
    global _first_terminal_size, _terminal_size_state
1497
    if _terminal_size_state == 'no_data':
1498
        _first_terminal_size = os_size
1499
        _terminal_size_state = 'unchanged'
1500
    elif (_terminal_size_state == 'unchanged' and
1501
          _first_terminal_size != os_size):
1502
        _terminal_size_state = 'changed'
1503
1504
    # If the OS claims to know how wide the terminal is, and this value has
1505
    # ever changed, use that.
1506
    if _terminal_size_state == 'changed':
1507
        if width is not None and width > 0:
1508
            return width
4797.54.1 by Andrew Bennetts
Remove the SIGWINCH signal handler.
1509
1510
    # If COLUMNS is set, use it.
4747.4.3 by Vincent Ladeuil
Re-fix the priority order since there is a known valid case.
1511
    try:
1512
        return int(os.environ['COLUMNS'])
1513
    except (KeyError, ValueError):
1514
        pass
1515
4797.54.2 by Andrew Bennetts
Try to preserve the 'use COLUMNS until SIGWINCH' behaviour without using SIGWINCH, to keep the behaviour in 2.1 as stable as possible.
1516
    # Finally, use an unchanged size from the OS, if we have one.
1517
    if _terminal_size_state == 'unchanged':
1518
        if width is not None and width > 0:
1519
            return width
1520
1521
    # The width could not be determined.
4797.54.1 by Andrew Bennetts
Remove the SIGWINCH signal handler.
1522
    return None
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1523
1524
1525
def _win32_terminal_size(width, height):
1526
    width, height = win32utils.get_console_size(defaultx=width, defaulty=height)
1527
    return width, height
1528
1529
1530
def _ioctl_terminal_size(width, height):
1185.33.60 by Martin Pool
Use full terminal width for verbose test output.
1531
    try:
1704.2.2 by Martin Pool
Detect terminal width using ioctl
1532
        import struct, fcntl, termios
1533
        s = struct.pack('HHHH', 0, 0, 0, 0)
1534
        x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
4747.4.6 by Vincent Ladeuil
Fix parameter order.
1535
        height, width = struct.unpack('HHHH', x)[0:2]
4747.3.4 by Vincent Ladeuil
Add tests, introduce explicit default values, always respect COLUMNS.
1536
    except (IOError, AttributeError):
4747.4.5 by Vincent Ladeuil
More robusts tests for osutils.terminal_width().
1537
        pass
1538
    return width, height
1539
1540
_terminal_size = None
1541
"""Returns the terminal size as (width, height).
1542
1543
:param width: Default value for width.
1544
:param height: Default value for height.
1545
1546
This is defined specifically for each OS and query the size of the controlling
1547
terminal. If any error occurs, the provided default values should be returned.
1548
"""
1549
if sys.platform == 'win32':
1550
    _terminal_size = _win32_terminal_size
1551
else:
1552
    _terminal_size = _ioctl_terminal_size
1534.7.25 by Aaron Bentley
Added set_executability
1553
1963.1.5 by John Arbash Meinel
Create an osutils helper function for modifying the environment
1554
1534.7.25 by Aaron Bentley
Added set_executability
1555
def supports_executable():
1534.7.160 by Aaron Bentley
Changed implementation of supports_executable
1556
    return sys.platform != "win32"
1551.2.53 by abentley
Strip trailing slashes in a platform-sensible way
1557
1558
1551.10.4 by Aaron Bentley
Update to skip on win32
1559
def supports_posix_readonly():
1560
    """Return True if 'readonly' has POSIX semantics, False otherwise.
1561
1562
    Notably, a win32 readonly file cannot be deleted, unlike POSIX where the
1563
    directory controls creation/deletion, etc.
1564
1565
    And under win32, readonly means that the directory itself cannot be
1566
    deleted.  The contents of a readonly directory can be changed, unlike POSIX
1567
    where files in readonly directories cannot be added, deleted or renamed.
1568
    """
1569
    return sys.platform != "win32"
1570
1571
1963.1.5 by John Arbash Meinel
Create an osutils helper function for modifying the environment
1572
def set_or_unset_env(env_variable, value):
1573
    """Modify the environment, setting or removing the env_variable.
1574
1575
    :param env_variable: The environment variable in question
1576
    :param value: The value to set the environment to. If None, then
1577
        the variable will be removed.
1578
    :return: The original value of the environment variable.
1579
    """
1580
    orig_val = os.environ.get(env_variable)
1581
    if value is None:
1582
        if orig_val is not None:
1583
            del os.environ[env_variable]
1584
    else:
1585
        if isinstance(value, unicode):
3224.5.4 by Andrew Bennetts
Fix test suite, mainly weeding out uses of bzrlib.user_encoding.
1586
            value = value.encode(get_user_encoding())
1963.1.5 by John Arbash Meinel
Create an osutils helper function for modifying the environment
1587
        os.environ[env_variable] = value
1588
    return orig_val
1589
1590
1551.2.56 by Aaron Bentley
Better illegal pathname check for Windows
1591
_validWin32PathRE = re.compile(r'^([A-Za-z]:[/\\])?[^:<>*"?\|]*$')
1592
1593
1594
def check_legal_path(path):
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1595
    """Check whether the supplied path is legal.
1551.2.56 by Aaron Bentley
Better illegal pathname check for Windows
1596
    This is only required on Windows, so we don't test on other platforms
1597
    right now.
1598
    """
1599
    if sys.platform != "win32":
1600
        return
1601
    if _validWin32PathRE.match(path) is None:
1996.3.25 by John Arbash Meinel
Make importing errors lazy for osutils
1602
        raise errors.IllegalPath(path)
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1603
1604
3596.2.2 by John Arbash Meinel
Factor out the common exception handling looking for ENOTDIR and use it
1605
_WIN32_ERROR_DIRECTORY = 267 # Similar to errno.ENOTDIR
1606
1607
def _is_error_enotdir(e):
1608
    """Check if this exception represents ENOTDIR.
1609
1610
    Unfortunately, python is very inconsistent about the exception
1611
    here. The cases are:
1612
      1) Linux, Mac OSX all versions seem to set errno == ENOTDIR
1613
      2) Windows, Python2.4, uses errno == ERROR_DIRECTORY (267)
1614
         which is the windows error code.
1615
      3) Windows, Python2.5 uses errno == EINVAL and
1616
         winerror == ERROR_DIRECTORY
1617
1618
    :param e: An Exception object (expected to be OSError with an errno
1619
        attribute, but we should be able to cope with anything)
1620
    :return: True if this represents an ENOTDIR error. False otherwise.
1621
    """
1622
    en = getattr(e, 'errno', None)
1623
    if (en == errno.ENOTDIR
1624
        or (sys.platform == 'win32'
1625
            and (en == _WIN32_ERROR_DIRECTORY
1626
                 or (en == errno.EINVAL
1627
                     and getattr(e, 'winerror', None) == _WIN32_ERROR_DIRECTORY)
1628
        ))):
1629
        return True
1630
    return False
1631
1632
1757.2.8 by Robert Collins
Teach walkdirs to walk a subdir of a tree.
1633
def walkdirs(top, prefix=""):
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1634
    """Yield data about all the directories in a tree.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1635
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1636
    This yields all the data about the contents of a directory at a time.
1637
    After each directory has been yielded, if the caller has mutated the list
1638
    to exclude some directories, they are then not descended into.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1639
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1640
    The data yielded is of the form:
1897.1.2 by Robert Collins
cleanup osutils.walkdirs changes after review.
1641
    ((directory-relpath, directory-path-from-top),
2694.4.1 by Alexander Belchenko
trivial fix for docstring of osutils.walkdirs()
1642
    [(relpath, basename, kind, lstat, path-from-top), ...]),
1897.1.2 by Robert Collins
cleanup osutils.walkdirs changes after review.
1643
     - directory-relpath is the relative path of the directory being returned
1644
       with respect to top. prefix is prepended to this.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1645
     - directory-path-from-root is the path including top for this directory.
1897.1.2 by Robert Collins
cleanup osutils.walkdirs changes after review.
1646
       It is suitable for use with os functions.
1897.1.1 by Robert Collins
Add some useful summary data to osutils.walkdirs output.
1647
     - relpath is the relative path within the subtree being walked.
1648
     - basename is the basename of the path
1897.1.2 by Robert Collins
cleanup osutils.walkdirs changes after review.
1649
     - kind is the kind of the file now. If unknown then the file is not
1897.1.1 by Robert Collins
Add some useful summary data to osutils.walkdirs output.
1650
       present within the tree - but it may be recorded as versioned. See
1651
       versioned_kind.
1652
     - lstat is the stat data *if* the file was statted.
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1653
     - planned, not implemented:
1897.1.1 by Robert Collins
Add some useful summary data to osutils.walkdirs output.
1654
       path_from_tree_root is the path from the root of the tree.
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1655
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1656
    :param prefix: Prefix the relpaths that are yielded with 'prefix'. This
1757.2.16 by Robert Collins
Review comments.
1657
        allows one to walk a subtree but get paths that are relative to a tree
1658
        rooted higher up.
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1659
    :return: an iterator over the dirs.
1660
    """
1897.1.1 by Robert Collins
Add some useful summary data to osutils.walkdirs output.
1661
    #TODO there is a bit of a smell where the results of the directory-
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1662
    # summary in this, and the path from the root, may not agree
1897.1.1 by Robert Collins
Add some useful summary data to osutils.walkdirs output.
1663
    # depending on top and prefix - i.e. ./foo and foo as a pair leads to
1664
    # potentially confusing output. We should make this more robust - but
1897.1.2 by Robert Collins
cleanup osutils.walkdirs changes after review.
1665
    # not at a speed cost. RBC 20060731
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1666
    _lstat = os.lstat
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1667
    _directory = _directory_kind
1996.3.14 by John Arbash Meinel
lazy_import osutils and sign_my_commits
1668
    _listdir = os.listdir
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1669
    _kind_from_mode = file_kind_from_stat_mode
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1670
    pending = [(safe_unicode(prefix), "", _directory, None, safe_unicode(top))]
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1671
    while pending:
1672
        # 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1673
        relroot, _, _, _, top = pending.pop()
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1674
        if relroot:
1675
            relprefix = relroot + u'/'
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1676
        else:
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1677
            relprefix = ''
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1678
        top_slash = top + u'/'
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1679
1680
        dirblock = []
1681
        append = dirblock.append
3585.2.4 by Robert Collins
* Deleting directories by hand before running ``bzr rm`` will not
1682
        try:
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
1683
            names = sorted(map(decode_filename, _listdir(top)))
3596.2.2 by John Arbash Meinel
Factor out the common exception handling looking for ENOTDIR and use it
1684
        except OSError, e:
1685
            if not _is_error_enotdir(e):
3585.2.4 by Robert Collins
* Deleting directories by hand before running ``bzr rm`` will not
1686
                raise
1687
        else:
1688
            for name in names:
1689
                abspath = top_slash + name
1690
                statvalue = _lstat(abspath)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1691
                kind = _kind_from_mode(statvalue.st_mode)
3585.2.4 by Robert Collins
* Deleting directories by hand before running ``bzr rm`` will not
1692
                append((relprefix + name, name, kind, statvalue, abspath))
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1693
        yield (relroot, top), dirblock
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1694
1753.1.1 by Robert Collins
(rbc, jam, mbp)Add bzrlib.osutils.walkdirs, an optimised walk-and-stat routine.
1695
        # push the user specified dirs from dirblock
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1696
        pending.extend(d for d in reversed(dirblock) if d[2] == _directory)
1773.3.1 by Robert Collins
Add path_prefix_key and compare_paths_prefix_order utility functions.
1697
1698
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1699
class DirReader(object):
1700
    """An interface for reading directories."""
1701
1702
    def top_prefix_to_starting_dir(self, top, prefix=""):
1703
        """Converts top and prefix to a starting dir entry
1704
1705
        :param top: A utf8 path
1706
        :param prefix: An optional utf8 path to prefix output relative paths
1707
            with.
1708
        :return: A tuple starting with prefix, and ending with the native
1709
            encoding of top.
1710
        """
1711
        raise NotImplementedError(self.top_prefix_to_starting_dir)
1712
1713
    def read_dir(self, prefix, top):
1714
        """Read a specific dir.
1715
1716
        :param prefix: A utf8 prefix to be preprended to the path basenames.
1717
        :param top: A natively encoded path to read.
3696.3.10 by Robert Collins
Review feedback.
1718
        :return: A list of the directories contents. Each item contains:
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1719
            (utf8_relpath, utf8_name, kind, lstatvalue, native_abspath)
1720
        """
1721
        raise NotImplementedError(self.read_dir)
1722
1723
1724
_selected_dir_reader = None
1725
3557.2.3 by John Arbash Meinel
Change the logic for selecting a real _walkdirs_utf8 implementation,
1726
2255.7.27 by John Arbash Meinel
Add a _walkdirs_utf8 which returns utf8 paths instead of Unicode. Approx 20% faster in walking utf8 filesystems
1727
def _walkdirs_utf8(top, prefix=""):
1728
    """Yield data about all the directories in a tree.
1729
1730
    This yields the same information as walkdirs() only each entry is yielded
1731
    in utf-8. On platforms which have a filesystem encoding of utf8 the paths
1732
    are returned as exact byte-strings.
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1733
1734
    :return: yields a tuple of (dir_info, [file_info])
1735
        dir_info is (utf8_relpath, path-from-top)
1736
        file_info is (utf8_relpath, utf8_name, kind, lstat, path-from-top)
1737
        if top is an absolute path, path-from-top is also an absolute path.
1738
        path-from-top might be unicode or utf8, but it is the correct path to
1739
        pass to os functions to affect the file in question. (such as os.lstat)
1740
    """
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1741
    global _selected_dir_reader
1742
    if _selected_dir_reader is None:
3557.2.3 by John Arbash Meinel
Change the logic for selecting a real _walkdirs_utf8 implementation,
1743
        fs_encoding = _fs_enc.upper()
3224.5.17 by Andrew Bennetts
Avoid importing win32utils when sys.platform != win32
1744
        if sys.platform == "win32" and win32utils.winver == 'Windows NT':
3557.2.4 by John Arbash Meinel
Cleanup the tests a bit, and add a test that we downgrade if os.name isn't 'nt'
1745
            # Win98 doesn't have unicode apis like FindFirstFileW
1746
            # TODO: We possibly could support Win98 by falling back to the
1747
            #       original FindFirstFile, and using TCHAR instead of WCHAR,
1748
            #       but that gets a bit tricky, and requires custom compiling
1749
            #       for win98 anyway.
3557.2.3 by John Arbash Meinel
Change the logic for selecting a real _walkdirs_utf8 implementation,
1750
            try:
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1751
                from bzrlib._walkdirs_win32 import Win32ReadDir
1752
                _selected_dir_reader = Win32ReadDir()
4241.14.6 by Vincent Ladeuil
Start DirReader parametrized tests.
1753
            except ImportError:
1754
                pass
1755
        elif fs_encoding in ('UTF-8', 'US-ASCII', 'ANSI_X3.4-1968'):
3557.2.3 by John Arbash Meinel
Change the logic for selecting a real _walkdirs_utf8 implementation,
1756
            # ANSI_X3.4-1968 is a form of ASCII
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1757
            try:
1758
                from bzrlib._readdir_pyx import UTF8DirReader
1759
                _selected_dir_reader = UTF8DirReader()
4574.3.6 by Martin Pool
More warnings when failing to load extensions
1760
            except ImportError, e:
4574.3.8 by Martin Pool
Only mutter extension load errors when they occur, and record for later
1761
                failed_to_load_extension(e)
4241.14.6 by Vincent Ladeuil
Start DirReader parametrized tests.
1762
                pass
1763
1764
    if _selected_dir_reader is None:
1765
        # Fallback to the python version
1766
        _selected_dir_reader = UnicodeDirReader()
1767
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1768
    # 0 - relpath, 1- basename, 2- kind, 3- stat, 4-toppath
1769
    # But we don't actually uses 1-3 in pending, so set them to None
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1770
    pending = [[_selected_dir_reader.top_prefix_to_starting_dir(top, prefix)]]
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1771
    read_dir = _selected_dir_reader.read_dir
1772
    _directory = _directory_kind
2255.7.27 by John Arbash Meinel
Add a _walkdirs_utf8 which returns utf8 paths instead of Unicode. Approx 20% faster in walking utf8 filesystems
1773
    while pending:
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1774
        relroot, _, _, _, top = pending[-1].pop()
1775
        if not pending[-1]:
1776
            pending.pop()
1777
        dirblock = sorted(read_dir(relroot, top))
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1778
        yield (relroot, top), dirblock
1779
        # push the user specified dirs from dirblock
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1780
        next = [d for d in reversed(dirblock) if d[2] == _directory]
1781
        if next:
1782
            pending.append(next)
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1783
1784
1785
class UnicodeDirReader(DirReader):
1786
    """A dir reader for non-utf8 file systems, which transcodes."""
1787
1788
    __slots__ = ['_utf8_encode']
1789
1790
    def __init__(self):
1791
        self._utf8_encode = codecs.getencoder('utf8')
1792
1793
    def top_prefix_to_starting_dir(self, top, prefix=""):
1794
        """See DirReader.top_prefix_to_starting_dir."""
1795
        return (safe_utf8(prefix), None, None, None, safe_unicode(top))
1796
1797
    def read_dir(self, prefix, top):
1798
        """Read a single directory from a non-utf8 file system.
1799
1800
        top, and the abspath element in the output are unicode, all other paths
1801
        are utf8. Local disk IO is done via unicode calls to listdir etc.
1802
1803
        This is currently the fallback code path when the filesystem encoding is
1804
        not UTF-8. It may be better to implement an alternative so that we can
1805
        safely handle paths that are not properly decodable in the current
1806
        encoding.
1807
1808
        See DirReader.read_dir for details.
1809
        """
1810
        _utf8_encode = self._utf8_encode
1811
        _lstat = os.lstat
1812
        _listdir = os.listdir
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1813
        _kind_from_mode = file_kind_from_stat_mode
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1814
1815
        if prefix:
1816
            relprefix = prefix + '/'
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1817
        else:
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1818
            relprefix = ''
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1819
        top_slash = top + u'/'
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1820
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1821
        dirblock = []
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1822
        append = dirblock.append
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1823
        for name in sorted(_listdir(top)):
3696.3.12 by Robert Collins
Fix PQM test failure.
1824
            try:
1825
                name_utf8 = _utf8_encode(name)[0]
1826
            except UnicodeDecodeError:
1827
                raise errors.BadFilenameEncoding(
1828
                    _utf8_encode(relprefix)[0] + name, _fs_enc)
2255.7.32 by John Arbash Meinel
Add tests that the walkdirs variants work on unicode paths.
1829
            abspath = top_slash + name
1830
            statvalue = _lstat(abspath)
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
1831
            kind = _kind_from_mode(statvalue.st_mode)
2255.7.33 by John Arbash Meinel
More inner loop tuning of walkdirs, can save as much as 5%
1832
            append((relprefix + name_utf8, name_utf8, kind, statvalue, abspath))
3696.3.1 by Robert Collins
Refactor bzrlib.osutils._walkdirs_utf8 to aid API migration in future.
1833
        return dirblock
2255.7.27 by John Arbash Meinel
Add a _walkdirs_utf8 which returns utf8 paths instead of Unicode. Approx 20% faster in walking utf8 filesystems
1834
1835
1907.3.2 by John Arbash Meinel
Updated the copy_tree function to allow overriding functionality.
1836
def copy_tree(from_path, to_path, handlers={}):
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1837
    """Copy all of the entries in from_path into to_path.
1838
3943.8.1 by Marius Kruger
remove all trailing whitespace from bzr source
1839
    :param from_path: The base directory to copy.
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1840
    :param to_path: The target directory. If it does not exist, it will
1841
        be created.
1907.3.2 by John Arbash Meinel
Updated the copy_tree function to allow overriding functionality.
1842
    :param handlers: A dictionary of functions, which takes a source and
1843
        destinations for files, directories, etc.
1844
        It is keyed on the file kind, such as 'directory', 'symlink', or 'file'
1845
        'file', 'directory', and 'symlink' should always exist.
1846
        If they are missing, they will be replaced with 'os.mkdir()',
1847
        'os.readlink() + os.symlink()', and 'shutil.copy2()', respectively.
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1848
    """
1849
    # Now, just copy the existing cached tree to the new location
1850
    # We use a cheap trick here.
1851
    # Absolute paths are prefixed with the first parameter
1852
    # relative paths are prefixed with the second.
1853
    # So we can get both the source and target returned
1854
    # without any extra work.
1855
1907.3.2 by John Arbash Meinel
Updated the copy_tree function to allow overriding functionality.
1856
    def copy_dir(source, dest):
1857
        os.mkdir(dest)
1858
1859
    def copy_link(source, dest):
1860
        """Copy the contents of a symlink"""
1861
        link_to = os.readlink(source)
1862
        os.symlink(link_to, dest)
1863
1864
    real_handlers = {'file':shutil.copy2,
1865
                     'symlink':copy_link,
1866
                     'directory':copy_dir,
1867
                    }
1868
    real_handlers.update(handlers)
1869
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1870
    if not os.path.exists(to_path):
1907.3.2 by John Arbash Meinel
Updated the copy_tree function to allow overriding functionality.
1871
        real_handlers['directory'](from_path, to_path)
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1872
1873
    for dir_info, entries in walkdirs(from_path, prefix=to_path):
1874
        for relpath, name, kind, st, abspath in entries:
1907.3.2 by John Arbash Meinel
Updated the copy_tree function to allow overriding functionality.
1875
            real_handlers[kind](abspath, relpath)
1907.3.1 by John Arbash Meinel
create a copy_tree wrapper around walkdirs()
1876
1877
5116.2.6 by Parth Malwankar
renamed copy_ownership to copy_ownership_from_path.
1878
def copy_ownership_from_path(dst, src=None):
5051.4.11 by Parth Malwankar
closed Martins review comments.
1879
    """Copy usr/grp ownership from src file/dir to dst file/dir.
1880
1881
    If src is None, the containing directory is used as source. If chown
1882
    fails, the error is ignored and a warning is printed.
1883
    """
5074.4.6 by John Arbash Meinel
Unbreak bzr on windows.
1884
    chown = getattr(os, 'chown', None)
1885
    if chown is None:
1886
        return
5051.4.9 by Parth Malwankar
removed parent_dir.
1887
1888
    if src == None:
1889
        src = os.path.dirname(dst)
1890
        if src == '':
1891
            src = '.'
1892
4634.143.4 by Parth Malwankar
added parent_dir and mkdir to osutils. osutils.mkdir optionally
1893
    try:
4634.143.1 by Parth Malwankar
default .bazaar, .bzr.log and .bazaar/bazaar.conf retain
1894
        s = os.stat(src)
5074.4.6 by John Arbash Meinel
Unbreak bzr on windows.
1895
        chown(dst, s.st_uid, s.st_gid)
4634.143.4 by Parth Malwankar
added parent_dir and mkdir to osutils. osutils.mkdir optionally
1896
    except OSError, e:
5523.2.3 by Parth Malwankar
message is now shown to the user but is not too scary.
1897
        trace.warning(
1898
            'Unable to copy ownership from "%s" to "%s". '
1899
            'You may want to set it manually.', src, dst)
1900
        trace.log_exception_quietly()
4634.143.4 by Parth Malwankar
added parent_dir and mkdir to osutils. osutils.mkdir optionally
1901
1902
1773.3.1 by Robert Collins
Add path_prefix_key and compare_paths_prefix_order utility functions.
1903
def path_prefix_key(path):
1904
    """Generate a prefix-order path key for path.
1905
1906
    This can be used to sort paths in the same way that walkdirs does.
1907
    """
1773.3.2 by Robert Collins
New corner case from John Meinel, showing up the need to check the directory lexographically outside of a single tree's root. Fixed.
1908
    return (dirname(path) , path)
1773.3.1 by Robert Collins
Add path_prefix_key and compare_paths_prefix_order utility functions.
1909
1910
1911
def compare_paths_prefix_order(path_a, path_b):
1912
    """Compare path_a and path_b to generate the same order walkdirs uses."""
1913
    key_a = path_prefix_key(path_a)
1914
    key_b = path_prefix_key(path_b)
1915
    return cmp(key_a, key_b)
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1916
1917
1918
_cached_user_encoding = None
1919
1920
2192.1.3 by Alexander Belchenko
Tests for osutils.get_user_encoding
1921
def get_user_encoding(use_cache=True):
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1922
    """Find out what the preferred user encoding is.
1923
1924
    This is generally the encoding that is used for command line parameters
1925
    and file contents. This may be different from the terminal encoding
1926
    or the filesystem encoding.
1927
2192.1.3 by Alexander Belchenko
Tests for osutils.get_user_encoding
1928
    :param  use_cache:  Enable cache for detected encoding.
1929
                        (This parameter is turned on by default,
1930
                        and required only for selftesting)
1931
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1932
    :return: A string defining the preferred user encoding
1933
    """
1934
    global _cached_user_encoding
2192.1.3 by Alexander Belchenko
Tests for osutils.get_user_encoding
1935
    if _cached_user_encoding is not None and use_cache:
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1936
        return _cached_user_encoding
1937
1938
    if sys.platform == 'darwin':
3638.3.10 by Vincent Ladeuil
Provides a better default encoding on OSX.
1939
        # python locale.getpreferredencoding() always return
1940
        # 'mac-roman' on darwin. That's a lie.
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1941
        sys.platform = 'posix'
1942
        try:
3638.3.10 by Vincent Ladeuil
Provides a better default encoding on OSX.
1943
            if os.environ.get('LANG', None) is None:
1944
                # If LANG is not set, we end up with 'ascii', which is bad
1945
                # ('mac-roman' is more than ascii), so we set a default which
1946
                # will give us UTF-8 (which appears to work in all cases on
1947
                # OSX). Users are still free to override LANG of course, as
1948
                # long as it give us something meaningful. This work-around
1949
                # *may* not be needed with python 3k and/or OSX 10.5, but will
1950
                # work with them too -- vila 20080908
1951
                os.environ['LANG'] = 'en_US.UTF-8'
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1952
            import locale
1953
        finally:
1954
            sys.platform = 'darwin'
1955
    else:
1956
        import locale
1957
1958
    try:
2192.1.3 by Alexander Belchenko
Tests for osutils.get_user_encoding
1959
        user_encoding = locale.getpreferredencoding()
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1960
    except locale.Error, e:
1955.2.3 by John Arbash Meinel
Change error message text
1961
        sys.stderr.write('bzr: warning: %s\n'
2001.2.1 by Jelmer Vernooij
Fix typo in encoding warning.
1962
                         '  Could not determine what text encoding to use.\n'
1955.2.3 by John Arbash Meinel
Change error message text
1963
                         '  This error usually means your Python interpreter\n'
1964
                         '  doesn\'t support the locale set by $LANG (%s)\n'
1965
                         "  Continuing with ascii encoding.\n"
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1966
                         % (e, os.environ.get('LANG')))
2192.1.7 by Alexander Belchenko
get_user_encoding: if locale.Error raised we need to set user_encoding to 'ascii' as warning says
1967
        user_encoding = 'ascii'
1955.2.2 by John Arbash Meinel
Change the name of the test classes (test_lang => test_locale), move the function into osutils.py
1968
2127.4.1 by Alexander Belchenko
(jam, bialix) Workaround for cp0 console encoding on Windows
1969
    # Windows returns 'cp0' to indicate there is no code page. So we'll just
1970
    # treat that as ASCII, and not support printing unicode characters to the
1971
    # console.
3405.3.1 by Neil Martinsen-Burrell
accept for an encoding to mean ascii
1972
    #
1973
    # For python scripts run under vim, we get '', so also treat that as ASCII
1974
    if user_encoding in (None, 'cp0', ''):
2192.1.3 by Alexander Belchenko
Tests for osutils.get_user_encoding
1975
        user_encoding = 'ascii'
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
1976
    else:
1977
        # check encoding
1978
        try:
2192.1.3 by Alexander Belchenko
Tests for osutils.get_user_encoding
1979
            codecs.lookup(user_encoding)
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
1980
        except LookupError:
1981
            sys.stderr.write('bzr: warning:'
1982
                             ' unknown encoding %s.'
1983
                             ' Continuing with ascii encoding.\n'
2192.1.3 by Alexander Belchenko
Tests for osutils.get_user_encoding
1984
                             % user_encoding
2192.1.1 by Alexander Belchenko
Before actually using encoding need to check that Python has corresponding codec
1985
                            )
2192.1.3 by Alexander Belchenko
Tests for osutils.get_user_encoding
1986
            user_encoding = 'ascii'
1987
1988
    if use_cache:
1989
        _cached_user_encoding = user_encoding
1990
1991
    return user_encoding
2091.1.1 by Martin Pool
Avoid MSG_WAITALL as it doesn't work on Windows
1992
1993
4797.57.10 by Alexander Belchenko
path_encoding selection logic extracted as helper function
1994
def get_diff_header_encoding():
5258.1.5 by Alexander Belchenko
tweak requested by Martin Pool: use terminal encoding for diff headers on all platforms
1995
    return get_terminal_encoding()
4797.57.10 by Alexander Belchenko
path_encoding selection logic extracted as helper function
1996
1997
3626.1.1 by Mark Hammond
Add osutils.get_host_name() to return a unicode hostname to prevent
1998
def get_host_name():
3626.1.4 by John Arbash Meinel
Document the difference in get_host_name, per Robert's request.
1999
    """Return the current unicode host name.
2000
2001
    This is meant to be used in place of socket.gethostname() because that
2002
    behaves inconsistently on different platforms.
2003
    """
3626.1.1 by Mark Hammond
Add osutils.get_host_name() to return a unicode hostname to prevent
2004
    if sys.platform == "win32":
2005
        import win32utils
2006
        return win32utils.get_host_name()
2007
    else:
2008
        import socket
2009
        return socket.gethostname().decode(get_user_encoding())
2010
2011
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2012
# We must not read/write any more than 64k at a time from/to a socket so we
2013
# don't risk "no buffer space available" errors on some platforms.  Windows in
2014
# particular is likely to throw WSAECONNABORTED or WSAENOBUFS if given too much
2015
# data at once.
2016
MAX_SOCKET_CHUNK = 64 * 1024
2017
5599.3.1 by John Arbash Meinel
Consider WSAECONNABORTED to be an end-of-stream as well as WSAECONNRESET.
2018
_end_of_stream_errors = [errno.ECONNRESET]
2019
for _eno in ['WSAECONNRESET', 'WSAECONNABORTED']:
2020
    _eno = getattr(errno, _eno, None)
2021
    if _eno is not None:
2022
        _end_of_stream_errors.append(_eno)
2023
del _eno
2024
2025
5011.3.12 by Andrew Bennetts
Make report_activity param of read_bytes_from_socket optional.
2026
def read_bytes_from_socket(sock, report_activity=None,
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2027
        max_read_size=MAX_SOCKET_CHUNK):
2028
    """Read up to max_read_size of bytes from sock and notify of progress.
2029
2030
    Translates "Connection reset by peer" into file-like EOF (return an
2031
    empty string rather than raise an error), and repeats the recv if
2032
    interrupted by a signal.
2033
    """
2034
    while 1:
2035
        try:
2036
            bytes = sock.recv(max_read_size)
2037
        except socket.error, e:
2038
            eno = e.args[0]
5599.3.1 by John Arbash Meinel
Consider WSAECONNABORTED to be an end-of-stream as well as WSAECONNRESET.
2039
            if eno in _end_of_stream_errors:
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2040
                # The connection was closed by the other side.  Callers expect
2041
                # an empty string to signal end-of-stream.
2042
                return ""
2043
            elif eno == errno.EINTR:
2044
                # Retry the interrupted recv.
2045
                continue
2046
            raise
2047
        else:
5011.3.12 by Andrew Bennetts
Make report_activity param of read_bytes_from_socket optional.
2048
            if report_activity is not None:
2049
                report_activity(len(bytes), 'read')
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2050
            return bytes
2051
2052
2053
def recv_all(socket, count):
2054
    """Receive an exact number of bytes.
2055
2056
    Regular Socket.recv() may return less than the requested number of bytes,
2057
    depending on what's in the OS buffer.  MSG_WAITALL is not available
2058
    on all platforms, but this should work everywhere.  This will return
2059
    less than the requested amount if the remote end closes.
2060
2061
    This isn't optimized and is intended mostly for use in testing.
2062
    """
2063
    b = ''
2064
    while len(b) < count:
5011.3.12 by Andrew Bennetts
Make report_activity param of read_bytes_from_socket optional.
2065
        new = read_bytes_from_socket(socket, None, count - len(b))
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2066
        if new == '':
2067
            break # eof
2068
        b += new
2069
    return b
2070
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2071
2072
def send_all(sock, bytes, report_activity=None):
2073
    """Send all bytes on a socket.
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
2074
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2075
    Breaks large blocks in smaller chunks to avoid buffering limitations on
2076
    some platforms, and catches EINTR which may be thrown if the send is
2077
    interrupted by a signal.
2078
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2079
    This is preferred to socket.sendall(), because it avoids portability bugs
2080
    and provides activity reporting.
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
2081
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2082
    :param report_activity: Call this as bytes are read, see
2083
        Transport._report_activity
2084
    """
2085
    sent_total = 0
2086
    byte_count = len(bytes)
2087
    while sent_total < byte_count:
2088
        try:
5011.3.11 by Andrew Bennetts
Consolidate changes, try to minimise unnecessary changes and tidy up those that kept.
2089
            sent = sock.send(buffer(bytes, sent_total, MAX_SOCKET_CHUNK))
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2090
        except socket.error, e:
2091
            if e.args[0] != errno.EINTR:
2092
                raise
2093
        else:
2094
            sent_total += sent
2095
            report_activity(sent, 'write')
3118.2.1 by Andrew Bennetts
(andrew) Fix #115781 by passing no more than 64k at a time to socket.sendall.
2096
5247.5.14 by Vincent Ladeuil
Fix the helper again, the python one is bogus :-/
2097
5247.5.29 by Vincent Ladeuil
Fixed as per jam's review.
2098
def connect_socket(address):
2099
    # Slight variation of the socket.create_connection() function (provided by
2100
    # python-2.6) that can fail if getaddrinfo returns an empty list. We also
2101
    # provide it for previous python versions. Also, we don't use the timeout
2102
    # parameter (provided by the python implementation) so we don't implement
2103
    # it either).
5247.5.14 by Vincent Ladeuil
Fix the helper again, the python one is bogus :-/
2104
    err = socket.error('getaddrinfo returns an empty list')
2105
    host, port = address
2106
    for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
2107
        af, socktype, proto, canonname, sa = res
2108
        sock = None
2109
        try:
2110
            sock = socket.socket(af, socktype, proto)
2111
            sock.connect(sa)
2112
            return sock
2113
5247.5.15 by Vincent Ladeuil
Hopefully, the right fix this time.
2114
        except socket.error, err:
5247.5.14 by Vincent Ladeuil
Fix the helper again, the python one is bogus :-/
2115
            # 'err' is now the most recent error
2116
            if sock is not None:
2117
                sock.close()
2118
    raise err
5247.3.7 by Vincent Ladeuil
Provide connect_socket (socket.create_connection) for pythons older than 2.6.
2119
3118.2.1 by Andrew Bennetts
(andrew) Fix #115781 by passing no more than 64k at a time to socket.sendall.
2120
2091.3.7 by Aaron Bentley
Rename real_parent to dereferenced_path
2121
def dereference_path(path):
2122
    """Determine the real path to a file.
2123
2124
    All parent elements are dereferenced.  But the file itself is not
2125
    dereferenced.
2126
    :param path: The original path.  May be absolute or relative.
2127
    :return: the real path *to* the file
2128
    """
2091.3.5 by Aaron Bentley
Move realpath functionality into osutils
2129
    parent, base = os.path.split(path)
2130
    # The pathjoin for '.' is a workaround for Python bug #1213894.
2131
    # (initial path components aren't dereferenced)
2132
    return pathjoin(realpath(pathjoin('.', parent)), base)
2681.3.4 by Lukáš Lalinsky
- Rename 'windows' to 'mapi'
2133
2134
2135
def supports_mapi():
2136
    """Return True if we can use MAPI to launch a mail client."""
2137
    return sys.platform == "win32"
3089.3.8 by Ian Clatworthy
move resource loading into a reusable function
2138
2139
2140
def resource_string(package, resource_name):
2141
    """Load a resource from a package and return it as a string.
2142
2143
    Note: Only packages that start with bzrlib are currently supported.
2144
2145
    This is designed to be a lightweight implementation of resource
2146
    loading in a way which is API compatible with the same API from
2147
    pkg_resources. See
2148
    http://peak.telecommunity.com/DevCenter/PkgResources#basic-resource-access.
2149
    If and when pkg_resources becomes a standard library, this routine
2150
    can delegate to it.
2151
    """
2152
    # Check package name is within bzrlib
2153
    if package == "bzrlib":
2154
        resource_relpath = resource_name
2155
    elif package.startswith("bzrlib."):
2156
        package = package[len("bzrlib."):].replace('.', os.sep)
2157
        resource_relpath = pathjoin(package, resource_name)
2158
    else:
2159
        raise errors.BzrError('resource package %s not in bzrlib' % package)
2160
2161
    # Map the resource to a file and read its contents
2162
    base = dirname(bzrlib.__file__)
2163
    if getattr(sys, 'frozen', None):    # bzr.exe
2164
        base = abspath(pathjoin(base, '..', '..'))
4708.2.2 by Martin
Workingtree changes sitting around since November, more explict closing of files in bzrlib
2165
    f = file(pathjoin(base, resource_relpath), "rU")
2166
    try:
2167
        return f.read()
2168
    finally:
2169
        f.close()
1739.2.7 by Robert Collins
Update readdir pyrex source files and usage in line with current practice.
2170
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
2171
def file_kind_from_stat_mode_thunk(mode):
2172
    global file_kind_from_stat_mode
2173
    if file_kind_from_stat_mode is file_kind_from_stat_mode_thunk:
2174
        try:
2175
            from bzrlib._readdir_pyx import UTF8DirReader
2176
            file_kind_from_stat_mode = UTF8DirReader().kind_from_mode
4574.3.6 by Martin Pool
More warnings when failing to load extensions
2177
        except ImportError, e:
4694.2.1 by John Arbash Meinel
Fix bug #430645, don't issue a warning when failing to import _readdir_pyx the second time.
2178
            # This is one time where we won't warn that an extension failed to
2179
            # load. The extension is never available on Windows anyway.
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
2180
            from bzrlib._readdir_py import (
3696.4.8 by Robert Collins
Fix up inter_changes with dirstate both C and python.
2181
                _kind_from_mode as file_kind_from_stat_mode
3696.3.5 by Robert Collins
Streamline _walkdirs_utf8 for utf8 file systems, reducing time to traverse a mozilla tree from 1s to .6 seconds. (Robert Collins)
2182
                )
2183
    return file_kind_from_stat_mode(mode)
2184
file_kind_from_stat_mode = file_kind_from_stat_mode_thunk
2185
2186
2187
def file_kind(f, _lstat=os.lstat):
2188
    try:
2189
        return file_kind_from_stat_mode(_lstat(f).st_mode)
2190
    except OSError, e:
2191
        if getattr(e, 'errno', None) in (errno.ENOENT, errno.ENOTDIR):
2192
            raise errors.NoSuchFile(f)
2193
        raise
2194
3923.3.1 by Andrew Bennetts
Quick attempt at adding some EINTR-proofing to smart protocol code.
2195
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2196
def until_no_eintr(f, *a, **kw):
2197
    """Run f(*a, **kw), retrying if an EINTR error occurs.
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
2198
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2199
    WARNING: you must be certain that it is safe to retry the call repeatedly
2200
    if EINTR does occur.  This is typically only true for low-level operations
2201
    like os.read.  If in any doubt, don't use this.
5011.3.5 by Andrew Bennetts
Expand until_no_eintr's docstring more with some explanation for why it is not a complete solution.
2202
2203
    Keep in mind that this is not a complete solution to EINTR.  There is
2204
    probably code in the Python standard library and other dependencies that
2205
    may encounter EINTR if a signal arrives (and there is signal handler for
2206
    that signal).  So this function can reduce the impact for IO that bzrlib
2207
    directly controls, but it is not a complete solution.
5011.3.4 by Andrew Bennetts
Reinstate osutils.until_no_eintr and .send_all, reapply until_no_eintr in SmartSimplePipesClientMedium.read_bytes.
2208
    """
2209
    # Borrowed from Twisted's twisted.python.util.untilConcludes function.
2210
    while True:
2211
        try:
2212
            return f(*a, **kw)
2213
        except (IOError, OSError), e:
2214
            if e.errno == errno.EINTR:
2215
                continue
2216
            raise
2217
2218
5326.2.6 by Parth Malwankar
deprecate re_compile_checked rather than remove it.
2219
@deprecated_function(deprecated_in((2, 2, 0)))
2220
def re_compile_checked(re_string, flags=0, where=""):
2221
    """Return a compiled re, or raise a sensible error.
2222
2223
    This should only be used when compiling user-supplied REs.
2224
2225
    :param re_string: Text form of regular expression.
2226
    :param flags: eg re.IGNORECASE
2227
    :param where: Message explaining to the user the context where
2228
        it occurred, eg 'log search filter'.
2229
    """
2230
    # from https://bugs.launchpad.net/bzr/+bug/251352
2231
    try:
2232
        re_obj = re.compile(re_string, flags)
2233
        re_obj.search("")
2234
        return re_obj
2235
    except errors.InvalidPattern, e:
2236
        if where:
2237
            where = ' in ' + where
2238
        # despite the name 'error' is a type
2239
        raise errors.BzrCommandError('Invalid regular expression%s: %s'
5339.1.1 by Parth Malwankar
fixes errors.InvalidPattern to work on Python2.5
2240
            % (where, e.msg))
5326.2.6 by Parth Malwankar
deprecate re_compile_checked rather than remove it.
2241
2242
0.16.79 by Aaron Bentley
Remove dependencies on bzrtools
2243
if sys.platform == "win32":
2244
    import msvcrt
2245
    def getchar():
2246
        return msvcrt.getch()
2247
else:
2248
    import tty
2249
    import termios
2250
    def getchar():
2251
        fd = sys.stdin.fileno()
2252
        settings = termios.tcgetattr(fd)
2253
        try:
2254
            tty.setraw(fd)
2255
            ch = sys.stdin.read(1)
2256
        finally:
2257
            termios.tcsetattr(fd, termios.TCSADRAIN, settings)
2258
        return ch
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2259
5677.1.3 by Jelmer Vernooij
Use try/import rather than checking python version strings.
2260
if sys.platform == 'linux2':
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2261
    def _local_concurrency():
5677.1.2 by Jelmer Vernooij
Use os.sysconf to get the number of CPUs on Linux. Use sysctl to get the number of CPUs on all *BSDs, not just FreeBSD.
2262
        try:
2263
            return os.sysconf('SC_NPROCESSORS_ONLN')
2264
        except (ValueError, OSError, AttributeError):
2265
            return None
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2266
elif sys.platform == 'darwin':
2267
    def _local_concurrency():
2268
        return subprocess.Popen(['sysctl', '-n', 'hw.availcpu'],
2269
                                stdout=subprocess.PIPE).communicate()[0]
5677.1.2 by Jelmer Vernooij
Use os.sysconf to get the number of CPUs on Linux. Use sysctl to get the number of CPUs on all *BSDs, not just FreeBSD.
2270
elif "bsd" in sys.platform:
4413.1.1 by Matthew Fuller
Catch the number of cores on FreeBSD too.
2271
    def _local_concurrency():
2272
        return subprocess.Popen(['sysctl', '-n', 'hw.ncpu'],
2273
                                stdout=subprocess.PIPE).communicate()[0]
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2274
elif sys.platform == 'sunos5':
2275
    def _local_concurrency():
2276
        return subprocess.Popen(['psrinfo', '-p',],
2277
                                stdout=subprocess.PIPE).communicate()[0]
2278
elif sys.platform == "win32":
2279
    def _local_concurrency():
4398.4.4 by Vincent Ladeuil
Fixed as per John's review.
2280
        # This appears to return the number of cores.
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2281
        return os.environ.get('NUMBER_OF_PROCESSORS')
2282
else:
2283
    def _local_concurrency():
2284
        # Who knows ?
2285
        return None
2286
2287
4398.4.4 by Vincent Ladeuil
Fixed as per John's review.
2288
_cached_local_concurrency = None
2289
2290
def local_concurrency(use_cache=True):
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2291
    """Return how many processes can be run concurrently.
2292
2293
    Rely on platform specific implementations and default to 1 (one) if
2294
    anything goes wrong.
2295
    """
4398.4.4 by Vincent Ladeuil
Fixed as per John's review.
2296
    global _cached_local_concurrency
4766.3.4 by Matt Nordhoff
Change the environment variable to a global option.
2297
4398.4.4 by Vincent Ladeuil
Fixed as per John's review.
2298
    if _cached_local_concurrency is not None and use_cache:
2299
        return _cached_local_concurrency
2300
4766.3.7 by Vincent Ladeuil
Mix BZR_CONCURRENCY and --concurrency so both are available.
2301
    concurrency = os.environ.get('BZR_CONCURRENCY', None)
2302
    if concurrency is None:
2303
        try:
5677.1.3 by Jelmer Vernooij
Use try/import rather than checking python version strings.
2304
            import multiprocessing
2305
        except ImportError:
2306
            # multiprocessing is only available on Python >= 2.6
2307
            try:
2308
                concurrency = _local_concurrency()
2309
            except (OSError, IOError):
2310
                pass
2311
        else:
2312
            concurrency = multiprocessing.cpu_count()
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2313
    try:
2314
        concurrency = int(concurrency)
2315
    except (TypeError, ValueError):
2316
        concurrency = 1
4398.4.4 by Vincent Ladeuil
Fixed as per John's review.
2317
    if use_cache:
2318
        _cached_concurrency = concurrency
4398.4.3 by Vincent Ladeuil
Detect # cores on win32 and Solaris too.
2319
    return concurrency
4794.1.12 by Robert Collins
Create a StreamWriter helper that doesn't trigger implicit decode('ascii') on write(a_str).
2320
2321
4794.1.15 by Robert Collins
Review feedback.
2322
class UnicodeOrBytesToBytesWriter(codecs.StreamWriter):
4794.1.12 by Robert Collins
Create a StreamWriter helper that doesn't trigger implicit decode('ascii') on write(a_str).
2323
    """A stream writer that doesn't decode str arguments."""
2324
4794.1.21 by Robert Collins
Python 2.4 doesn't use CodecInfo, so do a type check on the result of codecs.lookup.
2325
    def __init__(self, encode, stream, errors='strict'):
4794.1.12 by Robert Collins
Create a StreamWriter helper that doesn't trigger implicit decode('ascii') on write(a_str).
2326
        codecs.StreamWriter.__init__(self, stream, errors)
4794.1.21 by Robert Collins
Python 2.4 doesn't use CodecInfo, so do a type check on the result of codecs.lookup.
2327
        self.encode = encode
4794.1.12 by Robert Collins
Create a StreamWriter helper that doesn't trigger implicit decode('ascii') on write(a_str).
2328
2329
    def write(self, object):
4794.1.15 by Robert Collins
Review feedback.
2330
        if type(object) is str:
4794.1.12 by Robert Collins
Create a StreamWriter helper that doesn't trigger implicit decode('ascii') on write(a_str).
2331
            self.stream.write(object)
2332
        else:
2333
            data, _ = self.encode(object, self.errors)
2334
            self.stream.write(data)
4797.2.27 by Vincent Ladeuil
Merge 2.0 into 2.1 including fix for #524560
2335
4634.140.4 by INADA Naoki
Fix easy miss in previous commit.
2336
if sys.platform == 'win32':
4634.140.10 by INADA Naoki
Change name from osutils.open to osutils.open_file
2337
    def open_file(filename, mode='r', bufsize=-1):
4634.140.13 by Vincent Ladeuil
Fix some typos and add a NEWS entry.
2338
        """This function is used to override the ``open`` builtin.
5279.2.7 by Eric Moritz
1. deleted trailing whitespace
2339
4634.140.13 by Vincent Ladeuil
Fix some typos and add a NEWS entry.
2340
        But it uses O_NOINHERIT flag so the file handle is not inherited by
2341
        child processes.  Deleting or renaming a closed file opened with this
2342
        function is not blocking child processes.
4634.140.6 by INADA Naoki
Add comment to osutils.open()
2343
        """
4634.140.9 by INADA Naoki
Revert to previous implementation using os.fdopen(os.open())
2344
        writing = 'w' in mode
2345
        appending = 'a' in mode
2346
        updating = '+' in mode
2347
        binary = 'b' in mode
2348
4634.140.12 by INADA Naoki
small clean up.
2349
        flags = O_NOINHERIT
4634.140.9 by INADA Naoki
Revert to previous implementation using os.fdopen(os.open())
2350
        # see http://msdn.microsoft.com/en-us/library/yeby3zcb%28VS.71%29.aspx
2351
        # for flags for each modes.
2352
        if binary:
4634.140.12 by INADA Naoki
small clean up.
2353
            flags |= O_BINARY
4634.140.9 by INADA Naoki
Revert to previous implementation using os.fdopen(os.open())
2354
        else:
4634.140.12 by INADA Naoki
small clean up.
2355
            flags |= O_TEXT
4634.140.9 by INADA Naoki
Revert to previous implementation using os.fdopen(os.open())
2356
2357
        if writing:
2358
            if updating:
2359
                flags |= os.O_RDWR
2360
            else:
2361
                flags |= os.O_WRONLY
2362
            flags |= os.O_CREAT | os.O_TRUNC
2363
        elif appending:
2364
            if updating:
2365
                flags |= os.O_RDWR
2366
            else:
2367
                flags |= os.O_WRONLY
2368
            flags |= os.O_CREAT | os.O_APPEND
2369
        else: #reading
2370
            if updating:
2371
                flags |= os.O_RDWR
2372
            else:
2373
                flags |= os.O_RDONLY
2374
2375
        return os.fdopen(os.open(filename, flags), mode, bufsize)
4634.140.2 by INADA Naoki
Add osutils.open() that uses O_NOINHERIT on Win32.
2376
else:
4634.140.10 by INADA Naoki
Change name from osutils.open to osutils.open_file
2377
    open_file = open
5187.2.7 by Parth Malwankar
moved getuser_unicode to osutils.
2378
2379
2380
def getuser_unicode():
2381
    """Return the username as unicode.
2382
    """
2383
    try:
2384
        user_encoding = get_user_encoding()
2385
        username = getpass.getuser().decode(user_encoding)
2386
    except UnicodeDecodeError:
2387
        raise errors.BzrError("Can't decode username as %s." % \
2388
                user_encoding)
2389
    return username
5409.1.7 by Vincent Ladeuil
First orphaning implementation (some tests lacking).
2390
2391
5409.5.3 by Vincent Ladeuil
Implement osutils.available_backup_name.
2392
def available_backup_name(base, exists):
2393
    """Find a non-existing backup file name.
2394
5409.5.6 by Vincent Ladeuil
Add NEWS entry, tweak osutils.available_backup_name docstring.
2395
    This will *not* create anything, this only return a 'free' entry.  This
2396
    should be used for checking names in a directory below a locked
5409.5.8 by Vincent Ladeuil
Be more explicit about race conditions and LBYL being discouraged
2397
    tree/branch/repo to avoid race conditions. This is LBYL (Look Before You
2398
    Leap) and generally discouraged.
5409.5.3 by Vincent Ladeuil
Implement osutils.available_backup_name.
2399
2400
    :param base: The base name.
5409.5.8 by Vincent Ladeuil
Be more explicit about race conditions and LBYL being discouraged
2401
2402
    :param exists: A callable returning True if the path parameter exists.
5409.5.3 by Vincent Ladeuil
Implement osutils.available_backup_name.
2403
    """
2404
    counter = 1
2405
    name = "%s.~%d~" % (base, counter)
2406
    while exists(name):
2407
        counter += 1
5409.1.7 by Vincent Ladeuil
First orphaning implementation (some tests lacking).
2408
        name = "%s.~%d~" % (base, counter)
5409.5.3 by Vincent Ladeuil
Implement osutils.available_backup_name.
2409
    return name
5321.1.79 by Gordon Tyler
Added is_executable_on_path to osutils based on _probe from ExecutableFeature.
2410
2411
5582.6.1 by Max Bowsher
Fix socketpair-based SSH transport leaking socket into other child processes.
2412
def set_fd_cloexec(fd):
2413
    """Set a Unix file descriptor's FD_CLOEXEC flag.  Do nothing if platform
2414
    support for this is not available.
2415
    """
2416
    try:
2417
        import fcntl
2418
        old = fcntl.fcntl(fd, fcntl.F_GETFD)
2419
        fcntl.fcntl(fd, fcntl.F_SETFD, old | fcntl.FD_CLOEXEC)
2420
    except (ImportError, AttributeError):
2421
        # Either the fcntl module or specific constants are not present
2422
        pass
5321.1.114 by Gordon Tyler
Merged from bzr.dev.
2423
2424
5321.1.80 by Gordon Tyler
Changed is_executable_on_path to find_executable_on_path to make it more useful.
2425
def find_executable_on_path(name):
2426
    """Finds an executable on the PATH.
5321.1.79 by Gordon Tyler
Added is_executable_on_path to osutils based on _probe from ExecutableFeature.
2427
    
2428
    On Windows, this will try to append each extension in the PATHEXT
2429
    environment variable to the name, if it cannot be found with the name
2430
    as given.
2431
    
2432
    :param name: The base name of the executable.
5321.1.80 by Gordon Tyler
Changed is_executable_on_path to find_executable_on_path to make it more useful.
2433
    :return: The path to the executable found or None.
5321.1.79 by Gordon Tyler
Added is_executable_on_path to osutils based on _probe from ExecutableFeature.
2434
    """
2435
    path = os.environ.get('PATH')
2436
    if path is None:
5321.1.81 by Gordon Tyler
find_executable_on_path should return None when the PATH is not defined.
2437
        return None
5321.1.106 by Gordon Tyler
Fixed find_executable_on_path to properly test for executable-ness on win32 and not split the PATH for each extension in PATHEXT.
2438
    path = path.split(os.pathsep)
5321.1.105 by Gordon Tyler
Fixed find_executable_on_path to only check PATHEXT on win32.
2439
    if sys.platform == 'win32':
2440
        exts = os.environ.get('PATHEXT', '').split(os.pathsep)
5321.1.106 by Gordon Tyler
Fixed find_executable_on_path to properly test for executable-ness on win32 and not split the PATH for each extension in PATHEXT.
2441
        exts = [ext.lower() for ext in exts]
2442
        base, ext = os.path.splitext(name)
2443
        if ext != '':
2444
            if ext.lower() not in exts:
2445
                return None
2446
            name = base
2447
            exts = [ext]
5321.1.105 by Gordon Tyler
Fixed find_executable_on_path to only check PATHEXT on win32.
2448
    else:
2449
        exts = ['']
5321.1.79 by Gordon Tyler
Added is_executable_on_path to osutils based on _probe from ExecutableFeature.
2450
    for ext in exts:
5321.1.106 by Gordon Tyler
Fixed find_executable_on_path to properly test for executable-ness on win32 and not split the PATH for each extension in PATHEXT.
2451
        for d in path:
2452
            f = os.path.join(d, name) + ext
2453
            if os.access(f, os.X_OK):
2454
                return f
5321.1.80 by Gordon Tyler
Changed is_executable_on_path to find_executable_on_path to make it more useful.
2455
    return None