~bzr-pqm/bzr/bzr.dev

1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
1
# Copyright (C) 2005, 2006 Canonical Ltd
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
2
#
1185.11.19 by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings.
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
7
#
1185.11.19 by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
1887.1.1 by Adeodato Simó
Do not separate paragraphs in the copyright statement with blank lines,
12
#
1185.11.19 by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
1185.16.72 by Martin Pool
[merge] from robert and fix up tests
16
17
"""Transport for the local filesystem.
18
1755.1.3 by Robert Collins
Fix regression in LocalTransport to allow merging.
19
This is a fairly thin wrapper on regular file IO.
20
"""
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
21
1996.3.17 by John Arbash Meinel
lazy_import plugin and transport/local
22
import os
23
from stat import ST_MODE, S_ISDIR, ST_SIZE, S_IMODE
24
import sys
25
26
from bzrlib.lazy_import import lazy_import
27
lazy_import(globals(), """
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
28
import errno
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
29
import shutil
30
1908.4.2 by John Arbash Meinel
Delay evaluating PathError.extra, and use fstat() instead of seek + tell, and we can check if we need to chmod(). Saves about 3/90 seconds of commit time
31
from bzrlib import (
1955.3.6 by John Arbash Meinel
Lots of deprecation warnings, but no errors
32
    atomicfile,
1908.4.2 by John Arbash Meinel
Delay evaluating PathError.extra, and use fstat() instead of seek + tell, and we can check if we need to chmod(). Saves about 3/90 seconds of commit time
33
    osutils,
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
34
    urlutils,
1996.3.17 by John Arbash Meinel
lazy_import plugin and transport/local
35
    symbol_versioning,
2671.3.2 by Robert Collins
Start open_file_stream logic.
36
    transport,
1908.4.2 by John Arbash Meinel
Delay evaluating PathError.extra, and use fstat() instead of seek + tell, and we can check if we need to chmod(). Saves about 3/90 seconds of commit time
37
    )
1685.1.45 by John Arbash Meinel
Moved url functions into bzrlib.urlutils
38
from bzrlib.trace import mutter
2052.6.2 by Robert Collins
Merge bzr.dev.
39
from bzrlib.transport import LateReadError
1996.3.17 by John Arbash Meinel
lazy_import plugin and transport/local
40
""")
41
1685.1.45 by John Arbash Meinel
Moved url functions into bzrlib.urlutils
42
from bzrlib.transport import Transport, Server
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
43
44
45
_append_flags = os.O_CREAT | os.O_APPEND | os.O_WRONLY | osutils.O_BINARY
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
46
_put_non_atomic_flags = os.O_CREAT | os.O_TRUNC | os.O_WRONLY | osutils.O_BINARY
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
47
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
48
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
49
class LocalTransport(Transport):
50
    """This is the transport agent for local filesystem access."""
51
52
    def __init__(self, base):
53
        """Set the base path where files will be stored."""
1685.1.9 by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url
54
        if not base.startswith('file://'):
1996.3.17 by John Arbash Meinel
lazy_import plugin and transport/local
55
            symbol_versioning.warn(
56
                "Instantiating LocalTransport with a filesystem path"
1685.1.9 by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url
57
                " is deprecated as of bzr 0.8."
58
                " Please use bzrlib.transport.get_transport()"
59
                " or pass in a file:// url.",
60
                 DeprecationWarning,
61
                 stacklevel=2
62
                 )
1685.1.45 by John Arbash Meinel
Moved url functions into bzrlib.urlutils
63
            base = urlutils.local_path_to_url(base)
1530.1.3 by Robert Collins
transport implementations now tested consistently.
64
        if base[-1] != '/':
65
            base = base + '/'
3503.1.2 by adwi2
Permits Windows to serve all paths on all drives.
66
67
        # Special case : windows has no "root", but does have
68
        # multiple lettered drives inside it. #240910
69
        if sys.platform == 'win32' and base == 'file:///':
70
            base = ''
71
            self._local_base = ''
72
            super(LocalTransport, self).__init__(base)
73
            return
74
            
1530.1.3 by Robert Collins
transport implementations now tested consistently.
75
        super(LocalTransport, self).__init__(base)
1685.1.45 by John Arbash Meinel
Moved url functions into bzrlib.urlutils
76
        self._local_base = urlutils.local_path_from_url(base)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
77
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
78
    def clone(self, offset=None):
79
        """Return a new LocalTransport with root at self.base + offset
80
        Because the local filesystem does not require a connection, 
81
        we can just return a new object.
82
        """
83
        if offset is None:
84
            return LocalTransport(self.base)
85
        else:
2245.6.1 by Alexander Belchenko
win32 UNC path: recursive cloning UNC path to root stops on //HOST, not on //
86
            abspath = self.abspath(offset)
87
            if abspath == 'file://':
88
                # fix upwalk for UNC path
89
                # when clone from //HOST/path updir recursively
90
                # we should stop at least at //HOST part
91
                abspath = self.base
92
            return LocalTransport(abspath)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
93
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
94
    def _abspath(self, relative_reference):
95
        """Return a path for use in os calls.
96
97
        Several assumptions are made:
98
         - relative_reference does not contain '..'
99
         - relative_reference is url escaped.
100
        """
1755.1.3 by Robert Collins
Fix regression in LocalTransport to allow merging.
101
        if relative_reference in ('.', ''):
102
            return self._local_base
1755.1.2 by Robert Collins
(robertc, ab)Merge some commit and fetch tuning steps.
103
        return self._local_base + urlutils.unescape(relative_reference)
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
104
907.1.8 by John Arbash Meinel
Changed the format for abspath. Updated branch to use a hidden _transport
105
    def abspath(self, relpath):
1636.1.1 by Robert Collins
Fix calling relpath() and abspath() on transports at their root.
106
        """Return the full url to the given relative URL."""
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
107
        # TODO: url escape the result. RBC 20060523.
1685.1.12 by John Arbash Meinel
Some more work to get LocalTransport to only support URLs
108
        # jam 20060426 Using normpath on the real path, because that ensures
109
        #       proper handling of stuff like
1996.3.17 by John Arbash Meinel
lazy_import plugin and transport/local
110
        path = osutils.normpath(osutils.pathjoin(
111
                    self._local_base, urlutils.unescape(relpath)))
3693.2.1 by Mark Hammond
ensure LocalTransport.abspath() returns an absolute URL with the same
112
        # on windows, our _local_base may or may not have a drive specified
113
        # (ie, it may be "/" or "c:/foo").
114
        # If 'relpath' is '/' we *always* get back an abspath without
115
        # the drive letter - but if our transport already has a drive letter,
116
        # we want our abspaths to have a drive letter too - so handle that
117
        # here.
118
        if (sys.platform == "win32" and self._local_base[1:2] == ":"
119
            and path == '/'):
120
            path = self._local_base[:3]
121
1685.1.45 by John Arbash Meinel
Moved url functions into bzrlib.urlutils
122
        return urlutils.local_path_to_url(path)
1685.1.9 by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url
123
124
    def local_abspath(self, relpath):
125
        """Transform the given relative path URL into the actual path on disk
126
127
        This function only exists for the LocalTransport, since it is
128
        the only one that has direct local access.
129
        This is mostly for stuff like WorkingTree which needs to know
3616.2.10 by Mark Hammond
docstring for transport.local.local_abspath() avoids potential confusion
130
        the local working directory.  The returned path will always contain
131
        forward slashes as the path separator, regardless of the platform.
132
1725.2.9 by Robert Collins
Merge current head.
133
        This function is quite expensive: it calls realpath which resolves
134
        symlinks.
1685.1.9 by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url
135
        """
136
        absurl = self.abspath(relpath)
137
        # mutter(u'relpath %s => base: %s, absurl %s', relpath, self.base, absurl)
1685.1.45 by John Arbash Meinel
Moved url functions into bzrlib.urlutils
138
        return urlutils.local_path_from_url(absurl)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
139
907.1.24 by John Arbash Meinel
Remote functionality work.
140
    def relpath(self, abspath):
141
        """Return the local path portion from a given absolute path.
142
        """
1442.1.64 by Robert Collins
Branch.open_containing now returns a tuple (Branch, relative-path).
143
        if abspath is None:
1185.33.66 by Martin Pool
[patch] use unicode literals for all hardcoded paths (Alexander Belchenko)
144
            abspath = u'.'
1551.2.53 by abentley
Strip trailing slashes in a platform-sensible way
145
1685.1.45 by John Arbash Meinel
Moved url functions into bzrlib.urlutils
146
        return urlutils.file_relpath(
2661.2.2 by Robert Collins
* ``bzrlib.pack.make_readv_reader`` allows readv based access to pack
147
            urlutils.strip_trailing_slash(self.base),
1685.1.45 by John Arbash Meinel
Moved url functions into bzrlib.urlutils
148
            urlutils.strip_trailing_slash(abspath))
907.1.24 by John Arbash Meinel
Remote functionality work.
149
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
150
    def has(self, relpath):
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
151
        return os.access(self._abspath(relpath), os.F_OK)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
152
2164.2.15 by Vincent Ladeuil
Http redirections are not followed by default. Do not use hints
153
    def get(self, relpath):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
154
        """Get the file at the given relative path.
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
155
156
        :param relpath: The relative path to the file
157
        """
2671.3.4 by Robert Collins
Sync up with open file streams on get/get_bytes.
158
        canonical_url = self.abspath(relpath)
159
        if canonical_url in transport._file_streams:
160
            transport._file_streams[canonical_url].flush()
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
161
        try:
1908.4.11 by John Arbash Meinel
reverting changes to errors.py and local transport.
162
            path = self._abspath(relpath)
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
163
            return open(path, 'rb')
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
164
        except (IOError, OSError),e:
2052.6.1 by Robert Collins
``Transport.get`` has had its interface made more clear for ease of use.
165
            if e.errno == errno.EISDIR:
166
                return LateReadError(relpath)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
167
            self._translate_error(e, path)
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
168
1955.3.6 by John Arbash Meinel
Lots of deprecation warnings, but no errors
169
    def put_file(self, relpath, f, mode=None):
1946.1.4 by John Arbash Meinel
Basic implementation for local transport
170
        """Copy the file-like object into the location.
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
171
172
        :param relpath: Location to put the contents, relative to base.
1946.1.4 by John Arbash Meinel
Basic implementation for local transport
173
        :param f:       File-like object.
174
        :param mode: The mode for the newly created file, 
175
                     None means just use the default
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
176
        """
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
177
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
178
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
179
        try:
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
180
            path = self._abspath(relpath)
1996.3.17 by John Arbash Meinel
lazy_import plugin and transport/local
181
            osutils.check_legal_path(path)
1955.3.6 by John Arbash Meinel
Lots of deprecation warnings, but no errors
182
            fp = atomicfile.AtomicFile(path, 'wb', new_mode=mode)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
183
        except (IOError, OSError),e:
184
            self._translate_error(e, path)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
185
        try:
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
186
            length = self._pump(f, fp)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
187
            fp.commit()
188
        finally:
189
            fp.close()
2745.5.2 by Robert Collins
* ``bzrlib.transport.Transport.put_file`` now returns the number of bytes
190
        return length
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
191
1955.3.6 by John Arbash Meinel
Lots of deprecation warnings, but no errors
192
    def put_bytes(self, relpath, bytes, mode=None):
193
        """Copy the string into the location.
194
195
        :param relpath: Location to put the contents, relative to base.
196
        :param bytes:   String
197
        """
198
199
        path = relpath
200
        try:
201
            path = self._abspath(relpath)
1996.3.17 by John Arbash Meinel
lazy_import plugin and transport/local
202
            osutils.check_legal_path(path)
1955.3.6 by John Arbash Meinel
Lots of deprecation warnings, but no errors
203
            fp = atomicfile.AtomicFile(path, 'wb', new_mode=mode)
204
        except (IOError, OSError),e:
205
            self._translate_error(e, path)
206
        try:
207
            fp.write(bytes)
208
            fp.commit()
209
        finally:
210
            fp.close()
211
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
212
    def _put_non_atomic_helper(self, relpath, writer,
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
213
                               mode=None,
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
214
                               create_parent_dir=False,
215
                               dir_mode=None):
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
216
        """Common functionality information for the put_*_non_atomic.
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
217
218
        This tracks all the create_parent_dir stuff.
219
220
        :param relpath: the path we are putting to.
221
        :param writer: A function that takes an os level file descriptor
222
            and writes whatever data it needs to write there.
223
        :param mode: The final file mode.
224
        :param create_parent_dir: Should we be creating the parent directory
225
            if it doesn't exist?
1946.1.4 by John Arbash Meinel
Basic implementation for local transport
226
        """
227
        abspath = self._abspath(relpath)
228
        if mode is None:
229
            # os.open() will automatically use the umask
230
            local_mode = 0666
231
        else:
232
            local_mode = mode
233
        try:
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
234
            fd = os.open(abspath, _put_non_atomic_flags, local_mode)
1946.1.4 by John Arbash Meinel
Basic implementation for local transport
235
        except (IOError, OSError),e:
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
236
            # We couldn't create the file, maybe we need to create
237
            # the parent directory, and try again
238
            if (not create_parent_dir
239
                or e.errno not in (errno.ENOENT,errno.ENOTDIR)):
240
                self._translate_error(e, relpath)
241
            parent_dir = os.path.dirname(abspath)
242
            if not parent_dir:
243
                self._translate_error(e, relpath)
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
244
            self._mkdir(parent_dir, mode=dir_mode)
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
245
            # We created the parent directory, lets try to open the
246
            # file again
247
            try:
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
248
                fd = os.open(abspath, _put_non_atomic_flags, local_mode)
1946.1.8 by John Arbash Meinel
Update non_atomic_put to have a create_parent_dir flag
249
            except (IOError, OSError), e:
250
                self._translate_error(e, relpath)
1946.1.4 by John Arbash Meinel
Basic implementation for local transport
251
        try:
252
            st = os.fstat(fd)
253
            if mode is not None and mode != S_IMODE(st.st_mode):
254
                # Because of umask, we may still need to chmod the file.
255
                # But in the general case, we won't have to
256
                os.chmod(abspath, mode)
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
257
            writer(fd)
1946.1.4 by John Arbash Meinel
Basic implementation for local transport
258
        finally:
259
            os.close(fd)
260
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
261
    def put_file_non_atomic(self, relpath, f, mode=None,
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
262
                            create_parent_dir=False,
263
                            dir_mode=None):
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
264
        """Copy the file-like object into the target location.
265
266
        This function is not strictly safe to use. It is only meant to
267
        be used when you already know that the target does not exist.
268
        It is not safe, because it will open and truncate the remote
269
        file. So there may be a time when the file has invalid contents.
270
271
        :param relpath: The remote location to put the contents.
272
        :param f:       File-like object.
273
        :param mode:    Possible access permissions for new file.
274
                        None means do not set remote permissions.
275
        :param create_parent_dir: If we cannot create the target file because
276
                        the parent directory does not exist, go ahead and
277
                        create it, and then try again.
278
        """
279
        def writer(fd):
280
            self._pump_to_fd(f, fd)
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
281
        self._put_non_atomic_helper(relpath, writer, mode=mode,
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
282
                                    create_parent_dir=create_parent_dir,
283
                                    dir_mode=dir_mode)
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
284
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
285
    def put_bytes_non_atomic(self, relpath, bytes, mode=None,
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
286
                             create_parent_dir=False, dir_mode=None):
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
287
        def writer(fd):
288
            os.write(fd, bytes)
1955.3.27 by John Arbash Meinel
rename non_atomic_put_* to put_*non_atomic, and re-order the functions
289
        self._put_non_atomic_helper(relpath, writer, mode=mode,
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
290
                                    create_parent_dir=create_parent_dir,
291
                                    dir_mode=dir_mode)
1955.3.21 by John Arbash Meinel
Update the LocalTransport and SftpTransport to implement non_atomic_*
292
1442.1.44 by Robert Collins
Many transport related tweaks:
293
    def iter_files_recursive(self):
294
        """Iter the relative paths of files in the transports sub-tree."""
1185.33.66 by Martin Pool
[patch] use unicode literals for all hardcoded paths (Alexander Belchenko)
295
        queue = list(self.list_dir(u'.'))
1442.1.44 by Robert Collins
Many transport related tweaks:
296
        while queue:
1608.1.1 by Martin Pool
[patch] LocalTransport.list_dir should return url-quoted strings (ddaa)
297
            relpath = queue.pop(0)
1442.1.44 by Robert Collins
Many transport related tweaks:
298
            st = self.stat(relpath)
299
            if S_ISDIR(st[ST_MODE]):
300
                for i, basename in enumerate(self.list_dir(relpath)):
301
                    queue.insert(i, relpath+'/'+basename)
302
            else:
303
                yield relpath
304
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
305
    def _mkdir(self, abspath, mode=None):
306
        """Create a real directory, filtering through mode"""
307
        if mode is None:
308
            # os.mkdir() will filter through umask
309
            local_mode = 0777
310
        else:
311
            local_mode = mode
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
312
        try:
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
313
            os.mkdir(abspath, local_mode)
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
314
            if mode is not None:
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
315
                # It is probably faster to just do the chmod, rather than
316
                # doing a stat, and then trying to compare
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
317
                os.chmod(abspath, mode)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
318
        except (IOError, OSError),e:
1946.2.12 by John Arbash Meinel
Add ability to pass a directory mode to non_atomic_put
319
            self._translate_error(e, abspath)
320
321
    def mkdir(self, relpath, mode=None):
322
        """Create a directory at the given path."""
323
        self._mkdir(self._abspath(relpath), mode=mode)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
324
2671.3.9 by Robert Collins
Review feedback and fix VFat emulated transports to not claim to have unix permissions.
325
    def open_write_stream(self, relpath, mode=None):
326
        """See Transport.open_write_stream."""
2671.3.2 by Robert Collins
Start open_file_stream logic.
327
        # initialise the file
2671.3.3 by Robert Collins
Add mode parameter to Transport.open_file_stream.
328
        self.put_bytes_non_atomic(relpath, "", mode=mode)
3010.1.10 by Robert Collins
Honour file modes for write streams.
329
        abspath = self._abspath(relpath)
330
        handle = open(abspath, 'wb')
331
        if mode is not None:
332
            self._check_mode_and_size(abspath, handle.fileno(), mode)
2671.3.2 by Robert Collins
Start open_file_stream logic.
333
        transport._file_streams[self.abspath(relpath)] = handle
2671.3.6 by Robert Collins
Review feedback.
334
        return transport.FileFileStream(self, relpath, handle)
2671.3.2 by Robert Collins
Start open_file_stream logic.
335
1955.3.15 by John Arbash Meinel
Deprecate 'Transport.append' in favor of Transport.append_file or Transport.append_bytes
336
    def _get_append_file(self, relpath, mode=None):
337
        """Call os.open() for the given relpath"""
1955.3.17 by John Arbash Meinel
Fix some bugs in Transport.append(mode!=None)
338
        file_abspath = self._abspath(relpath)
1755.3.3 by Robert Collins
allow None == 0666 for mode.
339
        if mode is None:
1755.3.9 by John Arbash Meinel
Make AtomicFile not do anything if not supplied a mode, clean up LocalTransport now that we do the right thing for None
340
            # os.open() will automatically use the umask
341
            local_mode = 0666
1755.3.7 by John Arbash Meinel
Clean up and write tests for permissions. Now we use fstat which should be cheap, and lets us check the permissions and the file size
342
        else:
343
            local_mode = mode
1530.1.4 by Robert Collins
integrate Memory tests into transport interface tests.
344
        try:
1955.3.17 by John Arbash Meinel
Fix some bugs in Transport.append(mode!=None)
345
            return file_abspath, os.open(file_abspath, _append_flags, local_mode)
1530.1.4 by Robert Collins
integrate Memory tests into transport interface tests.
346
        except (IOError, OSError),e:
347
            self._translate_error(e, relpath)
1955.3.15 by John Arbash Meinel
Deprecate 'Transport.append' in favor of Transport.append_file or Transport.append_bytes
348
1955.3.17 by John Arbash Meinel
Fix some bugs in Transport.append(mode!=None)
349
    def _check_mode_and_size(self, file_abspath, fd, mode=None):
1955.3.15 by John Arbash Meinel
Deprecate 'Transport.append' in favor of Transport.append_file or Transport.append_bytes
350
        """Check the mode of the file, and return the current size"""
351
        st = os.fstat(fd)
352
        if mode is not None and mode != S_IMODE(st.st_mode):
353
            # Because of umask, we may still need to chmod the file.
354
            # But in the general case, we won't have to
1955.3.17 by John Arbash Meinel
Fix some bugs in Transport.append(mode!=None)
355
            os.chmod(file_abspath, mode)
1955.3.15 by John Arbash Meinel
Deprecate 'Transport.append' in favor of Transport.append_file or Transport.append_bytes
356
        return st.st_size
357
358
    def append_file(self, relpath, f, mode=None):
359
        """Append the text in the file-like object into the final location."""
1955.3.17 by John Arbash Meinel
Fix some bugs in Transport.append(mode!=None)
360
        file_abspath, fd = self._get_append_file(relpath, mode=mode)
1755.3.1 by Robert Collins
Tune the time to build our kernel_like tree : make LocalTransport.put faster, AtomicFile faster, LocalTransport.append faster.
361
        try:
1955.3.17 by John Arbash Meinel
Fix some bugs in Transport.append(mode!=None)
362
            result = self._check_mode_and_size(file_abspath, fd, mode=mode)
1755.3.1 by Robert Collins
Tune the time to build our kernel_like tree : make LocalTransport.put faster, AtomicFile faster, LocalTransport.append faster.
363
            self._pump_to_fd(f, fd)
1711.7.25 by John Arbash Meinel
try/finally to close files, _KnitData was keeping a handle to a file it never used again, and using transport.rename() when it wanted transport.move()
364
        finally:
1755.3.1 by Robert Collins
Tune the time to build our kernel_like tree : make LocalTransport.put faster, AtomicFile faster, LocalTransport.append faster.
365
            os.close(fd)
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
366
        return result
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
367
1955.3.15 by John Arbash Meinel
Deprecate 'Transport.append' in favor of Transport.append_file or Transport.append_bytes
368
    def append_bytes(self, relpath, bytes, mode=None):
369
        """Append the text in the string into the final location."""
1955.3.17 by John Arbash Meinel
Fix some bugs in Transport.append(mode!=None)
370
        file_abspath, fd = self._get_append_file(relpath, mode=mode)
1955.3.15 by John Arbash Meinel
Deprecate 'Transport.append' in favor of Transport.append_file or Transport.append_bytes
371
        try:
1955.3.17 by John Arbash Meinel
Fix some bugs in Transport.append(mode!=None)
372
            result = self._check_mode_and_size(file_abspath, fd, mode=mode)
1955.3.15 by John Arbash Meinel
Deprecate 'Transport.append' in favor of Transport.append_file or Transport.append_bytes
373
            os.write(fd, bytes)
374
        finally:
375
            os.close(fd)
376
        return result
377
1755.3.1 by Robert Collins
Tune the time to build our kernel_like tree : make LocalTransport.put faster, AtomicFile faster, LocalTransport.append faster.
378
    def _pump_to_fd(self, fromfile, to_fd):
379
        """Copy contents of one file to another."""
380
        BUFSIZE = 32768
381
        while True:
382
            b = fromfile.read(BUFSIZE)
383
            if not b:
384
                break
385
            os.write(to_fd, b)
386
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
387
    def copy(self, rel_from, rel_to):
388
        """Copy the item at rel_from to the location at rel_to"""
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
389
        path_from = self._abspath(rel_from)
390
        path_to = self._abspath(rel_to)
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
391
        try:
392
            shutil.copy(path_from, path_to)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
393
        except (IOError, OSError),e:
394
            # TODO: What about path_to?
395
            self._translate_error(e, path_from)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
396
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
397
    def rename(self, rel_from, rel_to):
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
398
        path_from = self._abspath(rel_from)
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
399
        try:
400
            # *don't* call bzrlib.osutils.rename, because we want to 
401
            # detect errors on rename
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
402
            os.rename(path_from, self._abspath(rel_to))
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
403
        except (IOError, OSError),e:
404
            # TODO: What about path_to?
405
            self._translate_error(e, path_from)
406
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
407
    def move(self, rel_from, rel_to):
408
        """Move the item at rel_from to the location at rel_to"""
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
409
        path_from = self._abspath(rel_from)
410
        path_to = self._abspath(rel_to)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
411
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
412
        try:
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
413
            # this version will delete the destination if necessary
1996.3.17 by John Arbash Meinel
lazy_import plugin and transport/local
414
            osutils.rename(path_from, path_to)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
415
        except (IOError, OSError),e:
416
            # TODO: What about path_to?
417
            self._translate_error(e, path_from)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
418
419
    def delete(self, relpath):
420
        """Delete the item at relpath"""
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
421
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
422
        try:
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
423
            path = self._abspath(relpath)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
424
            os.remove(path)
425
        except (IOError, OSError),e:
426
            self._translate_error(e, path)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
427
2634.1.1 by Robert Collins
(robertc) Reinstate the accidentally backed out external_url patch.
428
    def external_url(self):
429
        """See bzrlib.transport.Transport.external_url."""
430
        # File URL's are externally usable.
431
        return self.base
432
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
433
    def copy_to(self, relpaths, other, mode=None, pb=None):
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
434
        """Copy a set of entries from self into another Transport.
435
436
        :param relpaths: A list/generator of entries to be copied.
437
        """
438
        if isinstance(other, LocalTransport):
439
            # Both from & to are on the local filesystem
440
            # Unfortunately, I can't think of anything faster than just
441
            # copying them across, one by one :(
442
            total = self._get_total(relpaths)
443
            count = 0
444
            for path in relpaths:
445
                self._update_pb(pb, 'copy-to', count, total)
1185.16.158 by John Arbash Meinel
Added a test that copy_to raises NoSuchFile when a directory is missing (not IOError)
446
                try:
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
447
                    mypath = self._abspath(path)
448
                    otherpath = other._abspath(path)
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
449
                    shutil.copy(mypath, otherpath)
450
                    if mode is not None:
451
                        os.chmod(otherpath, mode)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
452
                except (IOError, OSError),e:
453
                    self._translate_error(e, path)
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
454
                count += 1
455
            return count
456
        else:
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
457
            return super(LocalTransport, self).copy_to(relpaths, other, mode=mode, pb=pb)
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
458
1400.1.1 by Robert Collins
implement a basic test for the ui branch command from http servers
459
    def listable(self):
460
        """See Transport.listable."""
461
        return True
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
462
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
463
    def list_dir(self, relpath):
464
        """Return a list of all files at the given location.
465
        WARNING: many transports do not support this, so trying avoid using
466
        it if at all possible.
467
        """
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
468
        path = self._abspath(relpath)
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
469
        try:
1959.2.1 by John Arbash Meinel
David Allouche: Make transports return escaped paths
470
            entries = os.listdir(path)
1607.1.3 by Robert Collins
Apply David Allouches list_dir quoting fix.
471
        except (IOError, OSError), e:
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
472
            self._translate_error(e, path)
1959.2.1 by John Arbash Meinel
David Allouche: Make transports return escaped paths
473
        return [urlutils.escape(entry) for entry in entries]
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
474
475
    def stat(self, relpath):
476
        """Return the stat information for a file.
477
        """
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
478
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
479
        try:
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
480
            path = self._abspath(relpath)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
481
            return os.stat(path)
482
        except (IOError, OSError),e:
483
            self._translate_error(e, path)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
484
907.1.24 by John Arbash Meinel
Remote functionality work.
485
    def lock_read(self, relpath):
486
        """Lock the given file for shared (read) access.
487
        :return: A lock object, which should be passed to Transport.unlock()
488
        """
489
        from bzrlib.lock import ReadLock
1185.65.29 by Robert Collins
Implement final review suggestions.
490
        path = relpath
491
        try:
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
492
            path = self._abspath(relpath)
1185.65.29 by Robert Collins
Implement final review suggestions.
493
            return ReadLock(path)
494
        except (IOError, OSError), e:
495
            self._translate_error(e, path)
907.1.24 by John Arbash Meinel
Remote functionality work.
496
497
    def lock_write(self, relpath):
498
        """Lock the given file for exclusive (write) access.
499
        WARNING: many transports do not support this, so trying avoid using it
500
501
        :return: A lock object, which should be passed to Transport.unlock()
502
        """
503
        from bzrlib.lock import WriteLock
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
504
        return WriteLock(self._abspath(relpath))
907.1.24 by John Arbash Meinel
Remote functionality work.
505
1534.4.15 by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports.
506
    def rmdir(self, relpath):
507
        """See Transport.rmdir."""
508
        path = relpath
509
        try:
1725.2.1 by Robert Collins
Make LocalTransport faster by not normpathing every internal path translation.
510
            path = self._abspath(relpath)
1534.4.15 by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports.
511
            os.rmdir(path)
512
        except (IOError, OSError),e:
513
            self._translate_error(e, path)
1442.1.41 by Robert Collins
move duplicate scratch logic into a scratch transport
514
1608.2.7 by Martin Pool
Rename supports_unix_modebits to _can_roundtrip_unix_modebits for clarity
515
    def _can_roundtrip_unix_modebits(self):
1608.2.5 by Martin Pool
Add Transport.supports_unix_modebits, so tests can
516
        if sys.platform == 'win32':
517
            # anyone else?
518
            return False
519
        else:
520
            return True
521
522
2245.6.2 by Alexander Belchenko
Fix name of emulated Win32LocalTransport as Robert suggested.
523
class EmulatedWin32LocalTransport(LocalTransport):
2245.6.1 by Alexander Belchenko
win32 UNC path: recursive cloning UNC path to root stops on //HOST, not on //
524
    """Special transport for testing Win32 [UNC] paths on non-windows"""
525
526
    def __init__(self, base):
527
        if base[-1] != '/':
528
            base = base + '/'
529
        super(LocalTransport, self).__init__(base)
530
        self._local_base = urlutils._win32_local_path_from_url(base)
531
532
    def abspath(self, relpath):
533
        path = osutils.normpath(osutils.pathjoin(
534
                    self._local_base, urlutils.unescape(relpath)))
535
        return urlutils._win32_local_path_to_url(path)
536
2245.6.3 by Alexander Belchenko
EmulatedWin32LocalTransport should provide their own 'clone' method
537
    def clone(self, offset=None):
538
        """Return a new LocalTransport with root at self.base + offset
539
        Because the local filesystem does not require a connection, 
540
        we can just return a new object.
541
        """
542
        if offset is None:
543
            return EmulatedWin32LocalTransport(self.base)
544
        else:
545
            abspath = self.abspath(offset)
546
            if abspath == 'file://':
547
                # fix upwalk for UNC path
548
                # when clone from //HOST/path updir recursively
549
                # we should stop at least at //HOST part
550
                abspath = self.base
551
            return EmulatedWin32LocalTransport(abspath)
552
2245.6.1 by Alexander Belchenko
win32 UNC path: recursive cloning UNC path to root stops on //HOST, not on //
553
1530.1.3 by Robert Collins
transport implementations now tested consistently.
554
class LocalURLServer(Server):
1951.2.1 by Martin Pool
Change to using LocalURLServer for testing.
555
    """A pretend server for local transports, using file:// urls.
556
    
557
    Of course no actual server is required to access the local filesystem, so
558
    this just exists to tell the test code how to get to it.
559
    """
1530.1.3 by Robert Collins
transport implementations now tested consistently.
560
2018.5.114 by Robert Collins
Commit current test pass improvements.
561
    def setUp(self):
2018.5.44 by Andrew Bennetts
Small changes to help a couple more tests pass.
562
        """Setup the server to service requests.
563
        
564
        :param decorated_transport: ignored by this implementation.
565
        """
566
1530.1.3 by Robert Collins
transport implementations now tested consistently.
567
    def get_url(self):
568
        """See Transport.Server.get_url."""
1685.1.45 by John Arbash Meinel
Moved url functions into bzrlib.urlutils
569
        return urlutils.local_path_to_url('')
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
570
571
572
def get_test_permutations():
573
    """Return the permutations to be used in testing."""
1951.2.1 by Martin Pool
Change to using LocalURLServer for testing.
574
    return [
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
575
            (LocalTransport, LocalURLServer),
576
            ]