~bzr-pqm/bzr/bzr.dev

1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
1
# Copyright (C) 2005, 2006 Canonical Ltd
1185.11.19 by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings.
2
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
1185.16.72 by Martin Pool
[merge] from robert and fix up tests
16
17
"""Transport for the local filesystem.
18
19
This is a fairly thin wrapper on regular file IO."""
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
20
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
21
import os
22
import shutil
1608.2.5 by Martin Pool
Add Transport.supports_unix_modebits, so tests can
23
import sys
1442.1.44 by Robert Collins
Many transport related tweaks:
24
from stat import ST_MODE, S_ISDIR, ST_SIZE
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
25
import tempfile
1469 by Robert Collins
Change Transport.* to work with URL's.
26
import urllib
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
27
28
from bzrlib.trace import mutter
1530.1.3 by Robert Collins
transport implementations now tested consistently.
29
from bzrlib.transport import Transport, Server
1551.2.56 by Aaron Bentley
Better illegal pathname check for Windows
30
from bzrlib.osutils import (abspath, realpath, normpath, pathjoin, rename, 
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
31
                            check_legal_path, rmtree)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
32
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
33
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
34
class LocalTransport(Transport):
35
    """This is the transport agent for local filesystem access."""
36
37
    def __init__(self, base):
38
        """Set the base path where files will be stored."""
1185.11.1 by John Arbash Meinel
(broken) Transport work is merged in. Tests do not pass yet.
39
        if base.startswith('file://'):
1608.1.1 by Martin Pool
[patch] LocalTransport.list_dir should return url-quoted strings (ddaa)
40
            base = base[len('file://'):]
1092.2.24 by Robert Collins
merge from martins newformat branch - brings in transport abstraction
41
        # realpath is incompatible with symlinks. When we traverse
42
        # up we might be able to normpath stuff. RBC 20051003
1530.1.7 by Robert Collins
merge integration.
43
        base = normpath(abspath(base))
1530.1.3 by Robert Collins
transport implementations now tested consistently.
44
        if base[-1] != '/':
45
            base = base + '/'
46
        super(LocalTransport, self).__init__(base)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
47
907.1.32 by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate.
48
    def should_cache(self):
907.1.22 by John Arbash Meinel
Fixed some encoding issues, added is_remote function for Transport objects.
49
        return False
50
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
51
    def clone(self, offset=None):
52
        """Return a new LocalTransport with root at self.base + offset
53
        Because the local filesystem does not require a connection, 
54
        we can just return a new object.
55
        """
56
        if offset is None:
57
            return LocalTransport(self.base)
58
        else:
59
            return LocalTransport(self.abspath(offset))
60
907.1.8 by John Arbash Meinel
Changed the format for abspath. Updated branch to use a hidden _transport
61
    def abspath(self, relpath):
1636.1.1 by Robert Collins
Fix calling relpath() and abspath() on transports at their root.
62
        """Return the full url to the given relative URL."""
1185.12.70 by Aaron Bentley
Removed b
63
        assert isinstance(relpath, basestring), (type(relpath), relpath)
1636.1.1 by Robert Collins
Fix calling relpath() and abspath() on transports at their root.
64
        result = normpath(pathjoin(self.base, urllib.unquote(relpath)))
65
        #if result[-1] != '/':
66
        #    result += '/'
67
        return result
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
68
907.1.24 by John Arbash Meinel
Remote functionality work.
69
    def relpath(self, abspath):
70
        """Return the local path portion from a given absolute path.
71
        """
1551.2.53 by abentley
Strip trailing slashes in a platform-sensible way
72
        from bzrlib.osutils import relpath, strip_trailing_slash
1442.1.64 by Robert Collins
Branch.open_containing now returns a tuple (Branch, relative-path).
73
        if abspath is None:
1185.33.66 by Martin Pool
[patch] use unicode literals for all hardcoded paths (Alexander Belchenko)
74
            abspath = u'.'
1551.2.53 by abentley
Strip trailing slashes in a platform-sensible way
75
76
        return relpath(strip_trailing_slash(self.base), 
77
                       strip_trailing_slash(abspath))
907.1.24 by John Arbash Meinel
Remote functionality work.
78
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
79
    def has(self, relpath):
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
80
        return os.access(self.abspath(relpath), os.F_OK)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
81
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
82
    def get(self, relpath):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
83
        """Get the file at the given relative path.
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
84
85
        :param relpath: The relative path to the file
86
        """
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
87
        try:
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
88
            path = self.abspath(relpath)
89
            return open(path, 'rb')
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
90
        except (IOError, OSError),e:
91
            self._translate_error(e, path)
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
92
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
93
    def put(self, relpath, f, mode=None):
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
94
        """Copy the file-like or string object into the location.
95
96
        :param relpath: Location to put the contents, relative to base.
97
        :param f:       File-like or string object.
98
        """
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
99
        from bzrlib.atomicfile import AtomicFile
100
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
101
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
102
        try:
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
103
            path = self.abspath(relpath)
1551.2.56 by Aaron Bentley
Better illegal pathname check for Windows
104
            check_legal_path(path)
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
105
            fp = AtomicFile(path, 'wb', new_mode=mode)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
106
        except (IOError, OSError),e:
107
            self._translate_error(e, path)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
108
        try:
109
            self._pump(f, fp)
110
            fp.commit()
111
        finally:
112
            fp.close()
113
1442.1.44 by Robert Collins
Many transport related tweaks:
114
    def iter_files_recursive(self):
115
        """Iter the relative paths of files in the transports sub-tree."""
1185.33.66 by Martin Pool
[patch] use unicode literals for all hardcoded paths (Alexander Belchenko)
116
        queue = list(self.list_dir(u'.'))
1442.1.44 by Robert Collins
Many transport related tweaks:
117
        while queue:
1608.1.1 by Martin Pool
[patch] LocalTransport.list_dir should return url-quoted strings (ddaa)
118
            relpath = queue.pop(0)
1442.1.44 by Robert Collins
Many transport related tweaks:
119
            st = self.stat(relpath)
120
            if S_ISDIR(st[ST_MODE]):
121
                for i, basename in enumerate(self.list_dir(relpath)):
122
                    queue.insert(i, relpath+'/'+basename)
123
            else:
124
                yield relpath
125
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
126
    def mkdir(self, relpath, mode=None):
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
127
        """Create a directory at the given path."""
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
128
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
129
        try:
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
130
            path = self.abspath(relpath)
131
            os.mkdir(path)
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
132
            if mode is not None:
133
                os.chmod(path, mode)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
134
        except (IOError, OSError),e:
135
            self._translate_error(e, path)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
136
1666.1.6 by Robert Collins
Make knit the default format.
137
    def append(self, relpath, f, mode=None):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
138
        """Append the text in the file-like object into the final
139
        location.
140
        """
1530.1.4 by Robert Collins
integrate Memory tests into transport interface tests.
141
        try:
142
            fp = open(self.abspath(relpath), 'ab')
1666.1.6 by Robert Collins
Make knit the default format.
143
            if mode is not None:
144
                os.chmod(self.abspath(relpath), mode)
1530.1.4 by Robert Collins
integrate Memory tests into transport interface tests.
145
        except (IOError, OSError),e:
146
            self._translate_error(e, relpath)
1551.2.38 by abentley
Fixed win32 bug with tell
147
        # win32 workaround (tell on an unwritten file returns 0)
148
        fp.seek(0, 2)
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
149
        result = fp.tell()
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
150
        self._pump(f, fp)
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
151
        return result
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
152
153
    def copy(self, rel_from, rel_to):
154
        """Copy the item at rel_from to the location at rel_to"""
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
155
        path_from = self.abspath(rel_from)
156
        path_to = self.abspath(rel_to)
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
157
        try:
158
            shutil.copy(path_from, path_to)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
159
        except (IOError, OSError),e:
160
            # TODO: What about path_to?
161
            self._translate_error(e, path_from)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
162
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
163
    def rename(self, rel_from, rel_to):
164
        path_from = self.abspath(rel_from)
165
        try:
166
            # *don't* call bzrlib.osutils.rename, because we want to 
167
            # detect errors on rename
168
            os.rename(path_from, self.abspath(rel_to))
169
        except (IOError, OSError),e:
170
            # TODO: What about path_to?
171
            self._translate_error(e, path_from)
172
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
173
    def move(self, rel_from, rel_to):
174
        """Move the item at rel_from to the location at rel_to"""
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
175
        path_from = self.abspath(rel_from)
176
        path_to = self.abspath(rel_to)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
177
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
178
        try:
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
179
            # this version will delete the destination if necessary
1185.31.58 by John Arbash Meinel
Updating for new transport tests so that they pass on win32
180
            rename(path_from, path_to)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
181
        except (IOError, OSError),e:
182
            # TODO: What about path_to?
183
            self._translate_error(e, path_from)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
184
185
    def delete(self, relpath):
186
        """Delete the item at relpath"""
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
187
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
188
        try:
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
189
            path = self.abspath(relpath)
190
            os.remove(path)
191
        except (IOError, OSError),e:
192
            # TODO: What about path_to?
193
            self._translate_error(e, path)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
194
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
195
    def copy_to(self, relpaths, other, mode=None, pb=None):
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
196
        """Copy a set of entries from self into another Transport.
197
198
        :param relpaths: A list/generator of entries to be copied.
199
        """
200
        if isinstance(other, LocalTransport):
201
            # Both from & to are on the local filesystem
202
            # Unfortunately, I can't think of anything faster than just
203
            # copying them across, one by one :(
204
            total = self._get_total(relpaths)
205
            count = 0
206
            for path in relpaths:
207
                self._update_pb(pb, 'copy-to', count, total)
1185.16.158 by John Arbash Meinel
Added a test that copy_to raises NoSuchFile when a directory is missing (not IOError)
208
                try:
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
209
                    mypath = self.abspath(path)
210
                    otherpath = other.abspath(path)
211
                    shutil.copy(mypath, otherpath)
212
                    if mode is not None:
213
                        os.chmod(otherpath, mode)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
214
                except (IOError, OSError),e:
215
                    self._translate_error(e, path)
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
216
                count += 1
217
            return count
218
        else:
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
219
            return super(LocalTransport, self).copy_to(relpaths, other, mode=mode, pb=pb)
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
220
1400.1.1 by Robert Collins
implement a basic test for the ui branch command from http servers
221
    def listable(self):
222
        """See Transport.listable."""
223
        return True
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
224
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
225
    def list_dir(self, relpath):
226
        """Return a list of all files at the given location.
227
        WARNING: many transports do not support this, so trying avoid using
228
        it if at all possible.
229
        """
1608.1.1 by Martin Pool
[patch] LocalTransport.list_dir should return url-quoted strings (ddaa)
230
        path = self.abspath(relpath)
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
231
        try:
1608.1.1 by Martin Pool
[patch] LocalTransport.list_dir should return url-quoted strings (ddaa)
232
            return [urllib.quote(entry) for entry in os.listdir(path)]
1607.1.3 by Robert Collins
Apply David Allouches list_dir quoting fix.
233
        except (IOError, OSError), e:
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
234
            self._translate_error(e, path)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
235
236
    def stat(self, relpath):
237
        """Return the stat information for a file.
238
        """
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
239
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
240
        try:
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
241
            path = self.abspath(relpath)
242
            return os.stat(path)
243
        except (IOError, OSError),e:
244
            self._translate_error(e, path)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
245
907.1.24 by John Arbash Meinel
Remote functionality work.
246
    def lock_read(self, relpath):
247
        """Lock the given file for shared (read) access.
248
        :return: A lock object, which should be passed to Transport.unlock()
249
        """
250
        from bzrlib.lock import ReadLock
1185.65.29 by Robert Collins
Implement final review suggestions.
251
        path = relpath
252
        try:
253
            path = self.abspath(relpath)
254
            return ReadLock(path)
255
        except (IOError, OSError), e:
256
            self._translate_error(e, path)
907.1.24 by John Arbash Meinel
Remote functionality work.
257
258
    def lock_write(self, relpath):
259
        """Lock the given file for exclusive (write) access.
260
        WARNING: many transports do not support this, so trying avoid using it
261
262
        :return: A lock object, which should be passed to Transport.unlock()
263
        """
264
        from bzrlib.lock import WriteLock
265
        return WriteLock(self.abspath(relpath))
266
1534.4.15 by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports.
267
    def rmdir(self, relpath):
268
        """See Transport.rmdir."""
269
        path = relpath
270
        try:
271
            path = self.abspath(relpath)
272
            os.rmdir(path)
273
        except (IOError, OSError),e:
274
            self._translate_error(e, path)
1442.1.41 by Robert Collins
move duplicate scratch logic into a scratch transport
275
1608.2.7 by Martin Pool
Rename supports_unix_modebits to _can_roundtrip_unix_modebits for clarity
276
    def _can_roundtrip_unix_modebits(self):
1608.2.5 by Martin Pool
Add Transport.supports_unix_modebits, so tests can
277
        if sys.platform == 'win32':
278
            # anyone else?
279
            return False
280
        else:
281
            return True
282
283
1442.1.41 by Robert Collins
move duplicate scratch logic into a scratch transport
284
class ScratchTransport(LocalTransport):
285
    """A transport that works in a temporary dir and cleans up after itself.
286
    
287
    The dir only exists for the lifetime of the Python object.
288
    Obviously you should not put anything precious in it.
289
    """
290
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
291
    def __init__(self, base=None):
292
        if base is None:
293
            base = tempfile.mkdtemp()
1442.1.41 by Robert Collins
move duplicate scratch logic into a scratch transport
294
        super(ScratchTransport, self).__init__(base)
295
296
    def __del__(self):
1692.7.6 by Martin Pool
[patch] force deletion of trees containing readonly files (alexander)
297
        rmtree(self.base, ignore_errors=True)
1442.1.41 by Robert Collins
move duplicate scratch logic into a scratch transport
298
        mutter("%r destroyed" % self)
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
299
300
1530.1.3 by Robert Collins
transport implementations now tested consistently.
301
class LocalRelpathServer(Server):
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
302
    """A pretend server for local transports, using relpaths."""
303
1530.1.3 by Robert Collins
transport implementations now tested consistently.
304
    def get_url(self):
305
        """See Transport.Server.get_url."""
306
        return "."
307
308
309
class LocalAbspathServer(Server):
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
310
    """A pretend server for local transports, using absolute paths."""
311
1530.1.3 by Robert Collins
transport implementations now tested consistently.
312
    def get_url(self):
313
        """See Transport.Server.get_url."""
314
        return os.path.abspath("")
315
316
317
class LocalURLServer(Server):
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
318
    """A pretend server for local transports, using file:// urls."""
1530.1.3 by Robert Collins
transport implementations now tested consistently.
319
320
    def get_url(self):
321
        """See Transport.Server.get_url."""
322
        # FIXME: \ to / on windows
323
        return "file://%s" % os.path.abspath("")
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
324
325
326
def get_test_permutations():
327
    """Return the permutations to be used in testing."""
328
    return [(LocalTransport, LocalRelpathServer),
329
            (LocalTransport, LocalAbspathServer),
330
            (LocalTransport, LocalURLServer),
331
            ]