~bzr-pqm/bzr/bzr.dev

1185.11.19 by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings.
1
# Copyright (C) 2005 Canonical Ltd
2
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
1185.16.72 by Martin Pool
[merge] from robert and fix up tests
16
17
"""Transport for the local filesystem.
18
19
This is a fairly thin wrapper on regular file IO."""
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
20
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
21
import os
22
import shutil
1442.1.44 by Robert Collins
Many transport related tweaks:
23
from stat import ST_MODE, S_ISDIR, ST_SIZE
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
24
import tempfile
1469 by Robert Collins
Change Transport.* to work with URL's.
25
import urllib
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
26
27
from bzrlib.trace import mutter
1530.1.3 by Robert Collins
transport implementations now tested consistently.
28
from bzrlib.transport import Transport, Server
1185.31.58 by John Arbash Meinel
Updating for new transport tests so that they pass on win32
29
from bzrlib.osutils import abspath, realpath, normpath, pathjoin, rename
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
30
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
31
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
32
class LocalTransport(Transport):
33
    """This is the transport agent for local filesystem access."""
34
35
    def __init__(self, base):
36
        """Set the base path where files will be stored."""
1185.11.1 by John Arbash Meinel
(broken) Transport work is merged in. Tests do not pass yet.
37
        if base.startswith('file://'):
38
            base = base[7:]
1092.2.24 by Robert Collins
merge from martins newformat branch - brings in transport abstraction
39
        # realpath is incompatible with symlinks. When we traverse
40
        # up we might be able to normpath stuff. RBC 20051003
1530.1.7 by Robert Collins
merge integration.
41
        base = normpath(abspath(base))
1530.1.3 by Robert Collins
transport implementations now tested consistently.
42
        if base[-1] != '/':
43
            base = base + '/'
44
        super(LocalTransport, self).__init__(base)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
45
907.1.32 by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate.
46
    def should_cache(self):
907.1.22 by John Arbash Meinel
Fixed some encoding issues, added is_remote function for Transport objects.
47
        return False
48
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
49
    def clone(self, offset=None):
50
        """Return a new LocalTransport with root at self.base + offset
51
        Because the local filesystem does not require a connection, 
52
        we can just return a new object.
53
        """
54
        if offset is None:
55
            return LocalTransport(self.base)
56
        else:
57
            return LocalTransport(self.abspath(offset))
58
907.1.8 by John Arbash Meinel
Changed the format for abspath. Updated branch to use a hidden _transport
59
    def abspath(self, relpath):
1469 by Robert Collins
Change Transport.* to work with URL's.
60
        """Return the full url to the given relative URL.
907.1.8 by John Arbash Meinel
Changed the format for abspath. Updated branch to use a hidden _transport
61
        This can be supplied with a string or a list
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
62
        """
1185.12.70 by Aaron Bentley
Removed b
63
        assert isinstance(relpath, basestring), (type(relpath), relpath)
1185.31.33 by John Arbash Meinel
A couple more path.join statements needed changing.
64
        return pathjoin(self.base, urllib.unquote(relpath))
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
65
907.1.24 by John Arbash Meinel
Remote functionality work.
66
    def relpath(self, abspath):
67
        """Return the local path portion from a given absolute path.
68
        """
1457.1.2 by Robert Collins
move branch._relpath into osutils as relpath
69
        from bzrlib.osutils import relpath
1442.1.64 by Robert Collins
Branch.open_containing now returns a tuple (Branch, relative-path).
70
        if abspath is None:
1185.33.66 by Martin Pool
[patch] use unicode literals for all hardcoded paths (Alexander Belchenko)
71
            abspath = u'.'
1530.1.3 by Robert Collins
transport implementations now tested consistently.
72
        if abspath.endswith('/'):
73
            abspath = abspath[:-1]
74
        return relpath(self.base[:-1], abspath)
907.1.24 by John Arbash Meinel
Remote functionality work.
75
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
76
    def has(self, relpath):
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
77
        return os.access(self.abspath(relpath), os.F_OK)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
78
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
79
    def get(self, relpath):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
80
        """Get the file at the given relative path.
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
81
82
        :param relpath: The relative path to the file
83
        """
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
84
        try:
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
85
            path = self.abspath(relpath)
86
            return open(path, 'rb')
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
87
        except (IOError, OSError),e:
88
            self._translate_error(e, path)
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
89
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
90
    def put(self, relpath, f, mode=None):
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
91
        """Copy the file-like or string object into the location.
92
93
        :param relpath: Location to put the contents, relative to base.
94
        :param f:       File-like or string object.
95
        """
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
96
        from bzrlib.atomicfile import AtomicFile
97
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
98
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
99
        try:
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
100
            path = self.abspath(relpath)
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
101
            fp = AtomicFile(path, 'wb', new_mode=mode)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
102
        except (IOError, OSError),e:
103
            self._translate_error(e, path)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
104
        try:
105
            self._pump(f, fp)
106
            fp.commit()
107
        finally:
108
            fp.close()
109
1442.1.44 by Robert Collins
Many transport related tweaks:
110
    def iter_files_recursive(self):
111
        """Iter the relative paths of files in the transports sub-tree."""
1185.33.66 by Martin Pool
[patch] use unicode literals for all hardcoded paths (Alexander Belchenko)
112
        queue = list(self.list_dir(u'.'))
1442.1.44 by Robert Collins
Many transport related tweaks:
113
        while queue:
1479 by Robert Collins
More quoting at the transport layer bugfixes.
114
            relpath = urllib.quote(queue.pop(0))
1442.1.44 by Robert Collins
Many transport related tweaks:
115
            st = self.stat(relpath)
116
            if S_ISDIR(st[ST_MODE]):
117
                for i, basename in enumerate(self.list_dir(relpath)):
118
                    queue.insert(i, relpath+'/'+basename)
119
            else:
120
                yield relpath
121
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
122
    def mkdir(self, relpath, mode=None):
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
123
        """Create a directory at the given path."""
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
124
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
125
        try:
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
126
            path = self.abspath(relpath)
127
            os.mkdir(path)
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
128
            if mode is not None:
129
                os.chmod(path, mode)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
130
        except (IOError, OSError),e:
131
            self._translate_error(e, path)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
132
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
133
    def append(self, relpath, f):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
134
        """Append the text in the file-like object into the final
135
        location.
136
        """
1530.1.4 by Robert Collins
integrate Memory tests into transport interface tests.
137
        try:
138
            fp = open(self.abspath(relpath), 'ab')
139
        except (IOError, OSError),e:
140
            self._translate_error(e, relpath)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
141
        self._pump(f, fp)
142
143
    def copy(self, rel_from, rel_to):
144
        """Copy the item at rel_from to the location at rel_to"""
145
        import shutil
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
146
        path_from = self.abspath(rel_from)
147
        path_to = self.abspath(rel_to)
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
148
        try:
149
            shutil.copy(path_from, path_to)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
150
        except (IOError, OSError),e:
151
            # TODO: What about path_to?
152
            self._translate_error(e, path_from)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
153
154
    def move(self, rel_from, rel_to):
155
        """Move the item at rel_from to the location at rel_to"""
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
156
        path_from = self.abspath(rel_from)
157
        path_to = self.abspath(rel_to)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
158
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
159
        try:
1185.31.58 by John Arbash Meinel
Updating for new transport tests so that they pass on win32
160
            rename(path_from, path_to)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
161
        except (IOError, OSError),e:
162
            # TODO: What about path_to?
163
            self._translate_error(e, path_from)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
164
165
    def delete(self, relpath):
166
        """Delete the item at relpath"""
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
167
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
168
        try:
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
169
            path = self.abspath(relpath)
170
            os.remove(path)
171
        except (IOError, OSError),e:
172
            # TODO: What about path_to?
173
            self._translate_error(e, path)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
174
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
175
    def copy_to(self, relpaths, other, mode=None, pb=None):
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
176
        """Copy a set of entries from self into another Transport.
177
178
        :param relpaths: A list/generator of entries to be copied.
179
        """
180
        if isinstance(other, LocalTransport):
181
            # Both from & to are on the local filesystem
182
            # Unfortunately, I can't think of anything faster than just
183
            # copying them across, one by one :(
184
            import shutil
185
186
            total = self._get_total(relpaths)
187
            count = 0
188
            for path in relpaths:
189
                self._update_pb(pb, 'copy-to', count, total)
1185.16.158 by John Arbash Meinel
Added a test that copy_to raises NoSuchFile when a directory is missing (not IOError)
190
                try:
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
191
                    mypath = self.abspath(path)
192
                    otherpath = other.abspath(path)
193
                    shutil.copy(mypath, otherpath)
194
                    if mode is not None:
195
                        os.chmod(otherpath, mode)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
196
                except (IOError, OSError),e:
197
                    self._translate_error(e, path)
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
198
                count += 1
199
            return count
200
        else:
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
201
            return super(LocalTransport, self).copy_to(relpaths, other, mode=mode, pb=pb)
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
202
1400.1.1 by Robert Collins
implement a basic test for the ui branch command from http servers
203
    def listable(self):
204
        """See Transport.listable."""
205
        return True
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
206
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
207
    def list_dir(self, relpath):
208
        """Return a list of all files at the given location.
209
        WARNING: many transports do not support this, so trying avoid using
210
        it if at all possible.
211
        """
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
212
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
213
        try:
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
214
            path = self.abspath(relpath)
215
            return os.listdir(path)
216
        except (IOError, OSError),e:
217
            self._translate_error(e, path)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
218
219
    def stat(self, relpath):
220
        """Return the stat information for a file.
221
        """
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
222
        path = relpath
907.1.48 by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass.
223
        try:
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
224
            path = self.abspath(relpath)
225
            return os.stat(path)
226
        except (IOError, OSError),e:
227
            self._translate_error(e, path)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
228
907.1.24 by John Arbash Meinel
Remote functionality work.
229
    def lock_read(self, relpath):
230
        """Lock the given file for shared (read) access.
231
        :return: A lock object, which should be passed to Transport.unlock()
232
        """
233
        from bzrlib.lock import ReadLock
234
        return ReadLock(self.abspath(relpath))
235
236
    def lock_write(self, relpath):
237
        """Lock the given file for exclusive (write) access.
238
        WARNING: many transports do not support this, so trying avoid using it
239
240
        :return: A lock object, which should be passed to Transport.unlock()
241
        """
242
        from bzrlib.lock import WriteLock
243
        return WriteLock(self.abspath(relpath))
244
1442.1.41 by Robert Collins
move duplicate scratch logic into a scratch transport
245
246
class ScratchTransport(LocalTransport):
247
    """A transport that works in a temporary dir and cleans up after itself.
248
    
249
    The dir only exists for the lifetime of the Python object.
250
    Obviously you should not put anything precious in it.
251
    """
252
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
253
    def __init__(self, base=None):
254
        if base is None:
255
            base = tempfile.mkdtemp()
1442.1.41 by Robert Collins
move duplicate scratch logic into a scratch transport
256
        super(ScratchTransport, self).__init__(base)
257
258
    def __del__(self):
1442.1.42 by Robert Collins
rebuild ScratchBranch on top of ScratchTransport
259
        shutil.rmtree(self.base, ignore_errors=True)
1442.1.41 by Robert Collins
move duplicate scratch logic into a scratch transport
260
        mutter("%r destroyed" % self)
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
261
262
1530.1.3 by Robert Collins
transport implementations now tested consistently.
263
class LocalRelpathServer(Server):
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
264
    """A pretend server for local transports, using relpaths."""
265
1530.1.3 by Robert Collins
transport implementations now tested consistently.
266
    def get_url(self):
267
        """See Transport.Server.get_url."""
268
        return "."
269
270
271
class LocalAbspathServer(Server):
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
272
    """A pretend server for local transports, using absolute paths."""
273
1530.1.3 by Robert Collins
transport implementations now tested consistently.
274
    def get_url(self):
275
        """See Transport.Server.get_url."""
276
        return os.path.abspath("")
277
278
279
class LocalURLServer(Server):
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
280
    """A pretend server for local transports, using file:// urls."""
1530.1.3 by Robert Collins
transport implementations now tested consistently.
281
282
    def get_url(self):
283
        """See Transport.Server.get_url."""
284
        # FIXME: \ to / on windows
285
        return "file://%s" % os.path.abspath("")
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
286
287
288
def get_test_permutations():
289
    """Return the permutations to be used in testing."""
290
    return [(LocalTransport, LocalRelpathServer),
291
            (LocalTransport, LocalAbspathServer),
292
            (LocalTransport, LocalURLServer),
293
            ]