1185.11.19
by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings. |
1 |
# Copyright (C) 2005 Canonical Ltd
|
2 |
||
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
||
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
||
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
1185.16.72
by Martin Pool
[merge] from robert and fix up tests |
16 |
|
17 |
"""Transport for the local filesystem.
|
|
18 |
||
19 |
This is a fairly thin wrapper on regular file IO."""
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
20 |
|
1442.1.42
by Robert Collins
rebuild ScratchBranch on top of ScratchTransport |
21 |
import os |
22 |
import shutil |
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
23 |
from stat import ST_MODE, S_ISDIR, ST_SIZE |
1442.1.42
by Robert Collins
rebuild ScratchBranch on top of ScratchTransport |
24 |
import tempfile |
1469
by Robert Collins
Change Transport.* to work with URL's. |
25 |
import urllib |
1442.1.42
by Robert Collins
rebuild ScratchBranch on top of ScratchTransport |
26 |
|
27 |
from bzrlib.trace import mutter |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
28 |
from bzrlib.transport import Transport, Server |
1185.31.58
by John Arbash Meinel
Updating for new transport tests so that they pass on win32 |
29 |
from bzrlib.osutils import abspath, realpath, normpath, pathjoin, rename |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
30 |
|
1442.1.42
by Robert Collins
rebuild ScratchBranch on top of ScratchTransport |
31 |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
32 |
class LocalTransport(Transport): |
33 |
"""This is the transport agent for local filesystem access."""
|
|
34 |
||
35 |
def __init__(self, base): |
|
36 |
"""Set the base path where files will be stored."""
|
|
1185.11.1
by John Arbash Meinel
(broken) Transport work is merged in. Tests do not pass yet. |
37 |
if base.startswith('file://'): |
38 |
base = base[7:] |
|
1092.2.24
by Robert Collins
merge from martins newformat branch - brings in transport abstraction |
39 |
# realpath is incompatible with symlinks. When we traverse
|
40 |
# up we might be able to normpath stuff. RBC 20051003
|
|
1530.1.7
by Robert Collins
merge integration. |
41 |
base = normpath(abspath(base)) |
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
42 |
if base[-1] != '/': |
43 |
base = base + '/' |
|
44 |
super(LocalTransport, self).__init__(base) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
45 |
|
907.1.32
by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate. |
46 |
def should_cache(self): |
907.1.22
by John Arbash Meinel
Fixed some encoding issues, added is_remote function for Transport objects. |
47 |
return False |
48 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
49 |
def clone(self, offset=None): |
50 |
"""Return a new LocalTransport with root at self.base + offset
|
|
51 |
Because the local filesystem does not require a connection,
|
|
52 |
we can just return a new object.
|
|
53 |
"""
|
|
54 |
if offset is None: |
|
55 |
return LocalTransport(self.base) |
|
56 |
else: |
|
57 |
return LocalTransport(self.abspath(offset)) |
|
58 |
||
907.1.8
by John Arbash Meinel
Changed the format for abspath. Updated branch to use a hidden _transport |
59 |
def abspath(self, relpath): |
1469
by Robert Collins
Change Transport.* to work with URL's. |
60 |
"""Return the full url to the given relative URL.
|
907.1.8
by John Arbash Meinel
Changed the format for abspath. Updated branch to use a hidden _transport |
61 |
This can be supplied with a string or a list
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
62 |
"""
|
1185.12.70
by Aaron Bentley
Removed b |
63 |
assert isinstance(relpath, basestring), (type(relpath), relpath) |
1185.31.33
by John Arbash Meinel
A couple more path.join statements needed changing. |
64 |
return pathjoin(self.base, urllib.unquote(relpath)) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
65 |
|
907.1.24
by John Arbash Meinel
Remote functionality work. |
66 |
def relpath(self, abspath): |
67 |
"""Return the local path portion from a given absolute path.
|
|
68 |
"""
|
|
1457.1.2
by Robert Collins
move branch._relpath into osutils as relpath |
69 |
from bzrlib.osutils import relpath |
1442.1.64
by Robert Collins
Branch.open_containing now returns a tuple (Branch, relative-path). |
70 |
if abspath is None: |
1185.33.66
by Martin Pool
[patch] use unicode literals for all hardcoded paths (Alexander Belchenko) |
71 |
abspath = u'.' |
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
72 |
if abspath.endswith('/'): |
73 |
abspath = abspath[:-1] |
|
74 |
return relpath(self.base[:-1], abspath) |
|
907.1.24
by John Arbash Meinel
Remote functionality work. |
75 |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
76 |
def has(self, relpath): |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
77 |
return os.access(self.abspath(relpath), os.F_OK) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
78 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
79 |
def get(self, relpath): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
80 |
"""Get the file at the given relative path.
|
907.1.20
by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8 |
81 |
|
82 |
:param relpath: The relative path to the file
|
|
83 |
"""
|
|
907.1.48
by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass. |
84 |
try: |
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
85 |
path = self.abspath(relpath) |
86 |
return open(path, 'rb') |
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
87 |
except (IOError, OSError),e: |
88 |
self._translate_error(e, path) |
|
907.1.20
by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8 |
89 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
90 |
def put(self, relpath, f, mode=None): |
907.1.20
by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8 |
91 |
"""Copy the file-like or string object into the location.
|
92 |
||
93 |
:param relpath: Location to put the contents, relative to base.
|
|
94 |
:param f: File-like or string object.
|
|
95 |
"""
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
96 |
from bzrlib.atomicfile import AtomicFile |
97 |
||
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
98 |
path = relpath |
907.1.48
by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass. |
99 |
try: |
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
100 |
path = self.abspath(relpath) |
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
101 |
fp = AtomicFile(path, 'wb', new_mode=mode) |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
102 |
except (IOError, OSError),e: |
103 |
self._translate_error(e, path) |
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
104 |
try: |
105 |
self._pump(f, fp) |
|
106 |
fp.commit() |
|
107 |
finally: |
|
108 |
fp.close() |
|
109 |
||
1442.1.44
by Robert Collins
Many transport related tweaks: |
110 |
def iter_files_recursive(self): |
111 |
"""Iter the relative paths of files in the transports sub-tree."""
|
|
1185.33.66
by Martin Pool
[patch] use unicode literals for all hardcoded paths (Alexander Belchenko) |
112 |
queue = list(self.list_dir(u'.')) |
1442.1.44
by Robert Collins
Many transport related tweaks: |
113 |
while queue: |
1479
by Robert Collins
More quoting at the transport layer bugfixes. |
114 |
relpath = urllib.quote(queue.pop(0)) |
1442.1.44
by Robert Collins
Many transport related tweaks: |
115 |
st = self.stat(relpath) |
116 |
if S_ISDIR(st[ST_MODE]): |
|
117 |
for i, basename in enumerate(self.list_dir(relpath)): |
|
118 |
queue.insert(i, relpath+'/'+basename) |
|
119 |
else: |
|
120 |
yield relpath |
|
121 |
||
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
122 |
def mkdir(self, relpath, mode=None): |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
123 |
"""Create a directory at the given path."""
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
124 |
path = relpath |
907.1.48
by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass. |
125 |
try: |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
126 |
path = self.abspath(relpath) |
127 |
os.mkdir(path) |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
128 |
if mode is not None: |
129 |
os.chmod(path, mode) |
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
130 |
except (IOError, OSError),e: |
131 |
self._translate_error(e, path) |
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
132 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
133 |
def append(self, relpath, f): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
134 |
"""Append the text in the file-like object into the final
|
135 |
location.
|
|
136 |
"""
|
|
1530.1.4
by Robert Collins
integrate Memory tests into transport interface tests. |
137 |
try: |
138 |
fp = open(self.abspath(relpath), 'ab') |
|
139 |
except (IOError, OSError),e: |
|
140 |
self._translate_error(e, relpath) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
141 |
self._pump(f, fp) |
142 |
||
143 |
def copy(self, rel_from, rel_to): |
|
144 |
"""Copy the item at rel_from to the location at rel_to"""
|
|
145 |
import shutil |
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
146 |
path_from = self.abspath(rel_from) |
147 |
path_to = self.abspath(rel_to) |
|
907.1.48
by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass. |
148 |
try: |
149 |
shutil.copy(path_from, path_to) |
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
150 |
except (IOError, OSError),e: |
151 |
# TODO: What about path_to?
|
|
152 |
self._translate_error(e, path_from) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
153 |
|
154 |
def move(self, rel_from, rel_to): |
|
155 |
"""Move the item at rel_from to the location at rel_to"""
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
156 |
path_from = self.abspath(rel_from) |
157 |
path_to = self.abspath(rel_to) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
158 |
|
907.1.48
by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass. |
159 |
try: |
1185.31.58
by John Arbash Meinel
Updating for new transport tests so that they pass on win32 |
160 |
rename(path_from, path_to) |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
161 |
except (IOError, OSError),e: |
162 |
# TODO: What about path_to?
|
|
163 |
self._translate_error(e, path_from) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
164 |
|
165 |
def delete(self, relpath): |
|
166 |
"""Delete the item at relpath"""
|
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
167 |
path = relpath |
907.1.48
by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass. |
168 |
try: |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
169 |
path = self.abspath(relpath) |
170 |
os.remove(path) |
|
171 |
except (IOError, OSError),e: |
|
172 |
# TODO: What about path_to?
|
|
173 |
self._translate_error(e, path) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
174 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
175 |
def copy_to(self, relpaths, other, mode=None, pb=None): |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
176 |
"""Copy a set of entries from self into another Transport.
|
177 |
||
178 |
:param relpaths: A list/generator of entries to be copied.
|
|
179 |
"""
|
|
180 |
if isinstance(other, LocalTransport): |
|
181 |
# Both from & to are on the local filesystem
|
|
182 |
# Unfortunately, I can't think of anything faster than just
|
|
183 |
# copying them across, one by one :(
|
|
184 |
import shutil |
|
185 |
||
186 |
total = self._get_total(relpaths) |
|
187 |
count = 0 |
|
188 |
for path in relpaths: |
|
189 |
self._update_pb(pb, 'copy-to', count, total) |
|
1185.16.158
by John Arbash Meinel
Added a test that copy_to raises NoSuchFile when a directory is missing (not IOError) |
190 |
try: |
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
191 |
mypath = self.abspath(path) |
192 |
otherpath = other.abspath(path) |
|
193 |
shutil.copy(mypath, otherpath) |
|
194 |
if mode is not None: |
|
195 |
os.chmod(otherpath, mode) |
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
196 |
except (IOError, OSError),e: |
197 |
self._translate_error(e, path) |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
198 |
count += 1 |
199 |
return count |
|
200 |
else: |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
201 |
return super(LocalTransport, self).copy_to(relpaths, other, mode=mode, pb=pb) |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
202 |
|
1400.1.1
by Robert Collins
implement a basic test for the ui branch command from http servers |
203 |
def listable(self): |
204 |
"""See Transport.listable."""
|
|
205 |
return True |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
206 |
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
207 |
def list_dir(self, relpath): |
208 |
"""Return a list of all files at the given location.
|
|
209 |
WARNING: many transports do not support this, so trying avoid using
|
|
210 |
it if at all possible.
|
|
211 |
"""
|
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
212 |
path = relpath |
907.1.48
by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass. |
213 |
try: |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
214 |
path = self.abspath(relpath) |
215 |
return os.listdir(path) |
|
216 |
except (IOError, OSError),e: |
|
217 |
self._translate_error(e, path) |
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
218 |
|
219 |
def stat(self, relpath): |
|
220 |
"""Return the stat information for a file.
|
|
221 |
"""
|
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
222 |
path = relpath |
907.1.48
by John Arbash Meinel
Updated LocalTransport by passing it through the transport_test suite, and got it to pass. |
223 |
try: |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
224 |
path = self.abspath(relpath) |
225 |
return os.stat(path) |
|
226 |
except (IOError, OSError),e: |
|
227 |
self._translate_error(e, path) |
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
228 |
|
907.1.24
by John Arbash Meinel
Remote functionality work. |
229 |
def lock_read(self, relpath): |
230 |
"""Lock the given file for shared (read) access.
|
|
231 |
:return: A lock object, which should be passed to Transport.unlock()
|
|
232 |
"""
|
|
233 |
from bzrlib.lock import ReadLock |
|
234 |
return ReadLock(self.abspath(relpath)) |
|
235 |
||
236 |
def lock_write(self, relpath): |
|
237 |
"""Lock the given file for exclusive (write) access.
|
|
238 |
WARNING: many transports do not support this, so trying avoid using it
|
|
239 |
||
240 |
:return: A lock object, which should be passed to Transport.unlock()
|
|
241 |
"""
|
|
242 |
from bzrlib.lock import WriteLock |
|
243 |
return WriteLock(self.abspath(relpath)) |
|
244 |
||
1442.1.41
by Robert Collins
move duplicate scratch logic into a scratch transport |
245 |
|
246 |
class ScratchTransport(LocalTransport): |
|
247 |
"""A transport that works in a temporary dir and cleans up after itself.
|
|
248 |
|
|
249 |
The dir only exists for the lifetime of the Python object.
|
|
250 |
Obviously you should not put anything precious in it.
|
|
251 |
"""
|
|
252 |
||
1442.1.42
by Robert Collins
rebuild ScratchBranch on top of ScratchTransport |
253 |
def __init__(self, base=None): |
254 |
if base is None: |
|
255 |
base = tempfile.mkdtemp() |
|
1442.1.41
by Robert Collins
move duplicate scratch logic into a scratch transport |
256 |
super(ScratchTransport, self).__init__(base) |
257 |
||
258 |
def __del__(self): |
|
1442.1.42
by Robert Collins
rebuild ScratchBranch on top of ScratchTransport |
259 |
shutil.rmtree(self.base, ignore_errors=True) |
1442.1.41
by Robert Collins
move duplicate scratch logic into a scratch transport |
260 |
mutter("%r destroyed" % self) |
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
261 |
|
262 |
||
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
263 |
class LocalRelpathServer(Server): |
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
264 |
"""A pretend server for local transports, using relpaths."""
|
265 |
||
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
266 |
def get_url(self): |
267 |
"""See Transport.Server.get_url."""
|
|
268 |
return "." |
|
269 |
||
270 |
||
271 |
class LocalAbspathServer(Server): |
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
272 |
"""A pretend server for local transports, using absolute paths."""
|
273 |
||
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
274 |
def get_url(self): |
275 |
"""See Transport.Server.get_url."""
|
|
276 |
return os.path.abspath("") |
|
277 |
||
278 |
||
279 |
class LocalURLServer(Server): |
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
280 |
"""A pretend server for local transports, using file:// urls."""
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
281 |
|
282 |
def get_url(self): |
|
283 |
"""See Transport.Server.get_url."""
|
|
284 |
# FIXME: \ to / on windows
|
|
285 |
return "file://%s" % os.path.abspath("") |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
286 |
|
287 |
||
288 |
def get_test_permutations(): |
|
289 |
"""Return the permutations to be used in testing."""
|
|
290 |
return [(LocalTransport, LocalRelpathServer), |
|
291 |
(LocalTransport, LocalAbspathServer), |
|
292 |
(LocalTransport, LocalURLServer), |
|
293 |
]
|