1553.5.10
by Martin Pool
New DirectoryNotEmpty exception, and raise this from local and memory |
1 |
# Copyright (C) 2005, 2006 Canonical Ltd
|
1185.11.19
by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings. |
2 |
|
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
||
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
||
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
16 |
|
1185.11.19
by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings. |
17 |
"""Transport is an abstraction layer to handle file access.
|
18 |
||
19 |
The abstraction is to allow access from the local filesystem, as well
|
|
20 |
as remote (such as http or sftp).
|
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
21 |
|
22 |
Transports are constructed from a string, being a URL or (as a degenerate
|
|
23 |
case) a local filesystem path. This is typically the top directory of
|
|
24 |
a bzrdir, repository, or similar object we are interested in working with.
|
|
25 |
The Transport returned has methods to read, write and manipulate files within
|
|
26 |
it.
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
27 |
"""
|
28 |
||
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
29 |
import errno |
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
30 |
from collections import deque |
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
31 |
from copy import deepcopy |
1685.1.9
by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url |
32 |
import re |
1773.4.1
by Martin Pool
Add pyflakes makefile target; fix many warnings |
33 |
from stat import S_ISDIR |
1530.1.7
by Robert Collins
merge integration. |
34 |
import sys |
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
35 |
from unittest import TestSuite |
1685.1.70
by Wouter van Heyst
working on get_parent, set_parent and relative urls, broken |
36 |
import urllib |
1636.1.1
by Robert Collins
Fix calling relpath() and abspath() on transports at their root. |
37 |
import urlparse |
1773.4.1
by Martin Pool
Add pyflakes makefile target; fix many warnings |
38 |
import warnings |
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
39 |
|
1725.2.3
by Robert Collins
Remove import of pumpfile from inner loop of commit. |
40 |
import bzrlib |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
41 |
import bzrlib.errors as errors |
1540.3.8
by Martin Pool
Some support for falling back between transport implementations. |
42 |
from bzrlib.errors import DependencyNotPresent |
1685.1.9
by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url |
43 |
import bzrlib.osutils as osutils |
1725.2.3
by Robert Collins
Remove import of pumpfile from inner loop of commit. |
44 |
from bzrlib.osutils import pumpfile |
1773.4.1
by Martin Pool
Add pyflakes makefile target; fix many warnings |
45 |
from bzrlib.symbol_versioning import (deprecated_passed, deprecated_method, deprecated_function, |
46 |
DEPRECATED_PARAMETER, |
|
47 |
zero_eight) |
|
1685.1.9
by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url |
48 |
from bzrlib.trace import mutter, warning |
1685.1.45
by John Arbash Meinel
Moved url functions into bzrlib.urlutils |
49 |
import bzrlib.urlutils as urlutils |
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
50 |
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
51 |
# {prefix: [transport_classes]}
|
52 |
# Transports are inserted onto the list LIFO and tried in order; as a result
|
|
53 |
# transports provided by plugins are tried first, which is usually what we
|
|
54 |
# want.
|
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
55 |
_protocol_handlers = { |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
56 |
}
|
57 |
||
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
58 |
def register_transport(prefix, klass, override=DEPRECATED_PARAMETER): |
1540.3.8
by Martin Pool
Some support for falling back between transport implementations. |
59 |
"""Register a transport that can be used to open URLs
|
60 |
||
61 |
Normally you should use register_lazy_transport, which defers loading the
|
|
62 |
implementation until it's actually used, and so avoids pulling in possibly
|
|
63 |
large implementation libraries.
|
|
64 |
"""
|
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
65 |
# Note that this code runs very early in library setup -- trace may not be
|
66 |
# working, etc.
|
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
67 |
global _protocol_handlers |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
68 |
if deprecated_passed(override): |
1773.4.1
by Martin Pool
Add pyflakes makefile target; fix many warnings |
69 |
warnings.warn("register_transport(override) is deprecated") |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
70 |
_protocol_handlers.setdefault(prefix, []).insert(0, klass) |
71 |
||
72 |
||
73 |
def register_lazy_transport(scheme, module, classname): |
|
74 |
"""Register lazy-loaded transport class.
|
|
75 |
||
76 |
When opening a URL with the given scheme, load the module and then
|
|
77 |
instantiate the particular class.
|
|
78 |
||
79 |
If the module raises DependencyNotPresent when it's imported, it is
|
|
80 |
skipped and another implementation of the protocol is tried. This is
|
|
81 |
intended to be used when the implementation depends on an external
|
|
82 |
implementation that may not be present. If any other error is raised, it
|
|
83 |
propagates up and the attempt to open the url fails.
|
|
84 |
"""
|
|
85 |
# TODO: If no implementation of a protocol is available because of missing
|
|
86 |
# dependencies, we should perhaps show the message about what dependency
|
|
87 |
# was missing.
|
|
88 |
def _loader(base): |
|
89 |
mod = __import__(module, globals(), locals(), [classname]) |
|
90 |
klass = getattr(mod, classname) |
|
91 |
return klass(base) |
|
92 |
_loader.module = module |
|
93 |
register_transport(scheme, _loader) |
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
94 |
|
1442.1.24
by Robert Collins
Pull up _check_id and _relpath from Text and CompressedText stores into TransportStore |
95 |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
96 |
def _get_protocol_handlers(): |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
97 |
"""Return a dictionary of {urlprefix: [factory]}"""
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
98 |
return _protocol_handlers |
99 |
||
100 |
||
101 |
def _set_protocol_handlers(new_handlers): |
|
102 |
"""Replace the current protocol handlers dictionary.
|
|
103 |
||
104 |
WARNING this will remove all build in protocols. Use with care.
|
|
105 |
"""
|
|
106 |
global _protocol_handlers |
|
107 |
_protocol_handlers = new_handlers |
|
108 |
||
109 |
||
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
110 |
def _clear_protocol_handlers(): |
111 |
global _protocol_handlers |
|
112 |
_protocol_handlers = {} |
|
113 |
||
114 |
||
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
115 |
def _get_transport_modules(): |
116 |
"""Return a list of the modules providing transports."""
|
|
117 |
modules = set() |
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
118 |
for prefix, factory_list in _protocol_handlers.items(): |
119 |
for factory in factory_list: |
|
120 |
if factory.__module__ == "bzrlib.transport": |
|
121 |
# this is a lazy load transport, because no real ones
|
|
122 |
# are directlry in bzrlib.transport
|
|
123 |
modules.add(factory.module) |
|
124 |
else: |
|
125 |
modules.add(factory.__module__) |
|
1530.1.19
by Robert Collins
Make transport test adapter tests reliable. |
126 |
result = list(modules) |
127 |
result.sort() |
|
128 |
return result |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
129 |
|
130 |
||
1636.1.2
by Robert Collins
More review fixen to the relpath at '/' fixes. |
131 |
def register_urlparse_netloc_protocol(protocol): |
1636.1.1
by Robert Collins
Fix calling relpath() and abspath() on transports at their root. |
132 |
"""Ensure that protocol is setup to be used with urlparse netloc parsing."""
|
133 |
if protocol not in urlparse.uses_netloc: |
|
134 |
urlparse.uses_netloc.append(protocol) |
|
135 |
||
136 |
||
1707.3.4
by John Arbash Meinel
Moved most of sftp.split_url into a Transport function. |
137 |
def split_url(url): |
1685.1.69
by Wouter van Heyst
merge bzr.dev 1740 |
138 |
# TODO: jam 20060606 urls should only be ascii, or they should raise InvalidURL
|
1707.3.4
by John Arbash Meinel
Moved most of sftp.split_url into a Transport function. |
139 |
if isinstance(url, unicode): |
140 |
url = url.encode('utf-8') |
|
141 |
(scheme, netloc, path, params, |
|
142 |
query, fragment) = urlparse.urlparse(url, allow_fragments=False) |
|
143 |
username = password = host = port = None |
|
144 |
if '@' in netloc: |
|
145 |
username, host = netloc.split('@', 1) |
|
146 |
if ':' in username: |
|
147 |
username, password = username.split(':', 1) |
|
148 |
password = urllib.unquote(password) |
|
149 |
username = urllib.unquote(username) |
|
150 |
else: |
|
151 |
host = netloc |
|
152 |
||
153 |
if ':' in host: |
|
154 |
host, port = host.rsplit(':', 1) |
|
155 |
try: |
|
156 |
port = int(port) |
|
157 |
except ValueError: |
|
158 |
# TODO: Should this be ConnectionError?
|
|
159 |
raise errors.TransportError('%s: invalid port number' % port) |
|
160 |
host = urllib.unquote(host) |
|
161 |
||
162 |
path = urllib.unquote(path) |
|
163 |
||
164 |
return (scheme, username, password, host, port, path) |
|
165 |
||
166 |
||
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
167 |
class Transport(object): |
168 |
"""This class encapsulates methods for retrieving or putting a file
|
|
169 |
from/to a storage location.
|
|
170 |
||
171 |
Most functions have a _multi variant, which allows you to queue up
|
|
172 |
multiple requests. They generally have a dumb base implementation
|
|
173 |
which just iterates over the arguments, but smart Transport
|
|
174 |
implementations can do pipelining.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
175 |
In general implementations should support having a generator or a list
|
176 |
as an argument (ie always iterate, never index)
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
177 |
"""
|
178 |
||
179 |
def __init__(self, base): |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
180 |
super(Transport, self).__init__() |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
181 |
self.base = base |
182 |
||
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
183 |
def _translate_error(self, e, path, raise_generic=True): |
184 |
"""Translate an IOError or OSError into an appropriate bzr error.
|
|
185 |
||
186 |
This handles things like ENOENT, ENOTDIR, EEXIST, and EACCESS
|
|
187 |
"""
|
|
188 |
if hasattr(e, 'errno'): |
|
189 |
if e.errno in (errno.ENOENT, errno.ENOTDIR): |
|
190 |
raise errors.NoSuchFile(path, extra=e) |
|
1185.31.58
by John Arbash Meinel
Updating for new transport tests so that they pass on win32 |
191 |
# I would rather use errno.EFOO, but there doesn't seem to be
|
192 |
# any matching for 267
|
|
193 |
# This is the error when doing a listdir on a file:
|
|
194 |
# WindowsError: [Errno 267] The directory name is invalid
|
|
195 |
if sys.platform == 'win32' and e.errno in (errno.ESRCH, 267): |
|
196 |
raise errors.NoSuchFile(path, extra=e) |
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
197 |
if e.errno == errno.EEXIST: |
198 |
raise errors.FileExists(path, extra=e) |
|
199 |
if e.errno == errno.EACCES: |
|
200 |
raise errors.PermissionDenied(path, extra=e) |
|
1553.5.10
by Martin Pool
New DirectoryNotEmpty exception, and raise this from local and memory |
201 |
if e.errno == errno.ENOTEMPTY: |
202 |
raise errors.DirectoryNotEmpty(path, extra=e) |
|
1558.10.1
by Aaron Bentley
Handle lockdirs over NFS properly |
203 |
if e.errno == errno.EBUSY: |
204 |
raise errors.ResourceBusy(path, extra=e) |
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
205 |
if raise_generic: |
206 |
raise errors.TransportError(orig_error=e) |
|
207 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
208 |
def clone(self, offset=None): |
209 |
"""Return a new Transport object, cloned from the current location,
|
|
1185.11.6
by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently. |
210 |
using a subdirectory or parent directory. This allows connections
|
211 |
to be pooled, rather than a new one needed for each subdir.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
212 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
213 |
raise NotImplementedError(self.clone) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
214 |
|
907.1.32
by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate. |
215 |
def should_cache(self): |
216 |
"""Return True if the data pulled across should be cached locally.
|
|
907.1.22
by John Arbash Meinel
Fixed some encoding issues, added is_remote function for Transport objects. |
217 |
"""
|
907.1.32
by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate. |
218 |
return False |
907.1.22
by John Arbash Meinel
Fixed some encoding issues, added is_remote function for Transport objects. |
219 |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
220 |
def _pump(self, from_file, to_file): |
221 |
"""Most children will need to copy from one file-like
|
|
222 |
object or string to another one.
|
|
223 |
This just gives them something easy to call.
|
|
224 |
"""
|
|
225 |
if isinstance(from_file, basestring): |
|
907.1.8
by John Arbash Meinel
Changed the format for abspath. Updated branch to use a hidden _transport |
226 |
to_file.write(from_file) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
227 |
else: |
228 |
pumpfile(from_file, to_file) |
|
229 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
230 |
def _get_total(self, multi): |
231 |
"""Try to figure out how many entries are in multi,
|
|
232 |
but if not possible, return None.
|
|
233 |
"""
|
|
234 |
try: |
|
235 |
return len(multi) |
|
236 |
except TypeError: # We can't tell how many, because relpaths is a generator |
|
237 |
return None |
|
238 |
||
239 |
def _update_pb(self, pb, msg, count, total): |
|
240 |
"""Update the progress bar based on the current count
|
|
241 |
and total available, total may be None if it was
|
|
242 |
not possible to determine.
|
|
243 |
"""
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
244 |
if pb is None: |
245 |
return
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
246 |
if total is None: |
247 |
pb.update(msg, count, count+1) |
|
248 |
else: |
|
249 |
pb.update(msg, count, total) |
|
250 |
||
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
251 |
def _iterate_over(self, multi, func, pb, msg, expand=True): |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
252 |
"""Iterate over all entries in multi, passing them to func,
|
253 |
and update the progress bar as you go along.
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
254 |
|
255 |
:param expand: If True, the entries will be passed to the function
|
|
256 |
by expanding the tuple. If False, it will be passed
|
|
257 |
as a single parameter.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
258 |
"""
|
259 |
total = self._get_total(multi) |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
260 |
result = [] |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
261 |
count = 0 |
262 |
for entry in multi: |
|
263 |
self._update_pb(pb, msg, count, total) |
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
264 |
if expand: |
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
265 |
result.append(func(*entry)) |
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
266 |
else: |
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
267 |
result.append(func(entry)) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
268 |
count += 1 |
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
269 |
return tuple(result) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
270 |
|
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
271 |
def abspath(self, relpath): |
272 |
"""Return the full url to the given relative path.
|
|
273 |
This can be supplied with a string or a list
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
274 |
|
275 |
XXX: Robert Collins 20051016 - is this really needed in the public
|
|
276 |
interface ?
|
|
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
277 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
278 |
raise NotImplementedError(self.abspath) |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
279 |
|
280 |
def relpath(self, abspath): |
|
281 |
"""Return the local path portion from a given absolute path.
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
282 |
|
283 |
This default implementation is not suitable for filesystems with
|
|
284 |
aliasing, such as that given by symlinks, where a path may not
|
|
285 |
start with our base, but still be a relpath once aliasing is
|
|
286 |
resolved.
|
|
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
287 |
"""
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
288 |
# TODO: This might want to use bzrlib.osutils.relpath
|
289 |
# but we have to watch out because of the prefix issues
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
290 |
if not (abspath == self.base[:-1] or abspath.startswith(self.base)): |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
291 |
raise errors.PathNotChild(abspath, self.base) |
1442.1.44
by Robert Collins
Many transport related tweaks: |
292 |
pl = len(self.base) |
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
293 |
return abspath[pl:].strip('/') |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
294 |
|
1685.1.9
by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url |
295 |
def local_abspath(self, relpath): |
296 |
"""Return the absolute path on the local filesystem.
|
|
297 |
||
298 |
This function will only be defined for Transports which have a
|
|
299 |
physical local filesystem representation.
|
|
300 |
"""
|
|
1685.1.11
by John Arbash Meinel
Not all of the registered transports use :// as part of their path, specifically memory:/ |
301 |
# TODO: jam 20060426 Should this raise NotLocalUrl instead?
|
1685.1.9
by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url |
302 |
raise errors.TransportNotPossible('This is not a LocalTransport,' |
303 |
' so there is no local representation for a path') |
|
304 |
||
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
305 |
def has(self, relpath): |
1442.1.44
by Robert Collins
Many transport related tweaks: |
306 |
"""Does the file relpath exist?
|
307 |
|
|
308 |
Note that some transports MAY allow querying on directories, but this
|
|
1553.5.66
by Martin Pool
doc |
309 |
is not part of the protocol. In other words, the results of
|
310 |
t.has("a_directory_name") are undefined."
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
311 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
312 |
raise NotImplementedError(self.has) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
313 |
|
907.1.36
by John Arbash Meinel
Moving the multi-get functionality higher up into the Branch class. |
314 |
def has_multi(self, relpaths, pb=None): |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
315 |
"""Return True/False for each entry in relpaths"""
|
316 |
total = self._get_total(relpaths) |
|
317 |
count = 0 |
|
318 |
for relpath in relpaths: |
|
907.1.36
by John Arbash Meinel
Moving the multi-get functionality higher up into the Branch class. |
319 |
self._update_pb(pb, 'has', count, total) |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
320 |
yield self.has(relpath) |
321 |
count += 1 |
|
322 |
||
1185.16.155
by John Arbash Meinel
Added a has_any function to the Transport API |
323 |
def has_any(self, relpaths): |
324 |
"""Return True if any of the paths exist."""
|
|
325 |
for relpath in relpaths: |
|
326 |
if self.has(relpath): |
|
327 |
return True |
|
328 |
return False |
|
329 |
||
1442.1.44
by Robert Collins
Many transport related tweaks: |
330 |
def iter_files_recursive(self): |
331 |
"""Iter the relative paths of files in the transports sub-tree.
|
|
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
332 |
|
333 |
*NOTE*: This only lists *files*, not subdirectories!
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
334 |
|
335 |
As with other listing functions, only some transports implement this,.
|
|
336 |
you may check via is_listable to determine if it will.
|
|
337 |
"""
|
|
1530.1.4
by Robert Collins
integrate Memory tests into transport interface tests. |
338 |
raise errors.TransportNotPossible("This transport has not " |
1530.1.21
by Robert Collins
Review feedback fixes. |
339 |
"implemented iter_files_recursive "
|
1530.1.4
by Robert Collins
integrate Memory tests into transport interface tests. |
340 |
"(but must claim to be listable "
|
341 |
"to trigger this error).") |
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
342 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
343 |
def get(self, relpath): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
344 |
"""Get the file at the given relative path.
|
907.1.20
by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8 |
345 |
|
346 |
:param relpath: The relative path to the file
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
347 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
348 |
raise NotImplementedError(self.get) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
349 |
|
1594.2.5
by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support. |
350 |
def readv(self, relpath, offsets): |
351 |
"""Get parts of the file at the given relative path.
|
|
352 |
||
353 |
:offsets: A list of (offset, size) tuples.
|
|
354 |
:return: A list or generator of (offset, data) tuples
|
|
355 |
"""
|
|
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
356 |
def do_combined_read(combined_offsets): |
357 |
total_size = 0 |
|
358 |
for offset, size in combined_offsets: |
|
359 |
total_size += size |
|
360 |
mutter('readv coalesced %d reads.', len(combined_offsets)) |
|
361 |
offset = combined_offsets[0][0] |
|
362 |
fp.seek(offset) |
|
363 |
data = fp.read(total_size) |
|
364 |
pos = 0 |
|
365 |
for offset, size in combined_offsets: |
|
366 |
yield offset, data[pos:pos + size] |
|
367 |
pos += size |
|
368 |
||
1594.2.16
by Robert Collins
Coalesce readv requests on file based transports. |
369 |
if not len(offsets): |
370 |
return
|
|
1594.2.5
by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support. |
371 |
fp = self.get(relpath) |
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
372 |
pending_offsets = deque(offsets) |
373 |
combined_offsets = [] |
|
374 |
while len(pending_offsets): |
|
375 |
offset, size = pending_offsets.popleft() |
|
376 |
if not combined_offsets: |
|
377 |
combined_offsets = [[offset, size]] |
|
1594.2.16
by Robert Collins
Coalesce readv requests on file based transports. |
378 |
else: |
1594.2.19
by Robert Collins
More coalescing tweaks, and knit feedback. |
379 |
if (len (combined_offsets) < 50 and |
380 |
combined_offsets[-1][0] + combined_offsets[-1][1] == offset): |
|
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
381 |
# combatible offset:
|
382 |
combined_offsets.append([offset, size]) |
|
383 |
else: |
|
1594.2.19
by Robert Collins
More coalescing tweaks, and knit feedback. |
384 |
# incompatible, or over the threshold issue a read and yield
|
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
385 |
pending_offsets.appendleft((offset, size)) |
386 |
for result in do_combined_read(combined_offsets): |
|
387 |
yield result |
|
388 |
combined_offsets = [] |
|
389 |
# whatever is left is a single coalesced request
|
|
390 |
if len(combined_offsets): |
|
391 |
for result in do_combined_read(combined_offsets): |
|
392 |
yield result |
|
1594.2.5
by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support. |
393 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
394 |
def get_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
395 |
"""Get a list of file-like objects, one for each entry in relpaths.
|
396 |
||
397 |
:param relpaths: A list of relative paths.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
398 |
:param pb: An optional ProgressBar for indicating percent done.
|
399 |
:return: A list or generator of file-like objects
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
400 |
"""
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
401 |
# TODO: Consider having this actually buffer the requests,
|
402 |
# in the default mode, it probably won't give worse performance,
|
|
403 |
# and all children wouldn't have to implement buffering
|
|
907.1.16
by John Arbash Meinel
Fixing a few cut&paste typos. |
404 |
total = self._get_total(relpaths) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
405 |
count = 0 |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
406 |
for relpath in relpaths: |
907.1.16
by John Arbash Meinel
Fixing a few cut&paste typos. |
407 |
self._update_pb(pb, 'get', count, total) |
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
408 |
yield self.get(relpath) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
409 |
count += 1 |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
410 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
411 |
def put(self, relpath, f, mode=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
412 |
"""Copy the file-like or string object into the location.
|
907.1.20
by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8 |
413 |
|
414 |
:param relpath: Location to put the contents, relative to base.
|
|
415 |
:param f: File-like or string object.
|
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
416 |
:param mode: The mode for the newly created file,
|
417 |
None means just use the default
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
418 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
419 |
raise NotImplementedError(self.put) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
420 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
421 |
def put_multi(self, files, mode=None, pb=None): |
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
422 |
"""Put a set of files into the location.
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
423 |
|
424 |
:param files: A list of tuples of relpath, file object [(path1, file1), (path2, file2),...]
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
425 |
:param pb: An optional ProgressBar for indicating percent done.
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
426 |
:param mode: The mode for the newly created files
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
427 |
:return: The number of files copied.
|
428 |
"""
|
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
429 |
def put(path, f): |
430 |
self.put(path, f, mode=mode) |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
431 |
return len(self._iterate_over(files, put, pb, 'put', expand=True)) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
432 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
433 |
def mkdir(self, relpath, mode=None): |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
434 |
"""Create a directory at the given path."""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
435 |
raise NotImplementedError(self.mkdir) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
436 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
437 |
def mkdir_multi(self, relpaths, mode=None, pb=None): |
907.1.47
by John Arbash Meinel
Created mkdir_multi, modified branch to use _multi forms when initializing a branch, creating a full test routine for transports |
438 |
"""Create a group of directories"""
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
439 |
def mkdir(path): |
440 |
self.mkdir(path, mode=mode) |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
441 |
return len(self._iterate_over(relpaths, mkdir, pb, 'mkdir', expand=False)) |
907.1.47
by John Arbash Meinel
Created mkdir_multi, modified branch to use _multi forms when initializing a branch, creating a full test routine for transports |
442 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
443 |
def append(self, relpath, f): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
444 |
"""Append the text in the file-like or string object to
|
445 |
the supplied location.
|
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
446 |
|
447 |
returns the length of f before the content was written to it.
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
448 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
449 |
raise NotImplementedError(self.append) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
450 |
|
1185.11.19
by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings. |
451 |
def append_multi(self, files, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
452 |
"""Append the text in each file-like or string object to
|
453 |
the supplied location.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
454 |
|
455 |
:param files: A set of (path, f) entries
|
|
456 |
:param pb: An optional ProgressBar for indicating percent done.
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
457 |
"""
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
458 |
return self._iterate_over(files, self.append, pb, 'append', expand=True) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
459 |
|
460 |
def copy(self, rel_from, rel_to): |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
461 |
"""Copy the item at rel_from to the location at rel_to.
|
462 |
|
|
463 |
Override this for efficiency if a specific transport can do it
|
|
464 |
faster than this default implementation.
|
|
465 |
"""
|
|
466 |
self.put(rel_to, self.get(rel_from)) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
467 |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
468 |
def copy_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
469 |
"""Copy a bunch of entries.
|
470 |
|
|
471 |
:param relpaths: A list of tuples of the form [(from, to), (from, to),...]
|
|
472 |
"""
|
|
473 |
# This is the non-pipelined implementation, so that
|
|
474 |
# implementors don't have to implement everything.
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
475 |
return self._iterate_over(relpaths, self.copy, pb, 'copy', expand=True) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
476 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
477 |
def copy_to(self, relpaths, other, mode=None, pb=None): |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
478 |
"""Copy a set of entries from self into another Transport.
|
479 |
||
480 |
:param relpaths: A list/generator of entries to be copied.
|
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
481 |
:param mode: This is the target mode for the newly created files
|
1185.16.156
by John Arbash Meinel
Adding a note about changing copy_to's interface |
482 |
TODO: This interface needs to be updated so that the target location
|
483 |
can be different from the source location.
|
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
484 |
"""
|
485 |
# The dummy implementation just does a simple get + put
|
|
486 |
def copy_entry(path): |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
487 |
other.put(path, self.get(path), mode=mode) |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
488 |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
489 |
return len(self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False)) |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
490 |
|
1534.4.26
by Robert Collins
Move working tree initialisation out from Branch.initialize, deprecated Branch.initialize to Branch.create. |
491 |
def copy_tree(self, from_relpath, to_relpath): |
492 |
"""Copy a subtree from one relpath to another.
|
|
493 |
||
494 |
If a faster implementation is available, specific transports should
|
|
495 |
implement it.
|
|
496 |
"""
|
|
497 |
source = self.clone(from_relpath) |
|
498 |
self.mkdir(to_relpath) |
|
499 |
target = self.clone(to_relpath) |
|
500 |
files = [] |
|
501 |
directories = ['.'] |
|
502 |
while directories: |
|
503 |
dir = directories.pop() |
|
504 |
if dir != '.': |
|
505 |
target.mkdir(dir) |
|
506 |
for path in source.list_dir(dir): |
|
507 |
path = dir + '/' + path |
|
508 |
stat = source.stat(path) |
|
509 |
if S_ISDIR(stat.st_mode): |
|
510 |
directories.append(path) |
|
511 |
else: |
|
512 |
files.append(path) |
|
513 |
source.copy_to(files, target) |
|
514 |
||
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
515 |
def rename(self, rel_from, rel_to): |
516 |
"""Rename a file or directory.
|
|
517 |
||
518 |
This *must* fail if the destination is a nonempty directory - it must
|
|
519 |
not automatically remove it. It should raise DirectoryNotEmpty, or
|
|
520 |
some other PathError if the case can't be specifically detected.
|
|
521 |
||
522 |
If the destination is an empty directory or a file this function may
|
|
523 |
either fail or succeed, depending on the underlying transport. It
|
|
524 |
should not attempt to remove the destination if overwriting is not the
|
|
525 |
native transport behaviour. If at all possible the transport should
|
|
526 |
ensure that the rename either completes or not, without leaving the
|
|
527 |
destination deleted and the new file not moved in place.
|
|
528 |
||
529 |
This is intended mainly for use in implementing LockDir.
|
|
530 |
"""
|
|
531 |
# transports may need to override this
|
|
1553.5.17
by Martin Pool
Transport.rename should be unimplemented in base class |
532 |
raise NotImplementedError(self.rename) |
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
533 |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
534 |
def move(self, rel_from, rel_to): |
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
535 |
"""Move the item at rel_from to the location at rel_to.
|
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
536 |
|
537 |
The destination is deleted if possible, even if it's a non-empty
|
|
538 |
directory tree.
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
539 |
|
540 |
If a transport can directly implement this it is suggested that
|
|
541 |
it do so for efficiency.
|
|
542 |
"""
|
|
1534.4.26
by Robert Collins
Move working tree initialisation out from Branch.initialize, deprecated Branch.initialize to Branch.create. |
543 |
if S_ISDIR(self.stat(rel_from).st_mode): |
544 |
self.copy_tree(rel_from, rel_to) |
|
545 |
self.delete_tree(rel_from) |
|
546 |
else: |
|
547 |
self.copy(rel_from, rel_to) |
|
548 |
self.delete(rel_from) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
549 |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
550 |
def move_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
551 |
"""Move a bunch of entries.
|
552 |
|
|
553 |
:param relpaths: A list of tuples of the form [(from1, to1), (from2, to2),...]
|
|
554 |
"""
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
555 |
return self._iterate_over(relpaths, self.move, pb, 'move', expand=True) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
556 |
|
557 |
def move_multi_to(self, relpaths, rel_to): |
|
558 |
"""Move a bunch of entries to a single location.
|
|
559 |
This differs from move_multi in that you give a list of from, and
|
|
560 |
a single destination, rather than multiple destinations.
|
|
561 |
||
562 |
:param relpaths: A list of relative paths [from1, from2, from3, ...]
|
|
563 |
:param rel_to: A directory where each entry should be placed.
|
|
564 |
"""
|
|
565 |
# This is not implemented, because you need to do special tricks to
|
|
566 |
# extract the basename, and add it to rel_to
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
567 |
raise NotImplementedError(self.move_multi_to) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
568 |
|
569 |
def delete(self, relpath): |
|
570 |
"""Delete the item at relpath"""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
571 |
raise NotImplementedError(self.delete) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
572 |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
573 |
def delete_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
574 |
"""Queue up a bunch of deletes to be done.
|
575 |
"""
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
576 |
return self._iterate_over(relpaths, self.delete, pb, 'delete', expand=False) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
577 |
|
1534.4.15
by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports. |
578 |
def delete_tree(self, relpath): |
579 |
"""Delete an entire tree. This may require a listable transport."""
|
|
580 |
subtree = self.clone(relpath) |
|
581 |
files = [] |
|
582 |
directories = ['.'] |
|
583 |
pending_rmdirs = [] |
|
584 |
while directories: |
|
585 |
dir = directories.pop() |
|
586 |
if dir != '.': |
|
587 |
pending_rmdirs.append(dir) |
|
588 |
for path in subtree.list_dir(dir): |
|
589 |
path = dir + '/' + path |
|
590 |
stat = subtree.stat(path) |
|
591 |
if S_ISDIR(stat.st_mode): |
|
592 |
directories.append(path) |
|
593 |
else: |
|
594 |
files.append(path) |
|
595 |
subtree.delete_multi(files) |
|
596 |
pending_rmdirs.reverse() |
|
597 |
for dir in pending_rmdirs: |
|
598 |
subtree.rmdir(dir) |
|
599 |
self.rmdir(relpath) |
|
600 |
||
1534.5.7
by Robert Collins
Start factoring out the upgrade policy logic. |
601 |
def __repr__(self): |
602 |
return "<%s.%s url=%s>" % (self.__module__, self.__class__.__name__, self.base) |
|
603 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
604 |
def stat(self, relpath): |
605 |
"""Return the stat information for a file.
|
|
606 |
WARNING: This may not be implementable for all protocols, so use
|
|
607 |
sparingly.
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
608 |
NOTE: This returns an object with fields such as 'st_size'. It MAY
|
609 |
or MAY NOT return the literal result of an os.stat() call, so all
|
|
610 |
access should be via named fields.
|
|
611 |
ALSO NOTE: Stats of directories may not be supported on some
|
|
612 |
transports.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
613 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
614 |
raise NotImplementedError(self.stat) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
615 |
|
1534.4.15
by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports. |
616 |
def rmdir(self, relpath): |
617 |
"""Remove a directory at the given path."""
|
|
618 |
raise NotImplementedError |
|
619 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
620 |
def stat_multi(self, relpaths, pb=None): |
621 |
"""Stat multiple files and return the information.
|
|
622 |
"""
|
|
623 |
#TODO: Is it worth making this a generator instead of a
|
|
624 |
# returning a list?
|
|
625 |
stats = [] |
|
626 |
def gather(path): |
|
627 |
stats.append(self.stat(path)) |
|
628 |
||
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
629 |
count = self._iterate_over(relpaths, gather, pb, 'stat', expand=False) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
630 |
return stats |
631 |
||
1400.1.1
by Robert Collins
implement a basic test for the ui branch command from http servers |
632 |
def listable(self): |
633 |
"""Return True if this store supports listing."""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
634 |
raise NotImplementedError(self.listable) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
635 |
|
636 |
def list_dir(self, relpath): |
|
637 |
"""Return a list of all files at the given location.
|
|
638 |
WARNING: many transports do not support this, so trying avoid using
|
|
639 |
it if at all possible.
|
|
640 |
"""
|
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
641 |
raise errors.TransportNotPossible("This transport has not " |
1530.1.21
by Robert Collins
Review feedback fixes. |
642 |
"implemented list_dir "
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
643 |
"(but must claim to be listable "
|
644 |
"to trigger this error).") |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
645 |
|
907.1.24
by John Arbash Meinel
Remote functionality work. |
646 |
def lock_read(self, relpath): |
647 |
"""Lock the given file for shared (read) access.
|
|
648 |
WARNING: many transports do not support this, so trying avoid using it
|
|
649 |
||
650 |
:return: A lock object, which should contain an unlock() function.
|
|
651 |
"""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
652 |
raise NotImplementedError(self.lock_read) |
907.1.24
by John Arbash Meinel
Remote functionality work. |
653 |
|
654 |
def lock_write(self, relpath): |
|
655 |
"""Lock the given file for exclusive (write) access.
|
|
656 |
WARNING: many transports do not support this, so trying avoid using it
|
|
657 |
||
658 |
:return: A lock object, which should contain an unlock() function.
|
|
659 |
"""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
660 |
raise NotImplementedError(self.lock_write) |
907.1.24
by John Arbash Meinel
Remote functionality work. |
661 |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
662 |
def is_readonly(self): |
663 |
"""Return true if this connection cannot be written to."""
|
|
664 |
return False |
|
665 |
||
1608.2.7
by Martin Pool
Rename supports_unix_modebits to _can_roundtrip_unix_modebits for clarity |
666 |
def _can_roundtrip_unix_modebits(self): |
1608.2.5
by Martin Pool
Add Transport.supports_unix_modebits, so tests can |
667 |
"""Return true if this transport can store and retrieve unix modebits.
|
668 |
||
669 |
(For example, 0700 to make a directory owner-private.)
|
|
670 |
|
|
671 |
Note: most callers will not want to switch on this, but should rather
|
|
672 |
just try and set permissions and let them be either stored or not.
|
|
673 |
This is intended mainly for the use of the test suite.
|
|
674 |
|
|
675 |
Warning: this is not guaranteed to be accurate as sometimes we can't
|
|
676 |
be sure: for example with vfat mounted on unix, or a windows sftp
|
|
677 |
server."""
|
|
678 |
# TODO: Perhaps return a e.g. TransportCharacteristics that can answer
|
|
679 |
# several questions about the transport.
|
|
680 |
return False |
|
681 |
||
907.1.24
by John Arbash Meinel
Remote functionality work. |
682 |
|
1685.1.9
by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url |
683 |
# jam 20060426 For compatibility we copy the functions here
|
1685.1.45
by John Arbash Meinel
Moved url functions into bzrlib.urlutils |
684 |
# TODO: The should be marked as deprecated
|
685 |
urlescape = urlutils.escape |
|
686 |
urlunescape = urlutils.unescape |
|
1685.1.9
by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url |
687 |
_urlRE = re.compile(r'^(?P<proto>[^:/\\]+)://(?P<path>.*)$') |
688 |
||
689 |
||
1393.2.4
by John Arbash Meinel
All tests pass. |
690 |
def get_transport(base): |
1185.70.3
by Martin Pool
Various updates to make storage branch mergeable: |
691 |
"""Open a transport to access a URL or directory.
|
692 |
||
693 |
base is either a URL or a directory name.
|
|
694 |
"""
|
|
1540.3.8
by Martin Pool
Some support for falling back between transport implementations. |
695 |
# TODO: give a better error if base looks like a url but there's no
|
696 |
# handler for the scheme?
|
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
697 |
global _protocol_handlers |
907.1.13
by John Arbash Meinel
Fixed bzr root. |
698 |
if base is None: |
1685.1.11
by John Arbash Meinel
Not all of the registered transports use :// as part of their path, specifically memory:/ |
699 |
base = '.' |
1685.1.33
by John Arbash Meinel
Be more of a Nazi about URLs not being unicode |
700 |
|
1843.1.1
by John Arbash Meinel
Update get_transport to raise a nicer error which includes dependency info |
701 |
last_err = None |
702 |
||
1685.1.33
by John Arbash Meinel
Be more of a Nazi about URLs not being unicode |
703 |
def convert_path_to_url(base, error_str): |
704 |
m = _urlRE.match(base) |
|
705 |
if m: |
|
706 |
# This looks like a URL, but we weren't able to
|
|
707 |
# instantiate it as such raise an appropriate error
|
|
1843.1.1
by John Arbash Meinel
Update get_transport to raise a nicer error which includes dependency info |
708 |
raise errors.UnsupportedProtocol(base, last_err) |
1685.1.33
by John Arbash Meinel
Be more of a Nazi about URLs not being unicode |
709 |
# This doesn't look like a protocol, consider it a local path
|
1685.1.45
by John Arbash Meinel
Moved url functions into bzrlib.urlutils |
710 |
new_base = urlutils.local_path_to_url(base) |
1711.4.6
by John Arbash Meinel
Removing hacks for _win32_abspath, on real win32 abspath handles unicode just fine, it doesn't handle encoding into 'mbcs' |
711 |
mutter('converting os path %r => url %s', base, new_base) |
1685.1.33
by John Arbash Meinel
Be more of a Nazi about URLs not being unicode |
712 |
return new_base |
713 |
||
714 |
# Catch any URLs which are passing Unicode rather than ASCII
|
|
715 |
try: |
|
716 |
base = base.encode('ascii') |
|
717 |
except UnicodeError: |
|
718 |
# Only local paths can be Unicode
|
|
719 |
base = convert_path_to_url(base, |
|
720 |
'URLs must be properly escaped (protocol: %s)') |
|
1685.1.11
by John Arbash Meinel
Not all of the registered transports use :// as part of their path, specifically memory:/ |
721 |
|
722 |
for proto, factory_list in _protocol_handlers.iteritems(): |
|
723 |
if proto is not None and base.startswith(proto): |
|
1843.1.1
by John Arbash Meinel
Update get_transport to raise a nicer error which includes dependency info |
724 |
t, last_err = _try_transport_factories(base, factory_list) |
1685.1.11
by John Arbash Meinel
Not all of the registered transports use :// as part of their path, specifically memory:/ |
725 |
if t: |
726 |
return t |
|
1685.1.9
by John Arbash Meinel
Updated LocalTransport so that it's base is now a URL rather than a local path. This helps consistency with all other functions. To do so, I added local_abspath() which returns the local path, and local_path_to/from_url |
727 |
|
1685.1.33
by John Arbash Meinel
Be more of a Nazi about URLs not being unicode |
728 |
# We tried all the different protocols, now try one last time
|
729 |
# as a local protocol
|
|
730 |
base = convert_path_to_url(base, 'Unsupported protocol: %s') |
|
1685.1.11
by John Arbash Meinel
Not all of the registered transports use :// as part of their path, specifically memory:/ |
731 |
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
732 |
# The default handler is the filesystem handler, stored as protocol None
|
1843.1.1
by John Arbash Meinel
Update get_transport to raise a nicer error which includes dependency info |
733 |
return _try_transport_factories(base, _protocol_handlers[None])[0] |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
734 |
|
735 |
||
736 |
def _try_transport_factories(base, factory_list): |
|
1843.1.1
by John Arbash Meinel
Update get_transport to raise a nicer error which includes dependency info |
737 |
last_err = None |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
738 |
for factory in factory_list: |
739 |
try: |
|
1843.1.1
by John Arbash Meinel
Update get_transport to raise a nicer error which includes dependency info |
740 |
return factory(base), None |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
741 |
except DependencyNotPresent, e: |
742 |
mutter("failed to instantiate transport %r for %r: %r" % |
|
743 |
(factory, base, e)) |
|
1843.1.1
by John Arbash Meinel
Update get_transport to raise a nicer error which includes dependency info |
744 |
last_err = e |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
745 |
continue
|
1843.1.1
by John Arbash Meinel
Update get_transport to raise a nicer error which includes dependency info |
746 |
return None, last_err |
1185.16.81
by mbp at sourcefrog
[merge] robert |
747 |
|
1185.16.78
by Martin Pool
- load paramiko sftp transport by default |
748 |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
749 |
class Server(object): |
1530.1.21
by Robert Collins
Review feedback fixes. |
750 |
"""A Transport Server.
|
751 |
|
|
752 |
The Server interface provides a server for a given transport. We use
|
|
753 |
these servers as loopback testing tools. For any given transport the
|
|
754 |
Servers it provides must either allow writing, or serve the contents
|
|
755 |
of os.getcwdu() at the time setUp is called.
|
|
756 |
|
|
757 |
Note that these are real servers - they must implement all the things
|
|
758 |
that we want bzr transports to take advantage of.
|
|
759 |
"""
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
760 |
|
761 |
def setUp(self): |
|
762 |
"""Setup the server to service requests."""
|
|
763 |
||
764 |
def tearDown(self): |
|
765 |
"""Remove the server and cleanup any resources it owns."""
|
|
766 |
||
767 |
def get_url(self): |
|
1530.1.21
by Robert Collins
Review feedback fixes. |
768 |
"""Return a url for this server.
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
769 |
|
770 |
If the transport does not represent a disk directory (i.e. it is
|
|
1530.1.21
by Robert Collins
Review feedback fixes. |
771 |
a database like svn, or a memory only transport, it should return
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
772 |
a connection to a newly established resource for this Server.
|
1530.1.21
by Robert Collins
Review feedback fixes. |
773 |
Otherwise it should return a url that will provide access to the path
|
774 |
that was os.getcwdu() when setUp() was called.
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
775 |
|
776 |
Subsequent calls will return the same resource.
|
|
777 |
"""
|
|
778 |
raise NotImplementedError |
|
779 |
||
1530.1.9
by Robert Collins
Test bogus urls with http in the new infrastructure. |
780 |
def get_bogus_url(self): |
781 |
"""Return a url for this protocol, that will fail to connect."""
|
|
782 |
raise NotImplementedError |
|
783 |
||
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
784 |
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
785 |
class TransportTestProviderAdapter(object): |
1530.1.21
by Robert Collins
Review feedback fixes. |
786 |
"""A tool to generate a suite testing all transports for a single test.
|
787 |
||
788 |
This is done by copying the test once for each transport and injecting
|
|
789 |
the transport_class and transport_server classes into each copy. Each copy
|
|
790 |
is also given a new id() to make it easy to identify.
|
|
791 |
"""
|
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
792 |
|
793 |
def adapt(self, test): |
|
794 |
result = TestSuite() |
|
795 |
for klass, server_factory in self._test_permutations(): |
|
796 |
new_test = deepcopy(test) |
|
797 |
new_test.transport_class = klass |
|
798 |
new_test.transport_server = server_factory |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
799 |
def make_new_test_id(): |
800 |
new_id = "%s(%s)" % (new_test.id(), server_factory.__name__) |
|
801 |
return lambda: new_id |
|
802 |
new_test.id = make_new_test_id() |
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
803 |
result.addTest(new_test) |
804 |
return result |
|
805 |
||
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
806 |
def get_transport_test_permutations(self, module): |
807 |
"""Get the permutations module wants to have tested."""
|
|
1540.3.6
by Martin Pool
[merge] update from bzr.dev |
808 |
if not hasattr(module, 'get_test_permutations'): |
809 |
warning("transport module %s doesn't provide get_test_permutations()" |
|
810 |
% module.__name__) |
|
811 |
return [] |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
812 |
return module.get_test_permutations() |
813 |
||
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
814 |
def _test_permutations(self): |
815 |
"""Return a list of the klass, server_factory pairs to test."""
|
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
816 |
result = [] |
817 |
for module in _get_transport_modules(): |
|
1534.7.70
by abentley
Fix bug when no paramiko |
818 |
try: |
819 |
result.extend(self.get_transport_test_permutations(reduce(getattr, |
|
820 |
(module).split('.')[1:], |
|
1185.71.1
by John Arbash Meinel
Allow selftest to run, even if we can't load a transport. |
821 |
__import__(module)))) |
1185.62.24
by John Arbash Meinel
Changing the exception that sftp.py throws when it can't find paramiko, so that the test suite can handle it. |
822 |
except errors.DependencyNotPresent, e: |
823 |
# Continue even if a dependency prevents us
|
|
824 |
# from running this test
|
|
1534.7.70
by abentley
Fix bug when no paramiko |
825 |
pass
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
826 |
return result |
1594.2.24
by Robert Collins
Make use of the transaction finalisation warning support to implement in-knit caching. |
827 |
|
828 |
||
829 |
class TransportLogger(object): |
|
830 |
"""Adapt a transport to get clear logging data on api calls.
|
|
831 |
|
|
832 |
Feel free to extend to log whatever calls are of interest.
|
|
833 |
"""
|
|
834 |
||
835 |
def __init__(self, adapted): |
|
836 |
self._adapted = adapted |
|
837 |
self._calls = [] |
|
838 |
||
839 |
def get(self, name): |
|
840 |
self._calls.append((name,)) |
|
841 |
return self._adapted.get(name) |
|
842 |
||
843 |
def __getattr__(self, name): |
|
844 |
"""Thunk all undefined access through to self._adapted."""
|
|
845 |
# raise AttributeError, name
|
|
846 |
return getattr(self._adapted, name) |
|
847 |
||
848 |
def readv(self, name, offsets): |
|
849 |
self._calls.append((name, offsets)) |
|
850 |
return self._adapted.readv(name, offsets) |
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
851 |
|
852 |
||
1185.16.81
by mbp at sourcefrog
[merge] robert |
853 |
# None is the default transport, for things with no url scheme
|
1185.16.79
by Martin Pool
Load transports when they're first used. |
854 |
register_lazy_transport(None, 'bzrlib.transport.local', 'LocalTransport') |
855 |
register_lazy_transport('file://', 'bzrlib.transport.local', 'LocalTransport') |
|
856 |
register_lazy_transport('sftp://', 'bzrlib.transport.sftp', 'SFTPTransport') |
|
1540.3.23
by Martin Pool
Allow urls like http+pycurl://host/ to use a particular impl |
857 |
register_lazy_transport('http+urllib://', 'bzrlib.transport.http._urllib', |
1540.3.26
by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet |
858 |
'HttpTransport_urllib') |
1540.3.23
by Martin Pool
Allow urls like http+pycurl://host/ to use a particular impl |
859 |
register_lazy_transport('https+urllib://', 'bzrlib.transport.http._urllib', |
1540.3.26
by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet |
860 |
'HttpTransport_urllib') |
861 |
register_lazy_transport('http+pycurl://', 'bzrlib.transport.http._pycurl', |
|
862 |
'PyCurlTransport') |
|
863 |
register_lazy_transport('https+pycurl://', 'bzrlib.transport.http._pycurl', |
|
864 |
'PyCurlTransport') |
|
865 |
register_lazy_transport('http://', 'bzrlib.transport.http._urllib', |
|
866 |
'HttpTransport_urllib') |
|
867 |
register_lazy_transport('https://', 'bzrlib.transport.http._urllib', |
|
868 |
'HttpTransport_urllib') |
|
1540.3.7
by Martin Pool
Prepare to select a transport depending on what dependencies can be satisfied. |
869 |
register_lazy_transport('http://', 'bzrlib.transport.http._pycurl', 'PyCurlTransport') |
870 |
register_lazy_transport('https://', 'bzrlib.transport.http._pycurl', 'PyCurlTransport') |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
871 |
register_lazy_transport('ftp://', 'bzrlib.transport.ftp', 'FtpTransport') |
872 |
register_lazy_transport('aftp://', 'bzrlib.transport.ftp', 'FtpTransport') |
|
1685.1.41
by John Arbash Meinel
memory is now memory://, need to fix the test cases. |
873 |
register_lazy_transport('memory://', 'bzrlib.transport.memory', 'MemoryTransport') |
1534.4.9
by Robert Collins
Add a readonly decorator for transports. |
874 |
register_lazy_transport('readonly+', 'bzrlib.transport.readonly', 'ReadonlyTransportDecorator') |
1558.10.2
by Robert Collins
Refactor the FakeNFS support into a TransportDecorator. |
875 |
register_lazy_transport('fakenfs+', 'bzrlib.transport.fakenfs', 'FakeNFSTransportDecorator') |
1608.2.4
by Martin Pool
[broken] Add FakeFVATTransport |
876 |
register_lazy_transport('vfat+', |
877 |
'bzrlib.transport.fakevfat', |
|
878 |
'FakeVFATTransportDecorator') |