1553.5.10
by Martin Pool
New DirectoryNotEmpty exception, and raise this from local and memory |
1 |
# Copyright (C) 2005, 2006 Canonical Ltd
|
1185.11.19
by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings. |
2 |
|
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
||
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
||
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
16 |
|
1185.11.19
by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings. |
17 |
"""Transport is an abstraction layer to handle file access.
|
18 |
||
19 |
The abstraction is to allow access from the local filesystem, as well
|
|
20 |
as remote (such as http or sftp).
|
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
21 |
|
22 |
Transports are constructed from a string, being a URL or (as a degenerate
|
|
23 |
case) a local filesystem path. This is typically the top directory of
|
|
24 |
a bzrdir, repository, or similar object we are interested in working with.
|
|
25 |
The Transport returned has methods to read, write and manipulate files within
|
|
26 |
it.
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
27 |
"""
|
28 |
||
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
29 |
import errno |
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
30 |
from collections import deque |
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
31 |
from copy import deepcopy |
1534.4.15
by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports. |
32 |
from stat import * |
1530.1.7
by Robert Collins
merge integration. |
33 |
import sys |
1185.85.76
by John Arbash Meinel
Adding an InvalidURL so transports can report they expect utf-8 quoted paths. Updated tests |
34 |
import urllib |
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
35 |
from unittest import TestSuite |
1185.80.5
by John Arbash Meinel
Changing the escaping just a little bit. Now we can handle unicode characters. |
36 |
import urllib |
1636.1.1
by Robert Collins
Fix calling relpath() and abspath() on transports at their root. |
37 |
import urlparse |
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
38 |
|
1540.3.6
by Martin Pool
[merge] update from bzr.dev |
39 |
from bzrlib.trace import mutter, warning |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
40 |
import bzrlib.errors as errors |
1540.3.8
by Martin Pool
Some support for falling back between transport implementations. |
41 |
from bzrlib.errors import DependencyNotPresent |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
42 |
from bzrlib.symbol_versioning import * |
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
43 |
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
44 |
# {prefix: [transport_classes]}
|
45 |
# Transports are inserted onto the list LIFO and tried in order; as a result
|
|
46 |
# transports provided by plugins are tried first, which is usually what we
|
|
47 |
# want.
|
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
48 |
_protocol_handlers = { |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
49 |
}
|
50 |
||
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
51 |
def register_transport(prefix, klass, override=DEPRECATED_PARAMETER): |
1540.3.8
by Martin Pool
Some support for falling back between transport implementations. |
52 |
"""Register a transport that can be used to open URLs
|
53 |
||
54 |
Normally you should use register_lazy_transport, which defers loading the
|
|
55 |
implementation until it's actually used, and so avoids pulling in possibly
|
|
56 |
large implementation libraries.
|
|
57 |
"""
|
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
58 |
# Note that this code runs very early in library setup -- trace may not be
|
59 |
# working, etc.
|
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
60 |
global _protocol_handlers |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
61 |
if deprecated_passed(override): |
62 |
warn("register_transport(override) is deprecated") |
|
63 |
_protocol_handlers.setdefault(prefix, []).insert(0, klass) |
|
64 |
||
65 |
||
66 |
def register_lazy_transport(scheme, module, classname): |
|
67 |
"""Register lazy-loaded transport class.
|
|
68 |
||
69 |
When opening a URL with the given scheme, load the module and then
|
|
70 |
instantiate the particular class.
|
|
71 |
||
72 |
If the module raises DependencyNotPresent when it's imported, it is
|
|
73 |
skipped and another implementation of the protocol is tried. This is
|
|
74 |
intended to be used when the implementation depends on an external
|
|
75 |
implementation that may not be present. If any other error is raised, it
|
|
76 |
propagates up and the attempt to open the url fails.
|
|
77 |
"""
|
|
78 |
# TODO: If no implementation of a protocol is available because of missing
|
|
79 |
# dependencies, we should perhaps show the message about what dependency
|
|
80 |
# was missing.
|
|
81 |
def _loader(base): |
|
82 |
mod = __import__(module, globals(), locals(), [classname]) |
|
83 |
klass = getattr(mod, classname) |
|
84 |
return klass(base) |
|
85 |
_loader.module = module |
|
86 |
register_transport(scheme, _loader) |
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
87 |
|
1442.1.24
by Robert Collins
Pull up _check_id and _relpath from Text and CompressedText stores into TransportStore |
88 |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
89 |
def _get_protocol_handlers(): |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
90 |
"""Return a dictionary of {urlprefix: [factory]}"""
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
91 |
return _protocol_handlers |
92 |
||
93 |
||
94 |
def _set_protocol_handlers(new_handlers): |
|
95 |
"""Replace the current protocol handlers dictionary.
|
|
96 |
||
97 |
WARNING this will remove all build in protocols. Use with care.
|
|
98 |
"""
|
|
99 |
global _protocol_handlers |
|
100 |
_protocol_handlers = new_handlers |
|
101 |
||
102 |
||
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
103 |
def _clear_protocol_handlers(): |
104 |
global _protocol_handlers |
|
105 |
_protocol_handlers = {} |
|
106 |
||
107 |
||
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
108 |
def _get_transport_modules(): |
109 |
"""Return a list of the modules providing transports."""
|
|
110 |
modules = set() |
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
111 |
for prefix, factory_list in _protocol_handlers.items(): |
112 |
for factory in factory_list: |
|
113 |
if factory.__module__ == "bzrlib.transport": |
|
114 |
# this is a lazy load transport, because no real ones
|
|
115 |
# are directlry in bzrlib.transport
|
|
116 |
modules.add(factory.module) |
|
117 |
else: |
|
118 |
modules.add(factory.__module__) |
|
1530.1.19
by Robert Collins
Make transport test adapter tests reliable. |
119 |
result = list(modules) |
120 |
result.sort() |
|
121 |
return result |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
122 |
|
123 |
||
1636.1.2
by Robert Collins
More review fixen to the relpath at '/' fixes. |
124 |
def register_urlparse_netloc_protocol(protocol): |
1636.1.1
by Robert Collins
Fix calling relpath() and abspath() on transports at their root. |
125 |
"""Ensure that protocol is setup to be used with urlparse netloc parsing."""
|
126 |
if protocol not in urlparse.uses_netloc: |
|
127 |
urlparse.uses_netloc.append(protocol) |
|
128 |
||
129 |
||
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
130 |
class Transport(object): |
131 |
"""This class encapsulates methods for retrieving or putting a file
|
|
132 |
from/to a storage location.
|
|
133 |
||
134 |
Most functions have a _multi variant, which allows you to queue up
|
|
135 |
multiple requests. They generally have a dumb base implementation
|
|
136 |
which just iterates over the arguments, but smart Transport
|
|
137 |
implementations can do pipelining.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
138 |
In general implementations should support having a generator or a list
|
139 |
as an argument (ie always iterate, never index)
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
140 |
"""
|
141 |
||
142 |
def __init__(self, base): |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
143 |
super(Transport, self).__init__() |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
144 |
self.base = base |
145 |
||
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
146 |
def _translate_error(self, e, path, raise_generic=True): |
147 |
"""Translate an IOError or OSError into an appropriate bzr error.
|
|
148 |
||
149 |
This handles things like ENOENT, ENOTDIR, EEXIST, and EACCESS
|
|
150 |
"""
|
|
151 |
if hasattr(e, 'errno'): |
|
152 |
if e.errno in (errno.ENOENT, errno.ENOTDIR): |
|
153 |
raise errors.NoSuchFile(path, extra=e) |
|
1185.31.58
by John Arbash Meinel
Updating for new transport tests so that they pass on win32 |
154 |
# I would rather use errno.EFOO, but there doesn't seem to be
|
155 |
# any matching for 267
|
|
156 |
# This is the error when doing a listdir on a file:
|
|
157 |
# WindowsError: [Errno 267] The directory name is invalid
|
|
158 |
if sys.platform == 'win32' and e.errno in (errno.ESRCH, 267): |
|
159 |
raise errors.NoSuchFile(path, extra=e) |
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
160 |
if e.errno == errno.EEXIST: |
161 |
raise errors.FileExists(path, extra=e) |
|
162 |
if e.errno == errno.EACCES: |
|
163 |
raise errors.PermissionDenied(path, extra=e) |
|
1553.5.10
by Martin Pool
New DirectoryNotEmpty exception, and raise this from local and memory |
164 |
if e.errno == errno.ENOTEMPTY: |
165 |
raise errors.DirectoryNotEmpty(path, extra=e) |
|
1558.10.1
by Aaron Bentley
Handle lockdirs over NFS properly |
166 |
if e.errno == errno.EBUSY: |
167 |
raise errors.ResourceBusy(path, extra=e) |
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
168 |
if raise_generic: |
169 |
raise errors.TransportError(orig_error=e) |
|
170 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
171 |
def clone(self, offset=None): |
172 |
"""Return a new Transport object, cloned from the current location,
|
|
1185.11.6
by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently. |
173 |
using a subdirectory or parent directory. This allows connections
|
174 |
to be pooled, rather than a new one needed for each subdir.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
175 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
176 |
raise NotImplementedError(self.clone) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
177 |
|
907.1.32
by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate. |
178 |
def should_cache(self): |
179 |
"""Return True if the data pulled across should be cached locally.
|
|
907.1.22
by John Arbash Meinel
Fixed some encoding issues, added is_remote function for Transport objects. |
180 |
"""
|
907.1.32
by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate. |
181 |
return False |
907.1.22
by John Arbash Meinel
Fixed some encoding issues, added is_remote function for Transport objects. |
182 |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
183 |
def _pump(self, from_file, to_file): |
184 |
"""Most children will need to copy from one file-like
|
|
185 |
object or string to another one.
|
|
186 |
This just gives them something easy to call.
|
|
187 |
"""
|
|
188 |
if isinstance(from_file, basestring): |
|
907.1.8
by John Arbash Meinel
Changed the format for abspath. Updated branch to use a hidden _transport |
189 |
to_file.write(from_file) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
190 |
else: |
191 |
from bzrlib.osutils import pumpfile |
|
192 |
pumpfile(from_file, to_file) |
|
193 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
194 |
def _get_total(self, multi): |
195 |
"""Try to figure out how many entries are in multi,
|
|
196 |
but if not possible, return None.
|
|
197 |
"""
|
|
198 |
try: |
|
199 |
return len(multi) |
|
200 |
except TypeError: # We can't tell how many, because relpaths is a generator |
|
201 |
return None |
|
202 |
||
203 |
def _update_pb(self, pb, msg, count, total): |
|
204 |
"""Update the progress bar based on the current count
|
|
205 |
and total available, total may be None if it was
|
|
206 |
not possible to determine.
|
|
207 |
"""
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
208 |
if pb is None: |
209 |
return
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
210 |
if total is None: |
211 |
pb.update(msg, count, count+1) |
|
212 |
else: |
|
213 |
pb.update(msg, count, total) |
|
214 |
||
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
215 |
def _iterate_over(self, multi, func, pb, msg, expand=True): |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
216 |
"""Iterate over all entries in multi, passing them to func,
|
217 |
and update the progress bar as you go along.
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
218 |
|
219 |
:param expand: If True, the entries will be passed to the function
|
|
220 |
by expanding the tuple. If False, it will be passed
|
|
221 |
as a single parameter.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
222 |
"""
|
223 |
total = self._get_total(multi) |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
224 |
result = [] |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
225 |
count = 0 |
226 |
for entry in multi: |
|
227 |
self._update_pb(pb, msg, count, total) |
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
228 |
if expand: |
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
229 |
result.append(func(*entry)) |
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
230 |
else: |
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
231 |
result.append(func(entry)) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
232 |
count += 1 |
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
233 |
return tuple(result) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
234 |
|
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
235 |
def abspath(self, relpath): |
236 |
"""Return the full url to the given relative path.
|
|
237 |
This can be supplied with a string or a list
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
238 |
|
239 |
XXX: Robert Collins 20051016 - is this really needed in the public
|
|
240 |
interface ?
|
|
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
241 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
242 |
raise NotImplementedError(self.abspath) |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
243 |
|
244 |
def relpath(self, abspath): |
|
245 |
"""Return the local path portion from a given absolute path.
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
246 |
|
247 |
This default implementation is not suitable for filesystems with
|
|
248 |
aliasing, such as that given by symlinks, where a path may not
|
|
249 |
start with our base, but still be a relpath once aliasing is
|
|
250 |
resolved.
|
|
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
251 |
"""
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
252 |
# TODO: This might want to use bzrlib.osutils.relpath
|
253 |
# but we have to watch out because of the prefix issues
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
254 |
if not (abspath == self.base[:-1] or abspath.startswith(self.base)): |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
255 |
raise errors.PathNotChild(abspath, self.base) |
1442.1.44
by Robert Collins
Many transport related tweaks: |
256 |
pl = len(self.base) |
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
257 |
return abspath[pl:].strip('/') |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
258 |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
259 |
def has(self, relpath): |
1442.1.44
by Robert Collins
Many transport related tweaks: |
260 |
"""Does the file relpath exist?
|
261 |
|
|
262 |
Note that some transports MAY allow querying on directories, but this
|
|
1553.5.66
by Martin Pool
doc |
263 |
is not part of the protocol. In other words, the results of
|
264 |
t.has("a_directory_name") are undefined."
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
265 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
266 |
raise NotImplementedError(self.has) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
267 |
|
907.1.36
by John Arbash Meinel
Moving the multi-get functionality higher up into the Branch class. |
268 |
def has_multi(self, relpaths, pb=None): |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
269 |
"""Return True/False for each entry in relpaths"""
|
270 |
total = self._get_total(relpaths) |
|
271 |
count = 0 |
|
272 |
for relpath in relpaths: |
|
907.1.36
by John Arbash Meinel
Moving the multi-get functionality higher up into the Branch class. |
273 |
self._update_pb(pb, 'has', count, total) |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
274 |
yield self.has(relpath) |
275 |
count += 1 |
|
276 |
||
1185.16.155
by John Arbash Meinel
Added a has_any function to the Transport API |
277 |
def has_any(self, relpaths): |
278 |
"""Return True if any of the paths exist."""
|
|
279 |
for relpath in relpaths: |
|
280 |
if self.has(relpath): |
|
281 |
return True |
|
282 |
return False |
|
283 |
||
1442.1.44
by Robert Collins
Many transport related tweaks: |
284 |
def iter_files_recursive(self): |
285 |
"""Iter the relative paths of files in the transports sub-tree.
|
|
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
286 |
|
287 |
*NOTE*: This only lists *files*, not subdirectories!
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
288 |
|
289 |
As with other listing functions, only some transports implement this,.
|
|
290 |
you may check via is_listable to determine if it will.
|
|
291 |
"""
|
|
1530.1.4
by Robert Collins
integrate Memory tests into transport interface tests. |
292 |
raise errors.TransportNotPossible("This transport has not " |
1530.1.21
by Robert Collins
Review feedback fixes. |
293 |
"implemented iter_files_recursive "
|
1530.1.4
by Robert Collins
integrate Memory tests into transport interface tests. |
294 |
"(but must claim to be listable "
|
295 |
"to trigger this error).") |
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
296 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
297 |
def get(self, relpath): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
298 |
"""Get the file at the given relative path.
|
907.1.20
by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8 |
299 |
|
300 |
:param relpath: The relative path to the file
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
301 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
302 |
raise NotImplementedError(self.get) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
303 |
|
1594.2.5
by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support. |
304 |
def readv(self, relpath, offsets): |
305 |
"""Get parts of the file at the given relative path.
|
|
306 |
||
307 |
:offsets: A list of (offset, size) tuples.
|
|
308 |
:return: A list or generator of (offset, data) tuples
|
|
309 |
"""
|
|
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
310 |
def do_combined_read(combined_offsets): |
311 |
total_size = 0 |
|
312 |
for offset, size in combined_offsets: |
|
313 |
total_size += size |
|
314 |
mutter('readv coalesced %d reads.', len(combined_offsets)) |
|
315 |
offset = combined_offsets[0][0] |
|
316 |
fp.seek(offset) |
|
317 |
data = fp.read(total_size) |
|
318 |
pos = 0 |
|
319 |
for offset, size in combined_offsets: |
|
320 |
yield offset, data[pos:pos + size] |
|
321 |
pos += size |
|
322 |
||
1594.2.16
by Robert Collins
Coalesce readv requests on file based transports. |
323 |
if not len(offsets): |
324 |
return
|
|
1594.2.5
by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support. |
325 |
fp = self.get(relpath) |
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
326 |
pending_offsets = deque(offsets) |
327 |
combined_offsets = [] |
|
328 |
while len(pending_offsets): |
|
329 |
offset, size = pending_offsets.popleft() |
|
330 |
if not combined_offsets: |
|
331 |
combined_offsets = [[offset, size]] |
|
1594.2.16
by Robert Collins
Coalesce readv requests on file based transports. |
332 |
else: |
1594.2.19
by Robert Collins
More coalescing tweaks, and knit feedback. |
333 |
if (len (combined_offsets) < 50 and |
334 |
combined_offsets[-1][0] + combined_offsets[-1][1] == offset): |
|
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
335 |
# combatible offset:
|
336 |
combined_offsets.append([offset, size]) |
|
337 |
else: |
|
1594.2.19
by Robert Collins
More coalescing tweaks, and knit feedback. |
338 |
# incompatible, or over the threshold issue a read and yield
|
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
339 |
pending_offsets.appendleft((offset, size)) |
340 |
for result in do_combined_read(combined_offsets): |
|
341 |
yield result |
|
342 |
combined_offsets = [] |
|
343 |
# whatever is left is a single coalesced request
|
|
344 |
if len(combined_offsets): |
|
345 |
for result in do_combined_read(combined_offsets): |
|
346 |
yield result |
|
1594.2.5
by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support. |
347 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
348 |
def get_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
349 |
"""Get a list of file-like objects, one for each entry in relpaths.
|
350 |
||
351 |
:param relpaths: A list of relative paths.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
352 |
:param pb: An optional ProgressBar for indicating percent done.
|
353 |
:return: A list or generator of file-like objects
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
354 |
"""
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
355 |
# TODO: Consider having this actually buffer the requests,
|
356 |
# in the default mode, it probably won't give worse performance,
|
|
357 |
# and all children wouldn't have to implement buffering
|
|
907.1.16
by John Arbash Meinel
Fixing a few cut&paste typos. |
358 |
total = self._get_total(relpaths) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
359 |
count = 0 |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
360 |
for relpath in relpaths: |
907.1.16
by John Arbash Meinel
Fixing a few cut&paste typos. |
361 |
self._update_pb(pb, 'get', count, total) |
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
362 |
yield self.get(relpath) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
363 |
count += 1 |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
364 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
365 |
def put(self, relpath, f, mode=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
366 |
"""Copy the file-like or string object into the location.
|
907.1.20
by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8 |
367 |
|
368 |
:param relpath: Location to put the contents, relative to base.
|
|
369 |
:param f: File-like or string object.
|
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
370 |
:param mode: The mode for the newly created file,
|
371 |
None means just use the default
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
372 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
373 |
raise NotImplementedError(self.put) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
374 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
375 |
def put_multi(self, files, mode=None, pb=None): |
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
376 |
"""Put a set of files into the location.
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
377 |
|
378 |
:param files: A list of tuples of relpath, file object [(path1, file1), (path2, file2),...]
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
379 |
:param pb: An optional ProgressBar for indicating percent done.
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
380 |
:param mode: The mode for the newly created files
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
381 |
:return: The number of files copied.
|
382 |
"""
|
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
383 |
def put(path, f): |
384 |
self.put(path, f, mode=mode) |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
385 |
return len(self._iterate_over(files, put, pb, 'put', expand=True)) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
386 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
387 |
def mkdir(self, relpath, mode=None): |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
388 |
"""Create a directory at the given path."""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
389 |
raise NotImplementedError(self.mkdir) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
390 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
391 |
def mkdir_multi(self, relpaths, mode=None, pb=None): |
907.1.47
by John Arbash Meinel
Created mkdir_multi, modified branch to use _multi forms when initializing a branch, creating a full test routine for transports |
392 |
"""Create a group of directories"""
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
393 |
def mkdir(path): |
394 |
self.mkdir(path, mode=mode) |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
395 |
return len(self._iterate_over(relpaths, mkdir, pb, 'mkdir', expand=False)) |
907.1.47
by John Arbash Meinel
Created mkdir_multi, modified branch to use _multi forms when initializing a branch, creating a full test routine for transports |
396 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
397 |
def append(self, relpath, f): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
398 |
"""Append the text in the file-like or string object to
|
399 |
the supplied location.
|
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
400 |
|
401 |
returns the length of f before the content was written to it.
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
402 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
403 |
raise NotImplementedError(self.append) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
404 |
|
1185.11.19
by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings. |
405 |
def append_multi(self, files, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
406 |
"""Append the text in each file-like or string object to
|
407 |
the supplied location.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
408 |
|
409 |
:param files: A set of (path, f) entries
|
|
410 |
:param pb: An optional ProgressBar for indicating percent done.
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
411 |
"""
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
412 |
return self._iterate_over(files, self.append, pb, 'append', expand=True) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
413 |
|
414 |
def copy(self, rel_from, rel_to): |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
415 |
"""Copy the item at rel_from to the location at rel_to.
|
416 |
|
|
417 |
Override this for efficiency if a specific transport can do it
|
|
418 |
faster than this default implementation.
|
|
419 |
"""
|
|
420 |
self.put(rel_to, self.get(rel_from)) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
421 |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
422 |
def copy_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
423 |
"""Copy a bunch of entries.
|
424 |
|
|
425 |
:param relpaths: A list of tuples of the form [(from, to), (from, to),...]
|
|
426 |
"""
|
|
427 |
# This is the non-pipelined implementation, so that
|
|
428 |
# implementors don't have to implement everything.
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
429 |
return self._iterate_over(relpaths, self.copy, pb, 'copy', expand=True) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
430 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
431 |
def copy_to(self, relpaths, other, mode=None, pb=None): |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
432 |
"""Copy a set of entries from self into another Transport.
|
433 |
||
434 |
:param relpaths: A list/generator of entries to be copied.
|
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
435 |
:param mode: This is the target mode for the newly created files
|
1185.16.156
by John Arbash Meinel
Adding a note about changing copy_to's interface |
436 |
TODO: This interface needs to be updated so that the target location
|
437 |
can be different from the source location.
|
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
438 |
"""
|
439 |
# The dummy implementation just does a simple get + put
|
|
440 |
def copy_entry(path): |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
441 |
other.put(path, self.get(path), mode=mode) |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
442 |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
443 |
return len(self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False)) |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
444 |
|
1534.4.26
by Robert Collins
Move working tree initialisation out from Branch.initialize, deprecated Branch.initialize to Branch.create. |
445 |
def copy_tree(self, from_relpath, to_relpath): |
446 |
"""Copy a subtree from one relpath to another.
|
|
447 |
||
448 |
If a faster implementation is available, specific transports should
|
|
449 |
implement it.
|
|
450 |
"""
|
|
451 |
source = self.clone(from_relpath) |
|
452 |
self.mkdir(to_relpath) |
|
453 |
target = self.clone(to_relpath) |
|
454 |
files = [] |
|
455 |
directories = ['.'] |
|
456 |
while directories: |
|
457 |
dir = directories.pop() |
|
458 |
if dir != '.': |
|
459 |
target.mkdir(dir) |
|
460 |
for path in source.list_dir(dir): |
|
461 |
path = dir + '/' + path |
|
462 |
stat = source.stat(path) |
|
463 |
if S_ISDIR(stat.st_mode): |
|
464 |
directories.append(path) |
|
465 |
else: |
|
466 |
files.append(path) |
|
467 |
source.copy_to(files, target) |
|
468 |
||
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
469 |
def rename(self, rel_from, rel_to): |
470 |
"""Rename a file or directory.
|
|
471 |
||
472 |
This *must* fail if the destination is a nonempty directory - it must
|
|
473 |
not automatically remove it. It should raise DirectoryNotEmpty, or
|
|
474 |
some other PathError if the case can't be specifically detected.
|
|
475 |
||
476 |
If the destination is an empty directory or a file this function may
|
|
477 |
either fail or succeed, depending on the underlying transport. It
|
|
478 |
should not attempt to remove the destination if overwriting is not the
|
|
479 |
native transport behaviour. If at all possible the transport should
|
|
480 |
ensure that the rename either completes or not, without leaving the
|
|
481 |
destination deleted and the new file not moved in place.
|
|
482 |
||
483 |
This is intended mainly for use in implementing LockDir.
|
|
484 |
"""
|
|
485 |
# transports may need to override this
|
|
1553.5.17
by Martin Pool
Transport.rename should be unimplemented in base class |
486 |
raise NotImplementedError(self.rename) |
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
487 |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
488 |
def move(self, rel_from, rel_to): |
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
489 |
"""Move the item at rel_from to the location at rel_to.
|
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
490 |
|
491 |
The destination is deleted if possible, even if it's a non-empty
|
|
492 |
directory tree.
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
493 |
|
494 |
If a transport can directly implement this it is suggested that
|
|
495 |
it do so for efficiency.
|
|
496 |
"""
|
|
1534.4.26
by Robert Collins
Move working tree initialisation out from Branch.initialize, deprecated Branch.initialize to Branch.create. |
497 |
if S_ISDIR(self.stat(rel_from).st_mode): |
498 |
self.copy_tree(rel_from, rel_to) |
|
499 |
self.delete_tree(rel_from) |
|
500 |
else: |
|
501 |
self.copy(rel_from, rel_to) |
|
502 |
self.delete(rel_from) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
503 |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
504 |
def move_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
505 |
"""Move a bunch of entries.
|
506 |
|
|
507 |
:param relpaths: A list of tuples of the form [(from1, to1), (from2, to2),...]
|
|
508 |
"""
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
509 |
return self._iterate_over(relpaths, self.move, pb, 'move', expand=True) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
510 |
|
511 |
def move_multi_to(self, relpaths, rel_to): |
|
512 |
"""Move a bunch of entries to a single location.
|
|
513 |
This differs from move_multi in that you give a list of from, and
|
|
514 |
a single destination, rather than multiple destinations.
|
|
515 |
||
516 |
:param relpaths: A list of relative paths [from1, from2, from3, ...]
|
|
517 |
:param rel_to: A directory where each entry should be placed.
|
|
518 |
"""
|
|
519 |
# This is not implemented, because you need to do special tricks to
|
|
520 |
# extract the basename, and add it to rel_to
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
521 |
raise NotImplementedError(self.move_multi_to) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
522 |
|
523 |
def delete(self, relpath): |
|
524 |
"""Delete the item at relpath"""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
525 |
raise NotImplementedError(self.delete) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
526 |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
527 |
def delete_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
528 |
"""Queue up a bunch of deletes to be done.
|
529 |
"""
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
530 |
return self._iterate_over(relpaths, self.delete, pb, 'delete', expand=False) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
531 |
|
1534.4.15
by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports. |
532 |
def delete_tree(self, relpath): |
533 |
"""Delete an entire tree. This may require a listable transport."""
|
|
534 |
subtree = self.clone(relpath) |
|
535 |
files = [] |
|
536 |
directories = ['.'] |
|
537 |
pending_rmdirs = [] |
|
538 |
while directories: |
|
539 |
dir = directories.pop() |
|
540 |
if dir != '.': |
|
541 |
pending_rmdirs.append(dir) |
|
542 |
for path in subtree.list_dir(dir): |
|
543 |
path = dir + '/' + path |
|
544 |
stat = subtree.stat(path) |
|
545 |
if S_ISDIR(stat.st_mode): |
|
546 |
directories.append(path) |
|
547 |
else: |
|
548 |
files.append(path) |
|
549 |
subtree.delete_multi(files) |
|
550 |
pending_rmdirs.reverse() |
|
551 |
for dir in pending_rmdirs: |
|
552 |
subtree.rmdir(dir) |
|
553 |
self.rmdir(relpath) |
|
554 |
||
1534.5.7
by Robert Collins
Start factoring out the upgrade policy logic. |
555 |
def __repr__(self): |
556 |
return "<%s.%s url=%s>" % (self.__module__, self.__class__.__name__, self.base) |
|
557 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
558 |
def stat(self, relpath): |
559 |
"""Return the stat information for a file.
|
|
560 |
WARNING: This may not be implementable for all protocols, so use
|
|
561 |
sparingly.
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
562 |
NOTE: This returns an object with fields such as 'st_size'. It MAY
|
563 |
or MAY NOT return the literal result of an os.stat() call, so all
|
|
564 |
access should be via named fields.
|
|
565 |
ALSO NOTE: Stats of directories may not be supported on some
|
|
566 |
transports.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
567 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
568 |
raise NotImplementedError(self.stat) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
569 |
|
1534.4.15
by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports. |
570 |
def rmdir(self, relpath): |
571 |
"""Remove a directory at the given path."""
|
|
572 |
raise NotImplementedError |
|
573 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
574 |
def stat_multi(self, relpaths, pb=None): |
575 |
"""Stat multiple files and return the information.
|
|
576 |
"""
|
|
577 |
#TODO: Is it worth making this a generator instead of a
|
|
578 |
# returning a list?
|
|
579 |
stats = [] |
|
580 |
def gather(path): |
|
581 |
stats.append(self.stat(path)) |
|
582 |
||
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
583 |
count = self._iterate_over(relpaths, gather, pb, 'stat', expand=False) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
584 |
return stats |
585 |
||
1400.1.1
by Robert Collins
implement a basic test for the ui branch command from http servers |
586 |
def listable(self): |
587 |
"""Return True if this store supports listing."""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
588 |
raise NotImplementedError(self.listable) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
589 |
|
590 |
def list_dir(self, relpath): |
|
591 |
"""Return a list of all files at the given location.
|
|
592 |
WARNING: many transports do not support this, so trying avoid using
|
|
593 |
it if at all possible.
|
|
594 |
"""
|
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
595 |
raise errors.TransportNotPossible("This transport has not " |
1530.1.21
by Robert Collins
Review feedback fixes. |
596 |
"implemented list_dir "
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
597 |
"(but must claim to be listable "
|
598 |
"to trigger this error).") |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
599 |
|
907.1.24
by John Arbash Meinel
Remote functionality work. |
600 |
def lock_read(self, relpath): |
601 |
"""Lock the given file for shared (read) access.
|
|
602 |
WARNING: many transports do not support this, so trying avoid using it
|
|
603 |
||
604 |
:return: A lock object, which should contain an unlock() function.
|
|
605 |
"""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
606 |
raise NotImplementedError(self.lock_read) |
907.1.24
by John Arbash Meinel
Remote functionality work. |
607 |
|
608 |
def lock_write(self, relpath): |
|
609 |
"""Lock the given file for exclusive (write) access.
|
|
610 |
WARNING: many transports do not support this, so trying avoid using it
|
|
611 |
||
612 |
:return: A lock object, which should contain an unlock() function.
|
|
613 |
"""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
614 |
raise NotImplementedError(self.lock_write) |
907.1.24
by John Arbash Meinel
Remote functionality work. |
615 |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
616 |
def is_readonly(self): |
617 |
"""Return true if this connection cannot be written to."""
|
|
618 |
return False |
|
619 |
||
1608.2.7
by Martin Pool
Rename supports_unix_modebits to _can_roundtrip_unix_modebits for clarity |
620 |
def _can_roundtrip_unix_modebits(self): |
1608.2.5
by Martin Pool
Add Transport.supports_unix_modebits, so tests can |
621 |
"""Return true if this transport can store and retrieve unix modebits.
|
622 |
||
623 |
(For example, 0700 to make a directory owner-private.)
|
|
624 |
|
|
625 |
Note: most callers will not want to switch on this, but should rather
|
|
626 |
just try and set permissions and let them be either stored or not.
|
|
627 |
This is intended mainly for the use of the test suite.
|
|
628 |
|
|
629 |
Warning: this is not guaranteed to be accurate as sometimes we can't
|
|
630 |
be sure: for example with vfat mounted on unix, or a windows sftp
|
|
631 |
server."""
|
|
632 |
# TODO: Perhaps return a e.g. TransportCharacteristics that can answer
|
|
633 |
# several questions about the transport.
|
|
634 |
return False |
|
635 |
||
907.1.24
by John Arbash Meinel
Remote functionality work. |
636 |
|
1393.2.4
by John Arbash Meinel
All tests pass. |
637 |
def get_transport(base): |
1185.70.3
by Martin Pool
Various updates to make storage branch mergeable: |
638 |
"""Open a transport to access a URL or directory.
|
639 |
||
640 |
base is either a URL or a directory name.
|
|
641 |
"""
|
|
1540.3.8
by Martin Pool
Some support for falling back between transport implementations. |
642 |
# TODO: give a better error if base looks like a url but there's no
|
643 |
# handler for the scheme?
|
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
644 |
global _protocol_handlers |
907.1.13
by John Arbash Meinel
Fixed bzr root. |
645 |
if base is None: |
1185.33.66
by Martin Pool
[patch] use unicode literals for all hardcoded paths (Alexander Belchenko) |
646 |
base = u'.' |
1185.31.60
by John Arbash Meinel
get_transport() now forces unicode paths, which helps get proper filenames. |
647 |
else: |
648 |
base = unicode(base) |
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
649 |
for proto, factory_list in _protocol_handlers.iteritems(): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
650 |
if proto is not None and base.startswith(proto): |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
651 |
t = _try_transport_factories(base, factory_list) |
652 |
if t: |
|
653 |
return t |
|
654 |
# The default handler is the filesystem handler, stored as protocol None
|
|
655 |
return _try_transport_factories(base, _protocol_handlers[None]) |
|
656 |
||
657 |
||
658 |
def _try_transport_factories(base, factory_list): |
|
659 |
for factory in factory_list: |
|
660 |
try: |
|
661 |
return factory(base) |
|
662 |
except DependencyNotPresent, e: |
|
663 |
mutter("failed to instantiate transport %r for %r: %r" % |
|
664 |
(factory, base, e)) |
|
665 |
continue
|
|
666 |
return None |
|
1185.16.81
by mbp at sourcefrog
[merge] robert |
667 |
|
1185.16.78
by Martin Pool
- load paramiko sftp transport by default |
668 |
|
1469
by Robert Collins
Change Transport.* to work with URL's. |
669 |
def urlescape(relpath): |
670 |
"""Escape relpath to be a valid url."""
|
|
1185.80.5
by John Arbash Meinel
Changing the escaping just a little bit. Now we can handle unicode characters. |
671 |
if isinstance(relpath, unicode): |
672 |
relpath = relpath.encode('utf-8') |
|
1469
by Robert Collins
Change Transport.* to work with URL's. |
673 |
return urllib.quote(relpath) |
1470
by Robert Collins
merge from Martin, via newformat, and teach sftp about urlescaped paths |
674 |
|
1185.16.78
by Martin Pool
- load paramiko sftp transport by default |
675 |
|
1185.85.76
by John Arbash Meinel
Adding an InvalidURL so transports can report they expect utf-8 quoted paths. Updated tests |
676 |
def urlunescape(url): |
1685.1.1
by John Arbash Meinel
[merge] the old bzr-encoding changes, reparenting them on bzr.dev |
677 |
"""Unescape relpath from url format.
|
678 |
||
679 |
This returns a Unicode path from a URL
|
|
680 |
"""
|
|
1185.85.76
by John Arbash Meinel
Adding an InvalidURL so transports can report they expect utf-8 quoted paths. Updated tests |
681 |
unquoted = urllib.unquote(url) |
682 |
try: |
|
683 |
unicode_path = unquoted.decode('utf-8') |
|
684 |
except UnicodeError, e: |
|
685 |
raise errors.InvalidURL(url, e) |
|
686 |
return unicode_path |
|
1608.1.1
by Martin Pool
[patch] LocalTransport.list_dir should return url-quoted strings (ddaa) |
687 |
|
1185.85.76
by John Arbash Meinel
Adding an InvalidURL so transports can report they expect utf-8 quoted paths. Updated tests |
688 |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
689 |
class Server(object): |
1530.1.21
by Robert Collins
Review feedback fixes. |
690 |
"""A Transport Server.
|
691 |
|
|
692 |
The Server interface provides a server for a given transport. We use
|
|
693 |
these servers as loopback testing tools. For any given transport the
|
|
694 |
Servers it provides must either allow writing, or serve the contents
|
|
695 |
of os.getcwdu() at the time setUp is called.
|
|
696 |
|
|
697 |
Note that these are real servers - they must implement all the things
|
|
698 |
that we want bzr transports to take advantage of.
|
|
699 |
"""
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
700 |
|
701 |
def setUp(self): |
|
702 |
"""Setup the server to service requests."""
|
|
703 |
||
704 |
def tearDown(self): |
|
705 |
"""Remove the server and cleanup any resources it owns."""
|
|
706 |
||
707 |
def get_url(self): |
|
1530.1.21
by Robert Collins
Review feedback fixes. |
708 |
"""Return a url for this server.
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
709 |
|
710 |
If the transport does not represent a disk directory (i.e. it is
|
|
1530.1.21
by Robert Collins
Review feedback fixes. |
711 |
a database like svn, or a memory only transport, it should return
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
712 |
a connection to a newly established resource for this Server.
|
1530.1.21
by Robert Collins
Review feedback fixes. |
713 |
Otherwise it should return a url that will provide access to the path
|
714 |
that was os.getcwdu() when setUp() was called.
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
715 |
|
716 |
Subsequent calls will return the same resource.
|
|
717 |
"""
|
|
718 |
raise NotImplementedError |
|
719 |
||
1530.1.9
by Robert Collins
Test bogus urls with http in the new infrastructure. |
720 |
def get_bogus_url(self): |
721 |
"""Return a url for this protocol, that will fail to connect."""
|
|
722 |
raise NotImplementedError |
|
723 |
||
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
724 |
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
725 |
class TransportTestProviderAdapter(object): |
1530.1.21
by Robert Collins
Review feedback fixes. |
726 |
"""A tool to generate a suite testing all transports for a single test.
|
727 |
||
728 |
This is done by copying the test once for each transport and injecting
|
|
729 |
the transport_class and transport_server classes into each copy. Each copy
|
|
730 |
is also given a new id() to make it easy to identify.
|
|
731 |
"""
|
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
732 |
|
733 |
def adapt(self, test): |
|
734 |
result = TestSuite() |
|
735 |
for klass, server_factory in self._test_permutations(): |
|
736 |
new_test = deepcopy(test) |
|
737 |
new_test.transport_class = klass |
|
738 |
new_test.transport_server = server_factory |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
739 |
def make_new_test_id(): |
740 |
new_id = "%s(%s)" % (new_test.id(), server_factory.__name__) |
|
741 |
return lambda: new_id |
|
742 |
new_test.id = make_new_test_id() |
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
743 |
result.addTest(new_test) |
744 |
return result |
|
745 |
||
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
746 |
def get_transport_test_permutations(self, module): |
747 |
"""Get the permutations module wants to have tested."""
|
|
1540.3.6
by Martin Pool
[merge] update from bzr.dev |
748 |
if not hasattr(module, 'get_test_permutations'): |
749 |
warning("transport module %s doesn't provide get_test_permutations()" |
|
750 |
% module.__name__) |
|
751 |
return [] |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
752 |
return module.get_test_permutations() |
753 |
||
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
754 |
def _test_permutations(self): |
755 |
"""Return a list of the klass, server_factory pairs to test."""
|
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
756 |
result = [] |
757 |
for module in _get_transport_modules(): |
|
1534.7.70
by abentley
Fix bug when no paramiko |
758 |
try: |
759 |
result.extend(self.get_transport_test_permutations(reduce(getattr, |
|
760 |
(module).split('.')[1:], |
|
1185.71.1
by John Arbash Meinel
Allow selftest to run, even if we can't load a transport. |
761 |
__import__(module)))) |
1185.62.24
by John Arbash Meinel
Changing the exception that sftp.py throws when it can't find paramiko, so that the test suite can handle it. |
762 |
except errors.DependencyNotPresent, e: |
763 |
# Continue even if a dependency prevents us
|
|
764 |
# from running this test
|
|
1534.7.70
by abentley
Fix bug when no paramiko |
765 |
pass
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
766 |
return result |
1594.2.24
by Robert Collins
Make use of the transaction finalisation warning support to implement in-knit caching. |
767 |
|
768 |
||
769 |
class TransportLogger(object): |
|
770 |
"""Adapt a transport to get clear logging data on api calls.
|
|
771 |
|
|
772 |
Feel free to extend to log whatever calls are of interest.
|
|
773 |
"""
|
|
774 |
||
775 |
def __init__(self, adapted): |
|
776 |
self._adapted = adapted |
|
777 |
self._calls = [] |
|
778 |
||
779 |
def get(self, name): |
|
780 |
self._calls.append((name,)) |
|
781 |
return self._adapted.get(name) |
|
782 |
||
783 |
def __getattr__(self, name): |
|
784 |
"""Thunk all undefined access through to self._adapted."""
|
|
785 |
# raise AttributeError, name
|
|
786 |
return getattr(self._adapted, name) |
|
787 |
||
788 |
def readv(self, name, offsets): |
|
789 |
self._calls.append((name, offsets)) |
|
790 |
return self._adapted.readv(name, offsets) |
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
791 |
|
792 |
||
1185.16.81
by mbp at sourcefrog
[merge] robert |
793 |
# None is the default transport, for things with no url scheme
|
1185.16.79
by Martin Pool
Load transports when they're first used. |
794 |
register_lazy_transport(None, 'bzrlib.transport.local', 'LocalTransport') |
795 |
register_lazy_transport('file://', 'bzrlib.transport.local', 'LocalTransport') |
|
796 |
register_lazy_transport('sftp://', 'bzrlib.transport.sftp', 'SFTPTransport') |
|
1540.3.23
by Martin Pool
Allow urls like http+pycurl://host/ to use a particular impl |
797 |
register_lazy_transport('http+urllib://', 'bzrlib.transport.http._urllib', |
1540.3.26
by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet |
798 |
'HttpTransport_urllib') |
1540.3.23
by Martin Pool
Allow urls like http+pycurl://host/ to use a particular impl |
799 |
register_lazy_transport('https+urllib://', 'bzrlib.transport.http._urllib', |
1540.3.26
by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet |
800 |
'HttpTransport_urllib') |
801 |
register_lazy_transport('http+pycurl://', 'bzrlib.transport.http._pycurl', |
|
802 |
'PyCurlTransport') |
|
803 |
register_lazy_transport('https+pycurl://', 'bzrlib.transport.http._pycurl', |
|
804 |
'PyCurlTransport') |
|
805 |
register_lazy_transport('http://', 'bzrlib.transport.http._urllib', |
|
806 |
'HttpTransport_urllib') |
|
807 |
register_lazy_transport('https://', 'bzrlib.transport.http._urllib', |
|
808 |
'HttpTransport_urllib') |
|
1540.3.7
by Martin Pool
Prepare to select a transport depending on what dependencies can be satisfied. |
809 |
register_lazy_transport('http://', 'bzrlib.transport.http._pycurl', 'PyCurlTransport') |
810 |
register_lazy_transport('https://', 'bzrlib.transport.http._pycurl', 'PyCurlTransport') |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
811 |
register_lazy_transport('ftp://', 'bzrlib.transport.ftp', 'FtpTransport') |
812 |
register_lazy_transport('aftp://', 'bzrlib.transport.ftp', 'FtpTransport') |
|
1534.4.39
by Robert Collins
Basic BzrDir support. |
813 |
register_lazy_transport('memory:/', 'bzrlib.transport.memory', 'MemoryTransport') |
1534.4.9
by Robert Collins
Add a readonly decorator for transports. |
814 |
register_lazy_transport('readonly+', 'bzrlib.transport.readonly', 'ReadonlyTransportDecorator') |
1558.10.2
by Robert Collins
Refactor the FakeNFS support into a TransportDecorator. |
815 |
register_lazy_transport('fakenfs+', 'bzrlib.transport.fakenfs', 'FakeNFSTransportDecorator') |
1608.2.4
by Martin Pool
[broken] Add FakeFVATTransport |
816 |
register_lazy_transport('vfat+', |
817 |
'bzrlib.transport.fakevfat', |
|
818 |
'FakeVFATTransportDecorator') |