1553.5.10
by Martin Pool
New DirectoryNotEmpty exception, and raise this from local and memory |
1 |
# Copyright (C) 2005, 2006 Canonical Ltd
|
1185.11.19
by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings. |
2 |
|
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
||
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
||
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
16 |
|
1185.11.19
by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings. |
17 |
"""Transport is an abstraction layer to handle file access.
|
18 |
||
19 |
The abstraction is to allow access from the local filesystem, as well
|
|
20 |
as remote (such as http or sftp).
|
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
21 |
|
22 |
Transports are constructed from a string, being a URL or (as a degenerate
|
|
23 |
case) a local filesystem path. This is typically the top directory of
|
|
24 |
a bzrdir, repository, or similar object we are interested in working with.
|
|
25 |
The Transport returned has methods to read, write and manipulate files within
|
|
26 |
it.
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
27 |
"""
|
28 |
||
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
29 |
import errno |
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
30 |
from collections import deque |
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
31 |
from copy import deepcopy |
1534.4.15
by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports. |
32 |
from stat import * |
1530.1.7
by Robert Collins
merge integration. |
33 |
import sys |
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
34 |
from unittest import TestSuite |
1607.1.3
by Robert Collins
Apply David Allouches list_dir quoting fix. |
35 |
import urllib |
1636.1.1
by Robert Collins
Fix calling relpath() and abspath() on transports at their root. |
36 |
import urlparse |
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
37 |
|
1540.3.6
by Martin Pool
[merge] update from bzr.dev |
38 |
from bzrlib.trace import mutter, warning |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
39 |
import bzrlib.errors as errors |
1540.3.8
by Martin Pool
Some support for falling back between transport implementations. |
40 |
from bzrlib.errors import DependencyNotPresent |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
41 |
from bzrlib.symbol_versioning import * |
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
42 |
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
43 |
# {prefix: [transport_classes]}
|
44 |
# Transports are inserted onto the list LIFO and tried in order; as a result
|
|
45 |
# transports provided by plugins are tried first, which is usually what we
|
|
46 |
# want.
|
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
47 |
_protocol_handlers = { |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
48 |
}
|
49 |
||
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
50 |
def register_transport(prefix, klass, override=DEPRECATED_PARAMETER): |
1540.3.8
by Martin Pool
Some support for falling back between transport implementations. |
51 |
"""Register a transport that can be used to open URLs
|
52 |
||
53 |
Normally you should use register_lazy_transport, which defers loading the
|
|
54 |
implementation until it's actually used, and so avoids pulling in possibly
|
|
55 |
large implementation libraries.
|
|
56 |
"""
|
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
57 |
# Note that this code runs very early in library setup -- trace may not be
|
58 |
# working, etc.
|
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
59 |
global _protocol_handlers |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
60 |
if deprecated_passed(override): |
61 |
warn("register_transport(override) is deprecated") |
|
62 |
_protocol_handlers.setdefault(prefix, []).insert(0, klass) |
|
63 |
||
64 |
||
65 |
def register_lazy_transport(scheme, module, classname): |
|
66 |
"""Register lazy-loaded transport class.
|
|
67 |
||
68 |
When opening a URL with the given scheme, load the module and then
|
|
69 |
instantiate the particular class.
|
|
70 |
||
71 |
If the module raises DependencyNotPresent when it's imported, it is
|
|
72 |
skipped and another implementation of the protocol is tried. This is
|
|
73 |
intended to be used when the implementation depends on an external
|
|
74 |
implementation that may not be present. If any other error is raised, it
|
|
75 |
propagates up and the attempt to open the url fails.
|
|
76 |
"""
|
|
77 |
# TODO: If no implementation of a protocol is available because of missing
|
|
78 |
# dependencies, we should perhaps show the message about what dependency
|
|
79 |
# was missing.
|
|
80 |
def _loader(base): |
|
81 |
mod = __import__(module, globals(), locals(), [classname]) |
|
82 |
klass = getattr(mod, classname) |
|
83 |
return klass(base) |
|
84 |
_loader.module = module |
|
85 |
register_transport(scheme, _loader) |
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
86 |
|
1442.1.24
by Robert Collins
Pull up _check_id and _relpath from Text and CompressedText stores into TransportStore |
87 |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
88 |
def _get_protocol_handlers(): |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
89 |
"""Return a dictionary of {urlprefix: [factory]}"""
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
90 |
return _protocol_handlers |
91 |
||
92 |
||
93 |
def _set_protocol_handlers(new_handlers): |
|
94 |
"""Replace the current protocol handlers dictionary.
|
|
95 |
||
96 |
WARNING this will remove all build in protocols. Use with care.
|
|
97 |
"""
|
|
98 |
global _protocol_handlers |
|
99 |
_protocol_handlers = new_handlers |
|
100 |
||
101 |
||
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
102 |
def _clear_protocol_handlers(): |
103 |
global _protocol_handlers |
|
104 |
_protocol_handlers = {} |
|
105 |
||
106 |
||
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
107 |
def _get_transport_modules(): |
108 |
"""Return a list of the modules providing transports."""
|
|
109 |
modules = set() |
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
110 |
for prefix, factory_list in _protocol_handlers.items(): |
111 |
for factory in factory_list: |
|
112 |
if factory.__module__ == "bzrlib.transport": |
|
113 |
# this is a lazy load transport, because no real ones
|
|
114 |
# are directlry in bzrlib.transport
|
|
115 |
modules.add(factory.module) |
|
116 |
else: |
|
117 |
modules.add(factory.__module__) |
|
1530.1.19
by Robert Collins
Make transport test adapter tests reliable. |
118 |
result = list(modules) |
119 |
result.sort() |
|
120 |
return result |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
121 |
|
122 |
||
1636.1.2
by Robert Collins
More review fixen to the relpath at '/' fixes. |
123 |
def register_urlparse_netloc_protocol(protocol): |
1636.1.1
by Robert Collins
Fix calling relpath() and abspath() on transports at their root. |
124 |
"""Ensure that protocol is setup to be used with urlparse netloc parsing."""
|
125 |
if protocol not in urlparse.uses_netloc: |
|
126 |
urlparse.uses_netloc.append(protocol) |
|
127 |
||
128 |
||
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
129 |
class Transport(object): |
130 |
"""This class encapsulates methods for retrieving or putting a file
|
|
131 |
from/to a storage location.
|
|
132 |
||
133 |
Most functions have a _multi variant, which allows you to queue up
|
|
134 |
multiple requests. They generally have a dumb base implementation
|
|
135 |
which just iterates over the arguments, but smart Transport
|
|
136 |
implementations can do pipelining.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
137 |
In general implementations should support having a generator or a list
|
138 |
as an argument (ie always iterate, never index)
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
139 |
"""
|
140 |
||
141 |
def __init__(self, base): |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
142 |
super(Transport, self).__init__() |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
143 |
self.base = base |
144 |
||
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
145 |
def _translate_error(self, e, path, raise_generic=True): |
146 |
"""Translate an IOError or OSError into an appropriate bzr error.
|
|
147 |
||
148 |
This handles things like ENOENT, ENOTDIR, EEXIST, and EACCESS
|
|
149 |
"""
|
|
150 |
if hasattr(e, 'errno'): |
|
151 |
if e.errno in (errno.ENOENT, errno.ENOTDIR): |
|
152 |
raise errors.NoSuchFile(path, extra=e) |
|
1185.31.58
by John Arbash Meinel
Updating for new transport tests so that they pass on win32 |
153 |
# I would rather use errno.EFOO, but there doesn't seem to be
|
154 |
# any matching for 267
|
|
155 |
# This is the error when doing a listdir on a file:
|
|
156 |
# WindowsError: [Errno 267] The directory name is invalid
|
|
157 |
if sys.platform == 'win32' and e.errno in (errno.ESRCH, 267): |
|
158 |
raise errors.NoSuchFile(path, extra=e) |
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
159 |
if e.errno == errno.EEXIST: |
160 |
raise errors.FileExists(path, extra=e) |
|
161 |
if e.errno == errno.EACCES: |
|
162 |
raise errors.PermissionDenied(path, extra=e) |
|
1553.5.10
by Martin Pool
New DirectoryNotEmpty exception, and raise this from local and memory |
163 |
if e.errno == errno.ENOTEMPTY: |
164 |
raise errors.DirectoryNotEmpty(path, extra=e) |
|
1558.10.1
by Aaron Bentley
Handle lockdirs over NFS properly |
165 |
if e.errno == errno.EBUSY: |
166 |
raise errors.ResourceBusy(path, extra=e) |
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
167 |
if raise_generic: |
168 |
raise errors.TransportError(orig_error=e) |
|
169 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
170 |
def clone(self, offset=None): |
171 |
"""Return a new Transport object, cloned from the current location,
|
|
1185.11.6
by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently. |
172 |
using a subdirectory or parent directory. This allows connections
|
173 |
to be pooled, rather than a new one needed for each subdir.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
174 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
175 |
raise NotImplementedError(self.clone) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
176 |
|
907.1.32
by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate. |
177 |
def should_cache(self): |
178 |
"""Return True if the data pulled across should be cached locally.
|
|
907.1.22
by John Arbash Meinel
Fixed some encoding issues, added is_remote function for Transport objects. |
179 |
"""
|
907.1.32
by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate. |
180 |
return False |
907.1.22
by John Arbash Meinel
Fixed some encoding issues, added is_remote function for Transport objects. |
181 |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
182 |
def _pump(self, from_file, to_file): |
183 |
"""Most children will need to copy from one file-like
|
|
184 |
object or string to another one.
|
|
185 |
This just gives them something easy to call.
|
|
186 |
"""
|
|
187 |
if isinstance(from_file, basestring): |
|
907.1.8
by John Arbash Meinel
Changed the format for abspath. Updated branch to use a hidden _transport |
188 |
to_file.write(from_file) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
189 |
else: |
190 |
from bzrlib.osutils import pumpfile |
|
191 |
pumpfile(from_file, to_file) |
|
192 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
193 |
def _get_total(self, multi): |
194 |
"""Try to figure out how many entries are in multi,
|
|
195 |
but if not possible, return None.
|
|
196 |
"""
|
|
197 |
try: |
|
198 |
return len(multi) |
|
199 |
except TypeError: # We can't tell how many, because relpaths is a generator |
|
200 |
return None |
|
201 |
||
202 |
def _update_pb(self, pb, msg, count, total): |
|
203 |
"""Update the progress bar based on the current count
|
|
204 |
and total available, total may be None if it was
|
|
205 |
not possible to determine.
|
|
206 |
"""
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
207 |
if pb is None: |
208 |
return
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
209 |
if total is None: |
210 |
pb.update(msg, count, count+1) |
|
211 |
else: |
|
212 |
pb.update(msg, count, total) |
|
213 |
||
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
214 |
def _iterate_over(self, multi, func, pb, msg, expand=True): |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
215 |
"""Iterate over all entries in multi, passing them to func,
|
216 |
and update the progress bar as you go along.
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
217 |
|
218 |
:param expand: If True, the entries will be passed to the function
|
|
219 |
by expanding the tuple. If False, it will be passed
|
|
220 |
as a single parameter.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
221 |
"""
|
222 |
total = self._get_total(multi) |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
223 |
result = [] |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
224 |
count = 0 |
225 |
for entry in multi: |
|
226 |
self._update_pb(pb, msg, count, total) |
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
227 |
if expand: |
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
228 |
result.append(func(*entry)) |
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
229 |
else: |
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
230 |
result.append(func(entry)) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
231 |
count += 1 |
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
232 |
return tuple(result) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
233 |
|
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
234 |
def abspath(self, relpath): |
235 |
"""Return the full url to the given relative path.
|
|
236 |
This can be supplied with a string or a list
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
237 |
|
238 |
XXX: Robert Collins 20051016 - is this really needed in the public
|
|
239 |
interface ?
|
|
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
240 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
241 |
raise NotImplementedError(self.abspath) |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
242 |
|
243 |
def relpath(self, abspath): |
|
244 |
"""Return the local path portion from a given absolute path.
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
245 |
|
246 |
This default implementation is not suitable for filesystems with
|
|
247 |
aliasing, such as that given by symlinks, where a path may not
|
|
248 |
start with our base, but still be a relpath once aliasing is
|
|
249 |
resolved.
|
|
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
250 |
"""
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
251 |
# TODO: This might want to use bzrlib.osutils.relpath
|
252 |
# but we have to watch out because of the prefix issues
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
253 |
if not (abspath == self.base[:-1] or abspath.startswith(self.base)): |
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
254 |
raise errors.PathNotChild(abspath, self.base) |
1442.1.44
by Robert Collins
Many transport related tweaks: |
255 |
pl = len(self.base) |
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
256 |
return abspath[pl:].strip('/') |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
257 |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
258 |
def has(self, relpath): |
1442.1.44
by Robert Collins
Many transport related tweaks: |
259 |
"""Does the file relpath exist?
|
260 |
|
|
261 |
Note that some transports MAY allow querying on directories, but this
|
|
1553.5.66
by Martin Pool
doc |
262 |
is not part of the protocol. In other words, the results of
|
263 |
t.has("a_directory_name") are undefined."
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
264 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
265 |
raise NotImplementedError(self.has) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
266 |
|
907.1.36
by John Arbash Meinel
Moving the multi-get functionality higher up into the Branch class. |
267 |
def has_multi(self, relpaths, pb=None): |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
268 |
"""Return True/False for each entry in relpaths"""
|
269 |
total = self._get_total(relpaths) |
|
270 |
count = 0 |
|
271 |
for relpath in relpaths: |
|
907.1.36
by John Arbash Meinel
Moving the multi-get functionality higher up into the Branch class. |
272 |
self._update_pb(pb, 'has', count, total) |
907.1.34
by John Arbash Meinel
Fixing append(), cleaning up function locations. |
273 |
yield self.has(relpath) |
274 |
count += 1 |
|
275 |
||
1185.16.155
by John Arbash Meinel
Added a has_any function to the Transport API |
276 |
def has_any(self, relpaths): |
277 |
"""Return True if any of the paths exist."""
|
|
278 |
for relpath in relpaths: |
|
279 |
if self.has(relpath): |
|
280 |
return True |
|
281 |
return False |
|
282 |
||
1442.1.44
by Robert Collins
Many transport related tweaks: |
283 |
def iter_files_recursive(self): |
284 |
"""Iter the relative paths of files in the transports sub-tree.
|
|
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
285 |
|
286 |
*NOTE*: This only lists *files*, not subdirectories!
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
287 |
|
288 |
As with other listing functions, only some transports implement this,.
|
|
289 |
you may check via is_listable to determine if it will.
|
|
290 |
"""
|
|
1530.1.4
by Robert Collins
integrate Memory tests into transport interface tests. |
291 |
raise errors.TransportNotPossible("This transport has not " |
1530.1.21
by Robert Collins
Review feedback fixes. |
292 |
"implemented iter_files_recursive "
|
1530.1.4
by Robert Collins
integrate Memory tests into transport interface tests. |
293 |
"(but must claim to be listable "
|
294 |
"to trigger this error).") |
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
295 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
296 |
def get(self, relpath): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
297 |
"""Get the file at the given relative path.
|
907.1.20
by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8 |
298 |
|
299 |
:param relpath: The relative path to the file
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
300 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
301 |
raise NotImplementedError(self.get) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
302 |
|
1594.2.5
by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support. |
303 |
def readv(self, relpath, offsets): |
304 |
"""Get parts of the file at the given relative path.
|
|
305 |
||
306 |
:offsets: A list of (offset, size) tuples.
|
|
307 |
:return: A list or generator of (offset, data) tuples
|
|
308 |
"""
|
|
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
309 |
def do_combined_read(combined_offsets): |
310 |
total_size = 0 |
|
311 |
for offset, size in combined_offsets: |
|
312 |
total_size += size |
|
313 |
mutter('readv coalesced %d reads.', len(combined_offsets)) |
|
314 |
offset = combined_offsets[0][0] |
|
315 |
fp.seek(offset) |
|
316 |
data = fp.read(total_size) |
|
317 |
pos = 0 |
|
318 |
for offset, size in combined_offsets: |
|
319 |
yield offset, data[pos:pos + size] |
|
320 |
pos += size |
|
321 |
||
1594.2.16
by Robert Collins
Coalesce readv requests on file based transports. |
322 |
if not len(offsets): |
323 |
return
|
|
1594.2.5
by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support. |
324 |
fp = self.get(relpath) |
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
325 |
pending_offsets = deque(offsets) |
326 |
combined_offsets = [] |
|
327 |
while len(pending_offsets): |
|
328 |
offset, size = pending_offsets.popleft() |
|
329 |
if not combined_offsets: |
|
330 |
combined_offsets = [[offset, size]] |
|
1594.2.16
by Robert Collins
Coalesce readv requests on file based transports. |
331 |
else: |
1594.2.19
by Robert Collins
More coalescing tweaks, and knit feedback. |
332 |
if (len (combined_offsets) < 50 and |
333 |
combined_offsets[-1][0] + combined_offsets[-1][1] == offset): |
|
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
334 |
# combatible offset:
|
335 |
combined_offsets.append([offset, size]) |
|
336 |
else: |
|
1594.2.19
by Robert Collins
More coalescing tweaks, and knit feedback. |
337 |
# incompatible, or over the threshold issue a read and yield
|
1594.2.17
by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading. |
338 |
pending_offsets.appendleft((offset, size)) |
339 |
for result in do_combined_read(combined_offsets): |
|
340 |
yield result |
|
341 |
combined_offsets = [] |
|
342 |
# whatever is left is a single coalesced request
|
|
343 |
if len(combined_offsets): |
|
344 |
for result in do_combined_read(combined_offsets): |
|
345 |
yield result |
|
1594.2.5
by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support. |
346 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
347 |
def get_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
348 |
"""Get a list of file-like objects, one for each entry in relpaths.
|
349 |
||
350 |
:param relpaths: A list of relative paths.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
351 |
:param pb: An optional ProgressBar for indicating percent done.
|
352 |
:return: A list or generator of file-like objects
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
353 |
"""
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
354 |
# TODO: Consider having this actually buffer the requests,
|
355 |
# in the default mode, it probably won't give worse performance,
|
|
356 |
# and all children wouldn't have to implement buffering
|
|
907.1.16
by John Arbash Meinel
Fixing a few cut&paste typos. |
357 |
total = self._get_total(relpaths) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
358 |
count = 0 |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
359 |
for relpath in relpaths: |
907.1.16
by John Arbash Meinel
Fixing a few cut&paste typos. |
360 |
self._update_pb(pb, 'get', count, total) |
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
361 |
yield self.get(relpath) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
362 |
count += 1 |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
363 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
364 |
def put(self, relpath, f, mode=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
365 |
"""Copy the file-like or string object into the location.
|
907.1.20
by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8 |
366 |
|
367 |
:param relpath: Location to put the contents, relative to base.
|
|
368 |
:param f: File-like or string object.
|
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
369 |
:param mode: The mode for the newly created file,
|
370 |
None means just use the default
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
371 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
372 |
raise NotImplementedError(self.put) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
373 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
374 |
def put_multi(self, files, mode=None, pb=None): |
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
375 |
"""Put a set of files into the location.
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
376 |
|
377 |
:param files: A list of tuples of relpath, file object [(path1, file1), (path2, file2),...]
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
378 |
:param pb: An optional ProgressBar for indicating percent done.
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
379 |
:param mode: The mode for the newly created files
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
380 |
:return: The number of files copied.
|
381 |
"""
|
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
382 |
def put(path, f): |
383 |
self.put(path, f, mode=mode) |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
384 |
return len(self._iterate_over(files, put, pb, 'put', expand=True)) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
385 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
386 |
def mkdir(self, relpath, mode=None): |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
387 |
"""Create a directory at the given path."""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
388 |
raise NotImplementedError(self.mkdir) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
389 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
390 |
def mkdir_multi(self, relpaths, mode=None, pb=None): |
907.1.47
by John Arbash Meinel
Created mkdir_multi, modified branch to use _multi forms when initializing a branch, creating a full test routine for transports |
391 |
"""Create a group of directories"""
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
392 |
def mkdir(path): |
393 |
self.mkdir(path, mode=mode) |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
394 |
return len(self._iterate_over(relpaths, mkdir, pb, 'mkdir', expand=False)) |
907.1.47
by John Arbash Meinel
Created mkdir_multi, modified branch to use _multi forms when initializing a branch, creating a full test routine for transports |
395 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
396 |
def append(self, relpath, f): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
397 |
"""Append the text in the file-like or string object to
|
398 |
the supplied location.
|
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
399 |
|
400 |
returns the length of f before the content was written to it.
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
401 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
402 |
raise NotImplementedError(self.append) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
403 |
|
1185.11.19
by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings. |
404 |
def append_multi(self, files, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
405 |
"""Append the text in each file-like or string object to
|
406 |
the supplied location.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
407 |
|
408 |
:param files: A set of (path, f) entries
|
|
409 |
:param pb: An optional ProgressBar for indicating percent done.
|
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
410 |
"""
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
411 |
return self._iterate_over(files, self.append, pb, 'append', expand=True) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
412 |
|
413 |
def copy(self, rel_from, rel_to): |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
414 |
"""Copy the item at rel_from to the location at rel_to.
|
415 |
|
|
416 |
Override this for efficiency if a specific transport can do it
|
|
417 |
faster than this default implementation.
|
|
418 |
"""
|
|
419 |
self.put(rel_to, self.get(rel_from)) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
420 |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
421 |
def copy_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
422 |
"""Copy a bunch of entries.
|
423 |
|
|
424 |
:param relpaths: A list of tuples of the form [(from, to), (from, to),...]
|
|
425 |
"""
|
|
426 |
# This is the non-pipelined implementation, so that
|
|
427 |
# implementors don't have to implement everything.
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
428 |
return self._iterate_over(relpaths, self.copy, pb, 'copy', expand=True) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
429 |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
430 |
def copy_to(self, relpaths, other, mode=None, pb=None): |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
431 |
"""Copy a set of entries from self into another Transport.
|
432 |
||
433 |
:param relpaths: A list/generator of entries to be copied.
|
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
434 |
:param mode: This is the target mode for the newly created files
|
1185.16.156
by John Arbash Meinel
Adding a note about changing copy_to's interface |
435 |
TODO: This interface needs to be updated so that the target location
|
436 |
can be different from the source location.
|
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
437 |
"""
|
438 |
# The dummy implementation just does a simple get + put
|
|
439 |
def copy_entry(path): |
|
1185.58.2
by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work. |
440 |
other.put(path, self.get(path), mode=mode) |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
441 |
|
1563.2.3
by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content. |
442 |
return len(self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False)) |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
443 |
|
1534.4.26
by Robert Collins
Move working tree initialisation out from Branch.initialize, deprecated Branch.initialize to Branch.create. |
444 |
def copy_tree(self, from_relpath, to_relpath): |
445 |
"""Copy a subtree from one relpath to another.
|
|
446 |
||
447 |
If a faster implementation is available, specific transports should
|
|
448 |
implement it.
|
|
449 |
"""
|
|
450 |
source = self.clone(from_relpath) |
|
451 |
self.mkdir(to_relpath) |
|
452 |
target = self.clone(to_relpath) |
|
453 |
files = [] |
|
454 |
directories = ['.'] |
|
455 |
while directories: |
|
456 |
dir = directories.pop() |
|
457 |
if dir != '.': |
|
458 |
target.mkdir(dir) |
|
459 |
for path in source.list_dir(dir): |
|
460 |
path = dir + '/' + path |
|
461 |
stat = source.stat(path) |
|
462 |
if S_ISDIR(stat.st_mode): |
|
463 |
directories.append(path) |
|
464 |
else: |
|
465 |
files.append(path) |
|
466 |
source.copy_to(files, target) |
|
467 |
||
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
468 |
def rename(self, rel_from, rel_to): |
469 |
"""Rename a file or directory.
|
|
470 |
||
471 |
This *must* fail if the destination is a nonempty directory - it must
|
|
472 |
not automatically remove it. It should raise DirectoryNotEmpty, or
|
|
473 |
some other PathError if the case can't be specifically detected.
|
|
474 |
||
475 |
If the destination is an empty directory or a file this function may
|
|
476 |
either fail or succeed, depending on the underlying transport. It
|
|
477 |
should not attempt to remove the destination if overwriting is not the
|
|
478 |
native transport behaviour. If at all possible the transport should
|
|
479 |
ensure that the rename either completes or not, without leaving the
|
|
480 |
destination deleted and the new file not moved in place.
|
|
481 |
||
482 |
This is intended mainly for use in implementing LockDir.
|
|
483 |
"""
|
|
484 |
# transports may need to override this
|
|
1553.5.17
by Martin Pool
Transport.rename should be unimplemented in base class |
485 |
raise NotImplementedError(self.rename) |
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
486 |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
487 |
def move(self, rel_from, rel_to): |
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
488 |
"""Move the item at rel_from to the location at rel_to.
|
1553.5.13
by Martin Pool
New Transport.rename that mustn't overwrite |
489 |
|
490 |
The destination is deleted if possible, even if it's a non-empty
|
|
491 |
directory tree.
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
492 |
|
493 |
If a transport can directly implement this it is suggested that
|
|
494 |
it do so for efficiency.
|
|
495 |
"""
|
|
1534.4.26
by Robert Collins
Move working tree initialisation out from Branch.initialize, deprecated Branch.initialize to Branch.create. |
496 |
if S_ISDIR(self.stat(rel_from).st_mode): |
497 |
self.copy_tree(rel_from, rel_to) |
|
498 |
self.delete_tree(rel_from) |
|
499 |
else: |
|
500 |
self.copy(rel_from, rel_to) |
|
501 |
self.delete(rel_from) |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
502 |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
503 |
def move_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
504 |
"""Move a bunch of entries.
|
505 |
|
|
506 |
:param relpaths: A list of tuples of the form [(from1, to1), (from2, to2),...]
|
|
507 |
"""
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
508 |
return self._iterate_over(relpaths, self.move, pb, 'move', expand=True) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
509 |
|
510 |
def move_multi_to(self, relpaths, rel_to): |
|
511 |
"""Move a bunch of entries to a single location.
|
|
512 |
This differs from move_multi in that you give a list of from, and
|
|
513 |
a single destination, rather than multiple destinations.
|
|
514 |
||
515 |
:param relpaths: A list of relative paths [from1, from2, from3, ...]
|
|
516 |
:param rel_to: A directory where each entry should be placed.
|
|
517 |
"""
|
|
518 |
# This is not implemented, because you need to do special tricks to
|
|
519 |
# extract the basename, and add it to rel_to
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
520 |
raise NotImplementedError(self.move_multi_to) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
521 |
|
522 |
def delete(self, relpath): |
|
523 |
"""Delete the item at relpath"""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
524 |
raise NotImplementedError(self.delete) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
525 |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
526 |
def delete_multi(self, relpaths, pb=None): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
527 |
"""Queue up a bunch of deletes to be done.
|
528 |
"""
|
|
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
529 |
return self._iterate_over(relpaths, self.delete, pb, 'delete', expand=False) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
530 |
|
1534.4.15
by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports. |
531 |
def delete_tree(self, relpath): |
532 |
"""Delete an entire tree. This may require a listable transport."""
|
|
533 |
subtree = self.clone(relpath) |
|
534 |
files = [] |
|
535 |
directories = ['.'] |
|
536 |
pending_rmdirs = [] |
|
537 |
while directories: |
|
538 |
dir = directories.pop() |
|
539 |
if dir != '.': |
|
540 |
pending_rmdirs.append(dir) |
|
541 |
for path in subtree.list_dir(dir): |
|
542 |
path = dir + '/' + path |
|
543 |
stat = subtree.stat(path) |
|
544 |
if S_ISDIR(stat.st_mode): |
|
545 |
directories.append(path) |
|
546 |
else: |
|
547 |
files.append(path) |
|
548 |
subtree.delete_multi(files) |
|
549 |
pending_rmdirs.reverse() |
|
550 |
for dir in pending_rmdirs: |
|
551 |
subtree.rmdir(dir) |
|
552 |
self.rmdir(relpath) |
|
553 |
||
1534.5.7
by Robert Collins
Start factoring out the upgrade policy logic. |
554 |
def __repr__(self): |
555 |
return "<%s.%s url=%s>" % (self.__module__, self.__class__.__name__, self.base) |
|
556 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
557 |
def stat(self, relpath): |
558 |
"""Return the stat information for a file.
|
|
559 |
WARNING: This may not be implementable for all protocols, so use
|
|
560 |
sparingly.
|
|
1442.1.44
by Robert Collins
Many transport related tweaks: |
561 |
NOTE: This returns an object with fields such as 'st_size'. It MAY
|
562 |
or MAY NOT return the literal result of an os.stat() call, so all
|
|
563 |
access should be via named fields.
|
|
564 |
ALSO NOTE: Stats of directories may not be supported on some
|
|
565 |
transports.
|
|
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
566 |
"""
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
567 |
raise NotImplementedError(self.stat) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
568 |
|
1534.4.15
by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports. |
569 |
def rmdir(self, relpath): |
570 |
"""Remove a directory at the given path."""
|
|
571 |
raise NotImplementedError |
|
572 |
||
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
573 |
def stat_multi(self, relpaths, pb=None): |
574 |
"""Stat multiple files and return the information.
|
|
575 |
"""
|
|
576 |
#TODO: Is it worth making this a generator instead of a
|
|
577 |
# returning a list?
|
|
578 |
stats = [] |
|
579 |
def gather(path): |
|
580 |
stats.append(self.stat(path)) |
|
581 |
||
907.1.14
by John Arbash Meinel
Handling some transport functions which take only a single argument. |
582 |
count = self._iterate_over(relpaths, gather, pb, 'stat', expand=False) |
907.1.2
by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport. |
583 |
return stats |
584 |
||
1400.1.1
by Robert Collins
implement a basic test for the ui branch command from http servers |
585 |
def listable(self): |
586 |
"""Return True if this store supports listing."""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
587 |
raise NotImplementedError(self.listable) |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
588 |
|
589 |
def list_dir(self, relpath): |
|
590 |
"""Return a list of all files at the given location.
|
|
591 |
WARNING: many transports do not support this, so trying avoid using
|
|
592 |
it if at all possible.
|
|
593 |
"""
|
|
1185.31.44
by John Arbash Meinel
Cleaned up Exceptions for all transports. |
594 |
raise errors.TransportNotPossible("This transport has not " |
1530.1.21
by Robert Collins
Review feedback fixes. |
595 |
"implemented list_dir "
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
596 |
"(but must claim to be listable "
|
597 |
"to trigger this error).") |
|
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
598 |
|
907.1.24
by John Arbash Meinel
Remote functionality work. |
599 |
def lock_read(self, relpath): |
600 |
"""Lock the given file for shared (read) access.
|
|
601 |
WARNING: many transports do not support this, so trying avoid using it
|
|
602 |
||
603 |
:return: A lock object, which should contain an unlock() function.
|
|
604 |
"""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
605 |
raise NotImplementedError(self.lock_read) |
907.1.24
by John Arbash Meinel
Remote functionality work. |
606 |
|
607 |
def lock_write(self, relpath): |
|
608 |
"""Lock the given file for exclusive (write) access.
|
|
609 |
WARNING: many transports do not support this, so trying avoid using it
|
|
610 |
||
611 |
:return: A lock object, which should contain an unlock() function.
|
|
612 |
"""
|
|
1540.3.1
by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib. |
613 |
raise NotImplementedError(self.lock_write) |
907.1.24
by John Arbash Meinel
Remote functionality work. |
614 |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
615 |
def is_readonly(self): |
616 |
"""Return true if this connection cannot be written to."""
|
|
617 |
return False |
|
618 |
||
907.1.24
by John Arbash Meinel
Remote functionality work. |
619 |
|
1393.2.4
by John Arbash Meinel
All tests pass. |
620 |
def get_transport(base): |
1185.70.3
by Martin Pool
Various updates to make storage branch mergeable: |
621 |
"""Open a transport to access a URL or directory.
|
622 |
||
623 |
base is either a URL or a directory name.
|
|
624 |
"""
|
|
1540.3.8
by Martin Pool
Some support for falling back between transport implementations. |
625 |
# TODO: give a better error if base looks like a url but there's no
|
626 |
# handler for the scheme?
|
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
627 |
global _protocol_handlers |
907.1.13
by John Arbash Meinel
Fixed bzr root. |
628 |
if base is None: |
1185.33.66
by Martin Pool
[patch] use unicode literals for all hardcoded paths (Alexander Belchenko) |
629 |
base = u'.' |
1185.31.60
by John Arbash Meinel
get_transport() now forces unicode paths, which helps get proper filenames. |
630 |
else: |
631 |
base = unicode(base) |
|
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
632 |
for proto, factory_list in _protocol_handlers.iteritems(): |
907.1.1
by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer. |
633 |
if proto is not None and base.startswith(proto): |
1540.3.12
by Martin Pool
Multiple transports can be registered for any protocol, and they are |
634 |
t = _try_transport_factories(base, factory_list) |
635 |
if t: |
|
636 |
return t |
|
637 |
# The default handler is the filesystem handler, stored as protocol None
|
|
638 |
return _try_transport_factories(base, _protocol_handlers[None]) |
|
639 |
||
640 |
||
641 |
def _try_transport_factories(base, factory_list): |
|
642 |
for factory in factory_list: |
|
643 |
try: |
|
644 |
return factory(base) |
|
645 |
except DependencyNotPresent, e: |
|
646 |
mutter("failed to instantiate transport %r for %r: %r" % |
|
647 |
(factory, base, e)) |
|
648 |
continue
|
|
649 |
return None |
|
1185.16.81
by mbp at sourcefrog
[merge] robert |
650 |
|
1185.16.78
by Martin Pool
- load paramiko sftp transport by default |
651 |
|
1469
by Robert Collins
Change Transport.* to work with URL's. |
652 |
def urlescape(relpath): |
653 |
"""Escape relpath to be a valid url."""
|
|
654 |
# TODO utf8 it first. utf8relpath = relpath.encode('utf8')
|
|
655 |
return urllib.quote(relpath) |
|
1470
by Robert Collins
merge from Martin, via newformat, and teach sftp about urlescaped paths |
656 |
|
1185.16.78
by Martin Pool
- load paramiko sftp transport by default |
657 |
|
1608.1.1
by Martin Pool
[patch] LocalTransport.list_dir should return url-quoted strings (ddaa) |
658 |
def urlunescape(relpath): |
659 |
"""Unescape relpath from url format."""
|
|
660 |
return urllib.unquote(relpath) |
|
661 |
# TODO de-utf8 it last. relpath = utf8relpath.decode('utf8')
|
|
662 |
||
663 |
||
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
664 |
class Server(object): |
1530.1.21
by Robert Collins
Review feedback fixes. |
665 |
"""A Transport Server.
|
666 |
|
|
667 |
The Server interface provides a server for a given transport. We use
|
|
668 |
these servers as loopback testing tools. For any given transport the
|
|
669 |
Servers it provides must either allow writing, or serve the contents
|
|
670 |
of os.getcwdu() at the time setUp is called.
|
|
671 |
|
|
672 |
Note that these are real servers - they must implement all the things
|
|
673 |
that we want bzr transports to take advantage of.
|
|
674 |
"""
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
675 |
|
676 |
def setUp(self): |
|
677 |
"""Setup the server to service requests."""
|
|
678 |
||
679 |
def tearDown(self): |
|
680 |
"""Remove the server and cleanup any resources it owns."""
|
|
681 |
||
682 |
def get_url(self): |
|
1530.1.21
by Robert Collins
Review feedback fixes. |
683 |
"""Return a url for this server.
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
684 |
|
685 |
If the transport does not represent a disk directory (i.e. it is
|
|
1530.1.21
by Robert Collins
Review feedback fixes. |
686 |
a database like svn, or a memory only transport, it should return
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
687 |
a connection to a newly established resource for this Server.
|
1530.1.21
by Robert Collins
Review feedback fixes. |
688 |
Otherwise it should return a url that will provide access to the path
|
689 |
that was os.getcwdu() when setUp() was called.
|
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
690 |
|
691 |
Subsequent calls will return the same resource.
|
|
692 |
"""
|
|
693 |
raise NotImplementedError |
|
694 |
||
1530.1.9
by Robert Collins
Test bogus urls with http in the new infrastructure. |
695 |
def get_bogus_url(self): |
696 |
"""Return a url for this protocol, that will fail to connect."""
|
|
697 |
raise NotImplementedError |
|
698 |
||
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
699 |
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
700 |
class TransportTestProviderAdapter(object): |
1530.1.21
by Robert Collins
Review feedback fixes. |
701 |
"""A tool to generate a suite testing all transports for a single test.
|
702 |
||
703 |
This is done by copying the test once for each transport and injecting
|
|
704 |
the transport_class and transport_server classes into each copy. Each copy
|
|
705 |
is also given a new id() to make it easy to identify.
|
|
706 |
"""
|
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
707 |
|
708 |
def adapt(self, test): |
|
709 |
result = TestSuite() |
|
710 |
for klass, server_factory in self._test_permutations(): |
|
711 |
new_test = deepcopy(test) |
|
712 |
new_test.transport_class = klass |
|
713 |
new_test.transport_server = server_factory |
|
1530.1.3
by Robert Collins
transport implementations now tested consistently. |
714 |
def make_new_test_id(): |
715 |
new_id = "%s(%s)" % (new_test.id(), server_factory.__name__) |
|
716 |
return lambda: new_id |
|
717 |
new_test.id = make_new_test_id() |
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
718 |
result.addTest(new_test) |
719 |
return result |
|
720 |
||
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
721 |
def get_transport_test_permutations(self, module): |
722 |
"""Get the permutations module wants to have tested."""
|
|
1540.3.6
by Martin Pool
[merge] update from bzr.dev |
723 |
if not hasattr(module, 'get_test_permutations'): |
724 |
warning("transport module %s doesn't provide get_test_permutations()" |
|
725 |
% module.__name__) |
|
726 |
return [] |
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
727 |
return module.get_test_permutations() |
728 |
||
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
729 |
def _test_permutations(self): |
730 |
"""Return a list of the klass, server_factory pairs to test."""
|
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
731 |
result = [] |
732 |
for module in _get_transport_modules(): |
|
1534.7.70
by abentley
Fix bug when no paramiko |
733 |
try: |
734 |
result.extend(self.get_transport_test_permutations(reduce(getattr, |
|
735 |
(module).split('.')[1:], |
|
1185.71.1
by John Arbash Meinel
Allow selftest to run, even if we can't load a transport. |
736 |
__import__(module)))) |
1185.62.24
by John Arbash Meinel
Changing the exception that sftp.py throws when it can't find paramiko, so that the test suite can handle it. |
737 |
except errors.DependencyNotPresent, e: |
738 |
# Continue even if a dependency prevents us
|
|
739 |
# from running this test
|
|
1534.7.70
by abentley
Fix bug when no paramiko |
740 |
pass
|
1530.1.11
by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports. |
741 |
return result |
1594.2.24
by Robert Collins
Make use of the transaction finalisation warning support to implement in-knit caching. |
742 |
|
743 |
||
744 |
class TransportLogger(object): |
|
745 |
"""Adapt a transport to get clear logging data on api calls.
|
|
746 |
|
|
747 |
Feel free to extend to log whatever calls are of interest.
|
|
748 |
"""
|
|
749 |
||
750 |
def __init__(self, adapted): |
|
751 |
self._adapted = adapted |
|
752 |
self._calls = [] |
|
753 |
||
754 |
def get(self, name): |
|
755 |
self._calls.append((name,)) |
|
756 |
return self._adapted.get(name) |
|
757 |
||
758 |
def __getattr__(self, name): |
|
759 |
"""Thunk all undefined access through to self._adapted."""
|
|
760 |
# raise AttributeError, name
|
|
761 |
return getattr(self._adapted, name) |
|
762 |
||
763 |
def readv(self, name, offsets): |
|
764 |
self._calls.append((name, offsets)) |
|
765 |
return self._adapted.readv(name, offsets) |
|
1530.1.1
by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter. |
766 |
|
767 |
||
1185.16.81
by mbp at sourcefrog
[merge] robert |
768 |
# None is the default transport, for things with no url scheme
|
1185.16.79
by Martin Pool
Load transports when they're first used. |
769 |
register_lazy_transport(None, 'bzrlib.transport.local', 'LocalTransport') |
770 |
register_lazy_transport('file://', 'bzrlib.transport.local', 'LocalTransport') |
|
771 |
register_lazy_transport('sftp://', 'bzrlib.transport.sftp', 'SFTPTransport') |
|
1540.3.23
by Martin Pool
Allow urls like http+pycurl://host/ to use a particular impl |
772 |
register_lazy_transport('http+urllib://', 'bzrlib.transport.http._urllib', |
1540.3.26
by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet |
773 |
'HttpTransport_urllib') |
1540.3.23
by Martin Pool
Allow urls like http+pycurl://host/ to use a particular impl |
774 |
register_lazy_transport('https+urllib://', 'bzrlib.transport.http._urllib', |
1540.3.26
by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet |
775 |
'HttpTransport_urllib') |
776 |
register_lazy_transport('http+pycurl://', 'bzrlib.transport.http._pycurl', |
|
777 |
'PyCurlTransport') |
|
778 |
register_lazy_transport('https+pycurl://', 'bzrlib.transport.http._pycurl', |
|
779 |
'PyCurlTransport') |
|
780 |
register_lazy_transport('http://', 'bzrlib.transport.http._urllib', |
|
781 |
'HttpTransport_urllib') |
|
782 |
register_lazy_transport('https://', 'bzrlib.transport.http._urllib', |
|
783 |
'HttpTransport_urllib') |
|
1540.3.7
by Martin Pool
Prepare to select a transport depending on what dependencies can be satisfied. |
784 |
register_lazy_transport('http://', 'bzrlib.transport.http._pycurl', 'PyCurlTransport') |
785 |
register_lazy_transport('https://', 'bzrlib.transport.http._pycurl', 'PyCurlTransport') |
|
1185.36.4
by Daniel Silverstone
Add FTP transport |
786 |
register_lazy_transport('ftp://', 'bzrlib.transport.ftp', 'FtpTransport') |
787 |
register_lazy_transport('aftp://', 'bzrlib.transport.ftp', 'FtpTransport') |
|
1534.4.39
by Robert Collins
Basic BzrDir support. |
788 |
register_lazy_transport('memory:/', 'bzrlib.transport.memory', 'MemoryTransport') |
1534.4.9
by Robert Collins
Add a readonly decorator for transports. |
789 |
register_lazy_transport('readonly+', 'bzrlib.transport.readonly', 'ReadonlyTransportDecorator') |
1558.10.2
by Robert Collins
Refactor the FakeNFS support into a TransportDecorator. |
790 |
register_lazy_transport('fakenfs+', 'bzrlib.transport.fakenfs', 'FakeNFSTransportDecorator') |