~bzr-pqm/bzr/bzr.dev

1553.5.10 by Martin Pool
New DirectoryNotEmpty exception, and raise this from local and memory
1
# Copyright (C) 2005, 2006 Canonical Ltd
1185.11.19 by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings.
2
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
16
1185.11.19 by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings.
17
"""Transport is an abstraction layer to handle file access.
18
19
The abstraction is to allow access from the local filesystem, as well
20
as remote (such as http or sftp).
1540.3.12 by Martin Pool
Multiple transports can be registered for any protocol, and they are
21
22
Transports are constructed from a string, being a URL or (as a degenerate
23
case) a local filesystem path.  This is typically the top directory of
24
a bzrdir, repository, or similar object we are interested in working with.
25
The Transport returned has methods to read, write and manipulate files within
26
it.
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
27
"""
28
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
29
import errno
1594.2.17 by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading.
30
from collections import deque
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
31
from copy import deepcopy
1534.4.15 by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports.
32
from stat import *
1530.1.7 by Robert Collins
merge integration.
33
import sys
1185.85.76 by John Arbash Meinel
Adding an InvalidURL so transports can report they expect utf-8 quoted paths. Updated tests
34
import urllib
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
35
from unittest import TestSuite
1185.80.5 by John Arbash Meinel
Changing the escaping just a little bit. Now we can handle unicode characters.
36
import urllib
1636.1.1 by Robert Collins
Fix calling relpath() and abspath() on transports at their root.
37
import urlparse
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
38
1540.3.6 by Martin Pool
[merge] update from bzr.dev
39
from bzrlib.trace import mutter, warning
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
40
import bzrlib.errors as errors
1540.3.8 by Martin Pool
Some support for falling back between transport implementations.
41
from bzrlib.errors import DependencyNotPresent
1540.3.12 by Martin Pool
Multiple transports can be registered for any protocol, and they are
42
from bzrlib.symbol_versioning import *
907.1.45 by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary.
43
1540.3.12 by Martin Pool
Multiple transports can be registered for any protocol, and they are
44
# {prefix: [transport_classes]}
45
# Transports are inserted onto the list LIFO and tried in order; as a result
46
# transports provided by plugins are tried first, which is usually what we
47
# want.
907.1.45 by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary.
48
_protocol_handlers = {
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
49
}
50
1540.3.12 by Martin Pool
Multiple transports can be registered for any protocol, and they are
51
def register_transport(prefix, klass, override=DEPRECATED_PARAMETER):
1540.3.8 by Martin Pool
Some support for falling back between transport implementations.
52
    """Register a transport that can be used to open URLs
53
54
    Normally you should use register_lazy_transport, which defers loading the
55
    implementation until it's actually used, and so avoids pulling in possibly
56
    large implementation libraries.
57
    """
1540.3.12 by Martin Pool
Multiple transports can be registered for any protocol, and they are
58
    # Note that this code runs very early in library setup -- trace may not be
59
    # working, etc.
907.1.45 by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary.
60
    global _protocol_handlers
1540.3.12 by Martin Pool
Multiple transports can be registered for any protocol, and they are
61
    if deprecated_passed(override):
62
        warn("register_transport(override) is deprecated")
63
    _protocol_handlers.setdefault(prefix, []).insert(0, klass)
64
65
66
def register_lazy_transport(scheme, module, classname):
67
    """Register lazy-loaded transport class.
68
69
    When opening a URL with the given scheme, load the module and then
70
    instantiate the particular class.  
71
72
    If the module raises DependencyNotPresent when it's imported, it is
73
    skipped and another implementation of the protocol is tried.  This is
74
    intended to be used when the implementation depends on an external
75
    implementation that may not be present.  If any other error is raised, it
76
    propagates up and the attempt to open the url fails.
77
    """
78
    # TODO: If no implementation of a protocol is available because of missing
79
    # dependencies, we should perhaps show the message about what dependency
80
    # was missing.
81
    def _loader(base):
82
        mod = __import__(module, globals(), locals(), [classname])
83
        klass = getattr(mod, classname)
84
        return klass(base)
85
    _loader.module = module
86
    register_transport(scheme, _loader)
907.1.45 by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary.
87
1442.1.24 by Robert Collins
Pull up _check_id and _relpath from Text and CompressedText stores into TransportStore
88
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
89
def _get_protocol_handlers():
1540.3.12 by Martin Pool
Multiple transports can be registered for any protocol, and they are
90
    """Return a dictionary of {urlprefix: [factory]}"""
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
91
    return _protocol_handlers
92
93
94
def _set_protocol_handlers(new_handlers):
95
    """Replace the current protocol handlers dictionary.
96
97
    WARNING this will remove all build in protocols. Use with care.
98
    """
99
    global _protocol_handlers
100
    _protocol_handlers = new_handlers
101
102
1540.3.12 by Martin Pool
Multiple transports can be registered for any protocol, and they are
103
def _clear_protocol_handlers():
104
    global _protocol_handlers
105
    _protocol_handlers = {}
106
107
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
108
def _get_transport_modules():
109
    """Return a list of the modules providing transports."""
110
    modules = set()
1540.3.12 by Martin Pool
Multiple transports can be registered for any protocol, and they are
111
    for prefix, factory_list in _protocol_handlers.items():
112
        for factory in factory_list:
113
            if factory.__module__ == "bzrlib.transport":
114
                # this is a lazy load transport, because no real ones
115
                # are directlry in bzrlib.transport
116
                modules.add(factory.module)
117
            else:
118
                modules.add(factory.__module__)
1530.1.19 by Robert Collins
Make transport test adapter tests reliable.
119
    result = list(modules)
120
    result.sort()
121
    return result
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
122
123
1636.1.2 by Robert Collins
More review fixen to the relpath at '/' fixes.
124
def register_urlparse_netloc_protocol(protocol):
1636.1.1 by Robert Collins
Fix calling relpath() and abspath() on transports at their root.
125
    """Ensure that protocol is setup to be used with urlparse netloc parsing."""
126
    if protocol not in urlparse.uses_netloc:
127
        urlparse.uses_netloc.append(protocol)
128
129
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
130
class Transport(object):
131
    """This class encapsulates methods for retrieving or putting a file
132
    from/to a storage location.
133
134
    Most functions have a _multi variant, which allows you to queue up
135
    multiple requests. They generally have a dumb base implementation 
136
    which just iterates over the arguments, but smart Transport
137
    implementations can do pipelining.
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
138
    In general implementations should support having a generator or a list
139
    as an argument (ie always iterate, never index)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
140
    """
141
142
    def __init__(self, base):
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
143
        super(Transport, self).__init__()
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
144
        self.base = base
145
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
146
    def _translate_error(self, e, path, raise_generic=True):
147
        """Translate an IOError or OSError into an appropriate bzr error.
148
149
        This handles things like ENOENT, ENOTDIR, EEXIST, and EACCESS
150
        """
151
        if hasattr(e, 'errno'):
152
            if e.errno in (errno.ENOENT, errno.ENOTDIR):
153
                raise errors.NoSuchFile(path, extra=e)
1185.31.58 by John Arbash Meinel
Updating for new transport tests so that they pass on win32
154
            # I would rather use errno.EFOO, but there doesn't seem to be
155
            # any matching for 267
156
            # This is the error when doing a listdir on a file:
157
            # WindowsError: [Errno 267] The directory name is invalid
158
            if sys.platform == 'win32' and e.errno in (errno.ESRCH, 267):
159
                raise errors.NoSuchFile(path, extra=e)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
160
            if e.errno == errno.EEXIST:
161
                raise errors.FileExists(path, extra=e)
162
            if e.errno == errno.EACCES:
163
                raise errors.PermissionDenied(path, extra=e)
1553.5.10 by Martin Pool
New DirectoryNotEmpty exception, and raise this from local and memory
164
            if e.errno == errno.ENOTEMPTY:
165
                raise errors.DirectoryNotEmpty(path, extra=e)
1558.10.1 by Aaron Bentley
Handle lockdirs over NFS properly
166
            if e.errno == errno.EBUSY:
167
                raise errors.ResourceBusy(path, extra=e)
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
168
        if raise_generic:
169
            raise errors.TransportError(orig_error=e)
170
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
171
    def clone(self, offset=None):
172
        """Return a new Transport object, cloned from the current location,
1185.11.6 by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently.
173
        using a subdirectory or parent directory. This allows connections 
174
        to be pooled, rather than a new one needed for each subdir.
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
175
        """
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
176
        raise NotImplementedError(self.clone)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
177
907.1.32 by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate.
178
    def should_cache(self):
179
        """Return True if the data pulled across should be cached locally.
907.1.22 by John Arbash Meinel
Fixed some encoding issues, added is_remote function for Transport objects.
180
        """
907.1.32 by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate.
181
        return False
907.1.22 by John Arbash Meinel
Fixed some encoding issues, added is_remote function for Transport objects.
182
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
183
    def _pump(self, from_file, to_file):
184
        """Most children will need to copy from one file-like 
185
        object or string to another one.
186
        This just gives them something easy to call.
187
        """
188
        if isinstance(from_file, basestring):
907.1.8 by John Arbash Meinel
Changed the format for abspath. Updated branch to use a hidden _transport
189
            to_file.write(from_file)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
190
        else:
191
            from bzrlib.osutils import pumpfile
192
            pumpfile(from_file, to_file)
193
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
194
    def _get_total(self, multi):
195
        """Try to figure out how many entries are in multi,
196
        but if not possible, return None.
197
        """
198
        try:
199
            return len(multi)
200
        except TypeError: # We can't tell how many, because relpaths is a generator
201
            return None
202
203
    def _update_pb(self, pb, msg, count, total):
204
        """Update the progress bar based on the current count
205
        and total available, total may be None if it was
206
        not possible to determine.
207
        """
907.1.14 by John Arbash Meinel
Handling some transport functions which take only a single argument.
208
        if pb is None:
209
            return
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
210
        if total is None:
211
            pb.update(msg, count, count+1)
212
        else:
213
            pb.update(msg, count, total)
214
907.1.14 by John Arbash Meinel
Handling some transport functions which take only a single argument.
215
    def _iterate_over(self, multi, func, pb, msg, expand=True):
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
216
        """Iterate over all entries in multi, passing them to func,
217
        and update the progress bar as you go along.
907.1.14 by John Arbash Meinel
Handling some transport functions which take only a single argument.
218
219
        :param expand:  If True, the entries will be passed to the function
220
                        by expanding the tuple. If False, it will be passed
221
                        as a single parameter.
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
222
        """
223
        total = self._get_total(multi)
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
224
        result = []
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
225
        count = 0
226
        for entry in multi:
227
            self._update_pb(pb, msg, count, total)
907.1.14 by John Arbash Meinel
Handling some transport functions which take only a single argument.
228
            if expand:
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
229
                result.append(func(*entry))
907.1.14 by John Arbash Meinel
Handling some transport functions which take only a single argument.
230
            else:
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
231
                result.append(func(entry))
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
232
            count += 1
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
233
        return tuple(result)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
234
907.1.34 by John Arbash Meinel
Fixing append(), cleaning up function locations.
235
    def abspath(self, relpath):
236
        """Return the full url to the given relative path.
237
        This can be supplied with a string or a list
1442.1.44 by Robert Collins
Many transport related tweaks:
238
239
        XXX: Robert Collins 20051016 - is this really needed in the public
240
             interface ?
907.1.34 by John Arbash Meinel
Fixing append(), cleaning up function locations.
241
        """
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
242
        raise NotImplementedError(self.abspath)
907.1.34 by John Arbash Meinel
Fixing append(), cleaning up function locations.
243
244
    def relpath(self, abspath):
245
        """Return the local path portion from a given absolute path.
1442.1.44 by Robert Collins
Many transport related tweaks:
246
247
        This default implementation is not suitable for filesystems with
248
        aliasing, such as that given by symlinks, where a path may not 
249
        start with our base, but still be a relpath once aliasing is 
250
        resolved.
907.1.34 by John Arbash Meinel
Fixing append(), cleaning up function locations.
251
        """
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
252
        # TODO: This might want to use bzrlib.osutils.relpath
253
        #       but we have to watch out because of the prefix issues
1530.1.3 by Robert Collins
transport implementations now tested consistently.
254
        if not (abspath == self.base[:-1] or abspath.startswith(self.base)):
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
255
            raise errors.PathNotChild(abspath, self.base)
1442.1.44 by Robert Collins
Many transport related tweaks:
256
        pl = len(self.base)
1530.1.3 by Robert Collins
transport implementations now tested consistently.
257
        return abspath[pl:].strip('/')
907.1.34 by John Arbash Meinel
Fixing append(), cleaning up function locations.
258
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
259
    def has(self, relpath):
1442.1.44 by Robert Collins
Many transport related tweaks:
260
        """Does the file relpath exist?
261
        
262
        Note that some transports MAY allow querying on directories, but this
1553.5.66 by Martin Pool
doc
263
        is not part of the protocol.  In other words, the results of 
264
        t.has("a_directory_name") are undefined."
1442.1.44 by Robert Collins
Many transport related tweaks:
265
        """
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
266
        raise NotImplementedError(self.has)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
267
907.1.36 by John Arbash Meinel
Moving the multi-get functionality higher up into the Branch class.
268
    def has_multi(self, relpaths, pb=None):
907.1.34 by John Arbash Meinel
Fixing append(), cleaning up function locations.
269
        """Return True/False for each entry in relpaths"""
270
        total = self._get_total(relpaths)
271
        count = 0
272
        for relpath in relpaths:
907.1.36 by John Arbash Meinel
Moving the multi-get functionality higher up into the Branch class.
273
            self._update_pb(pb, 'has', count, total)
907.1.34 by John Arbash Meinel
Fixing append(), cleaning up function locations.
274
            yield self.has(relpath)
275
            count += 1
276
1185.16.155 by John Arbash Meinel
Added a has_any function to the Transport API
277
    def has_any(self, relpaths):
278
        """Return True if any of the paths exist."""
279
        for relpath in relpaths:
280
            if self.has(relpath):
281
                return True
282
        return False
283
1442.1.44 by Robert Collins
Many transport related tweaks:
284
    def iter_files_recursive(self):
285
        """Iter the relative paths of files in the transports sub-tree.
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
286
287
        *NOTE*: This only lists *files*, not subdirectories!
1442.1.44 by Robert Collins
Many transport related tweaks:
288
        
289
        As with other listing functions, only some transports implement this,.
290
        you may check via is_listable to determine if it will.
291
        """
1530.1.4 by Robert Collins
integrate Memory tests into transport interface tests.
292
        raise errors.TransportNotPossible("This transport has not "
1530.1.21 by Robert Collins
Review feedback fixes.
293
                                          "implemented iter_files_recursive "
1530.1.4 by Robert Collins
integrate Memory tests into transport interface tests.
294
                                          "(but must claim to be listable "
295
                                          "to trigger this error).")
1442.1.44 by Robert Collins
Many transport related tweaks:
296
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
297
    def get(self, relpath):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
298
        """Get the file at the given relative path.
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
299
300
        :param relpath: The relative path to the file
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
301
        """
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
302
        raise NotImplementedError(self.get)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
303
1594.2.5 by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support.
304
    def readv(self, relpath, offsets):
305
        """Get parts of the file at the given relative path.
306
307
        :offsets: A list of (offset, size) tuples.
308
        :return: A list or generator of (offset, data) tuples
309
        """
1594.2.17 by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading.
310
        def do_combined_read(combined_offsets):
311
            total_size = 0
312
            for offset, size in combined_offsets:
313
                total_size += size
314
            mutter('readv coalesced %d reads.', len(combined_offsets))
315
            offset = combined_offsets[0][0]
316
            fp.seek(offset)
317
            data = fp.read(total_size)
318
            pos = 0
319
            for offset, size in combined_offsets:
320
                yield offset, data[pos:pos + size]
321
                pos += size
322
1594.2.16 by Robert Collins
Coalesce readv requests on file based transports.
323
        if not len(offsets):
324
            return
1594.2.5 by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support.
325
        fp = self.get(relpath)
1594.2.17 by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading.
326
        pending_offsets = deque(offsets)
327
        combined_offsets = []
328
        while len(pending_offsets):
329
            offset, size = pending_offsets.popleft()
330
            if not combined_offsets:
331
                combined_offsets = [[offset, size]]
1594.2.16 by Robert Collins
Coalesce readv requests on file based transports.
332
            else:
1594.2.19 by Robert Collins
More coalescing tweaks, and knit feedback.
333
                if (len (combined_offsets) < 50 and
334
                    combined_offsets[-1][0] + combined_offsets[-1][1] == offset):
1594.2.17 by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading.
335
                    # combatible offset:
336
                    combined_offsets.append([offset, size])
337
                else:
1594.2.19 by Robert Collins
More coalescing tweaks, and knit feedback.
338
                    # incompatible, or over the threshold issue a read and yield
1594.2.17 by Robert Collins
Better readv coalescing, now with test, and progress during knit index reading.
339
                    pending_offsets.appendleft((offset, size))
340
                    for result in do_combined_read(combined_offsets):
341
                        yield result
342
                    combined_offsets = []
343
        # whatever is left is a single coalesced request
344
        if len(combined_offsets):
345
            for result in do_combined_read(combined_offsets):
346
                yield result
1594.2.5 by Robert Collins
Readv patch from Johan Rydberg giving knits partial download support.
347
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
348
    def get_multi(self, relpaths, pb=None):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
349
        """Get a list of file-like objects, one for each entry in relpaths.
350
351
        :param relpaths: A list of relative paths.
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
352
        :param pb:  An optional ProgressBar for indicating percent done.
353
        :return: A list or generator of file-like objects
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
354
        """
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
355
        # TODO: Consider having this actually buffer the requests,
356
        # in the default mode, it probably won't give worse performance,
357
        # and all children wouldn't have to implement buffering
907.1.16 by John Arbash Meinel
Fixing a few cut&paste typos.
358
        total = self._get_total(relpaths)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
359
        count = 0
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
360
        for relpath in relpaths:
907.1.16 by John Arbash Meinel
Fixing a few cut&paste typos.
361
            self._update_pb(pb, 'get', count, total)
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
362
            yield self.get(relpath)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
363
            count += 1
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
364
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
365
    def put(self, relpath, f, mode=None):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
366
        """Copy the file-like or string object into the location.
907.1.20 by John Arbash Meinel
Removed Transport.open(), making get + put encode/decode to utf-8
367
368
        :param relpath: Location to put the contents, relative to base.
369
        :param f:       File-like or string object.
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
370
        :param mode: The mode for the newly created file, 
371
                     None means just use the default
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
372
        """
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
373
        raise NotImplementedError(self.put)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
374
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
375
    def put_multi(self, files, mode=None, pb=None):
1530.1.3 by Robert Collins
transport implementations now tested consistently.
376
        """Put a set of files into the location.
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
377
378
        :param files: A list of tuples of relpath, file object [(path1, file1), (path2, file2),...]
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
379
        :param pb:  An optional ProgressBar for indicating percent done.
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
380
        :param mode: The mode for the newly created files
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
381
        :return: The number of files copied.
382
        """
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
383
        def put(path, f):
384
            self.put(path, f, mode=mode)
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
385
        return len(self._iterate_over(files, put, pb, 'put', expand=True))
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
386
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
387
    def mkdir(self, relpath, mode=None):
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
388
        """Create a directory at the given path."""
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
389
        raise NotImplementedError(self.mkdir)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
390
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
391
    def mkdir_multi(self, relpaths, mode=None, pb=None):
907.1.47 by John Arbash Meinel
Created mkdir_multi, modified branch to use _multi forms when initializing a branch, creating a full test routine for transports
392
        """Create a group of directories"""
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
393
        def mkdir(path):
394
            self.mkdir(path, mode=mode)
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
395
        return len(self._iterate_over(relpaths, mkdir, pb, 'mkdir', expand=False))
907.1.47 by John Arbash Meinel
Created mkdir_multi, modified branch to use _multi forms when initializing a branch, creating a full test routine for transports
396
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
397
    def append(self, relpath, f):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
398
        """Append the text in the file-like or string object to 
399
        the supplied location.
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
400
401
        returns the length of f before the content was written to it.
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
402
        """
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
403
        raise NotImplementedError(self.append)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
404
1185.11.19 by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings.
405
    def append_multi(self, files, pb=None):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
406
        """Append the text in each file-like or string object to
407
        the supplied location.
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
408
409
        :param files: A set of (path, f) entries
410
        :param pb:  An optional ProgressBar for indicating percent done.
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
411
        """
907.1.14 by John Arbash Meinel
Handling some transport functions which take only a single argument.
412
        return self._iterate_over(files, self.append, pb, 'append', expand=True)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
413
414
    def copy(self, rel_from, rel_to):
1530.1.3 by Robert Collins
transport implementations now tested consistently.
415
        """Copy the item at rel_from to the location at rel_to.
416
        
417
        Override this for efficiency if a specific transport can do it 
418
        faster than this default implementation.
419
        """
420
        self.put(rel_to, self.get(rel_from))
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
421
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
422
    def copy_multi(self, relpaths, pb=None):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
423
        """Copy a bunch of entries.
424
        
425
        :param relpaths: A list of tuples of the form [(from, to), (from, to),...]
426
        """
427
        # This is the non-pipelined implementation, so that
428
        # implementors don't have to implement everything.
907.1.14 by John Arbash Meinel
Handling some transport functions which take only a single argument.
429
        return self._iterate_over(relpaths, self.copy, pb, 'copy', expand=True)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
430
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
431
    def copy_to(self, relpaths, other, mode=None, pb=None):
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
432
        """Copy a set of entries from self into another Transport.
433
434
        :param relpaths: A list/generator of entries to be copied.
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
435
        :param mode: This is the target mode for the newly created files
1185.16.156 by John Arbash Meinel
Adding a note about changing copy_to's interface
436
        TODO: This interface needs to be updated so that the target location
437
              can be different from the source location.
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
438
        """
439
        # The dummy implementation just does a simple get + put
440
        def copy_entry(path):
1185.58.2 by John Arbash Meinel
Added mode to the appropriate transport functions, and tests to make sure they work.
441
            other.put(path, self.get(path), mode=mode)
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
442
1563.2.3 by Robert Collins
Change the return signature of transport.append and append_multi to return the length of the pre-append content.
443
        return len(self._iterate_over(relpaths, copy_entry, pb, 'copy_to', expand=False))
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
444
1534.4.26 by Robert Collins
Move working tree initialisation out from Branch.initialize, deprecated Branch.initialize to Branch.create.
445
    def copy_tree(self, from_relpath, to_relpath):
446
        """Copy a subtree from one relpath to another.
447
448
        If a faster implementation is available, specific transports should 
449
        implement it.
450
        """
451
        source = self.clone(from_relpath)
452
        self.mkdir(to_relpath)
453
        target = self.clone(to_relpath)
454
        files = []
455
        directories = ['.']
456
        while directories:
457
            dir = directories.pop()
458
            if dir != '.':
459
                target.mkdir(dir)
460
            for path in source.list_dir(dir):
461
                path = dir + '/' + path
462
                stat = source.stat(path)
463
                if S_ISDIR(stat.st_mode):
464
                    directories.append(path)
465
                else:
466
                    files.append(path)
467
        source.copy_to(files, target)
468
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
469
    def rename(self, rel_from, rel_to):
470
        """Rename a file or directory.
471
472
        This *must* fail if the destination is a nonempty directory - it must
473
        not automatically remove it.  It should raise DirectoryNotEmpty, or
474
        some other PathError if the case can't be specifically detected.
475
476
        If the destination is an empty directory or a file this function may
477
        either fail or succeed, depending on the underlying transport.  It
478
        should not attempt to remove the destination if overwriting is not the
479
        native transport behaviour.  If at all possible the transport should
480
        ensure that the rename either completes or not, without leaving the
481
        destination deleted and the new file not moved in place.
482
483
        This is intended mainly for use in implementing LockDir.
484
        """
485
        # transports may need to override this
1553.5.17 by Martin Pool
Transport.rename should be unimplemented in base class
486
        raise NotImplementedError(self.rename)
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
487
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
488
    def move(self, rel_from, rel_to):
1530.1.3 by Robert Collins
transport implementations now tested consistently.
489
        """Move the item at rel_from to the location at rel_to.
1553.5.13 by Martin Pool
New Transport.rename that mustn't overwrite
490
491
        The destination is deleted if possible, even if it's a non-empty
492
        directory tree.
1530.1.3 by Robert Collins
transport implementations now tested consistently.
493
        
494
        If a transport can directly implement this it is suggested that
495
        it do so for efficiency.
496
        """
1534.4.26 by Robert Collins
Move working tree initialisation out from Branch.initialize, deprecated Branch.initialize to Branch.create.
497
        if S_ISDIR(self.stat(rel_from).st_mode):
498
            self.copy_tree(rel_from, rel_to)
499
            self.delete_tree(rel_from)
500
        else:
501
            self.copy(rel_from, rel_to)
502
            self.delete(rel_from)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
503
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
504
    def move_multi(self, relpaths, pb=None):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
505
        """Move a bunch of entries.
506
        
507
        :param relpaths: A list of tuples of the form [(from1, to1), (from2, to2),...]
508
        """
907.1.14 by John Arbash Meinel
Handling some transport functions which take only a single argument.
509
        return self._iterate_over(relpaths, self.move, pb, 'move', expand=True)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
510
511
    def move_multi_to(self, relpaths, rel_to):
512
        """Move a bunch of entries to a single location.
513
        This differs from move_multi in that you give a list of from, and
514
        a single destination, rather than multiple destinations.
515
516
        :param relpaths: A list of relative paths [from1, from2, from3, ...]
517
        :param rel_to: A directory where each entry should be placed.
518
        """
519
        # This is not implemented, because you need to do special tricks to
520
        # extract the basename, and add it to rel_to
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
521
        raise NotImplementedError(self.move_multi_to)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
522
523
    def delete(self, relpath):
524
        """Delete the item at relpath"""
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
525
        raise NotImplementedError(self.delete)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
526
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
527
    def delete_multi(self, relpaths, pb=None):
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
528
        """Queue up a bunch of deletes to be done.
529
        """
907.1.14 by John Arbash Meinel
Handling some transport functions which take only a single argument.
530
        return self._iterate_over(relpaths, self.delete, pb, 'delete', expand=False)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
531
1534.4.15 by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports.
532
    def delete_tree(self, relpath):
533
        """Delete an entire tree. This may require a listable transport."""
534
        subtree = self.clone(relpath)
535
        files = []
536
        directories = ['.']
537
        pending_rmdirs = []
538
        while directories:
539
            dir = directories.pop()
540
            if dir != '.':
541
                pending_rmdirs.append(dir)
542
            for path in subtree.list_dir(dir):
543
                path = dir + '/' + path
544
                stat = subtree.stat(path)
545
                if S_ISDIR(stat.st_mode):
546
                    directories.append(path)
547
                else:
548
                    files.append(path)
549
        subtree.delete_multi(files)
550
        pending_rmdirs.reverse()
551
        for dir in pending_rmdirs:
552
            subtree.rmdir(dir)
553
        self.rmdir(relpath)
554
1534.5.7 by Robert Collins
Start factoring out the upgrade policy logic.
555
    def __repr__(self):
556
        return "<%s.%s url=%s>" % (self.__module__, self.__class__.__name__, self.base)
557
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
558
    def stat(self, relpath):
559
        """Return the stat information for a file.
560
        WARNING: This may not be implementable for all protocols, so use
561
        sparingly.
1442.1.44 by Robert Collins
Many transport related tweaks:
562
        NOTE: This returns an object with fields such as 'st_size'. It MAY
563
        or MAY NOT return the literal result of an os.stat() call, so all
564
        access should be via named fields.
565
        ALSO NOTE: Stats of directories may not be supported on some 
566
        transports.
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
567
        """
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
568
        raise NotImplementedError(self.stat)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
569
1534.4.15 by Robert Collins
Remove shutil dependency in upgrade - create a delete_tree method for transports.
570
    def rmdir(self, relpath):
571
        """Remove a directory at the given path."""
572
        raise NotImplementedError
573
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
574
    def stat_multi(self, relpaths, pb=None):
575
        """Stat multiple files and return the information.
576
        """
577
        #TODO:  Is it worth making this a generator instead of a
578
        #       returning a list?
579
        stats = []
580
        def gather(path):
581
            stats.append(self.stat(path))
582
907.1.14 by John Arbash Meinel
Handling some transport functions which take only a single argument.
583
        count = self._iterate_over(relpaths, gather, pb, 'stat', expand=False)
907.1.2 by John Arbash Meinel
Working on making Branch() do all of it's work over a Transport.
584
        return stats
585
1400.1.1 by Robert Collins
implement a basic test for the ui branch command from http servers
586
    def listable(self):
587
        """Return True if this store supports listing."""
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
588
        raise NotImplementedError(self.listable)
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
589
590
    def list_dir(self, relpath):
591
        """Return a list of all files at the given location.
592
        WARNING: many transports do not support this, so trying avoid using
593
        it if at all possible.
594
        """
1185.31.44 by John Arbash Meinel
Cleaned up Exceptions for all transports.
595
        raise errors.TransportNotPossible("This transport has not "
1530.1.21 by Robert Collins
Review feedback fixes.
596
                                          "implemented list_dir "
1530.1.3 by Robert Collins
transport implementations now tested consistently.
597
                                          "(but must claim to be listable "
598
                                          "to trigger this error).")
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
599
907.1.24 by John Arbash Meinel
Remote functionality work.
600
    def lock_read(self, relpath):
601
        """Lock the given file for shared (read) access.
602
        WARNING: many transports do not support this, so trying avoid using it
603
604
        :return: A lock object, which should contain an unlock() function.
605
        """
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
606
        raise NotImplementedError(self.lock_read)
907.1.24 by John Arbash Meinel
Remote functionality work.
607
608
    def lock_write(self, relpath):
609
        """Lock the given file for exclusive (write) access.
610
        WARNING: many transports do not support this, so trying avoid using it
611
612
        :return: A lock object, which should contain an unlock() function.
613
        """
1540.3.1 by Martin Pool
First-cut implementation of pycurl. Substantially faster than using urllib.
614
        raise NotImplementedError(self.lock_write)
907.1.24 by John Arbash Meinel
Remote functionality work.
615
1530.1.3 by Robert Collins
transport implementations now tested consistently.
616
    def is_readonly(self):
617
        """Return true if this connection cannot be written to."""
618
        return False
619
1608.2.7 by Martin Pool
Rename supports_unix_modebits to _can_roundtrip_unix_modebits for clarity
620
    def _can_roundtrip_unix_modebits(self):
1608.2.5 by Martin Pool
Add Transport.supports_unix_modebits, so tests can
621
        """Return true if this transport can store and retrieve unix modebits.
622
623
        (For example, 0700 to make a directory owner-private.)
624
        
625
        Note: most callers will not want to switch on this, but should rather 
626
        just try and set permissions and let them be either stored or not.
627
        This is intended mainly for the use of the test suite.
628
        
629
        Warning: this is not guaranteed to be accurate as sometimes we can't 
630
        be sure: for example with vfat mounted on unix, or a windows sftp
631
        server."""
632
        # TODO: Perhaps return a e.g. TransportCharacteristics that can answer
633
        # several questions about the transport.
634
        return False
635
907.1.24 by John Arbash Meinel
Remote functionality work.
636
1393.2.4 by John Arbash Meinel
All tests pass.
637
def get_transport(base):
1185.70.3 by Martin Pool
Various updates to make storage branch mergeable:
638
    """Open a transport to access a URL or directory.
639
640
    base is either a URL or a directory name.  
641
    """
1540.3.8 by Martin Pool
Some support for falling back between transport implementations.
642
    # TODO: give a better error if base looks like a url but there's no
643
    # handler for the scheme?
907.1.45 by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary.
644
    global _protocol_handlers
907.1.13 by John Arbash Meinel
Fixed bzr root.
645
    if base is None:
1185.33.66 by Martin Pool
[patch] use unicode literals for all hardcoded paths (Alexander Belchenko)
646
        base = u'.'
1185.31.60 by John Arbash Meinel
get_transport() now forces unicode paths, which helps get proper filenames.
647
    else:
648
        base = unicode(base)
1540.3.12 by Martin Pool
Multiple transports can be registered for any protocol, and they are
649
    for proto, factory_list in _protocol_handlers.iteritems():
907.1.1 by John Arbash Meinel
Reworking the Branch and Store code to support an abstracted filesystem layer.
650
        if proto is not None and base.startswith(proto):
1540.3.12 by Martin Pool
Multiple transports can be registered for any protocol, and they are
651
            t = _try_transport_factories(base, factory_list)
652
            if t:
653
                return t
654
    # The default handler is the filesystem handler, stored as protocol None
655
    return _try_transport_factories(base, _protocol_handlers[None])
656
657
658
def _try_transport_factories(base, factory_list):
659
    for factory in factory_list:
660
        try:
661
            return factory(base)
662
        except DependencyNotPresent, e:
663
            mutter("failed to instantiate transport %r for %r: %r" %
664
                    (factory, base, e))
665
            continue
666
    return None
1185.16.81 by mbp at sourcefrog
[merge] robert
667
1185.16.78 by Martin Pool
- load paramiko sftp transport by default
668
1469 by Robert Collins
Change Transport.* to work with URL's.
669
def urlescape(relpath):
670
    """Escape relpath to be a valid url."""
1185.80.5 by John Arbash Meinel
Changing the escaping just a little bit. Now we can handle unicode characters.
671
    if isinstance(relpath, unicode):
672
        relpath = relpath.encode('utf-8')
1469 by Robert Collins
Change Transport.* to work with URL's.
673
    return urllib.quote(relpath)
1470 by Robert Collins
merge from Martin, via newformat, and teach sftp about urlescaped paths
674
1185.16.78 by Martin Pool
- load paramiko sftp transport by default
675
1185.85.76 by John Arbash Meinel
Adding an InvalidURL so transports can report they expect utf-8 quoted paths. Updated tests
676
def urlunescape(url):
1685.1.1 by John Arbash Meinel
[merge] the old bzr-encoding changes, reparenting them on bzr.dev
677
    """Unescape relpath from url format.
678
679
    This returns a Unicode path from a URL
680
    """
1185.85.76 by John Arbash Meinel
Adding an InvalidURL so transports can report they expect utf-8 quoted paths. Updated tests
681
    unquoted = urllib.unquote(url)
682
    try:
683
        unicode_path = unquoted.decode('utf-8')
684
    except UnicodeError, e:
685
        raise errors.InvalidURL(url, e)
686
    return unicode_path
1608.1.1 by Martin Pool
[patch] LocalTransport.list_dir should return url-quoted strings (ddaa)
687
1185.85.76 by John Arbash Meinel
Adding an InvalidURL so transports can report they expect utf-8 quoted paths. Updated tests
688
1530.1.3 by Robert Collins
transport implementations now tested consistently.
689
class Server(object):
1530.1.21 by Robert Collins
Review feedback fixes.
690
    """A Transport Server.
691
    
692
    The Server interface provides a server for a given transport. We use
693
    these servers as loopback testing tools. For any given transport the
694
    Servers it provides must either allow writing, or serve the contents
695
    of os.getcwdu() at the time setUp is called.
696
    
697
    Note that these are real servers - they must implement all the things
698
    that we want bzr transports to take advantage of.
699
    """
1530.1.3 by Robert Collins
transport implementations now tested consistently.
700
701
    def setUp(self):
702
        """Setup the server to service requests."""
703
704
    def tearDown(self):
705
        """Remove the server and cleanup any resources it owns."""
706
707
    def get_url(self):
1530.1.21 by Robert Collins
Review feedback fixes.
708
        """Return a url for this server.
1530.1.3 by Robert Collins
transport implementations now tested consistently.
709
        
710
        If the transport does not represent a disk directory (i.e. it is 
1530.1.21 by Robert Collins
Review feedback fixes.
711
        a database like svn, or a memory only transport, it should return
1530.1.3 by Robert Collins
transport implementations now tested consistently.
712
        a connection to a newly established resource for this Server.
1530.1.21 by Robert Collins
Review feedback fixes.
713
        Otherwise it should return a url that will provide access to the path
714
        that was os.getcwdu() when setUp() was called.
1530.1.3 by Robert Collins
transport implementations now tested consistently.
715
        
716
        Subsequent calls will return the same resource.
717
        """
718
        raise NotImplementedError
719
1530.1.9 by Robert Collins
Test bogus urls with http in the new infrastructure.
720
    def get_bogus_url(self):
721
        """Return a url for this protocol, that will fail to connect."""
722
        raise NotImplementedError
723
1530.1.3 by Robert Collins
transport implementations now tested consistently.
724
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
725
class TransportTestProviderAdapter(object):
1530.1.21 by Robert Collins
Review feedback fixes.
726
    """A tool to generate a suite testing all transports for a single test.
727
728
    This is done by copying the test once for each transport and injecting
729
    the transport_class and transport_server classes into each copy. Each copy
730
    is also given a new id() to make it easy to identify.
731
    """
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
732
733
    def adapt(self, test):
734
        result = TestSuite()
735
        for klass, server_factory in self._test_permutations():
736
            new_test = deepcopy(test)
737
            new_test.transport_class = klass
738
            new_test.transport_server = server_factory
1530.1.3 by Robert Collins
transport implementations now tested consistently.
739
            def make_new_test_id():
740
                new_id = "%s(%s)" % (new_test.id(), server_factory.__name__)
741
                return lambda: new_id
742
            new_test.id = make_new_test_id()
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
743
            result.addTest(new_test)
744
        return result
745
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
746
    def get_transport_test_permutations(self, module):
747
        """Get the permutations module wants to have tested."""
1540.3.6 by Martin Pool
[merge] update from bzr.dev
748
        if not hasattr(module, 'get_test_permutations'):
749
            warning("transport module %s doesn't provide get_test_permutations()"
750
                    % module.__name__)
751
            return []
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
752
        return module.get_test_permutations()
753
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
754
    def _test_permutations(self):
755
        """Return a list of the klass, server_factory pairs to test."""
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
756
        result = []
757
        for module in _get_transport_modules():
1534.7.70 by abentley
Fix bug when no paramiko
758
            try:
759
                result.extend(self.get_transport_test_permutations(reduce(getattr, 
760
                    (module).split('.')[1:],
1185.71.1 by John Arbash Meinel
Allow selftest to run, even if we can't load a transport.
761
                     __import__(module))))
1185.62.24 by John Arbash Meinel
Changing the exception that sftp.py throws when it can't find paramiko, so that the test suite can handle it.
762
            except errors.DependencyNotPresent, e:
763
                # Continue even if a dependency prevents us 
764
                # from running this test
1534.7.70 by abentley
Fix bug when no paramiko
765
                pass
1530.1.11 by Robert Collins
Push the transport permutations list into each transport module allowing for automatic testing of new modules that are registered as transports.
766
        return result
1594.2.24 by Robert Collins
Make use of the transaction finalisation warning support to implement in-knit caching.
767
768
769
class TransportLogger(object):
770
    """Adapt a transport to get clear logging data on api calls.
771
    
772
    Feel free to extend to log whatever calls are of interest.
773
    """
774
775
    def __init__(self, adapted):
776
        self._adapted = adapted
777
        self._calls = []
778
779
    def get(self, name):
780
        self._calls.append((name,))
781
        return self._adapted.get(name)
782
783
    def __getattr__(self, name):
784
        """Thunk all undefined access through to self._adapted."""
785
        # raise AttributeError, name 
786
        return getattr(self._adapted, name)
787
788
    def readv(self, name, offsets):
789
        self._calls.append((name, offsets))
790
        return self._adapted.readv(name, offsets)
1530.1.1 by Robert Collins
Minimal infrastructure to test TransportTestProviderAdapter.
791
        
792
1185.16.81 by mbp at sourcefrog
[merge] robert
793
# None is the default transport, for things with no url scheme
1185.16.79 by Martin Pool
Load transports when they're first used.
794
register_lazy_transport(None, 'bzrlib.transport.local', 'LocalTransport')
795
register_lazy_transport('file://', 'bzrlib.transport.local', 'LocalTransport')
796
register_lazy_transport('sftp://', 'bzrlib.transport.sftp', 'SFTPTransport')
1540.3.23 by Martin Pool
Allow urls like http+pycurl://host/ to use a particular impl
797
register_lazy_transport('http+urllib://', 'bzrlib.transport.http._urllib',
1540.3.26 by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet
798
                        'HttpTransport_urllib')
1540.3.23 by Martin Pool
Allow urls like http+pycurl://host/ to use a particular impl
799
register_lazy_transport('https+urllib://', 'bzrlib.transport.http._urllib',
1540.3.26 by Martin Pool
[merge] bzr.dev; pycurl not updated for readv yet
800
                        'HttpTransport_urllib')
801
register_lazy_transport('http+pycurl://', 'bzrlib.transport.http._pycurl',
802
                        'PyCurlTransport')
803
register_lazy_transport('https+pycurl://', 'bzrlib.transport.http._pycurl',
804
                        'PyCurlTransport')
805
register_lazy_transport('http://', 'bzrlib.transport.http._urllib',
806
                        'HttpTransport_urllib')
807
register_lazy_transport('https://', 'bzrlib.transport.http._urllib',
808
                        'HttpTransport_urllib')
1540.3.7 by Martin Pool
Prepare to select a transport depending on what dependencies can be satisfied.
809
register_lazy_transport('http://', 'bzrlib.transport.http._pycurl', 'PyCurlTransport')
810
register_lazy_transport('https://', 'bzrlib.transport.http._pycurl', 'PyCurlTransport')
1185.36.4 by Daniel Silverstone
Add FTP transport
811
register_lazy_transport('ftp://', 'bzrlib.transport.ftp', 'FtpTransport')
812
register_lazy_transport('aftp://', 'bzrlib.transport.ftp', 'FtpTransport')
1534.4.39 by Robert Collins
Basic BzrDir support.
813
register_lazy_transport('memory:/', 'bzrlib.transport.memory', 'MemoryTransport')
1534.4.9 by Robert Collins
Add a readonly decorator for transports.
814
register_lazy_transport('readonly+', 'bzrlib.transport.readonly', 'ReadonlyTransportDecorator')
1558.10.2 by Robert Collins
Refactor the FakeNFS support into a TransportDecorator.
815
register_lazy_transport('fakenfs+', 'bzrlib.transport.fakenfs', 'FakeNFSTransportDecorator')
1608.2.4 by Martin Pool
[broken] Add FakeFVATTransport
816
register_lazy_transport('vfat+', 
817
                        'bzrlib.transport.fakevfat',
818
                        'FakeVFATTransportDecorator')