~bzr-pqm/bzr/bzr.dev

1185.11.19 by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings.
1
# Copyright (C) 2005 Canonical Ltd
2
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
7
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
# GNU General Public License for more details.
12
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
16
"""Implementation of Transport over http.
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
17
"""
18
1393.2.3 by John Arbash Meinel
Fixing typos, updating stores, getting tests to pass.
19
from bzrlib.transport import Transport, register_transport
20
from bzrlib.errors import (TransportNotPossible, NoSuchFile, 
21
                           NonRelativePath, TransportError)
907.1.57 by John Arbash Meinel
Trying to get pipelined http library working + tests.
22
import os, errno
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
23
from cStringIO import StringIO
24
import urllib2
1185.11.6 by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently.
25
import urlparse
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
26
1185.11.1 by John Arbash Meinel
(broken) Transport work is merged in. Tests do not pass yet.
27
from bzrlib.errors import BzrError, BzrCheckError
1393.2.3 by John Arbash Meinel
Fixing typos, updating stores, getting tests to pass.
28
from bzrlib.branch import Branch
1185.11.1 by John Arbash Meinel
(broken) Transport work is merged in. Tests do not pass yet.
29
from bzrlib.trace import mutter
907.1.57 by John Arbash Meinel
Trying to get pipelined http library working + tests.
30
31
1185.11.14 by John Arbash Meinel
Working on getting tests to run. TestFetch only works if named runTest
32
def get_url(url):
33
    import urllib2
34
    mutter("get_url %s" % url)
35
    url_f = urllib2.urlopen(url)
36
    return url_f
37
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
38
class HttpTransportError(TransportError):
39
    pass
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
40
41
class HttpTransport(Transport):
907.1.36 by John Arbash Meinel
Moving the multi-get functionality higher up into the Branch class.
42
    """This is the transport agent for http:// access.
43
    
44
    TODO: Implement pipelined versions of all of the *_multi() functions.
45
    """
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
46
47
    def __init__(self, base):
48
        """Set the base path where files will be stored."""
49
        assert base.startswith('http://') or base.startswith('https://')
50
        super(HttpTransport, self).__init__(base)
907.1.57 by John Arbash Meinel
Trying to get pipelined http library working + tests.
51
        # In the future we might actually connect to the remote host
52
        # rather than using get_url
53
        # self._connection = None
1185.11.6 by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently.
54
        (self._proto, self._host,
55
            self._path, self._parameters,
56
            self._query, self._fragment) = urlparse.urlparse(self.base)
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
57
907.1.32 by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate.
58
    def should_cache(self):
59
        """Return True if the data pulled across should be cached locally.
60
        """
61
        return True
62
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
63
    def clone(self, offset=None):
64
        """Return a new HttpTransport with root at self.base + offset
65
        For now HttpTransport does not actually connect, so just return
66
        a new HttpTransport object.
67
        """
68
        if offset is None:
69
            return HttpTransport(self.base)
70
        else:
71
            return HttpTransport(self.abspath(offset))
72
73
    def abspath(self, relpath):
74
        """Return the full url to the given relative path.
75
        This can be supplied with a string or a list
76
        """
1469 by Robert Collins
Change Transport.* to work with URL's.
77
        assert isinstance(relpath, basestring)
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
78
        if isinstance(relpath, basestring):
1185.16.68 by Martin Pool
- http url fixes suggested by Robey Pointer, and tests
79
            relpath_parts = relpath.split('/')
80
        else:
81
            # TODO: Don't call this with an array - no magic interfaces
82
            relpath_parts = relpath[:]
83
        if len(relpath_parts) > 1:
84
            if relpath_parts[0] == '':
85
                raise ValueError("path %r within branch %r seems to be absolute"
86
                                 % (relpath, self._path))
87
            if relpath_parts[-1] == '':
88
                raise ValueError("path %r within branch %r seems to be a directory"
89
                                 % (relpath, self._path))
1185.11.6 by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently.
90
        basepath = self._path.split('/')
1185.11.8 by John Arbash Meinel
Fixed typo
91
        if len(basepath) > 0 and basepath[-1] == '':
1185.11.6 by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently.
92
            basepath = basepath[:-1]
1185.16.68 by Martin Pool
- http url fixes suggested by Robey Pointer, and tests
93
        for p in relpath_parts:
1185.11.6 by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently.
94
            if p == '..':
1185.16.68 by Martin Pool
- http url fixes suggested by Robey Pointer, and tests
95
                if len(basepath) == 0:
1185.11.7 by John Arbash Meinel
HttpTransport just returns root when parent is requested.
96
                    # In most filesystems, a request for the parent
97
                    # of root, just returns root.
98
                    continue
1185.16.68 by Martin Pool
- http url fixes suggested by Robey Pointer, and tests
99
                basepath.pop()
100
            elif p == '.' or p == '':
1185.11.6 by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently.
101
                continue # No-op
102
            else:
103
                basepath.append(p)
104
        # Possibly, we could use urlparse.urljoin() here, but
105
        # I'm concerned about when it chooses to strip the last
106
        # portion of the path, and when it doesn't.
107
        path = '/'.join(basepath)
1185.11.9 by John Arbash Meinel
Most tests pass, some problems with unavailable socket recv
108
        return urlparse.urlunparse((self._proto,
109
                self._host, path, '', '', ''))
907.1.24 by John Arbash Meinel
Remote functionality work.
110
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
111
    def has(self, relpath):
907.1.35 by John Arbash Meinel
updating TODO for http_transport.
112
        """Does the target location exist?
113
114
        TODO: HttpTransport.has() should use a HEAD request,
115
        not a full GET request.
1185.11.15 by John Arbash Meinel
Got HttpTransport tests to pass. Check for EAGAIN, pass permit_failure around, etc
116
117
        TODO: This should be changed so that we don't use
118
        urllib2 and get an exception, the code path would be
119
        cleaner if we just do an http HEAD request, and parse
120
        the return code.
907.1.35 by John Arbash Meinel
updating TODO for http_transport.
121
        """
907.1.57 by John Arbash Meinel
Trying to get pipelined http library working + tests.
122
        try:
123
            f = get_url(self.abspath(relpath))
1185.11.15 by John Arbash Meinel
Got HttpTransport tests to pass. Check for EAGAIN, pass permit_failure around, etc
124
            # Without the read and then close()
125
            # we tend to have busy sockets.
126
            f.read()
127
            f.close()
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
128
            return True
1468 by Robert Collins
The HTTP transport would return NoSuchFile inappropriately.
129
        except urllib2.URLError, e:
130
            if e.code == 404:
131
                return False
132
            raise
907.1.57 by John Arbash Meinel
Trying to get pipelined http library working + tests.
133
        except IOError, e:
134
            if e.errno == errno.ENOENT:
135
                return False
136
            raise HttpTransportError(orig_error=e)
907.1.55 by John Arbash Meinel
Adding pipelined http support.
137
907.1.57 by John Arbash Meinel
Trying to get pipelined http library working + tests.
138
    def get(self, relpath, decode=False):
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
139
        """Get the file at the given relative path.
140
141
        :param relpath: The relative path to the file
142
        """
907.1.57 by John Arbash Meinel
Trying to get pipelined http library working + tests.
143
        try:
144
            return get_url(self.abspath(relpath))
1468 by Robert Collins
The HTTP transport would return NoSuchFile inappropriately.
145
        except urllib2.URLError, e:
1185.16.152 by Martin Pool
Handle URLError without http error code
146
            if getattr(e, 'code', None) == 404:
1468 by Robert Collins
The HTTP transport would return NoSuchFile inappropriately.
147
                raise NoSuchFile(msg = "Error retrieving %s: %s" 
148
                                 % (self.abspath(relpath), str(e)),
149
                                 orig_error=e)
150
            raise
151
        except (BzrError, IOError), e:
1185.16.2 by Martin Pool
- try to get better reporting of http errors
152
            raise NoSuchFile(msg = "Error retrieving %s: %s" 
153
                             % (self.abspath(relpath), str(e)),
1428 by Robert Collins
report the url location on failed http requests
154
                             orig_error=e)
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
155
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
156
    def put(self, relpath, f):
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
157
        """Copy the file-like or string object into the location.
158
159
        :param relpath: Location to put the contents, relative to base.
160
        :param f:       File-like or string object.
161
        """
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
162
        raise TransportNotPossible('http PUT not supported')
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
163
164
    def mkdir(self, relpath):
165
        """Create a directory at the given path."""
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
166
        raise TransportNotPossible('http does not support mkdir()')
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
167
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
168
    def append(self, relpath, f):
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
169
        """Append the text in the file-like object into the final
170
        location.
171
        """
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
172
        raise TransportNotPossible('http does not support append()')
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
173
174
    def copy(self, rel_from, rel_to):
175
        """Copy the item at rel_from to the location at rel_to"""
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
176
        raise TransportNotPossible('http does not support copy()')
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
177
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
178
    def copy_to(self, relpaths, other, pb=None):
179
        """Copy a set of entries from self into another Transport.
180
181
        :param relpaths: A list/generator of entries to be copied.
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
182
183
        TODO: if other is LocalTransport, is it possible to
184
              do better than put(get())?
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
185
        """
907.1.29 by John Arbash Meinel
Fixing small bug in HttpTransport.copy_to
186
        # At this point HttpTransport might be able to check and see if
187
        # the remote location is the same, and rather than download, and
188
        # then upload, it could just issue a remote copy_this command.
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
189
        if isinstance(other, HttpTransport):
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
190
            raise TransportNotPossible('http cannot be the target of copy_to()')
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
191
        else:
907.1.29 by John Arbash Meinel
Fixing small bug in HttpTransport.copy_to
192
            return super(HttpTransport, self).copy_to(relpaths, other, pb=pb)
907.1.28 by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function.
193
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
194
    def move(self, rel_from, rel_to):
195
        """Move the item at rel_from to the location at rel_to"""
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
196
        raise TransportNotPossible('http does not support move()')
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
197
198
    def delete(self, relpath):
199
        """Delete the item at relpath"""
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
200
        raise TransportNotPossible('http does not support delete()')
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
201
1400.1.1 by Robert Collins
implement a basic test for the ui branch command from http servers
202
    def listable(self):
203
        """See Transport.listable."""
204
        return False
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
205
206
    def stat(self, relpath):
207
        """Return the stat information for a file.
208
        """
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
209
        raise TransportNotPossible('http does not support stat()')
907.1.21 by John Arbash Meinel
Adding http transport as a valid transport protocol.
210
907.1.24 by John Arbash Meinel
Remote functionality work.
211
    def lock_read(self, relpath):
212
        """Lock the given file for shared (read) access.
213
        :return: A lock object, which should be passed to Transport.unlock()
214
        """
215
        # The old RemoteBranch ignore lock for reading, so we will
216
        # continue that tradition and return a bogus lock object.
217
        class BogusLock(object):
218
            def __init__(self, path):
219
                self.path = path
220
            def unlock(self):
221
                pass
222
        return BogusLock(relpath)
223
224
    def lock_write(self, relpath):
225
        """Lock the given file for exclusive (write) access.
226
        WARNING: many transports do not support this, so trying avoid using it
227
228
        :return: A lock object, which should be passed to Transport.unlock()
229
        """
907.1.50 by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown.
230
        raise TransportNotPossible('http does not support lock_write()')