1185.11.19
by John Arbash Meinel
Testing put and append, also testing agaist file-like objects as well as strings. |
1 |
# Copyright (C) 2005 Canonical Ltd
|
2 |
||
3 |
# This program is free software; you can redistribute it and/or modify
|
|
4 |
# it under the terms of the GNU General Public License as published by
|
|
5 |
# the Free Software Foundation; either version 2 of the License, or
|
|
6 |
# (at your option) any later version.
|
|
7 |
||
8 |
# This program is distributed in the hope that it will be useful,
|
|
9 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
10 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
11 |
# GNU General Public License for more details.
|
|
12 |
||
13 |
# You should have received a copy of the GNU General Public License
|
|
14 |
# along with this program; if not, write to the Free Software
|
|
15 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
16 |
"""Implementation of Transport over http.
|
|
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
17 |
"""
|
18 |
||
1393.2.3
by John Arbash Meinel
Fixing typos, updating stores, getting tests to pass. |
19 |
from bzrlib.transport import Transport, register_transport |
20 |
from bzrlib.errors import (TransportNotPossible, NoSuchFile, |
|
21 |
NonRelativePath, TransportError) |
|
907.1.57
by John Arbash Meinel
Trying to get pipelined http library working + tests. |
22 |
import os, errno |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
23 |
from cStringIO import StringIO |
24 |
import urllib2 |
|
1185.11.6
by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently. |
25 |
import urlparse |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
26 |
|
1185.11.1
by John Arbash Meinel
(broken) Transport work is merged in. Tests do not pass yet. |
27 |
from bzrlib.errors import BzrError, BzrCheckError |
1393.2.3
by John Arbash Meinel
Fixing typos, updating stores, getting tests to pass. |
28 |
from bzrlib.branch import Branch |
1185.11.1
by John Arbash Meinel
(broken) Transport work is merged in. Tests do not pass yet. |
29 |
from bzrlib.trace import mutter |
907.1.57
by John Arbash Meinel
Trying to get pipelined http library working + tests. |
30 |
|
31 |
# velocitynet.com.au transparently proxies connections and thereby
|
|
32 |
# breaks keep-alive -- sucks!
|
|
33 |
||
34 |
||
1185.11.14
by John Arbash Meinel
Working on getting tests to run. TestFetch only works if named runTest |
35 |
def get_url(url): |
36 |
import urllib2 |
|
37 |
mutter("get_url %s" % url) |
|
38 |
url_f = urllib2.urlopen(url) |
|
39 |
return url_f |
|
40 |
||
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
41 |
class HttpTransportError(TransportError): |
42 |
pass
|
|
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
43 |
|
44 |
class HttpTransport(Transport): |
|
907.1.36
by John Arbash Meinel
Moving the multi-get functionality higher up into the Branch class. |
45 |
"""This is the transport agent for http:// access.
|
46 |
|
|
47 |
TODO: Implement pipelined versions of all of the *_multi() functions.
|
|
48 |
"""
|
|
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
49 |
|
50 |
def __init__(self, base): |
|
51 |
"""Set the base path where files will be stored."""
|
|
52 |
assert base.startswith('http://') or base.startswith('https://') |
|
53 |
super(HttpTransport, self).__init__(base) |
|
907.1.57
by John Arbash Meinel
Trying to get pipelined http library working + tests. |
54 |
# In the future we might actually connect to the remote host
|
55 |
# rather than using get_url
|
|
56 |
# self._connection = None
|
|
1185.11.6
by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently. |
57 |
(self._proto, self._host, |
58 |
self._path, self._parameters, |
|
59 |
self._query, self._fragment) = urlparse.urlparse(self.base) |
|
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
60 |
|
907.1.32
by John Arbash Meinel
Renaming is_remote to should_cache as it is more appropriate. |
61 |
def should_cache(self): |
62 |
"""Return True if the data pulled across should be cached locally.
|
|
63 |
"""
|
|
64 |
return True |
|
65 |
||
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
66 |
def clone(self, offset=None): |
67 |
"""Return a new HttpTransport with root at self.base + offset
|
|
68 |
For now HttpTransport does not actually connect, so just return
|
|
69 |
a new HttpTransport object.
|
|
70 |
"""
|
|
71 |
if offset is None: |
|
72 |
return HttpTransport(self.base) |
|
73 |
else: |
|
74 |
return HttpTransport(self.abspath(offset)) |
|
75 |
||
76 |
def abspath(self, relpath): |
|
77 |
"""Return the full url to the given relative path.
|
|
78 |
This can be supplied with a string or a list
|
|
79 |
"""
|
|
80 |
if isinstance(relpath, basestring): |
|
81 |
relpath = [relpath] |
|
1185.11.6
by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently. |
82 |
basepath = self._path.split('/') |
1185.11.8
by John Arbash Meinel
Fixed typo |
83 |
if len(basepath) > 0 and basepath[-1] == '': |
1185.11.6
by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently. |
84 |
basepath = basepath[:-1] |
85 |
||
86 |
for p in relpath: |
|
87 |
if p == '..': |
|
88 |
if len(basepath) < 0: |
|
1185.11.7
by John Arbash Meinel
HttpTransport just returns root when parent is requested. |
89 |
# In most filesystems, a request for the parent
|
90 |
# of root, just returns root.
|
|
91 |
continue
|
|
1185.12.18
by Aaron Bentley
Fixed error handling when NotBranch on HTTP |
92 |
if len(basepath) > 0: |
93 |
basepath.pop() |
|
1185.11.6
by John Arbash Meinel
Made HttpTransport handle a request for a parent directory differently. |
94 |
elif p == '.': |
95 |
continue # No-op |
|
96 |
else: |
|
97 |
basepath.append(p) |
|
98 |
||
99 |
# Possibly, we could use urlparse.urljoin() here, but
|
|
100 |
# I'm concerned about when it chooses to strip the last
|
|
101 |
# portion of the path, and when it doesn't.
|
|
102 |
path = '/'.join(basepath) |
|
1185.11.9
by John Arbash Meinel
Most tests pass, some problems with unavailable socket recv |
103 |
return urlparse.urlunparse((self._proto, |
104 |
self._host, path, '', '', '')) |
|
907.1.24
by John Arbash Meinel
Remote functionality work. |
105 |
|
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
106 |
def has(self, relpath): |
907.1.35
by John Arbash Meinel
updating TODO for http_transport. |
107 |
"""Does the target location exist?
|
108 |
||
109 |
TODO: HttpTransport.has() should use a HEAD request,
|
|
110 |
not a full GET request.
|
|
1185.11.15
by John Arbash Meinel
Got HttpTransport tests to pass. Check for EAGAIN, pass permit_failure around, etc |
111 |
|
112 |
TODO: This should be changed so that we don't use
|
|
113 |
urllib2 and get an exception, the code path would be
|
|
114 |
cleaner if we just do an http HEAD request, and parse
|
|
115 |
the return code.
|
|
907.1.35
by John Arbash Meinel
updating TODO for http_transport. |
116 |
"""
|
907.1.57
by John Arbash Meinel
Trying to get pipelined http library working + tests. |
117 |
try: |
118 |
f = get_url(self.abspath(relpath)) |
|
1185.11.15
by John Arbash Meinel
Got HttpTransport tests to pass. Check for EAGAIN, pass permit_failure around, etc |
119 |
# Without the read and then close()
|
120 |
# we tend to have busy sockets.
|
|
121 |
f.read() |
|
122 |
f.close() |
|
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
123 |
return True |
907.1.57
by John Arbash Meinel
Trying to get pipelined http library working + tests. |
124 |
except BzrError: |
125 |
return False |
|
126 |
except urllib2.URLError: |
|
127 |
return False |
|
128 |
except IOError, e: |
|
129 |
if e.errno == errno.ENOENT: |
|
130 |
return False |
|
131 |
raise HttpTransportError(orig_error=e) |
|
907.1.55
by John Arbash Meinel
Adding pipelined http support. |
132 |
|
907.1.57
by John Arbash Meinel
Trying to get pipelined http library working + tests. |
133 |
def get(self, relpath, decode=False): |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
134 |
"""Get the file at the given relative path.
|
135 |
||
136 |
:param relpath: The relative path to the file
|
|
137 |
"""
|
|
907.1.57
by John Arbash Meinel
Trying to get pipelined http library working + tests. |
138 |
try: |
139 |
return get_url(self.abspath(relpath)) |
|
1185.11.21
by John Arbash Meinel
Added implementations and tests for get_partial |
140 |
except (BzrError, urllib2.URLError, IOError), e: |
1185.16.2
by Martin Pool
- try to get better reporting of http errors |
141 |
raise NoSuchFile(msg = "Error retrieving %s: %s" |
142 |
% (self.abspath(relpath), str(e)), |
|
1428
by Robert Collins
report the url location on failed http requests |
143 |
orig_error=e) |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
144 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
145 |
def put(self, relpath, f): |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
146 |
"""Copy the file-like or string object into the location.
|
147 |
||
148 |
:param relpath: Location to put the contents, relative to base.
|
|
149 |
:param f: File-like or string object.
|
|
150 |
"""
|
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
151 |
raise TransportNotPossible('http PUT not supported') |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
152 |
|
153 |
def mkdir(self, relpath): |
|
154 |
"""Create a directory at the given path."""
|
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
155 |
raise TransportNotPossible('http does not support mkdir()') |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
156 |
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
157 |
def append(self, relpath, f): |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
158 |
"""Append the text in the file-like object into the final
|
159 |
location.
|
|
160 |
"""
|
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
161 |
raise TransportNotPossible('http does not support append()') |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
162 |
|
163 |
def copy(self, rel_from, rel_to): |
|
164 |
"""Copy the item at rel_from to the location at rel_to"""
|
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
165 |
raise TransportNotPossible('http does not support copy()') |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
166 |
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
167 |
def copy_to(self, relpaths, other, pb=None): |
168 |
"""Copy a set of entries from self into another Transport.
|
|
169 |
||
170 |
:param relpaths: A list/generator of entries to be copied.
|
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
171 |
|
172 |
TODO: if other is LocalTransport, is it possible to
|
|
173 |
do better than put(get())?
|
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
174 |
"""
|
907.1.29
by John Arbash Meinel
Fixing small bug in HttpTransport.copy_to |
175 |
# At this point HttpTransport might be able to check and see if
|
176 |
# the remote location is the same, and rather than download, and
|
|
177 |
# then upload, it could just issue a remote copy_this command.
|
|
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
178 |
if isinstance(other, HttpTransport): |
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
179 |
raise TransportNotPossible('http cannot be the target of copy_to()') |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
180 |
else: |
907.1.29
by John Arbash Meinel
Fixing small bug in HttpTransport.copy_to |
181 |
return super(HttpTransport, self).copy_to(relpaths, other, pb=pb) |
907.1.28
by John Arbash Meinel
Added pb to function that were missing, implemented a basic double-dispatch copy_to function. |
182 |
|
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
183 |
def move(self, rel_from, rel_to): |
184 |
"""Move the item at rel_from to the location at rel_to"""
|
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
185 |
raise TransportNotPossible('http does not support move()') |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
186 |
|
187 |
def delete(self, relpath): |
|
188 |
"""Delete the item at relpath"""
|
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
189 |
raise TransportNotPossible('http does not support delete()') |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
190 |
|
1400.1.1
by Robert Collins
implement a basic test for the ui branch command from http servers |
191 |
def listable(self): |
192 |
"""See Transport.listable."""
|
|
193 |
return False |
|
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
194 |
|
195 |
def stat(self, relpath): |
|
196 |
"""Return the stat information for a file.
|
|
197 |
"""
|
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
198 |
raise TransportNotPossible('http does not support stat()') |
907.1.21
by John Arbash Meinel
Adding http transport as a valid transport protocol. |
199 |
|
907.1.24
by John Arbash Meinel
Remote functionality work. |
200 |
def lock_read(self, relpath): |
201 |
"""Lock the given file for shared (read) access.
|
|
202 |
:return: A lock object, which should be passed to Transport.unlock()
|
|
203 |
"""
|
|
204 |
# The old RemoteBranch ignore lock for reading, so we will
|
|
205 |
# continue that tradition and return a bogus lock object.
|
|
206 |
class BogusLock(object): |
|
207 |
def __init__(self, path): |
|
208 |
self.path = path |
|
209 |
def unlock(self): |
|
210 |
pass
|
|
211 |
return BogusLock(relpath) |
|
212 |
||
213 |
def lock_write(self, relpath): |
|
214 |
"""Lock the given file for exclusive (write) access.
|
|
215 |
WARNING: many transports do not support this, so trying avoid using it
|
|
216 |
||
217 |
:return: A lock object, which should be passed to Transport.unlock()
|
|
218 |
"""
|
|
907.1.50
by John Arbash Meinel
Removed encode/decode from Transport.put/get, added more exceptions that can be thrown. |
219 |
raise TransportNotPossible('http does not support lock_write()') |
907.1.24
by John Arbash Meinel
Remote functionality work. |
220 |
|
907.1.45
by John Arbash Meinel
Switch to registering protocol handlers, rather than just updating a dictionary. |
221 |
register_transport('http://', HttpTransport) |
222 |
register_transport('https://', HttpTransport) |