16
16
"""Implementation of Transport over http.
19
from bzrlib.transport import Transport, register_transport
20
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
21
NonRelativePath, TransportError)
23
20
from cStringIO import StringIO
21
import urllib, urllib2
23
from warnings import warn
26
from bzrlib.transport import Transport, Server
27
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
28
TransportError, ConnectionError)
27
29
from bzrlib.errors import BzrError, BzrCheckError
28
30
from bzrlib.branch import Branch
29
31
from bzrlib.trace import mutter
31
# velocitynet.com.au transparently proxies connections and thereby
32
# breaks keep-alive -- sucks!
32
from bzrlib.ui import ui_factory
35
def extract_auth(url, password_manager):
37
Extract auth parameters from am HTTP/HTTPS url and add them to the given
38
password manager. Return the url, minus those auth parameters (which
41
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
42
assert (scheme == 'http') or (scheme == 'https')
45
auth, netloc = netloc.split('@', 1)
47
username, password = auth.split(':', 1)
49
username, password = auth, None
51
host = netloc.split(':', 1)[0]
54
username = urllib.unquote(username)
55
if password is not None:
56
password = urllib.unquote(password)
58
password = ui_factory.get_password(prompt='HTTP %(user)@%(host) password',
59
user=username, host=host)
60
password_manager.add_password(None, host, username, password)
61
url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
65
class Request(urllib2.Request):
66
"""Request object for urllib2 that allows the method to be overridden."""
71
if self.method is not None:
74
return urllib2.Request.get_method(self)
77
def get_url(url, method=None):
37
mutter("get_url %s" % url)
38
url_f = urllib2.urlopen(url)
41
class HttpTransportError(TransportError):
79
mutter("get_url %s", url)
80
manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
81
url = extract_auth(url, manager)
82
auth_handler = urllib2.HTTPBasicAuthHandler(manager)
83
opener = urllib2.build_opener(auth_handler)
85
request = Request(url)
86
request.method = method
87
request.add_header('User-Agent', 'bzr/%s' % bzrlib.__version__)
88
response = opener.open(request)
44
92
class HttpTransport(Transport):
45
93
"""This is the transport agent for http:// access.
77
127
"""Return the full url to the given relative path.
78
128
This can be supplied with a string or a list
130
assert isinstance(relpath, basestring)
80
131
if isinstance(relpath, basestring):
132
relpath_parts = relpath.split('/')
134
# TODO: Don't call this with an array - no magic interfaces
135
relpath_parts = relpath[:]
136
if len(relpath_parts) > 1:
137
if relpath_parts[0] == '':
138
raise ValueError("path %r within branch %r seems to be absolute"
139
% (relpath, self._path))
140
if relpath_parts[-1] == '':
141
raise ValueError("path %r within branch %r seems to be a directory"
142
% (relpath, self._path))
82
143
basepath = self._path.split('/')
83
144
if len(basepath) > 0 and basepath[-1] == '':
84
145
basepath = basepath[:-1]
146
for p in relpath_parts:
148
if len(basepath) == 0:
89
149
# In most filesystems, a request for the parent
90
150
# of root, just returns root.
153
elif p == '.' or p == '':
97
156
basepath.append(p)
99
157
# Possibly, we could use urlparse.urljoin() here, but
100
158
# I'm concerned about when it chooses to strip the last
101
159
# portion of the path, and when it doesn't.
103
161
return urlparse.urlunparse((self._proto,
104
162
self._host, path, '', '', ''))
106
def relpath(self, abspath):
107
if not abspath.startswith(self.base):
108
raise NonRelativePath('path %r is not under base URL %r'
109
% (abspath, self.base))
111
return abspath[pl:].lstrip('/')
113
164
def has(self, relpath):
114
165
"""Does the target location exist?
116
TODO: HttpTransport.has() should use a HEAD request,
117
not a full GET request.
119
167
TODO: This should be changed so that we don't use
120
168
urllib2 and get an exception, the code path would be
121
169
cleaner if we just do an http HEAD request, and parse
125
f = get_url(self.abspath(relpath))
174
path = self.abspath(relpath)
175
f = get_url(path, method='HEAD')
126
176
# Without the read and then close()
127
177
# we tend to have busy sockets.
133
except urllib2.URLError:
181
except urllib2.URLError, e:
182
mutter('url error code: %s for has url: %r', e.code, path)
135
186
except IOError, e:
187
mutter('io error: %s %s for has url: %r',
188
e.errno, errno.errorcode.get(e.errno), path)
136
189
if e.errno == errno.ENOENT:
138
raise HttpTransportError(orig_error=e)
191
raise TransportError(orig_error=e)
140
193
def get(self, relpath, decode=False):
141
194
"""Get the file at the given relative path.
143
196
:param relpath: The relative path to the file
146
return get_url(self.abspath(relpath))
147
except (BzrError, urllib2.URLError, IOError), e:
148
raise NoSuchFile(msg = "Error retrieving %s: %s"
200
path = self.abspath(relpath)
202
except urllib2.HTTPError, e:
203
mutter('url error code: %s for has url: %r', e.code, path)
205
raise NoSuchFile(path, extra=e)
207
except (BzrError, IOError), e:
208
if hasattr(e, 'errno'):
209
mutter('io error: %s %s for has url: %r',
210
e.errno, errno.errorcode.get(e.errno), path)
211
if e.errno == errno.ENOENT:
212
raise NoSuchFile(path, extra=e)
213
raise ConnectionError(msg = "Error retrieving %s: %s"
149
214
% (self.abspath(relpath), str(e)),
152
def get_partial(self, relpath, start, length=None):
153
"""Get just part of a file.
155
:param relpath: Path to the file, relative to base
156
:param start: The starting position to read from
157
:param length: The length to read. A length of None indicates
158
read to the end of the file.
159
:return: A file-like object containing at least the specified bytes.
160
Some implementations may return objects which can be read
161
past this length, but this is not guaranteed.
163
# TODO: You can make specialized http requests for just
164
# a portion of the file. Figure out how to do that.
165
# For now, urllib2 returns files that cannot seek() so
166
# we just read bytes off the beginning, until we
167
# get to the point that we care about.
168
f = self.get(relpath)
169
# TODO: read in smaller chunks, in case things are
170
# buffered internally.
174
def put(self, relpath, f):
217
def put(self, relpath, f, mode=None):
175
218
"""Copy the file-like or string object into the location.
177
220
:param relpath: Location to put the contents, relative to base.
248
299
raise TransportNotPossible('http does not support lock_write()')
250
register_transport('http://', HttpTransport)
251
register_transport('https://', HttpTransport)
302
#---------------- test server facilities ----------------
303
import BaseHTTPServer, SimpleHTTPServer, socket, time
307
class WebserverNotAvailable(Exception):
311
class BadWebserverPath(ValueError):
313
return 'path %s is not in %s' % self.args
316
class TestingHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
318
def log_message(self, format, *args):
319
self.server.test_case.log('webserver - %s - - [%s] %s "%s" "%s"',
320
self.address_string(),
321
self.log_date_time_string(),
323
self.headers.get('referer', '-'),
324
self.headers.get('user-agent', '-'))
326
def handle_one_request(self):
327
"""Handle a single HTTP request.
329
You normally don't need to override this method; see the class
330
__doc__ string for information on how to handle specific HTTP
331
commands such as GET and POST.
334
for i in xrange(1,11): # Don't try more than 10 times
336
self.raw_requestline = self.rfile.readline()
337
except socket.error, e:
338
if e.args[0] in (errno.EAGAIN, errno.EWOULDBLOCK):
339
# omitted for now because some tests look at the log of
340
# the server and expect to see no errors. see recent
341
# email thread. -- mbp 20051021.
342
## self.log_message('EAGAIN (%d) while reading from raw_requestline' % i)
348
if not self.raw_requestline:
349
self.close_connection = 1
351
if not self.parse_request(): # An error code has been sent, just exit
353
mname = 'do_' + self.command
354
if not hasattr(self, mname):
355
self.send_error(501, "Unsupported method (%r)" % self.command)
357
method = getattr(self, mname)
361
class TestingHTTPServer(BaseHTTPServer.HTTPServer):
362
def __init__(self, server_address, RequestHandlerClass, test_case):
363
BaseHTTPServer.HTTPServer.__init__(self, server_address,
365
self.test_case = test_case
368
class HttpServer(Server):
369
"""A test server for http transports."""
371
def _http_start(self):
373
httpd = TestingHTTPServer(('localhost', 0),
374
TestingHTTPRequestHandler,
376
host, port = httpd.socket.getsockname()
377
self._http_base_url = 'http://localhost:%s/' % port
378
self._http_starting.release()
379
httpd.socket.settimeout(0.1)
381
while self._http_running:
383
httpd.handle_request()
384
except socket.timeout:
387
def _get_remote_url(self, path):
388
path_parts = path.split(os.path.sep)
389
if os.path.isabs(path):
390
if path_parts[:len(self._local_path_parts)] != \
391
self._local_path_parts:
392
raise BadWebserverPath(path, self.test_dir)
393
remote_path = '/'.join(path_parts[len(self._local_path_parts):])
395
remote_path = '/'.join(path_parts)
397
self._http_starting.acquire()
398
self._http_starting.release()
399
return self._http_base_url + remote_path
401
def log(self, format, *args):
402
"""Capture Server log output."""
403
self.logs.append(format % args)
406
"""See bzrlib.transport.Server.setUp."""
407
self._home_dir = os.getcwdu()
408
self._local_path_parts = self._home_dir.split(os.path.sep)
409
self._http_starting = threading.Lock()
410
self._http_starting.acquire()
411
self._http_running = True
412
self._http_base_url = None
413
self._http_thread = threading.Thread(target=self._http_start)
414
self._http_thread.setDaemon(True)
415
self._http_thread.start()
416
self._http_proxy = os.environ.get("http_proxy")
417
if self._http_proxy is not None:
418
del os.environ["http_proxy"]
422
"""See bzrlib.transport.Server.tearDown."""
423
self._http_running = False
424
self._http_thread.join()
425
if self._http_proxy is not None:
427
os.environ["http_proxy"] = self._http_proxy
430
"""See bzrlib.transport.Server.get_url."""
431
return self._get_remote_url(self._home_dir)
433
def get_bogus_url(self):
434
"""See bzrlib.transport.Server.get_bogus_url."""
435
return 'http://jasldkjsalkdjalksjdkljasd'
438
def get_test_permutations():
439
"""Return the permutations to be used in testing."""
440
warn("There are no HTTPS transport provider tests yet.")
441
return [(HttpTransport, HttpServer),
added
removed
bzrlib/transport/http.py
