1
# Copyright (C) 2005 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over http.
20
from cStringIO import StringIO
21
import urllib, urllib2
23
from warnings import warn
26
from bzrlib.transport import Transport, Server
27
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
28
TransportError, ConnectionError)
29
from bzrlib.errors import BzrError, BzrCheckError
30
from bzrlib.branch import Branch
31
from bzrlib.trace import mutter
32
from bzrlib.ui import ui_factory
35
def extract_auth(url, password_manager):
37
Extract auth parameters from am HTTP/HTTPS url and add them to the given
38
password manager. Return the url, minus those auth parameters (which
41
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
42
assert (scheme == 'http') or (scheme == 'https')
45
auth, netloc = netloc.split('@', 1)
47
username, password = auth.split(':', 1)
49
username, password = auth, None
51
host = netloc.split(':', 1)[0]
54
username = urllib.unquote(username)
55
if password is not None:
56
password = urllib.unquote(password)
58
password = ui_factory.get_password(prompt='HTTP %(user)@%(host) password',
59
user=username, host=host)
60
password_manager.add_password(None, host, username, password)
61
url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
65
class Request(urllib2.Request):
66
"""Request object for urllib2 that allows the method to be overridden."""
71
if self.method is not None:
74
return urllib2.Request.get_method(self)
77
def get_url(url, method=None):
79
mutter("get_url %s", url)
80
manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
81
url = extract_auth(url, manager)
82
auth_handler = urllib2.HTTPBasicAuthHandler(manager)
83
opener = urllib2.build_opener(auth_handler)
85
request = Request(url)
86
request.method = method
87
request.add_header('User-Agent', 'bzr/%s' % bzrlib.__version__)
88
response = opener.open(request)
92
class HttpTransport(Transport):
93
"""This is the transport agent for http:// access.
95
TODO: Implement pipelined versions of all of the *_multi() functions.
98
def __init__(self, base):
99
"""Set the base path where files will be stored."""
100
assert base.startswith('http://') or base.startswith('https://')
103
super(HttpTransport, self).__init__(base)
104
# In the future we might actually connect to the remote host
105
# rather than using get_url
106
# self._connection = None
107
(self._proto, self._host,
108
self._path, self._parameters,
109
self._query, self._fragment) = urlparse.urlparse(self.base)
111
def should_cache(self):
112
"""Return True if the data pulled across should be cached locally.
116
def clone(self, offset=None):
117
"""Return a new HttpTransport with root at self.base + offset
118
For now HttpTransport does not actually connect, so just return
119
a new HttpTransport object.
122
return HttpTransport(self.base)
124
return HttpTransport(self.abspath(offset))
126
def abspath(self, relpath):
127
"""Return the full url to the given relative path.
128
This can be supplied with a string or a list
130
assert isinstance(relpath, basestring)
131
if isinstance(relpath, basestring):
132
relpath_parts = relpath.split('/')
134
# TODO: Don't call this with an array - no magic interfaces
135
relpath_parts = relpath[:]
136
if len(relpath_parts) > 1:
137
if relpath_parts[0] == '':
138
raise ValueError("path %r within branch %r seems to be absolute"
139
% (relpath, self._path))
140
if relpath_parts[-1] == '':
141
raise ValueError("path %r within branch %r seems to be a directory"
142
% (relpath, self._path))
143
basepath = self._path.split('/')
144
if len(basepath) > 0 and basepath[-1] == '':
145
basepath = basepath[:-1]
146
for p in relpath_parts:
148
if len(basepath) == 0:
149
# In most filesystems, a request for the parent
150
# of root, just returns root.
153
elif p == '.' or p == '':
157
# Possibly, we could use urlparse.urljoin() here, but
158
# I'm concerned about when it chooses to strip the last
159
# portion of the path, and when it doesn't.
160
path = '/'.join(basepath)
161
return urlparse.urlunparse((self._proto,
162
self._host, path, '', '', ''))
164
def has(self, relpath):
165
"""Does the target location exist?
167
TODO: This should be changed so that we don't use
168
urllib2 and get an exception, the code path would be
169
cleaner if we just do an http HEAD request, and parse
174
path = self.abspath(relpath)
175
f = get_url(path, method='HEAD')
176
# Without the read and then close()
177
# we tend to have busy sockets.
181
except urllib2.HTTPError, e:
182
mutter('url error code: %s for has url: %r', e.code, path)
187
mutter('io error: %s %s for has url: %r',
188
e.errno, errno.errorcode.get(e.errno), path)
189
if e.errno == errno.ENOENT:
191
raise TransportError(orig_error=e)
193
def get(self, relpath, decode=False):
194
"""Get the file at the given relative path.
196
:param relpath: The relative path to the file
200
path = self.abspath(relpath)
202
except urllib2.HTTPError, e:
203
mutter('url error code: %s for has url: %r', e.code, path)
205
raise NoSuchFile(path, extra=e)
207
except (BzrError, IOError), e:
208
if hasattr(e, 'errno'):
209
mutter('io error: %s %s for has url: %r',
210
e.errno, errno.errorcode.get(e.errno), path)
211
if e.errno == errno.ENOENT:
212
raise NoSuchFile(path, extra=e)
213
raise ConnectionError(msg = "Error retrieving %s: %s"
214
% (self.abspath(relpath), str(e)),
217
def put(self, relpath, f, mode=None):
218
"""Copy the file-like or string object into the location.
220
:param relpath: Location to put the contents, relative to base.
221
:param f: File-like or string object.
223
raise TransportNotPossible('http PUT not supported')
225
def mkdir(self, relpath, mode=None):
226
"""Create a directory at the given path."""
227
raise TransportNotPossible('http does not support mkdir()')
229
def rmdir(self, relpath):
230
"""See Transport.rmdir."""
231
raise TransportNotPossible('http does not support rmdir()')
233
def append(self, relpath, f):
234
"""Append the text in the file-like object into the final
237
raise TransportNotPossible('http does not support append()')
239
def copy(self, rel_from, rel_to):
240
"""Copy the item at rel_from to the location at rel_to"""
241
raise TransportNotPossible('http does not support copy()')
243
def copy_to(self, relpaths, other, mode=None, pb=None):
244
"""Copy a set of entries from self into another Transport.
246
:param relpaths: A list/generator of entries to be copied.
248
TODO: if other is LocalTransport, is it possible to
249
do better than put(get())?
251
# At this point HttpTransport might be able to check and see if
252
# the remote location is the same, and rather than download, and
253
# then upload, it could just issue a remote copy_this command.
254
if isinstance(other, HttpTransport):
255
raise TransportNotPossible('http cannot be the target of copy_to()')
257
return super(HttpTransport, self).copy_to(relpaths, other, mode=mode, pb=pb)
259
def move(self, rel_from, rel_to):
260
"""Move the item at rel_from to the location at rel_to"""
261
raise TransportNotPossible('http does not support move()')
263
def delete(self, relpath):
264
"""Delete the item at relpath"""
265
raise TransportNotPossible('http does not support delete()')
267
def is_readonly(self):
268
"""See Transport.is_readonly."""
272
"""See Transport.listable."""
275
def stat(self, relpath):
276
"""Return the stat information for a file.
278
raise TransportNotPossible('http does not support stat()')
280
def lock_read(self, relpath):
281
"""Lock the given file for shared (read) access.
282
:return: A lock object, which should be passed to Transport.unlock()
284
# The old RemoteBranch ignore lock for reading, so we will
285
# continue that tradition and return a bogus lock object.
286
class BogusLock(object):
287
def __init__(self, path):
291
return BogusLock(relpath)
293
def lock_write(self, relpath):
294
"""Lock the given file for exclusive (write) access.
295
WARNING: many transports do not support this, so trying avoid using it
297
:return: A lock object, which should be passed to Transport.unlock()
299
raise TransportNotPossible('http does not support lock_write()')
302
#---------------- test server facilities ----------------
303
import BaseHTTPServer, SimpleHTTPServer, socket, time
307
class WebserverNotAvailable(Exception):
311
class BadWebserverPath(ValueError):
313
return 'path %s is not in %s' % self.args
316
class TestingHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
318
def log_message(self, format, *args):
319
self.server.test_case.log('webserver - %s - - [%s] %s "%s" "%s"',
320
self.address_string(),
321
self.log_date_time_string(),
323
self.headers.get('referer', '-'),
324
self.headers.get('user-agent', '-'))
326
def handle_one_request(self):
327
"""Handle a single HTTP request.
329
You normally don't need to override this method; see the class
330
__doc__ string for information on how to handle specific HTTP
331
commands such as GET and POST.
334
for i in xrange(1,11): # Don't try more than 10 times
336
self.raw_requestline = self.rfile.readline()
337
except socket.error, e:
338
if e.args[0] in (errno.EAGAIN, errno.EWOULDBLOCK):
339
# omitted for now because some tests look at the log of
340
# the server and expect to see no errors. see recent
341
# email thread. -- mbp 20051021.
342
## self.log_message('EAGAIN (%d) while reading from raw_requestline' % i)
348
if not self.raw_requestline:
349
self.close_connection = 1
351
if not self.parse_request(): # An error code has been sent, just exit
353
mname = 'do_' + self.command
354
if not hasattr(self, mname):
355
self.send_error(501, "Unsupported method (%r)" % self.command)
357
method = getattr(self, mname)
361
class TestingHTTPServer(BaseHTTPServer.HTTPServer):
362
def __init__(self, server_address, RequestHandlerClass, test_case):
363
BaseHTTPServer.HTTPServer.__init__(self, server_address,
365
self.test_case = test_case
368
class HttpServer(Server):
369
"""A test server for http transports."""
371
def _http_start(self):
373
httpd = TestingHTTPServer(('localhost', 0),
374
TestingHTTPRequestHandler,
376
host, port = httpd.socket.getsockname()
377
self._http_base_url = 'http://localhost:%s/' % port
378
self._http_starting.release()
379
httpd.socket.settimeout(0.1)
381
while self._http_running:
383
httpd.handle_request()
384
except socket.timeout:
387
def _get_remote_url(self, path):
388
path_parts = path.split(os.path.sep)
389
if os.path.isabs(path):
390
if path_parts[:len(self._local_path_parts)] != \
391
self._local_path_parts:
392
raise BadWebserverPath(path, self.test_dir)
393
remote_path = '/'.join(path_parts[len(self._local_path_parts):])
395
remote_path = '/'.join(path_parts)
397
self._http_starting.acquire()
398
self._http_starting.release()
399
return self._http_base_url + remote_path
401
def log(self, format, *args):
402
"""Capture Server log output."""
403
self.logs.append(format % args)
406
"""See bzrlib.transport.Server.setUp."""
407
self._home_dir = os.getcwdu()
408
self._local_path_parts = self._home_dir.split(os.path.sep)
409
self._http_starting = threading.Lock()
410
self._http_starting.acquire()
411
self._http_running = True
412
self._http_base_url = None
413
self._http_thread = threading.Thread(target=self._http_start)
414
self._http_thread.setDaemon(True)
415
self._http_thread.start()
416
self._http_proxy = os.environ.get("http_proxy")
417
if self._http_proxy is not None:
418
del os.environ["http_proxy"]
422
"""See bzrlib.transport.Server.tearDown."""
423
self._http_running = False
424
self._http_thread.join()
425
if self._http_proxy is not None:
427
os.environ["http_proxy"] = self._http_proxy
430
"""See bzrlib.transport.Server.get_url."""
431
return self._get_remote_url(self._home_dir)
433
def get_bogus_url(self):
434
"""See bzrlib.transport.Server.get_bogus_url."""
435
return 'http://jasldkjsalkdjalksjdkljasd'
438
def get_test_permutations():
439
"""Return the permutations to be used in testing."""
440
warn("There are no HTTPS transport provider tests yet.")
441
return [(HttpTransport, HttpServer),
added
removed
bzrlib/transport/http.py
