1
# Copyright (C) 2005, 2006 Canonical Ltd
1
# Copyright (C) 2005 Canonical Ltd
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
6
6
# (at your option) any later version.
8
8
# This program is distributed in the hope that it will be useful,
9
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
11
# GNU General Public License for more details.
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Base implementation of Transport over http.
19
There are separate implementation modules for each http client implementation.
16
"""Implementation of Transport over http.
19
from bzrlib.transport import Transport, register_transport
20
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
21
NonRelativePath, TransportError)
22
23
from cStringIO import StringIO
31
from warnings import warn
33
# TODO: load these only when running http tests
34
import BaseHTTPServer, SimpleHTTPServer, socket, time
37
from bzrlib import errors
38
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
39
TransportError, ConnectionError, InvalidURL)
27
from bzrlib.errors import BzrError, BzrCheckError
40
28
from bzrlib.branch import Branch
41
29
from bzrlib.trace import mutter
42
from bzrlib.transport import (
49
from bzrlib.transport.http.response import (HttpMultipartRangeResponse,
51
from bzrlib.ui import ui_factory
54
def extract_auth(url, password_manager):
55
"""Extract auth parameters from am HTTP/HTTPS url and add them to the given
56
password manager. Return the url, minus those auth parameters (which
59
assert re.match(r'^(https?)(\+\w+)?://', url), \
60
'invalid absolute url %r' % url
61
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
31
# velocitynet.com.au transparently proxies connections and thereby
32
# breaks keep-alive -- sucks!
37
mutter("get_url %s" % url)
38
url_f = urllib2.urlopen(url)
41
class HttpTransportError(TransportError):
44
class HttpTransport(Transport):
45
"""This is the transport agent for http:// access.
64
auth, netloc = netloc.split('@', 1)
66
username, password = auth.split(':', 1)
68
username, password = auth, None
70
host = netloc.split(':', 1)[0]
73
username = urllib.unquote(username)
74
if password is not None:
75
password = urllib.unquote(password)
77
password = ui_factory.get_password(prompt='HTTP %(user)@%(host) password',
78
user=username, host=host)
79
password_manager.add_password(None, host, username, password)
80
url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
84
def _extract_headers(header_text, url):
85
"""Extract the mapping for an rfc2822 header
87
This is a helper function for the test suite and for _pycurl.
88
(urllib already parses the headers for us)
90
In the case that there are multiple headers inside the file,
91
the last one is returned.
93
:param header_text: A string of header information.
94
This expects that the first line of a header will always be HTTP ...
95
:param url: The url we are parsing, so we can raise nice errors
96
:return: mimetools.Message object, which basically acts like a case
97
insensitive dictionary.
100
remaining = header_text
103
raise errors.InvalidHttpResponse(url, 'Empty headers')
106
header_file = StringIO(remaining)
107
first_line = header_file.readline()
108
if not first_line.startswith('HTTP'):
109
if first_header: # The first header *must* start with HTTP
110
raise errors.InvalidHttpResponse(url,
111
'Opening header line did not start with HTTP: %s'
113
assert False, 'Opening header line was not HTTP'
115
break # We are done parsing
117
m = mimetools.Message(header_file)
119
# mimetools.Message parses the first header up to a blank line
120
# So while there is remaining data, it probably means there is
121
# another header to be parsed.
122
# Get rid of any preceeding whitespace, which if it is all whitespace
123
# will get rid of everything.
124
remaining = header_file.read().lstrip()
128
class HttpTransportBase(Transport, smart.SmartClientMedium):
129
"""Base class for http implementations.
131
Does URL parsing, etc, but not any network IO.
133
The protocol can be given as e.g. http+urllib://host/ to use a particular
137
# _proto: "http" or "https"
138
# _qualified_proto: may have "+pycurl", etc
47
TODO: Implement pipelined versions of all of the *_multi() functions.
140
50
def __init__(self, base):
141
51
"""Set the base path where files will be stored."""
142
proto_match = re.match(r'^(https?)(\+\w+)?://', base)
144
raise AssertionError("not a http url: %r" % base)
145
self._proto = proto_match.group(1)
146
impl_name = proto_match.group(2)
148
impl_name = impl_name[1:]
149
self._impl_name = impl_name
152
super(HttpTransportBase, self).__init__(base)
52
assert base.startswith('http://') or base.startswith('https://')
53
super(HttpTransport, self).__init__(base)
153
54
# In the future we might actually connect to the remote host
154
55
# rather than using get_url
155
56
# self._connection = None
156
(apparent_proto, self._host,
57
(self._proto, self._host,
157
58
self._path, self._parameters,
158
59
self._query, self._fragment) = urlparse.urlparse(self.base)
159
self._qualified_proto = apparent_proto
61
def should_cache(self):
62
"""Return True if the data pulled across should be cached locally.
66
def clone(self, offset=None):
67
"""Return a new HttpTransport with root at self.base + offset
68
For now HttpTransport does not actually connect, so just return
69
a new HttpTransport object.
72
return HttpTransport(self.base)
74
return HttpTransport(self.abspath(offset))
161
76
def abspath(self, relpath):
162
77
"""Return the full url to the given relative path.
164
This can be supplied with a string or a list.
166
The URL returned always has the protocol scheme originally used to
167
construct the transport, even if that includes an explicit
168
implementation qualifier.
78
This can be supplied with a string or a list
170
assert isinstance(relpath, basestring)
171
if isinstance(relpath, unicode):
172
raise InvalidURL(relpath, 'paths must not be unicode.')
173
80
if isinstance(relpath, basestring):
174
relpath_parts = relpath.split('/')
176
# TODO: Don't call this with an array - no magic interfaces
177
relpath_parts = relpath[:]
178
if relpath.startswith('/'):
181
# Except for the root, no trailing slashes are allowed
182
if len(relpath_parts) > 1 and relpath_parts[-1] == '':
183
raise ValueError("path %r within branch %r seems to be a directory"
184
% (relpath, self._path))
185
basepath = self._path.split('/')
186
if len(basepath) > 0 and basepath[-1] == '':
187
basepath = basepath[:-1]
82
basepath = self._path.split('/')
83
if len(basepath) > 0 and basepath[-1] == '':
84
basepath = basepath[:-1]
189
for p in relpath_parts:
191
if len(basepath) == 0:
192
89
# In most filesystems, a request for the parent
193
90
# of root, just returns root.
196
elif p == '.' or p == '':
199
97
basepath.append(p)
200
99
# Possibly, we could use urlparse.urljoin() here, but
201
100
# I'm concerned about when it chooses to strip the last
202
101
# portion of the path, and when it doesn't.
203
102
path = '/'.join(basepath)
206
result = urlparse.urlunparse((self._qualified_proto,
207
self._host, path, '', '', ''))
103
return urlparse.urlunparse((self._proto,
104
self._host, path, '', '', ''))
210
def _real_abspath(self, relpath):
211
"""Produce absolute path, adjusting protocol if needed"""
212
abspath = self.abspath(relpath)
213
qp = self._qualified_proto
215
if self._qualified_proto != self._proto:
216
abspath = rp + abspath[len(qp):]
217
if not isinstance(abspath, str):
218
# escaping must be done at a higher level
219
abspath = abspath.encode('ascii')
106
def relpath(self, abspath):
107
if not abspath.startswith(self.base):
108
raise NonRelativePath('path %r is not under base URL %r'
109
% (abspath, self.base))
111
return abspath[pl:].lstrip('/')
222
113
def has(self, relpath):
223
raise NotImplementedError("has() is abstract on %r" % self)
225
def get(self, relpath):
114
"""Does the target location exist?
116
TODO: HttpTransport.has() should use a HEAD request,
117
not a full GET request.
119
TODO: This should be changed so that we don't use
120
urllib2 and get an exception, the code path would be
121
cleaner if we just do an http HEAD request, and parse
125
f = get_url(self.abspath(relpath))
126
# Without the read and then close()
127
# we tend to have busy sockets.
133
except urllib2.URLError:
136
if e.errno == errno.ENOENT:
138
raise HttpTransportError(orig_error=e)
140
def get(self, relpath, decode=False):
226
141
"""Get the file at the given relative path.
228
143
:param relpath: The relative path to the file
230
code, response_file = self._get(relpath, None)
233
def _get(self, relpath, ranges):
234
"""Get a file, or part of a file.
236
:param relpath: Path relative to transport base URL
237
:param byte_range: None to get the whole file;
238
or [(start,end)] to fetch parts of a file.
240
:returns: (http_code, result_file)
242
Note that the current http implementations can only fetch one range at
243
a time through this call.
245
raise NotImplementedError(self._get)
247
def get_request(self):
248
return SmartClientHTTPMediumRequest(self)
250
def get_smart_medium(self):
251
"""See Transport.get_smart_medium.
253
HttpTransportBase directly implements the minimal interface of
254
SmartMediumClient, so this returns self.
258
def readv(self, relpath, offsets):
259
"""Get parts of the file at the given relative path.
261
:param offsets: A list of (offset, size) tuples.
262
:param return: A list or generator of (offset, data) tuples
264
ranges = self.offsets_to_ranges(offsets)
265
mutter('http readv of %s collapsed %s offsets => %s',
266
relpath, len(offsets), ranges)
267
code, f = self._get(relpath, ranges)
268
for start, size in offsets:
269
f.seek(start, (start < 0) and 2 or 0)
272
if len(data) != size:
273
raise errors.ShortReadvError(relpath, start, size,
278
def offsets_to_ranges(offsets):
279
"""Turn a list of offsets and sizes into a list of byte ranges.
281
:param offsets: A list of tuples of (start, size). An empty list
283
:return: a list of inclusive byte ranges (start, end)
284
Adjacent ranges will be combined.
286
# Make sure we process sorted offsets
287
offsets = sorted(offsets)
292
for start, size in offsets:
293
end = start + size - 1
295
combined.append([start, end])
296
elif start <= prev_end + 1:
297
combined[-1][1] = end
299
combined.append([start, end])
304
def _post(self, body_bytes):
305
"""POST body_bytes to .bzr/smart on this transport.
307
:returns: (response code, response body file-like object).
309
# TODO: Requiring all the body_bytes to be available at the beginning of
310
# the POST may require large client buffers. It would be nice to have
311
# an interface that allows streaming via POST when possible (and
312
# degrades to a local buffer when not).
313
raise NotImplementedError(self._post)
315
def put_file(self, relpath, f, mode=None):
316
"""Copy the file-like object into the location.
146
return get_url(self.abspath(relpath))
147
except (BzrError, urllib2.URLError, IOError), e:
148
raise NoSuchFile(msg = "Error retrieving %s: %s"
149
% (self.abspath(relpath), str(e)),
152
def get_partial(self, relpath, start, length=None):
153
"""Get just part of a file.
155
:param relpath: Path to the file, relative to base
156
:param start: The starting position to read from
157
:param length: The length to read. A length of None indicates
158
read to the end of the file.
159
:return: A file-like object containing at least the specified bytes.
160
Some implementations may return objects which can be read
161
past this length, but this is not guaranteed.
163
# TODO: You can make specialized http requests for just
164
# a portion of the file. Figure out how to do that.
165
# For now, urllib2 returns files that cannot seek() so
166
# we just read bytes off the beginning, until we
167
# get to the point that we care about.
168
f = self.get(relpath)
169
# TODO: read in smaller chunks, in case things are
170
# buffered internally.
174
def put(self, relpath, f):
175
"""Copy the file-like or string object into the location.
318
177
:param relpath: Location to put the contents, relative to base.
319
:param f: File-like object.
178
:param f: File-like or string object.
321
180
raise TransportNotPossible('http PUT not supported')
323
def mkdir(self, relpath, mode=None):
182
def mkdir(self, relpath):
324
183
"""Create a directory at the given path."""
325
184
raise TransportNotPossible('http does not support mkdir()')
327
def rmdir(self, relpath):
328
"""See Transport.rmdir."""
329
raise TransportNotPossible('http does not support rmdir()')
331
def append_file(self, relpath, f, mode=None):
186
def append(self, relpath, f):
332
187
"""Append the text in the file-like object into the final
398
248
raise TransportNotPossible('http does not support lock_write()')
400
def clone(self, offset=None):
401
"""Return a new HttpTransportBase with root at self.base + offset
403
We leave the daughter classes take advantage of the hint
404
that it's a cloning not a raw creation.
407
return self.__class__(self.base, self)
409
return self.__class__(self.abspath(offset), self)
412
def range_header(ranges, tail_amount):
413
"""Turn a list of bytes ranges into a HTTP Range header value.
415
:param offsets: A list of byte ranges, (start, end). An empty list
418
:return: HTTP range header string.
421
for start, end in ranges:
422
strings.append('%d-%d' % (start, end))
425
strings.append('-%d' % tail_amount)
427
return ','.join(strings)
429
def send_http_smart_request(self, bytes):
430
code, body_filelike = self._post(bytes)
431
assert code == 200, 'unexpected HTTP response code %r' % (code,)
435
class SmartClientHTTPMediumRequest(smart.SmartClientMediumRequest):
436
"""A SmartClientMediumRequest that works with an HTTP medium."""
438
def __init__(self, medium):
439
smart.SmartClientMediumRequest.__init__(self, medium)
442
def _accept_bytes(self, bytes):
443
self._buffer += bytes
445
def _finished_writing(self):
446
data = self._medium.send_http_smart_request(self._buffer)
447
self._response_body = data
449
def _read_bytes(self, count):
450
return self._response_body.read(count)
452
def _finished_reading(self):
453
"""See SmartClientMediumRequest._finished_reading."""
457
#---------------- test server facilities ----------------
458
# TODO: load these only when running tests
461
class WebserverNotAvailable(Exception):
465
class BadWebserverPath(ValueError):
467
return 'path %s is not in %s' % self.args
470
class TestingHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
472
def log_message(self, format, *args):
473
self.server.test_case.log('webserver - %s - - [%s] %s "%s" "%s"',
474
self.address_string(),
475
self.log_date_time_string(),
477
self.headers.get('referer', '-'),
478
self.headers.get('user-agent', '-'))
480
def handle_one_request(self):
481
"""Handle a single HTTP request.
483
You normally don't need to override this method; see the class
484
__doc__ string for information on how to handle specific HTTP
485
commands such as GET and POST.
488
for i in xrange(1,11): # Don't try more than 10 times
490
self.raw_requestline = self.rfile.readline()
491
except socket.error, e:
492
if e.args[0] in (errno.EAGAIN, errno.EWOULDBLOCK):
493
# omitted for now because some tests look at the log of
494
# the server and expect to see no errors. see recent
495
# email thread. -- mbp 20051021.
496
## self.log_message('EAGAIN (%d) while reading from raw_requestline' % i)
502
if not self.raw_requestline:
503
self.close_connection = 1
505
if not self.parse_request(): # An error code has been sent, just exit
507
mname = 'do_' + self.command
508
if getattr(self, mname, None) is None:
509
self.send_error(501, "Unsupported method (%r)" % self.command)
511
method = getattr(self, mname)
514
if sys.platform == 'win32':
515
# On win32 you cannot access non-ascii filenames without
516
# decoding them into unicode first.
517
# However, under Linux, you can access bytestream paths
518
# without any problems. If this function was always active
519
# it would probably break tests when LANG=C was set
520
def translate_path(self, path):
521
"""Translate a /-separated PATH to the local filename syntax.
523
For bzr, all url paths are considered to be utf8 paths.
524
On Linux, you can access these paths directly over the bytestream
525
request, but on win32, you must decode them, and access them
528
# abandon query parameters
529
path = urlparse.urlparse(path)[2]
530
path = posixpath.normpath(urllib.unquote(path))
531
path = path.decode('utf-8')
532
words = path.split('/')
533
words = filter(None, words)
536
drive, word = os.path.splitdrive(word)
537
head, word = os.path.split(word)
538
if word in (os.curdir, os.pardir): continue
539
path = os.path.join(path, word)
543
class TestingHTTPServer(BaseHTTPServer.HTTPServer):
544
def __init__(self, server_address, RequestHandlerClass, test_case):
545
BaseHTTPServer.HTTPServer.__init__(self, server_address,
547
self.test_case = test_case
550
class HttpServer(Server):
551
"""A test server for http transports."""
553
# used to form the url that connects to this server
554
_url_protocol = 'http'
556
# Subclasses can provide a specific request handler
557
def __init__(self, request_handler=TestingHTTPRequestHandler):
558
Server.__init__(self)
559
self.request_handler = request_handler
561
def _get_httpd(self):
562
return TestingHTTPServer(('localhost', 0),
563
self.request_handler,
566
def _http_start(self):
567
httpd = self._get_httpd()
568
host, port = httpd.socket.getsockname()
569
self._http_base_url = '%s://localhost:%s/' % (self._url_protocol, port)
570
self._http_starting.release()
571
httpd.socket.settimeout(0.1)
573
while self._http_running:
575
httpd.handle_request()
576
except socket.timeout:
579
def _get_remote_url(self, path):
580
path_parts = path.split(os.path.sep)
581
if os.path.isabs(path):
582
if path_parts[:len(self._local_path_parts)] != \
583
self._local_path_parts:
584
raise BadWebserverPath(path, self.test_dir)
585
remote_path = '/'.join(path_parts[len(self._local_path_parts):])
587
remote_path = '/'.join(path_parts)
589
self._http_starting.acquire()
590
self._http_starting.release()
591
return self._http_base_url + remote_path
593
def log(self, format, *args):
594
"""Capture Server log output."""
595
self.logs.append(format % args)
598
"""See bzrlib.transport.Server.setUp."""
599
self._home_dir = os.getcwdu()
600
self._local_path_parts = self._home_dir.split(os.path.sep)
601
self._http_starting = threading.Lock()
602
self._http_starting.acquire()
603
self._http_running = True
604
self._http_base_url = None
605
self._http_thread = threading.Thread(target=self._http_start)
606
self._http_thread.setDaemon(True)
607
self._http_thread.start()
608
self._http_proxy = os.environ.get("http_proxy")
609
if self._http_proxy is not None:
610
del os.environ["http_proxy"]
614
"""See bzrlib.transport.Server.tearDown."""
615
self._http_running = False
616
self._http_thread.join()
617
if self._http_proxy is not None:
619
os.environ["http_proxy"] = self._http_proxy
622
"""See bzrlib.transport.Server.get_url."""
623
return self._get_remote_url(self._home_dir)
625
def get_bogus_url(self):
626
"""See bzrlib.transport.Server.get_bogus_url."""
627
# this is chosen to try to prevent trouble with proxies, weird dns,
629
return 'http://127.0.0.1:1/'
632
class HTTPServerWithSmarts(HttpServer):
633
"""HTTPServerWithSmarts extends the HttpServer with POST methods that will
634
trigger a smart server to execute with a transport rooted at the rootdir of
639
HttpServer.__init__(self, SmartRequestHandler)
642
class SmartRequestHandler(TestingHTTPRequestHandler):
643
"""Extend TestingHTTPRequestHandler to support smart client POSTs."""
646
"""Hand the request off to a smart server instance."""
647
self.send_response(200)
648
self.send_header("Content-type", "application/octet-stream")
649
transport = get_transport(self.server.test_case._home_dir)
650
# TODO: We might like to support streaming responses. 1.0 allows no
651
# Content-length in this case, so for integrity we should perform our
652
# own chunking within the stream.
653
# 1.1 allows chunked responses, and in this case we could chunk using
654
# the HTTP chunking as this will allow HTTP persistence safely, even if
655
# we have to stop early due to error, but we would also have to use the
656
# HTTP trailer facility which may not be widely available.
657
out_buffer = StringIO()
658
smart_protocol_request = smart.SmartServerRequestProtocolOne(
659
transport, out_buffer.write)
660
# if this fails, we should return 400 bad request, but failure is
661
# failure for now - RBC 20060919
662
data_length = int(self.headers['Content-Length'])
663
# Perhaps there should be a SmartServerHTTPMedium that takes care of
664
# feeding the bytes in the http request to the smart_protocol_request,
665
# but for now it's simpler to just feed the bytes directly.
666
smart_protocol_request.accept_bytes(self.rfile.read(data_length))
667
assert smart_protocol_request.next_read_size() == 0, (
668
"not finished reading, but all data sent to protocol.")
669
self.send_header("Content-Length", str(len(out_buffer.getvalue())))
671
self.wfile.write(out_buffer.getvalue())
250
register_transport('http://', HttpTransport)
251
register_transport('https://', HttpTransport)
added
removed
bzrlib/transport/http.py
