« back to all changes in this revision
Viewing changes to bzrlib/transport/http/__init__.py
- browse files at revision 1607.1.4
- compare with another revision
- compare with revision 1607.1.10
- download diff
- download tarball
- view history from revision 1607.1.4
- Committer: Robert Collins
- Date: 2006-03-15 12:26:42 UTC
- mfrom: (1611 +trunk)
- mto: (1616.1.1 bzr.mbp.integration) (1666.1.5 integration)
- mto: This revision was merged to the branch mainline in revision 1621.
- Revision ID: robertc@robertcollins.net-20060315122642-05693b82bcd6c9f5
Merge bzr.dev
- files added:
- bzrlib/transport/http
- files removed:
- bzrlib/tests/test_reweave.py.moved
- files renamed:
- bzrlib/transport/http.py => bzrlib/transport/http/__init__.py
- files modified:
- NEWS
added
removed
bzrlib/transport/http/__init__.py
1
# Copyright (C) 2005 Canonical Ltd
2
1
# Copyright (C) 2005, 2006 Canonical Ltd
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
5
5
# the Free Software Foundation; either version 2 of the License, or
6
6
# (at your option) any later version.
7
7
#
8
8
# This program is distributed in the hope that it will be useful,
9
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
11
# GNU General Public License for more details.
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16
"""Implementation of Transport over http.
16
17
"""Base implementation of Transport over http.
18
19
There are separate implementation modules for each http client implementation.
17
20
"""
18
21
19
import os, errno
22
import errno
23
import os
20
24
from collections import deque
21
25
from cStringIO import StringIO
22
import urllib, urllib2
26
import re
23
27
import urlparse
28
import urllib
24
29
from warnings import warn
25
30
26
import bzrlib
27
from bzrlib.transport import Transport, Server
28
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
31
from bzrlib.transport import Transport, register_transport, Server
32
from bzrlib.errors import (TransportNotPossible, NoSuchFile,
29
33
TransportError, ConnectionError)
30
34
from bzrlib.errors import BzrError, BzrCheckError
31
35
from bzrlib.branch import Branch
32
36
from bzrlib.trace import mutter
37
# TODO: load these only when running http tests
38
import BaseHTTPServer, SimpleHTTPServer, socket, time
39
import threading
33
40
from bzrlib.ui import ui_factory
34
41
35
42
36
43
def extract_auth(url, password_manager):
37
"""
38
Extract auth parameters from am HTTP/HTTPS url and add them to the given
44
"""Extract auth parameters from am HTTP/HTTPS url and add them to the given
39
45
password manager. Return the url, minus those auth parameters (which
40
46
confuse urllib2).
41
47
"""
48
assert re.match(r'^(https?)(\+\w+)?://', url), \
49
'invalid absolute url %r' % url
42
50
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
43
assert (scheme == 'http') or (scheme == 'https')
44
51
45
52
if '@' in netloc:
46
53
auth, netloc = netloc.split('@', 1)
63
70
return url
64
71
65
72
66
class Request(urllib2.Request):
67
"""Request object for urllib2 that allows the method to be overridden."""
68
69
method = None
70
71
def get_method(self):
72
if self.method is not None:
73
return self.method
74
else:
75
return urllib2.Request.get_method(self)
76
77
78
def get_url(url, method=None, ranges=None):
79
import urllib2
80
if ranges:
81
rangestring = ranges
82
else:
83
rangestring = 'all'
84
mutter("get_url %s [%s]", url, rangestring)
85
manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
86
url = extract_auth(url, manager)
87
auth_handler = urllib2.HTTPBasicAuthHandler(manager)
88
opener = urllib2.build_opener(auth_handler)
89
90
request = Request(url)
91
request.method = method
92
request.add_header('User-Agent', 'bzr/%s' % bzrlib.__version__)
93
if ranges:
94
request.add_header('Range', ranges)
95
response = opener.open(request)
96
return response
97
98
99
class HttpTransport(Transport):
100
"""This is the transport agent for http:// access.
101
102
TODO: Implement pipelined versions of all of the *_multi() functions.
73
class HttpTransportBase(Transport):
74
"""Base class for http implementations.
75
76
Does URL parsing, etc, but not any network IO.
77
78
The protocol can be given as e.g. http+urllib://host/ to use a particular
79
implementation.
103
80
"""
104
81
82
# _proto: "http" or "https"
83
# _qualified_proto: may have "+pycurl", etc
84
105
85
def __init__(self, base):
106
86
"""Set the base path where files will be stored."""
107
assert base.startswith('http://') or base.startswith('https://')
87
proto_match = re.match(r'^(https?)(\+\w+)?://', base)
88
if not proto_match:
89
raise AssertionError("not a http url: %r" % base)
90
self._proto = proto_match.group(1)
91
impl_name = proto_match.group(2)
92
if impl_name:
93
impl_name = impl_name[1:]
94
self._impl_name = impl_name
108
95
if base[-1] != '/':
109
96
base = base + '/'
110
super(HttpTransport, self).__init__(base)
97
super(HttpTransportBase, self).__init__(base)
111
98
# In the future we might actually connect to the remote host
112
99
# rather than using get_url
113
100
# self._connection = None
114
(self._proto, self._host,
101
(apparent_proto, self._host,
115
102
self._path, self._parameters,
116
103
self._query, self._fragment) = urlparse.urlparse(self.base)
117
118
def should_cache(self):
119
"""Return True if the data pulled across should be cached locally.
120
"""
121
return True
122
123
def clone(self, offset=None):
124
"""Return a new HttpTransport with root at self.base + offset
125
For now HttpTransport does not actually connect, so just return
126
a new HttpTransport object.
127
"""
128
if offset is None:
129
return HttpTransport(self.base)
130
else:
131
return HttpTransport(self.abspath(offset))
104
self._qualified_proto = apparent_proto
132
105
133
106
def abspath(self, relpath):
134
107
"""Return the full url to the given relative path.
135
This can be supplied with a string or a list
108
109
This can be supplied with a string or a list.
110
111
The URL returned always has the protocol scheme originally used to
112
construct the transport, even if that includes an explicit
113
implementation qualifier.
136
114
"""
137
115
assert isinstance(relpath, basestring)
138
116
if isinstance(relpath, basestring):
165
143
# I'm concerned about when it chooses to strip the last
166
144
# portion of the path, and when it doesn't.
167
145
path = '/'.join(basepath)
168
return urlparse.urlunparse((self._proto,
169
self._host, path, '', '', ''))
146
return urlparse.urlunparse((self._qualified_proto,
147
self._host, path, '', '', ''))
148
149
def _real_abspath(self, relpath):
150
"""Produce absolute path, adjusting protocol if needed"""
151
abspath = self.abspath(relpath)
152
qp = self._qualified_proto
153
rp = self._proto
154
if self._qualified_proto != self._proto:
155
abspath = rp + abspath[len(qp):]
156
if not isinstance(abspath, str):
157
# escaping must be done at a higher level
158
abspath = abspath.encode('ascii')
159
return abspath
170
160
171
161
def has(self, relpath):
172
"""Does the target location exist?
173
174
TODO: This should be changed so that we don't use
175
urllib2 and get an exception, the code path would be
176
cleaner if we just do an http HEAD request, and parse
177
the return code.
178
"""
179
path = relpath
180
try:
181
path = self.abspath(relpath)
182
f = get_url(path, method='HEAD')
183
# Without the read and then close()
184
# we tend to have busy sockets.
185
f.read()
186
f.close()
187
return True
188
except urllib2.HTTPError, e:
189
mutter('url error code: %s for has url: %r', e.code, path)
190
if e.code == 404:
191
return False
192
raise
193
except IOError, e:
194
mutter('io error: %s %s for has url: %r',
195
e.errno, errno.errorcode.get(e.errno), path)
196
if e.errno == errno.ENOENT:
197
return False
198
raise TransportError(orig_error=e)
199
200
def _get(self, relpath, decode=False, ranges=None):
201
path = relpath
202
try:
203
path = self.abspath(relpath)
204
return get_url(path, ranges=ranges)
205
except urllib2.HTTPError, e:
206
mutter('url error code: %s for has url: %r', e.code, path)
207
if e.code == 404:
208
raise NoSuchFile(path, extra=e)
209
raise
210
except (BzrError, IOError), e:
211
if hasattr(e, 'errno'):
212
mutter('io error: %s %s for has url: %r',
213
e.errno, errno.errorcode.get(e.errno), path)
214
if e.errno == errno.ENOENT:
215
raise NoSuchFile(path, extra=e)
216
raise ConnectionError(msg = "Error retrieving %s: %s"
217
% (self.abspath(relpath), str(e)),
218
orig_error=e)
219
220
def get(self, relpath, decode=False):
162
raise NotImplementedError("has() is abstract on %r" % self)
163
164
def get(self, relpath):
221
165
"""Get the file at the given relative path.
222
166
223
167
:param relpath: The relative path to the file
224
168
"""
225
return self._get(relpath, decode=decode)
169
code, response_file = self._get(relpath, None)
170
return response_file
171
172
def _get(self, relpath, ranges):
173
"""Get a file, or part of a file.
174
175
:param relpath: Path relative to transport base URL
176
:param byte_range: None to get the whole file;
177
or [(start,end)] to fetch parts of a file.
178
179
:returns: (http_code, result_file)
180
181
Note that the current http implementations can only fetch one range at
182
a time through this call.
183
"""
184
raise NotImplementedError(self._get)
226
185
227
186
def readv(self, relpath, offsets):
228
187
"""Get parts of the file at the given relative path.
229
188
230
:offsets: A list of (offset, size) tuples.
231
:return: A list or generator of (offset, data) tuples
189
:param offsets: A list of (offset, size) tuples.
190
:param return: A list or generator of (offset, data) tuples
232
191
"""
233
# this is not quite regular enough to have a single driver routine and
192
# Ideally we would pass one big request asking for all the ranges in
193
# one go; however then the server will give a multipart mime response
194
# back, and we can't parse them yet. So instead we just get one range
195
# per region, and try to coallesce the regions as much as possible.
196
#
197
# The read-coallescing code is not quite regular enough to have a
198
# single driver routine and
234
199
# helper method in Transport.
235
200
def do_combined_read(combined_offsets):
236
201
# read one coalesced block
239
204
total_size += size
240
205
mutter('readv coalesced %d reads.', len(combined_offsets))
241
206
offset = combined_offsets[0][0]
242
ranges = 'bytes=%d-%d' % (offset, offset + total_size - 1)
243
response = self._get(relpath, ranges=ranges)
244
if response.code == 206:
207
byte_range = (offset, offset + total_size - 1)
208
code, result_file = self._get(relpath, [byte_range])
209
if code == 206:
245
210
for off, size in combined_offsets:
246
yield off, response.read(size)
247
elif response.code == 200:
248
data = response.read(offset + total_size)[offset:offset + total_size]
211
result_bytes = result_file.read(size)
212
assert len(result_bytes) == size
213
yield off, result_bytes
214
elif code == 200:
215
data = result_file.read(offset + total_size)[offset:offset + total_size]
249
216
pos = 0
250
217
for offset, size in combined_offsets:
251
218
yield offset, data[pos:pos + size]
252
219
pos += size
253
220
del data
254
255
221
if not len(offsets):
256
222
return
257
223
pending_offsets = deque(offsets)
313
279
# At this point HttpTransport might be able to check and see if
314
280
# the remote location is the same, and rather than download, and
315
281
# then upload, it could just issue a remote copy_this command.
316
if isinstance(other, HttpTransport):
282
if isinstance(other, HttpTransportBase):
317
283
raise TransportNotPossible('http cannot be the target of copy_to()')
318
284
else:
319
return super(HttpTransport, self).copy_to(relpaths, other, mode=mode, pb=pb)
285
return super(HttpTransportBase, self).\
286
copy_to(relpaths, other, mode=mode, pb=pb)
320
287
321
288
def move(self, rel_from, rel_to):
322
289
"""Move the item at rel_from to the location at rel_to"""
360
327
"""
361
328
raise TransportNotPossible('http does not support lock_write()')
362
329
330
def clone(self, offset=None):
331
"""Return a new HttpTransportBase with root at self.base + offset
332
For now HttpTransportBase does not actually connect, so just return
333
a new HttpTransportBase object.
334
"""
335
if offset is None:
336
return self.__class__(self.base)
337
else:
338
return self.__class__(self.abspath(offset))
363
339
364
340
#---------------- test server facilities ----------------
365
import BaseHTTPServer, SimpleHTTPServer, socket, time
366
import threading
367
341
# TODO: load these only when running tests
368
342
369
343
class WebserverNotAvailable(Exception):
370
344
pass
426
400
RequestHandlerClass)
427
401
self.test_case = test_case
428
402
429
430
403
class HttpServer(Server):
431
404
"""A test server for http transports."""
432
405
406
# used to form the url that connects to this server
407
_url_protocol = 'http'
408
433
409
def _http_start(self):
434
410
httpd = None
435
411
httpd = TestingHTTPServer(('localhost', 0),
436
412
TestingHTTPRequestHandler,
437
413
self)
438
414
host, port = httpd.socket.getsockname()
439
self._http_base_url = 'http://localhost:%s/' % port
415
self._http_base_url = '%s://localhost:%s/' % (self._url_protocol, port)
440
416
self._http_starting.release()
441
417
httpd.socket.settimeout(0.1)
442
418
494
470
495
471
def get_bogus_url(self):
496
472
"""See bzrlib.transport.Server.get_bogus_url."""
497
return 'http://jasldkjsalkdjalksjdkljasd'
498
499
500
def get_test_permutations():
501
"""Return the permutations to be used in testing."""
502
warn("There are no HTTPS transport provider tests yet.")
503
return [(HttpTransport, HttpServer),
504
]
473
# this is chosen to try to prevent trouble with proxies, wierd dns,
474
# etc
475
return 'http://127.0.0.1:1/'
476
Loggerhead is a web-based interface for Breezy
Version: 2.0.1