1
# Copyright (C) 2006 Canonical Ltd
3
# This program is free software; you can redistribute it and/or modify
4
# it under the terms of the GNU General Public License as published by
5
# the Free Software Foundation; either version 2 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU General Public License for more details.
13
# You should have received a copy of the GNU General Public License
14
# along with this program; if not, write to the Free Software
15
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
"""Implementaion of urllib2 tailored to bzr needs
19
This file complements the urllib2 class hierarchy with custom classes.
21
For instance, we create a new HTTPConnection and HTTPSConnection that inherit
22
from the original urllib2.HTTP(s)Connection objects, but also have a new base
23
which implements a custom getresponse and fake_close handlers.
25
And then we implement custom HTTPHandler and HTTPSHandler classes, that use
26
the custom HTTPConnection classes.
28
We have a custom Response class, which lets us maintain a keep-alive
29
connection even for requests that urllib2 doesn't expect to contain body data.
31
And a custom Request class that lets us track redirections, and
32
handle authentication schemes.
37
# TODO: It may be possible to share the password_manager across
38
# all transports by prefixing the realm by the protocol used
39
# (especially if other protocols do not use realms). See
40
# PasswordManager below.
42
# FIXME: Oversimplifying, two kind of exceptions should be
43
# raised, once a request is issued: URLError before we have been
44
# able to process the response, HTTPError after that. Process the
45
# response means we are able to leave the socket clean, so if we
46
# are not able to do that, we should close the connection. The
47
# actual code more or less do that, tests should be written to
61
from bzrlib import __version__ as bzrlib_version
69
# We define our own Response class to keep our httplib pipe clean
70
class Response(httplib.HTTPResponse):
71
"""Custom HTTPResponse, to avoid the need to decorate.
73
httplib prefers to decorate the returned objects, rather
74
than using a custom object.
77
# Some responses have bodies in which we have no interest
78
_body_ignored_responses = [301,302, 303, 307, 401, 403, 404]
80
def __init__(self, *args, **kwargs):
81
httplib.HTTPResponse.__init__(self, *args, **kwargs)
84
"""Begin to read the response from the server.
86
httplib assumes that some responses get no content and do
87
not even attempt to read the body in that case, leaving
88
the body in the socket, blocking the next request. Let's
89
try to workaround that.
91
httplib.HTTPResponse.begin(self)
92
if self.status in self._body_ignored_responses:
93
if self.debuglevel > 0:
94
print "For status: [%s]," % self.status,
95
print "will ready body, length: ",
96
if self.length is not None:
97
print "[%d]" % self.length
100
if not (self.length is None or self.will_close):
101
# In some cases, we just can't read the body not
102
# even try or we may encounter a 104, 'Connection
103
# reset by peer' error if there is indeed no body
104
# and the server closed the connection just after
105
# having issued the response headers (even if the
106
# headers indicate a Content-Type...)
107
body = self.fp.read(self.length)
108
if self.debuglevel > 0:
109
print "Consumed body: [%s]" % body
113
# Not inheriting from 'object' because httplib.HTTPConnection doesn't.
114
class AbstractHTTPConnection:
115
"""A custom HTTP(S) Connection, which can reset itself on a bad response"""
117
response_class = Response
118
strict = 1 # We don't support HTTP/0.9
120
def fake_close(self):
121
"""Make the connection believes the response have been fully handled.
123
That makes the httplib.HTTPConnection happy
125
# Preserve our preciousss
132
class HTTPConnection(AbstractHTTPConnection, httplib.HTTPConnection):
136
class HTTPSConnection(AbstractHTTPConnection, httplib.HTTPSConnection):
140
class Request(urllib2.Request):
141
"""A custom Request object.
143
urllib2 determines the request method heuristically (based on
144
the presence or absence of data). We set the method
147
The Request object tracks:
148
- the connection the request will be made on.
149
- the authentication parameters needed to preventively set
150
the authentication header once a first authentication have
154
def __init__(self, method, url, data=None, headers={},
155
origin_req_host=None, unverifiable=False,
156
connection=None, parent=None,):
157
urllib2.Request.__init__(self, url, data, headers,
158
origin_req_host, unverifiable)
160
self.connection = connection
161
# To handle redirections
163
self.redirected_to = None
164
# Unless told otherwise, redirections are not followed
165
self.follow_redirections = False
166
# auth and proxy_auth are dicts containing, at least
167
# (scheme, url, realm, user, password).
168
# The dict entries are mostly handled by the AuthHandler.
169
# Some authentication schemes may add more entries.
173
def get_method(self):
177
def extract_credentials(url):
178
"""Extracts credentials information from url.
180
Get user and password from url of the form: http://user:pass@host/path
181
:returns: (clean_url, user, password)
183
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
186
auth, netloc = netloc.split('@', 1)
188
user, password = auth.split(':', 1)
190
user, password = auth, None
191
user = urllib.unquote(user)
192
if password is not None:
193
password = urllib.unquote(password)
198
# Build the clean url
199
clean_url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
201
return clean_url, user, password
203
def extract_authentication_uri(url):
204
"""Extract the authentication uri from any url.
206
In the context of bzr, we simplified the authentication uri
207
to the host only. For the transport lifetime, we allow only
208
one user by realm on a given host. I.e. handling several
209
users for different paths for the same realm should be done
212
scheme, host, path, query, fragment = urlparse.urlsplit(url)
213
return '%s://%s' % (scheme, host)
216
# The urlib2.xxxAuthHandler handle the authentication of the
217
# requests, to do that, they need an urllib2 PasswordManager *at
218
# build time*. We also need one to reuse the passwords entered by
220
class PasswordManager(urllib2.HTTPPasswordMgrWithDefaultRealm):
223
urllib2.HTTPPasswordMgrWithDefaultRealm.__init__(self)
226
class ConnectionHandler(urllib2.BaseHandler):
227
"""Provides connection-sharing by pre-processing requests.
229
urllib2 provides no way to access the HTTPConnection object
230
internally used. But we need it in order to achieve
231
connection sharing. So, we add it to the request just before
232
it is processed, and then we override the do_open method for
233
http[s] requests in AbstractHTTPHandler.
236
handler_order = 1000 # after all pre-processings
238
def create_connection(self, request, http_connection_class):
239
host = request.get_host()
241
# Just a bit of paranoia here, this should have been
242
# handled in the higher levels
243
raise errors.InvalidURL(request.get_full_url(), 'no host given.')
245
# We create a connection (but it will not connect yet)
247
connection = http_connection_class(host)
248
except httplib.InvalidURL, exception:
249
# There is only one occurrence of InvalidURL in httplib
250
raise errors.InvalidURL(request.get_full_url(),
251
extra='nonnumeric port')
255
def capture_connection(self, request, http_connection_class):
256
"""Capture or inject the request connection.
259
- the request have no connection: create a new one,
261
- the request have a connection: this one have been used
262
already, let's capture it, so that we can give it to
263
another transport to be reused. We don't do that
264
ourselves: the Transport object get the connection from
265
a first request and then propagate it, from request to
266
request or to cloned transports.
268
connection = request.connection
269
if connection is None:
271
connection = self.create_connection(request, http_connection_class)
272
request.connection = connection
274
# All connections will pass here, propagate debug level
275
connection.set_debuglevel(DEBUG)
278
def http_request(self, request):
279
return self.capture_connection(request, HTTPConnection)
281
def https_request(self, request):
282
return self.capture_connection(request, HTTPSConnection)
285
class AbstractHTTPHandler(urllib2.AbstractHTTPHandler):
286
"""A custom handler for HTTP(S) requests.
288
We overrive urllib2.AbstractHTTPHandler to get a better
289
control of the connection, the ability to implement new
290
request types and return a response able to cope with
291
persistent connections.
294
# We change our order to be before urllib2 HTTP[S]Handlers
295
# and be chosen instead of them (the first http_open called
299
_default_headers = {'Pragma': 'no-cache',
300
'Cache-control': 'max-age=0',
301
'Connection': 'Keep-Alive',
302
# FIXME: Spell it User-*A*gent once we
303
# know how to properly avoid bogus
304
# urllib2 using capitalize() for headers
305
# instead of title(sp?).
306
'User-agent': 'bzr/%s (urllib)' % bzrlib_version,
311
urllib2.AbstractHTTPHandler.__init__(self, debuglevel=DEBUG)
313
def http_request(self, request):
314
"""Common headers setting"""
316
request.headers.update(self._default_headers.copy())
317
# FIXME: We may have to add the Content-Length header if
318
# we have data to send.
321
def retry_or_raise(self, http_class, request, first_try):
322
"""Retry the request (once) or raise the exception.
324
urllib2 raises exception of application level kind, we
325
just have to translate them.
327
httplib can raise exceptions of transport level (badly
328
formatted dialog, loss of connexion or socket level
329
problems). In that case we should issue the request again
330
(httplib will close and reopen a new connection if
333
# When an exception occurs, we give back the original
334
# Traceback or the bugs are hard to diagnose.
335
exc_type, exc_val, exc_tb = sys.exc_info()
336
if exc_type == socket.gaierror:
337
# No need to retry, that will not help
338
raise errors.ConnectionError("Couldn't resolve host '%s'"
339
% request.get_origin_req_host(),
343
if self._debuglevel > 0:
344
print 'Received exception: [%r]' % exc_val
345
print ' On connection: [%r]' % request.connection
346
method = request.get_method()
347
url = request.get_full_url()
348
print ' Will retry, %s %r' % (method, url)
349
request.connection.close()
350
response = self.do_open(http_class, request, False)
351
convert_to_addinfourl = False
353
if self._debuglevel > 0:
354
print 'Received second exception: [%r]' % exc_val
355
print ' On connection: [%r]' % request.connection
356
if exc_type in (httplib.BadStatusLine, httplib.UnknownProtocol):
357
# httplib.BadStatusLine and
358
# httplib.UnknownProtocol indicates that a
359
# bogus server was encountered or a bad
360
# connection (i.e. transient errors) is
361
# experimented, we have already retried once
362
# for that request so we raise the exception.
363
my_exception = errors.InvalidHttpResponse(
364
request.get_full_url(),
365
'Bad status line received',
368
# All other exception are considered connection related.
370
# httplib.HTTPException should indicate a bug
371
# in the urllib implementation, somewhow the
372
# httplib pipeline is in an incorrect state,
373
# we retry in hope that this will correct the
374
# problem but that may need investigation
375
# (note that no such bug is known as of
378
# socket errors generally occurs for reasons
379
# far outside our scope, so closing the
380
# connection and retrying is the best we can
383
# FIXME: and then there is HTTPError raised by:
384
# - HTTPDefaultErrorHandler (we define our own)
385
# - HTTPRedirectHandler.redirect_request
386
# - AbstractDigestAuthHandler.http_error_auth_reqed
388
my_exception = errors.ConnectionError(
389
msg= 'while sending %s %s:' % (request.get_method(),
390
request.get_selector()),
393
if self._debuglevel > 0:
394
print 'On connection: [%r]' % request.connection
395
method = request.get_method()
396
url = request.get_full_url()
397
print ' Failed again, %s %r' % (method, url)
398
print ' Will raise: [%r]' % my_exception
399
raise my_exception, None, exc_tb
400
return response, convert_to_addinfourl
402
def do_open(self, http_class, request, first_try=True):
403
"""See urllib2.AbstractHTTPHandler.do_open for the general idea.
405
The request will be retried once if it fails.
407
connection = request.connection
408
assert connection is not None, \
409
'Cannot process a request without a connection'
411
# Get all the headers
413
headers.update(request.header_items())
414
headers.update(request.unredirected_hdrs)
417
connection._send_request(request.get_method(),
418
request.get_selector(),
419
# FIXME: implements 100-continue
420
#None, # We don't send the body yet
423
if self._debuglevel > 0:
424
print 'Request sent: [%r]' % request
425
response = connection.getresponse()
426
convert_to_addinfourl = True
427
except (socket.gaierror, httplib.BadStatusLine, httplib.UnknownProtocol,
428
socket.error, httplib.HTTPException):
429
response, convert_to_addinfourl = self.retry_or_raise(http_class,
433
# FIXME: HTTPConnection does not fully support 100-continue (the
434
# server responses are just ignored)
437
# mutter('Will send the body')
438
# # We can send the body now
439
# body = request.get_data()
441
# raise URLError("No data given")
442
# connection.send(body)
443
# response = connection.getresponse()
445
if self._debuglevel > 0:
446
print 'Receives response: %r' % response
447
print ' For: %r(%r)' % (request.get_method(),
448
request.get_full_url())
450
if convert_to_addinfourl:
451
# Shamelessly copied from urllib2
455
fp = socket._fileobject(r)
456
resp = urllib2.addinfourl(fp, r.msg, req.get_full_url())
459
if self._debuglevel > 0:
460
print 'Create addinfourl: %r' % resp
461
print ' For: %r(%r)' % (request.get_method(),
462
request.get_full_url())
467
# # we need titled headers in a dict but
468
# # response.getheaders returns a list of (lower(header).
469
# # Let's title that because most of bzr handle titled
470
# # headers, but maybe we should switch to lowercased
472
# # jam 20060908: I think we actually expect the headers to
473
# # be similar to mimetools.Message object, which uses
474
# # case insensitive keys. It lowers() all requests.
475
# # My concern is that the code may not do perfect title case.
476
# # For example, it may use Content-type rather than Content-Type
478
# # When we get rid of addinfourl, we must ensure that bzr
479
# # always use titled headers and that any header received
480
# # from server is also titled.
483
# for header, value in (response.getheaders()):
484
# headers[header.title()] = value
485
# # FIXME: Implements a secured .read method
486
# response.code = response.status
487
# response.headers = headers
491
class HTTPHandler(AbstractHTTPHandler):
492
"""A custom handler that just thunks into HTTPConnection"""
494
def http_open(self, request):
495
return self.do_open(HTTPConnection, request)
498
class HTTPSHandler(AbstractHTTPHandler):
499
"""A custom handler that just thunks into HTTPSConnection"""
501
def https_open(self, request):
502
return self.do_open(HTTPSConnection, request)
505
class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
506
"""Handles redirect requests.
508
We have to implement our own scheme because we use a specific
509
Request object and because we want to implement a specific
513
# RFC2616 says that only read requests should be redirected
514
# without interacting with the user. But bzr use some
515
# shortcuts to optimize against roundtrips which can leads to
516
# write requests being issued before read requests of
517
# containing dirs can be redirected. So we redirect write
518
# requests in the same way which seems to respect the spirit
519
# of the RFC if not its letter.
521
def redirect_request(self, req, fp, code, msg, headers, newurl):
522
"""See urllib2.HTTPRedirectHandler.redirect_request"""
523
# We would have preferred to update the request instead
524
# of creating a new one, but the urllib2.Request object
525
# has a too complicated creation process to provide a
526
# simple enough equivalent update process. Instead, when
527
# redirecting, we only update the following request in
528
# the redirect chain with a reference to the parent
531
# Some codes make no sense in our context and are treated
534
# 300: Multiple choices for different representations of
535
# the URI. Using that mechanisn with bzr will violate the
536
# protocol neutrality of Transport.
538
# 304: Not modified (SHOULD only occurs with conditional
539
# GETs which are not used by our implementation)
541
# 305: Use proxy. I can't imagine this one occurring in
542
# our context-- vila/20060909
544
# 306: Unused (if the RFC says so...)
546
# If the code is 302 and the request is HEAD, some may
547
# think that it is a sufficent hint that the file exists
548
# and that we MAY avoid following the redirections. But
549
# if we want to be sure, we MUST follow them.
551
if code in (301, 302, 303, 307):
552
return Request(req.get_method(),newurl,
553
headers = req.headers,
554
origin_req_host = req.get_origin_req_host(),
556
# TODO: It will be nice to be able to
557
# detect virtual hosts sharing the same
558
# IP address, that will allow us to
559
# share the same connection...
564
raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)
566
def http_error_302(self, req, fp, code, msg, headers):
567
"""Requests the redirected to URI.
569
Copied from urllib2 to be able to fake_close the
570
associated connection, *before* issuing the redirected
571
request but *after* having eventually raised an error.
573
# Some servers (incorrectly) return multiple Location headers
574
# (so probably same goes for URI). Use first header.
576
# TODO: Once we get rid of addinfourl objects, the
577
# following will need to be updated to use correct case
579
if 'location' in headers:
580
newurl = headers.getheaders('location')[0]
581
elif 'uri' in headers:
582
newurl = headers.getheaders('uri')[0]
585
if self._debuglevel > 0:
586
print 'Redirected to: %s (followed: %r)' % (newurl,
587
req.follow_redirections)
588
if req.follow_redirections is False:
589
req.redirected_to = newurl
592
newurl = urlparse.urljoin(req.get_full_url(), newurl)
594
# This call succeeds or raise an error. urllib2 returns
595
# if redirect_request returns None, but our
596
# redirect_request never returns None.
597
redirected_req = self.redirect_request(req, fp, code, msg, headers,
601
# .redirect_dict has a key url if url was previously visited.
602
if hasattr(req, 'redirect_dict'):
603
visited = redirected_req.redirect_dict = req.redirect_dict
604
if (visited.get(newurl, 0) >= self.max_repeats or
605
len(visited) >= self.max_redirections):
606
raise urllib2.HTTPError(req.get_full_url(), code,
607
self.inf_msg + msg, headers, fp)
609
visited = redirected_req.redirect_dict = req.redirect_dict = {}
610
visited[newurl] = visited.get(newurl, 0) + 1
612
# We can close the fp now that we are sure that we won't
613
# use it with HTTPError.
615
# We have all we need already in the response
616
req.connection.fake_close()
618
return self.parent.open(redirected_req)
620
http_error_301 = http_error_303 = http_error_307 = http_error_302
623
class ProxyHandler(urllib2.ProxyHandler):
624
"""Handles proxy setting.
626
Copied and modified from urllib2 to be able to modify the
627
request during the request pre-processing instead of
628
modifying it at _open time. As we capture (or create) the
629
connection object during request processing, _open time was
632
Note that the proxy handling *may* modify the protocol used;
633
the request may be against an https server proxied through an
634
http proxy. So, https_request will be called, but later it's
635
really http_open that will be called. This explain why we
636
don't have to call self.parent.open as the urllib2 did.
639
# Proxies must be in front
643
def __init__(self, password_manager, proxies=None):
644
urllib2.ProxyHandler.__init__(self, proxies)
645
self.password_manager = password_manager
646
# First, let's get rid of urllib2 implementation
647
for type, proxy in self.proxies.items():
648
if self._debuglevel > 0:
649
print 'Will unbind %s_open for %r' % (type, proxy)
650
delattr(self, '%s_open' % type)
652
# We are interested only by the http[s] proxies
653
http_proxy = self.get_proxy_env_var('http')
654
https_proxy = self.get_proxy_env_var('https')
656
if http_proxy is not None:
657
if self._debuglevel > 0:
658
print 'Will bind http_request for %r' % http_proxy
659
setattr(self, 'http_request',
660
lambda request: self.set_proxy(request, 'http'))
662
if https_proxy is not None:
663
if self._debuglevel > 0:
664
print 'Will bind http_request for %r' % https_proxy
665
setattr(self, 'https_request',
666
lambda request: self.set_proxy(request, 'https'))
668
def get_proxy_env_var(self, name, default_to='all'):
669
"""Get a proxy env var.
671
Note that we indirectly rely on
672
urllib.getproxies_environment taking into account the
673
uppercased values for proxy variables.
676
return self.proxies[name.lower()]
678
if default_to is not None:
679
# Try to get the alternate environment variable
681
return self.proxies[default_to]
686
def proxy_bypass(self, host):
687
"""Check if host should be proxied or not"""
688
no_proxy = self.get_proxy_env_var('no', None)
691
hhost, hport = urllib.splitport(host)
692
# Does host match any of the domains mentioned in
693
# no_proxy ? The rules about what is authorized in no_proxy
694
# are fuzzy (to say the least). We try to allow most
695
# commonly seen values.
696
for domain in no_proxy.split(','):
697
dhost, dport = urllib.splitport(domain)
698
if hport == dport or dport is None:
700
dhost = dhost.replace(".", r"\.")
701
dhost = dhost.replace("*", r".*")
702
dhost = dhost.replace("?", r".")
703
if re.match(dhost, hhost, re.IGNORECASE):
705
# Nevertheless, there are platform-specific ways to
707
return urllib.proxy_bypass(host)
709
def set_proxy(self, request, type):
710
if self.proxy_bypass(request.get_host()):
713
proxy = self.get_proxy_env_var(type)
714
if self._debuglevel > 0:
715
print 'set_proxy %s_request for %r' % (type, proxy)
716
# Extract credentials from the url and store them in the
717
# password manager so that the proxy AuthHandler can use
719
proxy, user, password = extract_credentials(proxy)
720
if request.proxy_auth == {}:
721
# No proxy auth parameter are available, we are
722
# handling the first proxied request, intialize.
723
# scheme and realm will be set by the AuthHandler
724
authuri = extract_authentication_uri(proxy)
725
request.proxy_auth = {'user': user, 'password': password,
727
if user and password is not None: # '' is a valid password
728
# We default to a realm of None to catch them all.
729
self.password_manager.add_password(None, authuri,
731
orig_type = request.get_type()
732
scheme, r_scheme = urllib.splittype(proxy)
733
if self._debuglevel > 0:
734
print 'scheme: %s, r_scheme: %s' % (scheme, r_scheme)
735
host, XXX = urllib.splithost(r_scheme)
737
raise errors.InvalidURL(proxy,
738
'Invalid syntax in proxy env variable')
739
host = urllib.unquote(host)
740
request.set_proxy(host, type)
741
if self._debuglevel > 0:
742
print 'set_proxy: proxy set to %s://%s' % (type, host)
746
class AbstractAuthHandler(urllib2.BaseHandler):
747
"""A custom abstract authentication handler for all http authentications.
749
Provides the meat to handle authentication errors and
750
preventively set authentication headers after the first
751
successful authentication.
753
This can be used for http and proxy, as well as for basic and
754
digest authentications.
756
This provides an unified interface for all authentication handlers
757
(urllib2 provides far too many with different policies).
759
The interaction between this handler and the urllib2
760
framework is not obvious, it works as follow:
762
opener.open(request) is called:
764
- that may trigger http_request which will add an authentication header
765
(self.build_header) if enough info is available.
767
- the request is sent to the server,
769
- if an authentication error is received self.auth_required is called,
770
we acquire the authentication info in the error headers and call
771
self.auth_match to check that we are able to try the
772
authentication and complete the authentication parameters,
774
- we call parent.open(request), that may trigger http_request
775
and will add a header (self.build_header), but here we have
776
all the required info (keep in mind that the request and
777
authentication used in the recursive calls are really (and must be)
780
- if the call returns a response, the authentication have been
781
successful and the request authentication parameters have been updated.
784
# The following attributes should be defined by daughter
786
# - auth_required_header: the header received from the server
787
# - auth_header: the header sent in the request
789
def __init__(self, password_manager):
790
self.password_manager = password_manager
791
self.find_user_password = password_manager.find_user_password
792
self.add_password = password_manager.add_password
794
def update_auth(self, auth, key, value):
795
"""Update a value in auth marking the auth as modified if needed"""
796
old_value = auth.get(key, None)
797
if old_value != value:
799
auth['modified'] = True
801
def auth_required(self, request, headers):
802
"""Retry the request if the auth scheme is ours.
804
:param request: The request needing authentication.
805
:param headers: The headers for the authentication error response.
806
:return: None or the response for the authenticated request.
808
server_header = headers.get(self.auth_required_header, None)
809
if server_header is None:
810
# The http error MUST have the associated
811
# header. This must never happen in production code.
812
raise KeyError('%s not found' % self.auth_required_header)
814
auth = self.get_auth(request)
815
if auth.get('user', None) is None:
816
# Without a known user, we can't authenticate
819
auth['modified'] = False
820
if self.auth_match(server_header, auth):
821
# auth_match may have modified auth (by adding the
822
# password or changing the realm, for example)
823
if request.get_header(self.auth_header, None) is not None \
824
and not auth['modified']:
825
# We already tried that, give up
828
response = self.parent.open(request)
830
self.auth_successful(request, response)
832
# We are not qualified to handle the authentication.
833
# Note: the authentication error handling will try all
834
# available handlers. If one of them authenticates
835
# successfully, a response will be returned. If none of
836
# them succeeds, None will be returned and the error
837
# handler will raise the 401 'Unauthorized' or the 407
838
# 'Proxy Authentication Required' error.
841
def add_auth_header(self, request, header):
842
"""Add the authentication header to the request"""
843
request.add_unredirected_header(self.auth_header, header)
845
def auth_match(self, header, auth):
846
"""Check that we are able to handle that authentication scheme.
848
The request authentication parameters may need to be
849
updated with info from the server. Some of these
850
parameters, when combined, are considered to be the
851
authentication key, if one of them change the
852
authentication result may change. 'user' and 'password'
853
are exampls, but some auth schemes may have others
854
(digest's nonce is an example, digest's nonce_count is a
855
*counter-example*). Such parameters must be updated by
856
using the update_auth() method.
858
:param header: The authentication header sent by the server.
859
:param auth: The auth parameters already known. They may be
861
:returns: True if we can try to handle the authentication.
863
raise NotImplementedError(self.auth_match)
865
def build_auth_header(self, auth, request):
866
"""Build the value of the header used to authenticate.
868
:param auth: The auth parameters needed to build the header.
869
:param request: The request needing authentication.
871
:return: None or header.
873
raise NotImplementedError(self.build_auth_header)
875
def auth_successful(self, request, response):
876
"""The authentification was successful for the request.
878
Additional infos may be available in the response.
880
:param request: The succesfully authenticated request.
881
:param response: The server response (may contain auth info).
885
def get_password(self, user, authuri, realm=None):
886
"""Ask user for a password if none is already available."""
887
user_found, password = self.find_user_password(realm, authuri)
888
if user_found != user:
889
# FIXME: write a test for that case
893
# Prompt user only if we can't find a password
895
realm_prompt = " Realm: '%s'" % realm
898
scheme, host, path, query, fragment = urlparse.urlsplit(authuri)
899
password = ui.ui_factory.get_password(prompt=self.password_prompt,
900
user=user, host=host,
902
if password is not None:
903
self.add_password(realm, authuri, user, password)
906
def http_request(self, request):
907
"""Insert an authentication header if information is available"""
908
auth = self.get_auth(request)
909
if self.auth_params_reusable(auth):
910
self.add_auth_header(request, self.build_auth_header(auth, request))
913
https_request = http_request # FIXME: Need test
916
class BasicAuthHandler(AbstractAuthHandler):
917
"""A custom basic authentication handler."""
921
auth_regexp = re.compile('realm="([^"]*)"', re.I)
923
def build_auth_header(self, auth, request):
924
raw = '%s:%s' % (auth['user'], auth['password'])
925
auth_header = 'Basic ' + raw.encode('base64').strip()
928
def auth_match(self, header, auth):
929
scheme, raw_auth = header.split(None, 1)
930
scheme = scheme.lower()
931
if scheme != 'basic':
934
match = self.auth_regexp.search(raw_auth)
936
realm = match.groups()
937
if scheme != 'basic':
940
# Put useful info into auth
941
self.update_auth(auth, 'scheme', scheme)
942
self.update_auth(auth, 'realm', realm)
943
if auth.get('password',None) is None:
944
password = self.get_password(auth['user'], auth['authuri'],
946
self.update_auth(auth, 'password', password)
947
return match is not None
949
def auth_params_reusable(self, auth):
950
# If the auth scheme is known, it means a previous
951
# authentication was successful, all information is
952
# available, no further checks are needed.
953
return auth.get('scheme', None) == 'basic'
956
def get_digest_algorithm_impls(algorithm):
959
if algorithm == 'MD5':
960
H = lambda x: md5.new(x).hexdigest()
961
elif algorithm == 'SHA':
962
H = lambda x: sha.new(x).hexdigest()
964
KD = lambda secret, data: H("%s:%s" % (secret, data))
968
def get_new_cnonce(nonce, nonce_count):
969
raw = '%s:%d:%s:%s' % (nonce, nonce_count, time.ctime(),
970
urllib2.randombytes(8))
971
return sha.new(raw).hexdigest()[:16]
974
class DigestAuthHandler(AbstractAuthHandler):
975
"""A custom digest authentication handler."""
977
# Before basic as digest is a bit more secure
980
def auth_params_reusable(self, auth):
981
# If the auth scheme is known, it means a previous
982
# authentication was successful, all information is
983
# available, no further checks are needed.
984
return auth.get('scheme', None) == 'digest'
986
def auth_match(self, header, auth):
987
scheme, raw_auth = header.split(None, 1)
988
scheme = scheme.lower()
989
if scheme != 'digest':
992
# Put the requested authentication info into a dict
993
req_auth = urllib2.parse_keqv_list(urllib2.parse_http_list(raw_auth))
995
# Check that we can handle that authentication
996
qop = req_auth.get('qop', None)
997
if qop != 'auth': # No auth-int so far
1000
H, KD = get_digest_algorithm_impls(req_auth.get('algorithm', 'MD5'))
1004
realm = req_auth.get('realm', None)
1005
if auth.get('password',None) is None:
1006
auth['password'] = self.get_password(auth['user'],
1009
# Put useful info into auth
1011
self.update_auth(auth, 'scheme', scheme)
1012
if req_auth.get('algorithm', None) is not None:
1013
self.update_auth(auth, 'algorithm', req_auth.get('algorithm'))
1014
self.update_auth(auth, 'realm', realm)
1015
nonce = req_auth['nonce']
1016
if auth.get('nonce', None) != nonce:
1017
# A new nonce, never used
1018
self.update_auth(auth, 'nonce_count', 0)
1019
self.update_auth(auth, 'nonce', nonce)
1020
self.update_auth(auth, 'qop', qop)
1021
auth['opaque'] = req_auth.get('opaque', None)
1023
# Some required field is not there
1028
def build_auth_header(self, auth, request):
1029
url_scheme, url_selector = urllib.splittype(request.get_selector())
1030
sel_host, uri = urllib.splithost(url_selector)
1032
A1 = '%s:%s:%s' % (auth['user'], auth['realm'], auth['password'])
1033
A2 = '%s:%s' % (request.get_method(), uri)
1035
nonce = auth['nonce']
1038
nonce_count = auth['nonce_count'] + 1
1039
ncvalue = '%08x' % nonce_count
1040
cnonce = get_new_cnonce(nonce, nonce_count)
1042
H, KD = get_digest_algorithm_impls(auth.get('algorithm', 'MD5'))
1043
nonce_data = '%s:%s:%s:%s:%s' % (nonce, ncvalue, cnonce, qop, H(A2))
1044
request_digest = KD(H(A1), nonce_data)
1047
header += 'username="%s", realm="%s", nonce="%s"' % (auth['user'],
1050
header += ', uri="%s"' % uri
1051
header += ', cnonce="%s", nc=%s' % (cnonce, ncvalue)
1052
header += ', qop="%s"' % qop
1053
header += ', response="%s"' % request_digest
1054
# Append the optional fields
1055
opaque = auth.get('opaque', None)
1057
header += ', opaque="%s"' % opaque
1058
if auth.get('algorithm', None):
1059
header += ', algorithm="%s"' % auth.get('algorithm')
1061
# We have used the nonce once more, update the count
1062
auth['nonce_count'] = nonce_count
1067
class HTTPAuthHandler(AbstractAuthHandler):
1068
"""Custom http authentication handler.
1070
Send the authentication preventively to avoid the roundtrip
1071
associated with the 401 error and keep the revelant info in
1072
the auth request attribute.
1075
password_prompt = 'HTTP %(user)s@%(host)s%(realm)s password'
1076
auth_required_header = 'www-authenticate'
1077
auth_header = 'Authorization'
1079
def get_auth(self, request):
1080
"""Get the auth params from the request"""
1083
def set_auth(self, request, auth):
1084
"""Set the auth params for the request"""
1087
def http_error_401(self, req, fp, code, msg, headers):
1088
return self.auth_required(req, headers)
1091
class ProxyAuthHandler(AbstractAuthHandler):
1092
"""Custom proxy authentication handler.
1094
Send the authentication preventively to avoid the roundtrip
1095
associated with the 407 error and keep the revelant info in
1096
the proxy_auth request attribute..
1099
password_prompt = 'Proxy %(user)s@%(host)s%(realm)s password'
1100
auth_required_header = 'proxy-authenticate'
1101
# FIXME: the correct capitalization is Proxy-Authorization,
1102
# but python-2.4 urllib2.Request insist on using capitalize()
1103
# instead of title().
1104
auth_header = 'Proxy-authorization'
1106
def get_auth(self, request):
1107
"""Get the auth params from the request"""
1108
return request.proxy_auth
1110
def set_auth(self, request, auth):
1111
"""Set the auth params for the request"""
1112
request.proxy_auth = auth
1114
def http_error_407(self, req, fp, code, msg, headers):
1115
return self.auth_required(req, headers)
1118
class HTTPBasicAuthHandler(BasicAuthHandler, HTTPAuthHandler):
1119
"""Custom http basic authentication handler"""
1122
class ProxyBasicAuthHandler(BasicAuthHandler, ProxyAuthHandler):
1123
"""Custom proxy basic authentication handler"""
1126
class HTTPDigestAuthHandler(DigestAuthHandler, HTTPAuthHandler):
1127
"""Custom http basic authentication handler"""
1130
class ProxyDigestAuthHandler(DigestAuthHandler, ProxyAuthHandler):
1131
"""Custom proxy basic authentication handler"""
1134
class HTTPErrorProcessor(urllib2.HTTPErrorProcessor):
1135
"""Process HTTP error responses.
1137
We don't really process the errors, quite the contrary
1138
instead, we leave our Transport handle them.
1141
def http_response(self, request, response):
1142
code, msg, hdrs = response.code, response.msg, response.info()
1144
if code not in (200, # Ok
1145
206, # Partial content
1148
response = self.parent.error('http', request, response,
1152
https_response = http_response
1155
class HTTPDefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
1156
"""Translate common errors into bzr Exceptions"""
1158
def http_error_default(self, req, fp, code, msg, hdrs):
1160
raise errors.NoSuchFile(req.get_selector(),
1161
extra=HTTPError(req.get_full_url(),
1165
raise errors.TransportError('Server refuses to fullfil the request')
1167
# We don't know which, but one of the ranges we
1168
# specified was wrong. So we raise with 0 for a lack
1169
# of a better magic value.
1170
raise errors.InvalidRange(req.get_full_url(),0)
1172
raise errors.InvalidHttpResponse(req.get_full_url(),
1173
'Unable to handle http code %d: %s'
1176
class Opener(object):
1177
"""A wrapper around urllib2.build_opener
1179
Daughter classes can override to build their own specific opener
1181
# TODO: Provides hooks for daughter classes.
1184
connection=ConnectionHandler,
1185
redirect=HTTPRedirectHandler,
1186
error=HTTPErrorProcessor,):
1187
self.password_manager = PasswordManager()
1188
self._opener = urllib2.build_opener( \
1189
connection, redirect, error,
1190
ProxyHandler(self.password_manager),
1191
HTTPBasicAuthHandler(self.password_manager),
1192
HTTPDigestAuthHandler(self.password_manager),
1193
ProxyBasicAuthHandler(self.password_manager),
1194
ProxyDigestAuthHandler(self.password_manager),
1197
HTTPDefaultErrorHandler,
1199
self.open = self._opener.open
1201
# When dealing with handler order, it's easy to mess
1202
# things up, the following will help understand which
1203
# handler is used, when and for what.
1205
pprint.pprint(self._opener.__dict__)
added
removed
bzrlib/transport/http/_urllib2_wrappers.py
