« back to all changes in this revision
Viewing changes to bzrlib/transport/http/_urllib2_wrappers.py
- browse files at revision 2463
- compare with another revision
- compare with revision 1717
- download diff
- download tarball
- view history from revision 2463
- Committer: Canonical.com Patch Queue Manager
- Date: 2007-04-26 10:41:48 UTC
- mfrom: (2420.1.22 bzr.http.auth)
- Revision ID: pqm@pqm.ubuntu.com-20070426104148-4l5wq2zemlzv0shg
http authentication and other integrated fixes
- files modified:
- NEWS
added
removed
bzrlib/transport/http/_urllib2_wrappers.py
16
16
17
17
"""Implementaion of urllib2 tailored to bzr needs
18
18
19
This file re-implements the urllib2 class hierarchy with custom classes.
19
This file complements the urllib2 class hierarchy with custom classes.
20
20
21
21
For instance, we create a new HTTPConnection and HTTPSConnection that inherit
22
22
from the original urllib2.HTTP(s)Connection objects, but also have a new base
28
28
We have a custom Response class, which lets us maintain a keep-alive
29
29
connection even for requests that urllib2 doesn't expect to contain body data.
30
30
31
And a custom Request class that lets us track redirections, and send
32
authentication data without requiring an extra round trip to get rejected by
33
the server. We also create a Request hierarchy, to make it clear what type
34
of request is being made.
31
And a custom Request class that lets us track redirections, and
32
handle authentication schemes.
35
33
"""
36
34
37
35
DEBUG = 0
50
48
# ensure that.
51
49
52
50
import httplib
51
import md5
52
import sha
53
53
import socket
54
54
import urllib
55
55
import urllib2
56
56
import urlparse
57
57
import re
58
58
import sys
59
import time
59
60
60
61
from bzrlib import __version__ as bzrlib_version
61
from bzrlib import errors
62
from bzrlib import (
63
errors,
64
ui,
65
)
66
62
67
63
68
64
69
# We define our own Response class to keep our httplib pipe clean
139
144
the presence or absence of data). We set the method
140
145
statically.
141
146
142
Also, the Request object tracks the connection the request will
143
be made on.
147
The Request object tracks:
148
- the connection the request will be made on.
149
- the authentication parameters needed to preventively set
150
the authentication header once a first authentication have
151
been made.
144
152
"""
145
153
146
154
def __init__(self, method, url, data=None, headers={},
147
155
origin_req_host=None, unverifiable=False,
148
156
connection=None, parent=None,):
149
# urllib2.Request will be confused if we don't extract
150
# authentification info before building the request
151
url, self.user, self.password = self.extract_auth(url)
152
157
urllib2.Request.__init__(self, url, data, headers,
153
158
origin_req_host, unverifiable)
154
159
self.method = method
158
163
self.redirected_to = None
159
164
# Unless told otherwise, redirections are not followed
160
165
self.follow_redirections = False
161
162
def extract_auth(self, url):
163
"""Extracts authentification information from url.
164
165
Get user and password from url of the form: http://user:pass@host/path
166
"""
167
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
168
169
if '@' in netloc:
170
auth, netloc = netloc.split('@', 1)
171
if ':' in auth:
172
user, password = auth.split(':', 1)
173
else:
174
user, password = auth, None
175
user = urllib.unquote(user)
176
if password is not None:
177
password = urllib.unquote(password)
178
else:
179
user = None
180
password = None
181
182
url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
183
184
return url, user, password
166
# auth and proxy_auth are dicts containing, at least
167
# (scheme, url, realm, user, password).
168
# The dict entries are mostly handled by the AuthHandler.
169
# Some authentication schemes may add more entries.
170
self.auth = {}
171
self.proxy_auth = {}
185
172
186
173
def get_method(self):
187
174
return self.method
188
175
189
176
190
# The urlib2.xxxAuthHandler handle the authentification of the
177
def extract_credentials(url):
178
"""Extracts credentials information from url.
179
180
Get user and password from url of the form: http://user:pass@host/path
181
:returns: (clean_url, user, password)
182
"""
183
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
184
185
if '@' in netloc:
186
auth, netloc = netloc.split('@', 1)
187
if ':' in auth:
188
user, password = auth.split(':', 1)
189
else:
190
user, password = auth, None
191
user = urllib.unquote(user)
192
if password is not None:
193
password = urllib.unquote(password)
194
else:
195
user = None
196
password = None
197
198
# Build the clean url
199
clean_url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
200
201
return clean_url, user, password
202
203
def extract_authentication_uri(url):
204
"""Extract the authentication uri from any url.
205
206
In the context of bzr, we simplified the authentication uri
207
to the host only. For the transport lifetime, we allow only
208
one user by realm on a given host. I.e. handling several
209
users for different paths for the same realm should be done
210
at a higher level.
211
"""
212
scheme, host, path, query, fragment = urlparse.urlsplit(url)
213
return '%s://%s' % (scheme, host)
214
215
216
# The urlib2.xxxAuthHandler handle the authentication of the
191
217
# requests, to do that, they need an urllib2 PasswordManager *at
192
# build time*. We also need one to reuse the passwords already
193
# typed by the user.
218
# build time*. We also need one to reuse the passwords entered by
219
# the user.
194
220
class PasswordManager(urllib2.HTTPPasswordMgrWithDefaultRealm):
195
221
196
222
def __init__(self):
204
230
internally used. But we need it in order to achieve
205
231
connection sharing. So, we add it to the request just before
206
232
it is processed, and then we override the do_open method for
207
http[s] requests.
233
http[s] requests in AbstractHTTPHandler.
208
234
"""
209
235
210
236
handler_order = 1000 # after all pre-processings
211
237
212
def get_key(self, connection):
213
"""Returns the key for the connection in the cache"""
214
return '%s:%d' % (connection.host, connection.port)
215
216
238
def create_connection(self, request, http_connection_class):
217
239
host = request.get_host()
218
240
if not host:
282
304
# urllib2 using capitalize() for headers
283
305
# instead of title(sp?).
284
306
'User-agent': 'bzr/%s (urllib)' % bzrlib_version,
285
# FIXME: pycurl also set the following, understand why
286
307
'Accept': '*/*',
287
308
}
288
309
619
640
handler_order = 100
620
641
_debuglevel = DEBUG
621
642
622
def __init__(self, proxies=None):
643
def __init__(self, password_manager, proxies=None):
623
644
urllib2.ProxyHandler.__init__(self, proxies)
645
self.password_manager = password_manager
624
646
# First, let's get rid of urllib2 implementation
625
647
for type, proxy in self.proxies.items():
626
648
if self._debuglevel > 0:
691
713
proxy = self.get_proxy_env_var(type)
692
714
if self._debuglevel > 0:
693
715
print 'set_proxy %s_request for %r' % (type, proxy)
716
# Extract credentials from the url and store them in the
717
# password manager so that the proxy AuthHandler can use
718
# them later.
719
proxy, user, password = extract_credentials(proxy)
720
if request.proxy_auth == {}:
721
# No proxy auth parameter are available, we are
722
# handling the first proxied request, intialize.
723
# scheme and realm will be set by the AuthHandler
724
authuri = extract_authentication_uri(proxy)
725
request.proxy_auth = {'user': user, 'password': password,
726
'authuri': authuri}
727
if user and password is not None: # '' is a valid password
728
# We default to a realm of None to catch them all.
729
self.password_manager.add_password(None, authuri,
730
user, password)
694
731
orig_type = request.get_type()
695
732
scheme, r_scheme = urllib.splittype(proxy)
696
733
if self._debuglevel > 0:
699
736
if host is None:
700
737
raise errors.InvalidURL(proxy,
701
738
'Invalid syntax in proxy env variable')
702
elif '@' in host:
703
user_pass, host = host.split('@', 1)
704
if ':' in user_pass:
705
user, password = user_pass.split(':', 1)
706
else:
707
user = user_pass
708
password = ''
709
user_pass = '%s:%s' % (urllib.unquote(user),
710
urllib.unquote(password))
711
user_pass = user_pass.encode('base64').strip()
712
request.add_header('Proxy-authorization', 'Basic ' + user_pass)
713
739
host = urllib.unquote(host)
714
740
request.set_proxy(host, type)
715
741
if self._debuglevel > 0:
717
743
return request
718
744
719
745
720
class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
721
"""Custom basic authentification handler.
722
723
Send the authentification preventively to avoid the the
724
roundtrip associated with the 401 error.
725
"""
726
727
# def http_request(self, request):
728
# """Insert an authentification header if information is available"""
729
# if request.auth == 'basic' and request.password is not None:
730
#
731
# return request
746
class AbstractAuthHandler(urllib2.BaseHandler):
747
"""A custom abstract authentication handler for all http authentications.
748
749
Provides the meat to handle authentication errors and
750
preventively set authentication headers after the first
751
successful authentication.
752
753
This can be used for http and proxy, as well as for basic and
754
digest authentications.
755
756
This provides an unified interface for all authentication handlers
757
(urllib2 provides far too many with different policies).
758
759
The interaction between this handler and the urllib2
760
framework is not obvious, it works as follow:
761
762
opener.open(request) is called:
763
764
- that may trigger http_request which will add an authentication header
765
(self.build_header) if enough info is available.
766
767
- the request is sent to the server,
768
769
- if an authentication error is received self.auth_required is called,
770
we acquire the authentication info in the error headers and call
771
self.auth_match to check that we are able to try the
772
authentication and complete the authentication parameters,
773
774
- we call parent.open(request), that may trigger http_request
775
and will add a header (self.build_header), but here we have
776
all the required info (keep in mind that the request and
777
authentication used in the recursive calls are really (and must be)
778
the *same* objects).
779
780
- if the call returns a response, the authentication have been
781
successful and the request authentication parameters have been updated.
782
"""
783
784
# The following attributes should be defined by daughter
785
# classes:
786
# - auth_required_header: the header received from the server
787
# - auth_header: the header sent in the request
788
789
def __init__(self, password_manager):
790
self.password_manager = password_manager
791
self.find_user_password = password_manager.find_user_password
792
self.add_password = password_manager.add_password
793
794
def update_auth(self, auth, key, value):
795
"""Update a value in auth marking the auth as modified if needed"""
796
old_value = auth.get(key, None)
797
if old_value != value:
798
auth[key] = value
799
auth['modified'] = True
800
801
def auth_required(self, request, headers):
802
"""Retry the request if the auth scheme is ours.
803
804
:param request: The request needing authentication.
805
:param headers: The headers for the authentication error response.
806
:return: None or the response for the authenticated request.
807
"""
808
server_header = headers.get(self.auth_required_header, None)
809
if server_header is None:
810
# The http error MUST have the associated
811
# header. This must never happen in production code.
812
raise KeyError('%s not found' % self.auth_required_header)
813
814
auth = self.get_auth(request)
815
if auth.get('user', None) is None:
816
# Without a known user, we can't authenticate
817
return None
818
819
auth['modified'] = False
820
if self.auth_match(server_header, auth):
821
# auth_match may have modified auth (by adding the
822
# password or changing the realm, for example)
823
if request.get_header(self.auth_header, None) is not None \
824
and not auth['modified']:
825
# We already tried that, give up
826
return None
827
828
response = self.parent.open(request)
829
if response:
830
self.auth_successful(request, response)
831
return response
832
# We are not qualified to handle the authentication.
833
# Note: the authentication error handling will try all
834
# available handlers. If one of them authenticates
835
# successfully, a response will be returned. If none of
836
# them succeeds, None will be returned and the error
837
# handler will raise the 401 'Unauthorized' or the 407
838
# 'Proxy Authentication Required' error.
839
return None
840
841
def add_auth_header(self, request, header):
842
"""Add the authentication header to the request"""
843
request.add_unredirected_header(self.auth_header, header)
844
845
def auth_match(self, header, auth):
846
"""Check that we are able to handle that authentication scheme.
847
848
The request authentication parameters may need to be
849
updated with info from the server. Some of these
850
parameters, when combined, are considered to be the
851
authentication key, if one of them change the
852
authentication result may change. 'user' and 'password'
853
are exampls, but some auth schemes may have others
854
(digest's nonce is an example, digest's nonce_count is a
855
*counter-example*). Such parameters must be updated by
856
using the update_auth() method.
857
858
:param header: The authentication header sent by the server.
859
:param auth: The auth parameters already known. They may be
860
updated.
861
:returns: True if we can try to handle the authentication.
862
"""
863
raise NotImplementedError(self.auth_match)
864
865
def build_auth_header(self, auth, request):
866
"""Build the value of the header used to authenticate.
867
868
:param auth: The auth parameters needed to build the header.
869
:param request: The request needing authentication.
870
871
:return: None or header.
872
"""
873
raise NotImplementedError(self.build_auth_header)
874
875
def auth_successful(self, request, response):
876
"""The authentification was successful for the request.
877
878
Additional infos may be available in the response.
879
880
:param request: The succesfully authenticated request.
881
:param response: The server response (may contain auth info).
882
"""
883
pass
884
885
def get_password(self, user, authuri, realm=None):
886
"""Ask user for a password if none is already available."""
887
user_found, password = self.find_user_password(realm, authuri)
888
if user_found != user:
889
# FIXME: write a test for that case
890
password = None
891
892
if password is None:
893
# Prompt user only if we can't find a password
894
if realm:
895
realm_prompt = " Realm: '%s'" % realm
896
else:
897
realm_prompt = ''
898
scheme, host, path, query, fragment = urlparse.urlsplit(authuri)
899
password = ui.ui_factory.get_password(prompt=self.password_prompt,
900
user=user, host=host,
901
realm=realm_prompt)
902
if password is not None:
903
self.add_password(realm, authuri, user, password)
904
return password
905
906
def http_request(self, request):
907
"""Insert an authentication header if information is available"""
908
auth = self.get_auth(request)
909
if self.auth_params_reusable(auth):
910
self.add_auth_header(request, self.build_auth_header(auth, request))
911
return request
912
913
https_request = http_request # FIXME: Need test
914
915
916
class BasicAuthHandler(AbstractAuthHandler):
917
"""A custom basic authentication handler."""
918
919
auth_regexp = re.compile('realm="([^"]*)"', re.I)
920
921
def build_auth_header(self, auth, request):
922
raw = '%s:%s' % (auth['user'], auth['password'])
923
auth_header = 'Basic ' + raw.encode('base64').strip()
924
return auth_header
925
926
def auth_match(self, header, auth):
927
scheme, raw_auth = header.split(None, 1)
928
scheme = scheme.lower()
929
if scheme != 'basic':
930
return False
931
932
match = self.auth_regexp.search(raw_auth)
933
if match:
934
realm = match.groups()
935
if scheme != 'basic':
936
return False
937
938
# Put useful info into auth
939
self.update_auth(auth, 'scheme', scheme)
940
self.update_auth(auth, 'realm', realm)
941
if auth.get('password',None) is None:
942
password = self.get_password(auth['user'], auth['authuri'],
943
auth['realm'])
944
self.update_auth(auth, 'password', password)
945
return match is not None
946
947
def auth_params_reusable(self, auth):
948
# If the auth scheme is known, it means a previous
949
# authentication was successful, all information is
950
# available, no further checks are needed.
951
return auth.get('scheme', None) == 'basic'
952
953
954
def get_digest_algorithm_impls(algorithm):
955
H = None
956
KD = None
957
if algorithm == 'MD5':
958
H = lambda x: md5.new(x).hexdigest()
959
elif algorithm == 'SHA':
960
H = lambda x: sha.new(x).hexdigest()
961
if H is not None:
962
KD = lambda secret, data: H("%s:%s" % (secret, data))
963
return H, KD
964
965
966
def get_new_cnonce(nonce, nonce_count):
967
raw = '%s:%d:%s:%s' % (nonce, nonce_count, time.ctime(),
968
urllib2.randombytes(8))
969
return sha.new(raw).hexdigest()[:16]
970
971
972
class DigestAuthHandler(AbstractAuthHandler):
973
"""A custom digest authentication handler."""
974
975
def auth_params_reusable(self, auth):
976
# If the auth scheme is known, it means a previous
977
# authentication was successful, all information is
978
# available, no further checks are needed.
979
return auth.get('scheme', None) == 'digest'
980
981
def auth_match(self, header, auth):
982
scheme, raw_auth = header.split(None, 1)
983
scheme = scheme.lower()
984
if scheme != 'digest':
985
return False
986
987
# Put the requested authentication info into a dict
988
req_auth = urllib2.parse_keqv_list(urllib2.parse_http_list(raw_auth))
989
990
# Check that we can handle that authentication
991
qop = req_auth.get('qop', None)
992
if qop != 'auth': # No auth-int so far
993
return False
994
995
H, KD = get_digest_algorithm_impls(req_auth.get('algorithm', 'MD5'))
996
if H is None:
997
return False
998
999
realm = req_auth.get('realm', None)
1000
if auth.get('password',None) is None:
1001
auth['password'] = self.get_password(auth['user'],
1002
auth['authuri'],
1003
realm)
1004
# Put useful info into auth
1005
try:
1006
self.update_auth(auth, 'scheme', scheme)
1007
if req_auth.get('algorithm', None) is not None:
1008
self.update_auth(auth, 'algorithm', req_auth.get('algorithm'))
1009
self.update_auth(auth, 'realm', realm)
1010
nonce = req_auth['nonce']
1011
if auth.get('nonce', None) != nonce:
1012
# A new nonce, never used
1013
self.update_auth(auth, 'nonce_count', 0)
1014
self.update_auth(auth, 'nonce', nonce)
1015
self.update_auth(auth, 'qop', qop)
1016
auth['opaque'] = req_auth.get('opaque', None)
1017
except KeyError:
1018
# Some required field is not there
1019
return False
1020
1021
return True
1022
1023
def build_auth_header(self, auth, request):
1024
url_scheme, url_selector = urllib.splittype(request.get_selector())
1025
sel_host, uri = urllib.splithost(url_selector)
1026
1027
A1 = '%s:%s:%s' % (auth['user'], auth['realm'], auth['password'])
1028
A2 = '%s:%s' % (request.get_method(), uri)
1029
1030
nonce = auth['nonce']
1031
qop = auth['qop']
1032
1033
nonce_count = auth['nonce_count'] + 1
1034
ncvalue = '%08x' % nonce_count
1035
cnonce = get_new_cnonce(nonce, nonce_count)
1036
1037
H, KD = get_digest_algorithm_impls(auth.get('algorithm', 'MD5'))
1038
nonce_data = '%s:%s:%s:%s:%s' % (nonce, ncvalue, cnonce, qop, H(A2))
1039
request_digest = KD(H(A1), nonce_data)
1040
1041
header = 'Digest '
1042
header += 'username="%s", realm="%s", nonce="%s"' % (auth['user'],
1043
auth['realm'],
1044
nonce)
1045
header += ', uri="%s"' % uri
1046
header += ', cnonce="%s", nc=%s' % (cnonce, ncvalue)
1047
header += ', qop="%s"' % qop
1048
header += ', response="%s"' % request_digest
1049
# Append the optional fields
1050
opaque = auth.get('opaque', None)
1051
if opaque:
1052
header += ', opaque="%s"' % opaque
1053
if auth.get('algorithm', None):
1054
header += ', algorithm="%s"' % auth.get('algorithm')
1055
1056
# We have used the nonce once more, update the count
1057
auth['nonce_count'] = nonce_count
1058
1059
return header
1060
1061
1062
class HTTPAuthHandler(AbstractAuthHandler):
1063
"""Custom http authentication handler.
1064
1065
Send the authentication preventively to avoid the roundtrip
1066
associated with the 401 error and keep the revelant info in
1067
the auth request attribute.
1068
"""
1069
1070
password_prompt = 'HTTP %(user)s@%(host)s%(realm)s password'
1071
auth_required_header = 'www-authenticate'
1072
auth_header = 'Authorization'
1073
1074
def get_auth(self, request):
1075
"""Get the auth params from the request"""
1076
return request.auth
1077
1078
def set_auth(self, request, auth):
1079
"""Set the auth params for the request"""
1080
request.auth = auth
1081
1082
def http_error_401(self, req, fp, code, msg, headers):
1083
return self.auth_required(req, headers)
1084
1085
1086
class ProxyAuthHandler(AbstractAuthHandler):
1087
"""Custom proxy authentication handler.
1088
1089
Send the authentication preventively to avoid the roundtrip
1090
associated with the 407 error and keep the revelant info in
1091
the proxy_auth request attribute..
1092
"""
1093
1094
password_prompt = 'Proxy %(user)s@%(host)s%(realm)s password'
1095
auth_required_header = 'proxy-authenticate'
1096
# FIXME: the correct capitalization is Proxy-Authorization,
1097
# but python-2.4 urllib2.Request insist on using capitalize()
1098
# instead of title().
1099
auth_header = 'Proxy-authorization'
1100
1101
def get_auth(self, request):
1102
"""Get the auth params from the request"""
1103
return request.proxy_auth
1104
1105
def set_auth(self, request, auth):
1106
"""Set the auth params for the request"""
1107
request.proxy_auth = auth
1108
1109
def http_error_407(self, req, fp, code, msg, headers):
1110
return self.auth_required(req, headers)
1111
1112
1113
class HTTPBasicAuthHandler(BasicAuthHandler, HTTPAuthHandler):
1114
"""Custom http basic authentication handler"""
1115
1116
1117
class ProxyBasicAuthHandler(BasicAuthHandler, ProxyAuthHandler):
1118
"""Custom proxy basic authentication handler"""
1119
1120
1121
class HTTPDigestAuthHandler(DigestAuthHandler, HTTPAuthHandler):
1122
"""Custom http basic authentication handler"""
1123
1124
1125
class ProxyDigestAuthHandler(DigestAuthHandler, ProxyAuthHandler):
1126
"""Custom proxy basic authentication handler"""
732
1127
733
1128
734
1129
class HTTPErrorProcessor(urllib2.HTTPErrorProcessor):
737
1132
We don't really process the errors, quite the contrary
738
1133
instead, we leave our Transport handle them.
739
1134
"""
740
handler_order = 1000 # after all other processing
741
1135
742
1136
def http_response(self, request, response):
743
1137
code, msg, hdrs = response.code, response.msg, response.info()
770
1164
# of a better magic value.
771
1165
raise errors.InvalidRange(req.get_full_url(),0)
772
1166
else:
773
# TODO: A test is needed to exercise that code path
774
1167
raise errors.InvalidHttpResponse(req.get_full_url(),
775
1168
'Unable to handle http code %d: %s'
776
1169
% (code, msg))
787
1180
redirect=HTTPRedirectHandler,
788
1181
error=HTTPErrorProcessor,):
789
1182
self.password_manager = PasswordManager()
790
# TODO: Implements the necessary wrappers for the handlers
791
# commented out below
792
1183
self._opener = urllib2.build_opener( \
793
1184
connection, redirect, error,
794
ProxyHandler,
795
urllib2.HTTPBasicAuthHandler(self.password_manager),
796
#urllib2.HTTPDigestAuthHandler(self.password_manager),
797
#urllib2.ProxyBasicAuthHandler,
798
#urllib2.ProxyDigestAuthHandler,
1185
ProxyHandler(self.password_manager),
1186
HTTPBasicAuthHandler(self.password_manager),
1187
HTTPDigestAuthHandler(self.password_manager),
1188
ProxyBasicAuthHandler(self.password_manager),
1189
ProxyDigestAuthHandler(self.password_manager),
799
1190
HTTPHandler,
800
1191
HTTPSHandler,
801
1192
HTTPDefaultErrorHandler,
807
1198
# handler is used, when and for what.
808
1199
import pprint
809
1200
pprint.pprint(self._opener.__dict__)
810
Loggerhead is a web-based interface for Breezy
Version: 2.0.1