77
_ = (ssl.match_hostname, ssl.CertificateError)
78
except AttributeError:
79
# Provide fallbacks for python < 2.7.9
80
def match_hostname(cert, host):
82
'%s cannot be verified, https certificates verification is only'
83
' available for python versions >= 2.7.9' % (host,))
84
ssl.match_hostname = match_hostname
85
ssl.CertificateError = ValueError
88
# Note for packagers: if there is no package providing certs for your platform,
89
# the curl project produces http://curl.haxx.se/ca/cacert.pem weekly.
90
_ssl_ca_certs_known_locations = [
91
u'/etc/ssl/certs/ca-certificates.crt', # Ubuntu/debian/gentoo
92
u'/etc/pki/tls/certs/ca-bundle.crt', # Fedora/CentOS/RH
93
u'/etc/ssl/ca-bundle.pem', # OpenSuse
94
u'/etc/ssl/cert.pem', # OpenSuse
95
u"/usr/local/share/certs/ca-root-nss.crt", # FreeBSD
96
# XXX: Needs checking, can't trust the interweb ;) -- vila 2012-01-25
97
u'/etc/openssl/certs/ca-certificates.crt', # Solaris
101
def default_ca_certs():
102
if sys.platform == 'win32':
103
return os.path.join(os.path.dirname(sys.executable), u"cacert.pem")
104
elif sys.platform == 'darwin':
105
# FIXME: Needs some default value for osx, waiting for osx installers
106
# guys feedback -- vila 2012-01-25
109
# Try known locations for friendly OSes providing the root certificates
110
# without making them hard to use for any https client.
111
for path in _ssl_ca_certs_known_locations:
112
if os.path.exists(path):
115
# A default path that makes sense and will be mentioned in the error
116
# presented to the user, even if not correct for all platforms
117
return _ssl_ca_certs_known_locations[0]
120
def ca_certs_from_store(path):
121
if not os.path.exists(path):
122
raise ValueError("ca certs path %s does not exist" % path)
126
def cert_reqs_from_store(unicode_str):
129
return {"required": ssl.CERT_REQUIRED,
130
"none": ssl.CERT_NONE}[unicode_str]
132
raise ValueError("invalid value %s" % unicode_str)
135
def default_ca_reqs():
136
if sys.platform in ('win32', 'darwin'):
137
# FIXME: Once we get a native access to root certificates there, this
138
# won't needed anymore. See http://pad.lv/920455 -- vila 2012-02-15
143
opt_ssl_ca_certs = config.Option('ssl.ca_certs',
144
from_unicode=ca_certs_from_store,
145
default=default_ca_certs,
148
Path to certification authority certificates to trust.
150
This should be a valid path to a bundle containing all root Certificate
151
Authorities used to verify an https server certificate.
153
Use ssl.cert_reqs=none to disable certificate verification.
156
opt_ssl_cert_reqs = config.Option('ssl.cert_reqs',
157
default=default_ca_reqs,
158
from_unicode=cert_reqs_from_store,
161
Whether to require a certificate from the remote side. (default:required)
164
* none: Certificates ignored
165
* required: Certificates required and validated
168
checked_kerberos = False
172
class addinfourl(urllib2.addinfourl):
173
'''Replacement addinfourl class compatible with python-2.7's xmlrpclib
175
In python-2.7, xmlrpclib expects that the response object that it receives
176
has a getheader method. httplib.HTTPResponse provides this but
177
urllib2.addinfourl does not. Add the necessary functions here, ported to
178
use the internal data structures of addinfourl.
181
def getheader(self, name, default=None):
182
if self.headers is None:
183
raise httplib.ResponseNotReady()
184
return self.headers.getheader(name, default)
186
def getheaders(self):
187
if self.headers is None:
188
raise httplib.ResponseNotReady()
189
return self.headers.items()
192
class _ReportingFileSocket(object):
194
def __init__(self, filesock, report_activity=None):
195
self.filesock = filesock
196
self._report_activity = report_activity
198
def report_activity(self, size, direction):
199
if self._report_activity:
200
self._report_activity(size, direction)
202
def read(self, size=1):
203
s = self.filesock.read(size)
204
self.report_activity(len(s), 'read')
207
def readline(self, size=-1):
208
s = self.filesock.readline(size)
209
self.report_activity(len(s), 'read')
212
def __getattr__(self, name):
213
return getattr(self.filesock, name)
216
class _ReportingSocket(object):
218
def __init__(self, sock, report_activity=None):
71
class _BufferedMakefileSocket(object):
73
def __init__(self, sock):
220
self._report_activity = report_activity
222
def report_activity(self, size, direction):
223
if self._report_activity:
224
self._report_activity(size, direction)
226
def sendall(self, s, *args):
227
self.sock.sendall(s, *args)
228
self.report_activity(len(s), 'write')
230
def recv(self, *args):
231
s = self.sock.recv(*args)
232
self.report_activity(len(s), 'read')
235
76
def makefile(self, mode='r', bufsize=-1):
236
# httplib creates a fileobject that doesn't do buffering, which
237
# makes fp.readline() very expensive because it only reads one byte
238
# at a time. So we wrap the socket in an object that forces
239
# sock.makefile to make a buffered file.
240
fsock = self.sock.makefile(mode, 65536)
241
# And wrap that into a reporting kind of fileobject
242
return _ReportingFileSocket(fsock, self._report_activity)
77
return self.sock.makefile(mode, 65536)
244
79
def __getattr__(self, name):
245
80
return getattr(self.sock, name)
355
197
def cleanup_pipe(self):
356
198
"""Read the remaining bytes of the last response if any."""
357
199
if self._response is not None:
359
pending = self._response.finish()
360
# Warn the user (once)
361
if (self._ranges_received_whole_file is None
362
and self._response.status == 200
363
and pending and pending > self._range_warning_thresold
365
self._ranges_received_whole_file = True
367
'Got a 200 response when asking for multiple ranges,'
368
' does your server at %s:%s support range requests?',
369
self.host, self.port)
370
except socket.error, e:
371
# It's conceivable that the socket is in a bad state here
372
# (including some test cases) and in this case, it doesn't need
373
# cleaning anymore, so no need to fail, we just get rid of the
374
# socket and let callers reconnect
376
or e.args[0] not in (errno.ECONNRESET, errno.ECONNABORTED)):
200
pending = self._response.finish()
201
# Warn the user (once)
202
if (self._ranges_received_whole_file is None
203
and self._response.status == 200
204
and pending and pending > self._range_warning_thresold
206
self._ranges_received_whole_file = True
208
'Got a 200 response when asking for multiple ranges,'
209
' does your server at %s:%s support range requests?',
210
self.host, self.port)
379
211
self._response = None
380
212
# Preserve our preciousss
383
# Let httplib.HTTPConnection do its housekeeping
215
# Let httplib.HTTPConnection do its housekeeping
385
217
# Restore our preciousss
388
def _wrap_socket_for_reporting(self, sock):
389
"""Wrap the socket before anybody use it."""
390
self.sock = _ReportingSocket(sock, self._report_activity)
393
221
class HTTPConnection(AbstractHTTPConnection, httplib.HTTPConnection):
395
223
# XXX: Needs refactoring at the caller level.
396
def __init__(self, host, port=None, proxied_host=None,
397
report_activity=None, ca_certs=None):
398
AbstractHTTPConnection.__init__(self, report_activity=report_activity)
224
def __init__(self, host, port=None, proxied_host=None):
225
AbstractHTTPConnection.__init__(self)
399
226
# Use strict=True since we don't support HTTP/0.9
400
227
httplib.HTTPConnection.__init__(self, host, port, strict=True)
401
228
self.proxied_host = proxied_host
402
# ca_certs is ignored, it's only relevant for https
404
230
def connect(self):
405
231
if 'http' in debug.debug_flags:
406
232
self._mutter_connect()
407
233
httplib.HTTPConnection.connect(self)
408
self._wrap_socket_for_reporting(self.sock)
236
# FIXME: Should test for ssl availability
411
237
class HTTPSConnection(AbstractHTTPConnection, httplib.HTTPSConnection):
413
239
def __init__(self, host, port=None, key_file=None, cert_file=None,
415
report_activity=None, ca_certs=None):
416
AbstractHTTPConnection.__init__(self, report_activity=report_activity)
241
AbstractHTTPConnection.__init__(self)
417
242
# Use strict=True since we don't support HTTP/0.9
418
243
httplib.HTTPSConnection.__init__(self, host, port,
419
244
key_file, cert_file, strict=True)
420
245
self.proxied_host = proxied_host
421
self.ca_certs = ca_certs
423
247
def connect(self):
424
248
if 'http' in debug.debug_flags:
425
249
self._mutter_connect()
426
250
httplib.HTTPConnection.connect(self)
427
self._wrap_socket_for_reporting(self.sock)
428
251
if self.proxied_host is None:
429
252
self.connect_to_origin()
431
254
def connect_to_origin(self):
432
# FIXME JRV 2011-12-18: Use location config here?
433
config_stack = config.GlobalStack()
434
cert_reqs = config_stack.get('ssl.cert_reqs')
435
if self.proxied_host is not None:
436
host = self.proxied_host.split(":", 1)[0]
439
if cert_reqs == ssl.CERT_NONE:
440
ui.ui_factory.show_user_warning('not_checking_ssl_cert', host=host)
441
ui.ui_factory.suppressed_warnings.add('not_checking_ssl_cert')
444
if self.ca_certs is None:
445
ca_certs = config_stack.get('ssl.ca_certs')
447
ca_certs = self.ca_certs
450
"No valid trusted SSL CA certificates file set. See "
451
"'bzr help ssl.ca_certs' for more information on setting "
454
ssl_sock = ssl.wrap_socket(
455
self.sock, self.key_file, self.cert_file,
456
cert_reqs=cert_reqs, ca_certs=ca_certs)
460
"See `bzr help ssl.ca_certs` for how to specify trusted CA"
462
"Pass -Ossl.cert_reqs=none to disable certificate "
463
"verification entirely.\n")
465
if cert_reqs == ssl.CERT_REQUIRED:
466
peer_cert = ssl_sock.getpeercert()
467
ssl.match_hostname(peer_cert, host)
469
# Wrap the ssl socket before anybody use it
470
self._wrap_socket_for_reporting(ssl_sock)
255
ssl = socket.ssl(self.sock, self.key_file, self.cert_file)
256
self.sock = httplib.FakeSocket(self.sock, ssl)
473
259
class Request(urllib2.Request):
512
298
def set_proxy(self, proxy, type):
513
299
"""Set the proxy and remember the proxied host."""
514
host, port = urllib.splitport(self.get_host())
516
# We need to set the default port ourselves way before it gets set
517
# in the HTTP[S]Connection object at build time.
518
if self.type == 'https':
519
conn_class = HTTPSConnection
521
conn_class = HTTPConnection
522
port = conn_class.default_port
523
self.proxied_host = '%s:%s' % (host, port)
300
self.proxied_host = self.get_host()
524
301
urllib2.Request.set_proxy(self, proxy, type)
525
# When urllib2 makes a https request with our wrapper code and a proxy,
526
# it sets Host to the https proxy, not the host we want to talk to.
527
# I'm fairly sure this is our fault, but what is the cause is an open
528
# question. -- Robert Collins May 8 2010.
529
self.add_unredirected_header('Host', self.proxied_host)
532
304
class _ConnectRequest(Request):
534
306
def __init__(self, request):
537
309
:param request: the first request sent to the proxied host, already
538
310
processed by the opener (i.e. proxied_host is already set).
770
530
if 'http' in debug.debug_flags:
771
531
trace.mutter('> %s %s' % (method, url))
773
for k,v in headers.iteritems():
774
# People are often told to paste -Dhttp output to help
775
# debug. Don't compromise credentials.
776
if k in ('Authorization', 'Proxy-Authorization'):
778
hdrs.append('%s: %s' % (k, v))
532
hdrs = ['%s: %s' % (k, v) for k,v in headers.items()]
779
533
trace.mutter('> ' + '\n> '.join(hdrs) + '\n')
780
534
if self._debuglevel >= 1:
781
535
print 'Request sent: [%r] from (%s)' \
782
536
% (request, request.connection.sock.getsockname())
783
537
response = connection.getresponse()
784
538
convert_to_addinfourl = True
785
except (ssl.SSLError, ssl.CertificateError):
786
# Something is wrong with either the certificate or the hostname,
787
# re-trying won't help
789
539
except (socket.gaierror, httplib.BadStatusLine, httplib.UnknownProtocol,
790
540
socket.error, httplib.HTTPException):
791
541
response = self.retry_or_raise(http_class, request, first_try)
1031
781
print 'Will unbind %s_open for %r' % (type, proxy)
1032
782
delattr(self, '%s_open' % type)
1034
def bind_scheme_request(proxy, scheme):
1037
scheme_request = scheme + '_request'
1038
if self._debuglevel >= 3:
1039
print 'Will bind %s for %r' % (scheme_request, proxy)
1040
setattr(self, scheme_request,
1041
lambda request: self.set_proxy(request, scheme))
1042
784
# We are interested only by the http[s] proxies
1043
785
http_proxy = self.get_proxy_env_var('http')
1044
bind_scheme_request(http_proxy, 'http')
1045
786
https_proxy = self.get_proxy_env_var('https')
1046
bind_scheme_request(https_proxy, 'https')
788
if http_proxy is not None:
789
if self._debuglevel >= 3:
790
print 'Will bind http_request for %r' % http_proxy
791
setattr(self, 'http_request',
792
lambda request: self.set_proxy(request, 'http'))
794
if https_proxy is not None:
795
if self._debuglevel >= 3:
796
print 'Will bind http_request for %r' % https_proxy
797
setattr(self, 'https_request',
798
lambda request: self.set_proxy(request, 'https'))
1048
800
def get_proxy_env_var(self, name, default_to='all'):
1049
801
"""Get a proxy env var.
1121
849
# grok user:password@host:port as well as
1122
850
# http://user:password@host:port
1124
parsed_url = transport.ConnectedTransport._split_url(proxy)
1125
if not parsed_url.host:
1126
raise errors.InvalidURL(proxy, 'No host component')
852
(scheme, user, password,
853
host, port, path) = transport.ConnectedTransport._split_url(proxy)
1128
855
if request.proxy_auth == {}:
1129
856
# No proxy auth parameter are available, we are handling the first
1130
857
# proxied request, intialize. scheme (the authentication scheme)
1131
858
# and realm will be set by the AuthHandler
1132
859
request.proxy_auth = {
1133
'host': parsed_url.host,
1134
'port': parsed_url.port,
1135
'user': parsed_url.user,
1136
'password': parsed_url.password,
1137
'protocol': parsed_url.scheme,
860
'host': host, 'port': port,
861
'user': user, 'password': password,
1138
863
# We ignore path since we connect to a proxy
1140
if parsed_url.port is None:
1141
phost = parsed_url.host
1143
phost = parsed_url.host + ':%d' % parsed_url.port
868
phost = host + ':%d' % port
1144
869
request.set_proxy(phost, type)
1145
870
if self._debuglevel >= 3:
1146
871
print 'set_proxy: proxy set to %s://%s' % (type, phost)
1245
948
# Let's be ready for next round
1246
949
self._retry_count = None
1248
server_headers = headers.getheaders(self.auth_required_header)
1249
if not server_headers:
951
server_header = headers.get(self.auth_required_header, None)
952
if server_header is None:
1250
953
# The http error MUST have the associated
1251
954
# header. This must never happen in production code.
1252
955
raise KeyError('%s not found' % self.auth_required_header)
1254
957
auth = self.get_auth(request)
958
if auth.get('user', None) is None:
959
# Without a known user, we can't authenticate
1255
962
auth['modified'] = False
1256
# Put some common info in auth if the caller didn't
1257
if auth.get('path', None) is None:
1258
parsed_url = urlutils.URL.from_string(request.get_full_url())
1259
self.update_auth(auth, 'protocol', parsed_url.scheme)
1260
self.update_auth(auth, 'host', parsed_url.host)
1261
self.update_auth(auth, 'port', parsed_url.port)
1262
self.update_auth(auth, 'path', parsed_url.path)
1263
# FIXME: the auth handler should be selected at a single place instead
1264
# of letting all handlers try to match all headers, but the current
1265
# design doesn't allow a simple implementation.
1266
for server_header in server_headers:
1267
# Several schemes can be proposed by the server, try to match each
1269
matching_handler = self.auth_match(server_header, auth)
1270
if matching_handler:
1271
# auth_match may have modified auth (by adding the
1272
# password or changing the realm, for example)
1273
if (request.get_header(self.auth_header, None) is not None
1274
and not auth['modified']):
1275
# We already tried that, give up
1278
# Only the most secure scheme proposed by the server should be
1279
# used, since the handlers use 'handler_order' to describe that
1280
# property, the first handler tried takes precedence, the
1281
# others should not attempt to authenticate if the best one
1283
best_scheme = auth.get('best_scheme', None)
1284
if best_scheme is None:
1285
# At that point, if current handler should doesn't succeed
1286
# the credentials are wrong (or incomplete), but we know
1287
# that the associated scheme should be used.
1288
best_scheme = auth['best_scheme'] = self.scheme
1289
if best_scheme != self.scheme:
1292
if self.requires_username and auth.get('user', None) is None:
1293
# Without a known user, we can't authenticate
1297
request.connection.cleanup_pipe()
1298
# Retry the request with an authentication header added
1299
response = self.parent.open(request)
1301
self.auth_successful(request, response)
963
if self.auth_match(server_header, auth):
964
# auth_match may have modified auth (by adding the
965
# password or changing the realm, for example)
966
if (request.get_header(self.auth_header, None) is not None
967
and not auth['modified']):
968
# We already tried that, give up
972
request.connection.cleanup_pipe()
973
response = self.parent.open(request)
975
self.auth_successful(request, response)
1303
977
# We are not qualified to handle the authentication.
1304
978
# Note: the authentication error handling will try all
1305
979
# available handlers. If one of them authenticates
1355
1029
self._retry_count = None
1357
1031
def get_user_password(self, auth):
1358
"""Ask user for a password if none is already available.
1360
:param auth: authentication info gathered so far (from the initial url
1361
and then during dialog with the server).
1032
"""Ask user for a password if none is already available."""
1363
1033
auth_conf = config.AuthenticationConfig()
1364
user = auth.get('user', None)
1365
password = auth.get('password', None)
1035
password = auth['password']
1366
1036
realm = auth['realm']
1367
port = auth.get('port', None)
1369
1038
if user is None:
1370
user = auth_conf.get_user(auth['protocol'], auth['host'],
1371
port=port, path=auth['path'],
1372
realm=realm, ask=True,
1373
prompt=self.build_username_prompt(auth))
1374
if user is not None and password is None:
1039
user = auth.get_user(auth['protocol'], auth['host'],
1040
port=auth['port'], path=auth['path'],
1043
# Default to local user
1044
user = getpass.getuser()
1046
if password is None:
1375
1047
password = auth_conf.get_password(
1376
auth['protocol'], auth['host'], user,
1048
auth['protocol'], auth['host'], user, port=auth['port'],
1378
1049
path=auth['path'], realm=realm,
1379
1050
prompt=self.build_password_prompt(auth))
1426
1079
https_request = http_request # FIXME: Need test
1429
class NegotiateAuthHandler(AbstractAuthHandler):
1430
"""A authentication handler that handles WWW-Authenticate: Negotiate.
1432
At the moment this handler supports just Kerberos. In the future,
1433
NTLM support may also be added.
1436
scheme = 'negotiate'
1438
requires_username = False
1440
def auth_match(self, header, auth):
1441
scheme, raw_auth = self._parse_auth_header(header)
1442
if scheme != self.scheme:
1444
self.update_auth(auth, 'scheme', scheme)
1445
resp = self._auth_match_kerberos(auth)
1448
# Optionally should try to authenticate using NTLM here
1449
self.update_auth(auth, 'negotiate_response', resp)
1452
def _auth_match_kerberos(self, auth):
1453
"""Try to create a GSSAPI response for authenticating against a host."""
1454
global kerberos, checked_kerberos
1455
if kerberos is None and not checked_kerberos:
1460
checked_kerberos = True
1461
if kerberos is None:
1463
ret, vc = kerberos.authGSSClientInit("HTTP@%(host)s" % auth)
1465
trace.warning('Unable to create GSSAPI context for %s: %d',
1468
ret = kerberos.authGSSClientStep(vc, "")
1470
trace.mutter('authGSSClientStep failed: %d', ret)
1472
return kerberos.authGSSClientResponse(vc)
1474
def build_auth_header(self, auth, request):
1475
return "Negotiate %s" % auth['negotiate_response']
1477
def auth_params_reusable(self, auth):
1478
# If the auth scheme is known, it means a previous
1479
# authentication was successful, all information is
1480
# available, no further checks are needed.
1481
return (auth.get('scheme', None) == 'negotiate' and
1482
auth.get('negotiate_response', None) is not None)
1485
1082
class BasicAuthHandler(AbstractAuthHandler):
1486
1083
"""A custom basic authentication handler."""
1489
1085
handler_order = 500
1490
1087
auth_regexp = re.compile('realm="([^"]*)"', re.I)
1492
1089
def build_auth_header(self, auth, request):
1494
1091
auth_header = 'Basic ' + raw.encode('base64').strip()
1495
1092
return auth_header
1497
def extract_realm(self, header_value):
1498
match = self.auth_regexp.search(header_value)
1501
realm = match.group(1)
1504
1094
def auth_match(self, header, auth):
1505
scheme, raw_auth = self._parse_auth_header(header)
1506
if scheme != self.scheme:
1095
scheme, raw_auth = header.split(None, 1)
1096
scheme = scheme.lower()
1097
if scheme != 'basic':
1509
match, realm = self.extract_realm(raw_auth)
1100
match = self.auth_regexp.search(raw_auth)
1102
realm = match.groups()
1103
if scheme != 'basic':
1511
1106
# Put useful info into auth
1512
1107
self.update_auth(auth, 'scheme', scheme)
1513
1108
self.update_auth(auth, 'realm', realm)
1514
if (auth.get('user', None) is None
1515
or auth.get('password', None) is None):
1109
if auth['user'] is None or auth['password'] is None:
1516
1110
user, password = self.get_user_password(auth)
1517
1111
self.update_auth(auth, 'user', user)
1518
1112
self.update_auth(auth, 'password', password)
added
removed
bzrlib/transport/http/_urllib2_wrappers.py
