~bzr-pqm/bzr/bzr.dev

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
# Copyright (C) 2006-2010 Canonical Ltd
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA

"""Implementaion of urllib2 tailored to bzr needs

This file complements the urllib2 class hierarchy with custom classes.

For instance, we create a new HTTPConnection and HTTPSConnection that inherit
from the original urllib2.HTTP(s)Connection objects, but also have a new base
which implements a custom getresponse and cleanup_pipe handlers.

And then we implement custom HTTPHandler and HTTPSHandler classes, that use
the custom HTTPConnection classes.

We have a custom Response class, which lets us maintain a keep-alive
connection even for requests that urllib2 doesn't expect to contain body data.

And a custom Request class that lets us track redirections, and
handle authentication schemes.

For coherency with python libraries, we use capitalized header names throughout
the code, even if the header names will be titled just before sending the
request (see AbstractHTTPHandler.do_open).
"""

DEBUG = 0

# FIXME: Oversimplifying, two kind of exceptions should be
# raised, once a request is issued: URLError before we have been
# able to process the response, HTTPError after that. Process the
# response means we are able to leave the socket clean, so if we
# are not able to do that, we should close the connection. The
# actual code more or less do that, tests should be written to
# ensure that.

import errno
import httplib
try:
    import kerberos
except ImportError:
    have_kerberos = False
else:
    have_kerberos = True
import socket
import urllib
import urllib2
import urlparse
import re
import sys
import time

from bzrlib import __version__ as bzrlib_version
from bzrlib import (
    config,
    debug,
    errors,
    osutils,
    trace,
    transport,
    ui,
    urlutils,
    )


class _ReportingFileSocket(object):

    def __init__(self, filesock, report_activity=None):
        self.filesock = filesock
        self._report_activity = report_activity

    def report_activity(self, size, direction):
        if self._report_activity:
            self._report_activity(size, direction)

    def read(self, size=1):
        s = self.filesock.read(size)
        self.report_activity(len(s), 'read')
        return s

    def readline(self):
        # This should be readline(self, size=-1), but httplib in python 2.4 and
        #  2.5 defines a SSLFile wrapper whose readline method lacks the size
        #  parameter.  So until we drop support for 2.4 and 2.5 and since we
        #  don't *need* the size parameter we'll stay with readline(self)
        #  --  vila 20090209
        s = self.filesock.readline()
        self.report_activity(len(s), 'read')
        return s

    def __getattr__(self, name):
        return getattr(self.filesock, name)


class _ReportingSocket(object):

    def __init__(self, sock, report_activity=None):
        self.sock = sock
        self._report_activity = report_activity

    def report_activity(self, size, direction):
        if self._report_activity:
            self._report_activity(size, direction)

    def sendall(self, s, *args):
        self.sock.sendall(s, *args)
        self.report_activity(len(s), 'write')

    def recv(self, *args):
        s = self.sock.recv(*args)
        self.report_activity(len(s), 'read')
        return s

    def makefile(self, mode='r', bufsize=-1):
        # httplib creates a fileobject that doesn't do buffering, which
        # makes fp.readline() very expensive because it only reads one byte
        # at a time.  So we wrap the socket in an object that forces
        # sock.makefile to make a buffered file.
        fsock = self.sock.makefile(mode, 65536)
        # And wrap that into a reporting kind of fileobject
        return _ReportingFileSocket(fsock, self._report_activity)

    def __getattr__(self, name):
        return getattr(self.sock, name)


# We define our own Response class to keep our httplib pipe clean
class Response(httplib.HTTPResponse):
    """Custom HTTPResponse, to avoid the need to decorate.

    httplib prefers to decorate the returned objects, rather
    than using a custom object.
    """

    # Some responses have bodies in which we have no interest
    _body_ignored_responses = [301,302, 303, 307, 401, 403, 404]

    # in finish() below, we may have to discard several MB in the worst
    # case. To avoid buffering that much, we read and discard by chunks
    # instead. The underlying file is either a socket or a StringIO, so reading
    # 8k chunks should be fine.
    _discarded_buf_size = 8192

    def begin(self):
        """Begin to read the response from the server.

        httplib assumes that some responses get no content and do
        not even attempt to read the body in that case, leaving
        the body in the socket, blocking the next request. Let's
        try to workaround that.
        """
        httplib.HTTPResponse.begin(self)
        if self.status in self._body_ignored_responses:
            if self.debuglevel >= 2:
                print "For status: [%s]," % self.status,
                print "will ready body, length: %s" % self.length
            if not (self.length is None or self.will_close):
                # In some cases, we just can't read the body not
                # even try or we may encounter a 104, 'Connection
                # reset by peer' error if there is indeed no body
                # and the server closed the connection just after
                # having issued the response headers (even if the
                # headers indicate a Content-Type...)
                body = self.read(self.length)
                if self.debuglevel >= 9:
                    # This one can be huge and is generally not interesting
                    print "Consumed body: [%s]" % body
            self.close()
        elif self.status == 200:
            # Whatever the request is, it went ok, so we surely don't want to
            # close the connection. Some cases are not correctly detected by
            # httplib.HTTPConnection.getresponse (called by
            # httplib.HTTPResponse.begin). The CONNECT response for the https
            # through proxy case is one.  Note: the 'will_close' below refers
            # to the "true" socket between us and the server, whereas the
            # 'close()' above refers to the copy of that socket created by
            # httplib for the response itself. So, in the if above we close the
            # socket to indicate that we are done with the response whereas
            # below we keep the socket with the server opened.
            self.will_close = False

    def finish(self):
        """Finish reading the body.

        In some cases, the client may have left some bytes to read in the
        body. That will block the next request to succeed if we use a
        persistent connection. If we don't use a persistent connection, well,
        nothing will block the next request since a new connection will be
        issued anyway.

        :return: the number of bytes left on the socket (may be None)
        """
        pending = None
        if not self.isclosed():
            # Make sure nothing was left to be read on the socket
            pending = 0
            data = True
            while data and self.length:
                # read() will update self.length
                data = self.read(min(self.length, self._discarded_buf_size))
                pending += len(data)
            if pending:
                trace.mutter("%s bytes left on the HTTP socket", pending)
            self.close()
        return pending


# Not inheriting from 'object' because httplib.HTTPConnection doesn't.
class AbstractHTTPConnection:
    """A custom HTTP(S) Connection, which can reset itself on a bad response"""

    response_class = Response

    # When we detect a server responding with the whole file to range requests,
    # we want to warn. But not below a given thresold.
    _range_warning_thresold = 1024 * 1024

    def __init__(self, report_activity=None):
        self._response = None
        self._report_activity = report_activity
        self._ranges_received_whole_file = None

    def _mutter_connect(self):
        netloc = '%s:%s' % (self.host, self.port)
        if self.proxied_host is not None:
            netloc += '(proxy for %s)' % self.proxied_host
        trace.mutter('* About to connect() to %s' % netloc)

    def getresponse(self):
        """Capture the response to be able to cleanup"""
        self._response = httplib.HTTPConnection.getresponse(self)
        return self._response

    def cleanup_pipe(self):
        """Read the remaining bytes of the last response if any."""
        if self._response is not None:
            pending = self._response.finish()
            # Warn the user (once)
            if (self._ranges_received_whole_file is None
                and self._response.status == 200
                and pending and pending > self._range_warning_thresold
                ):
                self._ranges_received_whole_file = True
                trace.warning(
                    'Got a 200 response when asking for multiple ranges,'
                    ' does your server at %s:%s support range requests?',
                    self.host, self.port)
            self._response = None
        # Preserve our preciousss
        sock = self.sock
        self.sock = None
        # Let httplib.HTTPConnection do its housekeeping
        self.close()
        # Restore our preciousss
        self.sock = sock

    def _wrap_socket_for_reporting(self, sock):
        """Wrap the socket before anybody use it."""
        self.sock = _ReportingSocket(sock, self._report_activity)


class HTTPConnection(AbstractHTTPConnection, httplib.HTTPConnection):

    # XXX: Needs refactoring at the caller level.
    def __init__(self, host, port=None, proxied_host=None,
                 report_activity=None):
        AbstractHTTPConnection.__init__(self, report_activity=report_activity)
        # Use strict=True since we don't support HTTP/0.9
        httplib.HTTPConnection.__init__(self, host, port, strict=True)
        self.proxied_host = proxied_host

    def connect(self):
        if 'http' in debug.debug_flags:
            self._mutter_connect()
        httplib.HTTPConnection.connect(self)
        self._wrap_socket_for_reporting(self.sock)


# Build the appropriate socket wrapper for ssl
try:
    # python 2.6 introduced a better ssl package
    import ssl
    _ssl_wrap_socket = ssl.wrap_socket
except ImportError:
    # python versions prior to 2.6 don't have ssl and ssl.wrap_socket instead
    # they use httplib.FakeSocket
    def _ssl_wrap_socket(sock, key_file, cert_file):
        ssl_sock = socket.ssl(sock, key_file, cert_file)
        return httplib.FakeSocket(sock, ssl_sock)


class HTTPSConnection(AbstractHTTPConnection, httplib.HTTPSConnection):

    def __init__(self, host, port=None, key_file=None, cert_file=None,
                 proxied_host=None,
                 report_activity=None):
        AbstractHTTPConnection.__init__(self, report_activity=report_activity)
        # Use strict=True since we don't support HTTP/0.9
        httplib.HTTPSConnection.__init__(self, host, port,
                                         key_file, cert_file, strict=True)
        self.proxied_host = proxied_host

    def connect(self):
        if 'http' in debug.debug_flags:
            self._mutter_connect()
        httplib.HTTPConnection.connect(self)
        self._wrap_socket_for_reporting(self.sock)
        if self.proxied_host is None:
            self.connect_to_origin()

    def connect_to_origin(self):
        ssl_sock = _ssl_wrap_socket(self.sock, self.key_file, self.cert_file)
        # Wrap the ssl socket before anybody use it
        self._wrap_socket_for_reporting(ssl_sock)


class Request(urllib2.Request):
    """A custom Request object.

    urllib2 determines the request method heuristically (based on
    the presence or absence of data). We set the method
    statically.

    The Request object tracks:
    - the connection the request will be made on.
    - the authentication parameters needed to preventively set
      the authentication header once a first authentication have
       been made.
    """

    def __init__(self, method, url, data=None, headers={},
                 origin_req_host=None, unverifiable=False,
                 connection=None, parent=None,
                 accepted_errors=None):
        urllib2.Request.__init__(self, url, data, headers,
                                 origin_req_host, unverifiable)
        self.method = method
        self.connection = connection
        self.accepted_errors = accepted_errors
        # To handle redirections
        self.parent = parent
        self.redirected_to = None
        # Unless told otherwise, redirections are not followed
        self.follow_redirections = False
        # auth and proxy_auth are dicts containing, at least
        # (scheme, host, port, realm, user, password, protocol, path).
        # The dict entries are mostly handled by the AuthHandler.
        # Some authentication schemes may add more entries.
        self.auth = {}
        self.proxy_auth = {}
        self.proxied_host = None

    def get_method(self):
        return self.method

    def set_proxy(self, proxy, type):
        """Set the proxy and remember the proxied host."""
        host, port = urllib.splitport(self.get_host())
        if port is None:
            # We need to set the default port ourselves way before it gets set
            # in the HTTP[S]Connection object at build time.
            if self.type == 'https':
                conn_class = HTTPSConnection
            else:
                conn_class = HTTPConnection
            port = conn_class.default_port
        self.proxied_host = '%s:%s' % (host, port)
        urllib2.Request.set_proxy(self, proxy, type)


class _ConnectRequest(Request):

    def __init__(self, request):
        """Constructor

        :param request: the first request sent to the proxied host, already
            processed by the opener (i.e. proxied_host is already set).
        """
        # We give a fake url and redefine get_selector or urllib2 will be
        # confused
        Request.__init__(self, 'CONNECT', request.get_full_url(),
                         connection=request.connection)
        if request.proxied_host is None:
            raise AssertionError()
        self.proxied_host = request.proxied_host

    def get_selector(self):
        return self.proxied_host

    def set_proxy(self, proxy, type):
        """Set the proxy without remembering the proxied host.

        We already know the proxied host by definition, the CONNECT request
        occurs only when the connection goes through a proxy. The usual
        processing (masquerade the request so that the connection is done to
        the proxy while the request is targeted at another host) does not apply
        here. In fact, the connection is already established with proxy and we
        just want to enable the SSL tunneling.
        """
        urllib2.Request.set_proxy(self, proxy, type)


class ConnectionHandler(urllib2.BaseHandler):
    """Provides connection-sharing by pre-processing requests.

    urllib2 provides no way to access the HTTPConnection object
    internally used. But we need it in order to achieve
    connection sharing. So, we add it to the request just before
    it is processed, and then we override the do_open method for
    http[s] requests in AbstractHTTPHandler.
    """

    handler_order = 1000 # after all pre-processings

    def __init__(self, report_activity=None):
        self._report_activity = report_activity

    def create_connection(self, request, http_connection_class):
        host = request.get_host()
        if not host:
            # Just a bit of paranoia here, this should have been
            # handled in the higher levels
            raise errors.InvalidURL(request.get_full_url(), 'no host given.')

        # We create a connection (but it will not connect until the first
        # request is made)
        try:
            connection = http_connection_class(
                host, proxied_host=request.proxied_host,
                report_activity=self._report_activity)
        except httplib.InvalidURL, exception:
            # There is only one occurrence of InvalidURL in httplib
            raise errors.InvalidURL(request.get_full_url(),
                                    extra='nonnumeric port')

        return connection

    def capture_connection(self, request, http_connection_class):
        """Capture or inject the request connection.

        Two cases:
        - the request have no connection: create a new one,

        - the request have a connection: this one have been used
          already, let's capture it, so that we can give it to
          another transport to be reused. We don't do that
          ourselves: the Transport object get the connection from
          a first request and then propagate it, from request to
          request or to cloned transports.
        """
        connection = request.connection
        if connection is None:
            # Create a new one
            connection = self.create_connection(request, http_connection_class)
            request.connection = connection

        # All connections will pass here, propagate debug level
        connection.set_debuglevel(DEBUG)
        return request

    def http_request(self, request):
        return self.capture_connection(request, HTTPConnection)

    def https_request(self, request):
        return self.capture_connection(request, HTTPSConnection)


class AbstractHTTPHandler(urllib2.AbstractHTTPHandler):
    """A custom handler for HTTP(S) requests.

    We overrive urllib2.AbstractHTTPHandler to get a better
    control of the connection, the ability to implement new
    request types and return a response able to cope with
    persistent connections.
    """

    # We change our order to be before urllib2 HTTP[S]Handlers
    # and be chosen instead of them (the first http_open called
    # wins).
    handler_order = 400

    _default_headers = {'Pragma': 'no-cache',
                        'Cache-control': 'max-age=0',
                        'Connection': 'Keep-Alive',
                        'User-agent': 'bzr/%s (urllib)' % bzrlib_version,
                        'Accept': '*/*',
                        }

    def __init__(self):
        urllib2.AbstractHTTPHandler.__init__(self, debuglevel=DEBUG)

    def http_request(self, request):
        """Common headers setting"""

        request.headers.update(self._default_headers.copy())
        # FIXME: We may have to add the Content-Length header if
        # we have data to send.
        return request

    def retry_or_raise(self, http_class, request, first_try):
        """Retry the request (once) or raise the exception.

        urllib2 raises exception of application level kind, we
        just have to translate them.

        httplib can raise exceptions of transport level (badly
        formatted dialog, loss of connexion or socket level
        problems). In that case we should issue the request again
        (httplib will close and reopen a new connection if
        needed).
        """
        # When an exception occurs, we give back the original
        # Traceback or the bugs are hard to diagnose.
        exc_type, exc_val, exc_tb = sys.exc_info()
        if exc_type == socket.gaierror:
            # No need to retry, that will not help
            raise errors.ConnectionError("Couldn't resolve host '%s'"
                                         % request.get_origin_req_host(),
                                         orig_error=exc_val)
        elif isinstance(exc_val, httplib.ImproperConnectionState):
            # The httplib pipeline is in incorrect state, it's a bug in our
            # implementation.
            raise exc_type, exc_val, exc_tb
        else:
            if first_try:
                if self._debuglevel >= 2:
                    print 'Received exception: [%r]' % exc_val
                    print '  On connection: [%r]' % request.connection
                    method = request.get_method()
                    url = request.get_full_url()
                    print '  Will retry, %s %r' % (method, url)
                request.connection.close()
                response = self.do_open(http_class, request, False)
            else:
                if self._debuglevel >= 2:
                    print 'Received second exception: [%r]' % exc_val
                    print '  On connection: [%r]' % request.connection
                if exc_type in (httplib.BadStatusLine, httplib.UnknownProtocol):
                    # httplib.BadStatusLine and
                    # httplib.UnknownProtocol indicates that a
                    # bogus server was encountered or a bad
                    # connection (i.e. transient errors) is
                    # experimented, we have already retried once
                    # for that request so we raise the exception.
                    my_exception = errors.InvalidHttpResponse(
                        request.get_full_url(),
                        'Bad status line received',
                        orig_error=exc_val)
                elif (isinstance(exc_val, socket.error) and len(exc_val.args)
                      and exc_val.args[0] in (errno.ECONNRESET, 10054)):
                    raise errors.ConnectionReset(
                        "Connection lost while sending request.")
                else:
                    # All other exception are considered connection related.

                    # socket errors generally occurs for reasons
                    # far outside our scope, so closing the
                    # connection and retrying is the best we can
                    # do.

                    my_exception = errors.ConnectionError(
                        msg= 'while sending %s %s:' % (request.get_method(),
                                                       request.get_selector()),
                        orig_error=exc_val)

                if self._debuglevel >= 2:
                    print 'On connection: [%r]' % request.connection
                    method = request.get_method()
                    url = request.get_full_url()
                    print '  Failed again, %s %r' % (method, url)
                    print '  Will raise: [%r]' % my_exception
                raise my_exception, None, exc_tb
        return response

    def do_open(self, http_class, request, first_try=True):
        """See urllib2.AbstractHTTPHandler.do_open for the general idea.

        The request will be retried once if it fails.
        """
        connection = request.connection
        if connection is None:
            raise AssertionError(
                'Cannot process a request without a connection')

        # Get all the headers
        headers = {}
        headers.update(request.header_items())
        headers.update(request.unredirected_hdrs)
        # Some servers or proxies will choke on headers not properly
        # cased. httplib/urllib/urllib2 all use capitalize to get canonical
        # header names, but only python2.5 urllib2 use title() to fix them just
        # before sending the request. And not all versions of python 2.5 do
        # that. Since we replace urllib2.AbstractHTTPHandler.do_open we do it
        # ourself below.
        headers = dict((name.title(), val) for name, val in headers.iteritems())

        try:
            method = request.get_method()
            url = request.get_selector()
            connection._send_request(method, url,
                                     # FIXME: implements 100-continue
                                     #None, # We don't send the body yet
                                     request.get_data(),
                                     headers)
            if 'http' in debug.debug_flags:
                trace.mutter('> %s %s' % (method, url))
                hdrs = ['%s: %s' % (k, v) for k,v in headers.items()]
                trace.mutter('> ' + '\n> '.join(hdrs) + '\n')
            if self._debuglevel >= 1:
                print 'Request sent: [%r] from (%s)' \
                    % (request, request.connection.sock.getsockname())
            response = connection.getresponse()
            convert_to_addinfourl = True
        except (socket.gaierror, httplib.BadStatusLine, httplib.UnknownProtocol,
                socket.error, httplib.HTTPException):
            response = self.retry_or_raise(http_class, request, first_try)
            convert_to_addinfourl = False

# FIXME: HTTPConnection does not fully support 100-continue (the
# server responses are just ignored)

#        if code == 100:
#            mutter('Will send the body')
#            # We can send the body now
#            body = request.get_data()
#            if body is None:
#                raise URLError("No data given")
#            connection.send(body)
#            response = connection.getresponse()

        if self._debuglevel >= 2:
            print 'Receives response: %r' % response
            print '  For: %r(%r)' % (request.get_method(),
                                     request.get_full_url())

        if convert_to_addinfourl:
            # Shamelessly copied from urllib2
            req = request
            r = response
            r.recv = r.read
            fp = socket._fileobject(r, bufsize=65536)
            resp = urllib2.addinfourl(fp, r.msg, req.get_full_url())
            resp.code = r.status
            resp.msg = r.reason
            resp.version = r.version
            if self._debuglevel >= 2:
                print 'Create addinfourl: %r' % resp
                print '  For: %r(%r)' % (request.get_method(),
                                         request.get_full_url())
            if 'http' in debug.debug_flags:
                version = 'HTTP/%d.%d'
                try:
                    version = version % (resp.version / 10,
                                         resp.version % 10)
                except:
                    version = 'HTTP/%r' % resp.version
                trace.mutter('< %s %s %s' % (version, resp.code,
                                             resp.msg))
                # Use the raw header lines instead of treating resp.info() as a
                # dict since we may miss duplicated headers otherwise.
                hdrs = [h.rstrip('\r\n') for h in resp.info().headers]
                trace.mutter('< ' + '\n< '.join(hdrs) + '\n')
        else:
            resp = response
        return resp


class HTTPHandler(AbstractHTTPHandler):
    """A custom handler that just thunks into HTTPConnection"""

    def http_open(self, request):
        return self.do_open(HTTPConnection, request)


class HTTPSHandler(AbstractHTTPHandler):
    """A custom handler that just thunks into HTTPSConnection"""

    https_request = AbstractHTTPHandler.http_request

    def https_open(self, request):
        connection = request.connection
        if connection.sock is None and \
                connection.proxied_host is not None and \
                request.get_method() != 'CONNECT' : # Don't loop
            # FIXME: We need a gazillion connection tests here, but we still
            # miss a https server :-( :
            # - with and without proxy
            # - with and without certificate
            # - with self-signed certificate
            # - with and without authentication
            # - with good and bad credentials (especially the proxy auth around
            #   CONNECT)
            # - with basic and digest schemes
            # - reconnection on errors
            # - connection persistence behaviour (including reconnection)

            # We are about to connect for the first time via a proxy, we must
            # issue a CONNECT request first to establish the encrypted link
            connect = _ConnectRequest(request)
            response = self.parent.open(connect)
            if response.code != 200:
                raise ConnectionError("Can't connect to %s via proxy %s" % (
                        connect.proxied_host, self.host))
            # Housekeeping
            connection.cleanup_pipe()
            # Establish the connection encryption
            connection.connect_to_origin()
            # Propagate the connection to the original request
            request.connection = connection
        return self.do_open(HTTPSConnection, request)

class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
    """Handles redirect requests.

    We have to implement our own scheme because we use a specific
    Request object and because we want to implement a specific
    policy.
    """
    _debuglevel = DEBUG
    # RFC2616 says that only read requests should be redirected
    # without interacting with the user. But bzr use some
    # shortcuts to optimize against roundtrips which can leads to
    # write requests being issued before read requests of
    # containing dirs can be redirected. So we redirect write
    # requests in the same way which seems to respect the spirit
    # of the RFC if not its letter.

    def redirect_request(self, req, fp, code, msg, headers, newurl):
        """See urllib2.HTTPRedirectHandler.redirect_request"""
        # We would have preferred to update the request instead
        # of creating a new one, but the urllib2.Request object
        # has a too complicated creation process to provide a
        # simple enough equivalent update process. Instead, when
        # redirecting, we only update the following request in
        # the redirect chain with a reference to the parent
        # request .

        # Some codes make no sense in our context and are treated
        # as errors:

        # 300: Multiple choices for different representations of
        #      the URI. Using that mechanisn with bzr will violate the
        #      protocol neutrality of Transport.

        # 304: Not modified (SHOULD only occurs with conditional
        #      GETs which are not used by our implementation)

        # 305: Use proxy. I can't imagine this one occurring in
        #      our context-- vila/20060909

        # 306: Unused (if the RFC says so...)

        # If the code is 302 and the request is HEAD, some may
        # think that it is a sufficent hint that the file exists
        # and that we MAY avoid following the redirections. But
        # if we want to be sure, we MUST follow them.

        if code in (301, 302, 303, 307):
            return Request(req.get_method(),newurl,
                           headers = req.headers,
                           origin_req_host = req.get_origin_req_host(),
                           unverifiable = True,
                           # TODO: It will be nice to be able to
                           # detect virtual hosts sharing the same
                           # IP address, that will allow us to
                           # share the same connection...
                           connection = None,
                           parent = req,
                           )
        else:
            raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)

    def http_error_302(self, req, fp, code, msg, headers):
        """Requests the redirected to URI.

        Copied from urllib2 to be able to clean the pipe of the associated
        connection, *before* issuing the redirected request but *after* having
        eventually raised an error.
        """
        # Some servers (incorrectly) return multiple Location headers
        # (so probably same goes for URI).  Use first header.

        # TODO: Once we get rid of addinfourl objects, the
        # following will need to be updated to use correct case
        # for headers.
        if 'location' in headers:
            newurl = headers.getheaders('location')[0]
        elif 'uri' in headers:
            newurl = headers.getheaders('uri')[0]
        else:
            return
        if self._debuglevel >= 1:
            print 'Redirected to: %s (followed: %r)' % (newurl,
                                                        req.follow_redirections)
        if req.follow_redirections is False:
            req.redirected_to = newurl
            return fp

        newurl = urlparse.urljoin(req.get_full_url(), newurl)

        # This call succeeds or raise an error. urllib2 returns
        # if redirect_request returns None, but our
        # redirect_request never returns None.
        redirected_req = self.redirect_request(req, fp, code, msg, headers,
                                               newurl)

        # loop detection
        # .redirect_dict has a key url if url was previously visited.
        if hasattr(req, 'redirect_dict'):
            visited = redirected_req.redirect_dict = req.redirect_dict
            if (visited.get(newurl, 0) >= self.max_repeats or
                len(visited) >= self.max_redirections):
                raise urllib2.HTTPError(req.get_full_url(), code,
                                        self.inf_msg + msg, headers, fp)
        else:
            visited = redirected_req.redirect_dict = req.redirect_dict = {}
        visited[newurl] = visited.get(newurl, 0) + 1

        # We can close the fp now that we are sure that we won't
        # use it with HTTPError.
        fp.close()
        # We have all we need already in the response
        req.connection.cleanup_pipe()

        return self.parent.open(redirected_req)

    http_error_301 = http_error_303 = http_error_307 = http_error_302


class ProxyHandler(urllib2.ProxyHandler):
    """Handles proxy setting.

    Copied and modified from urllib2 to be able to modify the request during
    the request pre-processing instead of modifying it at _open time. As we
    capture (or create) the connection object during request processing, _open
    time was too late.

    The main task is to modify the request so that the connection is done to
    the proxy while the request still refers to the destination host.

    Note: the proxy handling *may* modify the protocol used; the request may be
    against an https server proxied through an http proxy. So, https_request
    will be called, but later it's really http_open that will be called. This
    explains why we don't have to call self.parent.open as the urllib2 did.
    """

    # Proxies must be in front
    handler_order = 100
    _debuglevel = DEBUG

    def __init__(self, proxies=None):
        urllib2.ProxyHandler.__init__(self, proxies)
        # First, let's get rid of urllib2 implementation
        for type, proxy in self.proxies.items():
            if self._debuglevel >= 3:
                print 'Will unbind %s_open for %r' % (type, proxy)
            delattr(self, '%s_open' % type)

        # We are interested only by the http[s] proxies
        http_proxy = self.get_proxy_env_var('http')
        https_proxy = self.get_proxy_env_var('https')

        if http_proxy is not None:
            if self._debuglevel >= 3:
                print 'Will bind http_request for %r' % http_proxy
            setattr(self, 'http_request',
                    lambda request: self.set_proxy(request, 'http'))

        if https_proxy is not None:
            if self._debuglevel >= 3:
                print 'Will bind http_request for %r' % https_proxy
            setattr(self, 'https_request',
                    lambda request: self.set_proxy(request, 'https'))

    def get_proxy_env_var(self, name, default_to='all'):
        """Get a proxy env var.

        Note that we indirectly rely on
        urllib.getproxies_environment taking into account the
        uppercased values for proxy variables.
        """
        try:
            return self.proxies[name.lower()]
        except KeyError:
            if default_to is not None:
                # Try to get the alternate environment variable
                try:
                    return self.proxies[default_to]
                except KeyError:
                    pass
        return None

    def proxy_bypass(self, host):
        """Check if host should be proxied or not"""
        no_proxy = self.get_proxy_env_var('no', default_to=None)
        if no_proxy is None:
            return False
        hhost, hport = urllib.splitport(host)
        # Does host match any of the domains mentioned in
        # no_proxy ? The rules about what is authorized in no_proxy
        # are fuzzy (to say the least). We try to allow most
        # commonly seen values.
        for domain in no_proxy.split(','):
            dhost, dport = urllib.splitport(domain)
            if hport == dport or dport is None:
                # Protect glob chars
                dhost = dhost.replace(".", r"\.")
                dhost = dhost.replace("*", r".*")
                dhost = dhost.replace("?", r".")
                if re.match(dhost, hhost, re.IGNORECASE):
                    return True
        # Nevertheless, there are platform-specific ways to
        # ignore proxies...
        return urllib.proxy_bypass(host)

    def set_proxy(self, request, type):
        if self.proxy_bypass(request.get_host()):
            return request

        proxy = self.get_proxy_env_var(type)
        if self._debuglevel >= 3:
            print 'set_proxy %s_request for %r' % (type, proxy)
        # FIXME: python 2.5 urlparse provides a better _parse_proxy which can
        # grok user:password@host:port as well as
        # http://user:password@host:port

        (scheme, user, password,
         host, port, path) = transport.ConnectedTransport._split_url(proxy)
        if not host:
            raise errors.InvalidURL(proxy, 'No host component')

        if request.proxy_auth == {}:
            # No proxy auth parameter are available, we are handling the first
            # proxied request, intialize.  scheme (the authentication scheme)
            # and realm will be set by the AuthHandler
            request.proxy_auth = {
                                  'host': host, 'port': port,
                                  'user': user, 'password': password,
                                  'protocol': scheme,
                                   # We ignore path since we connect to a proxy
                                  'path': None}
        if port is None:
            phost = host
        else:
            phost = host + ':%d' % port
        request.set_proxy(phost, type)
        if self._debuglevel >= 3:
            print 'set_proxy: proxy set to %s://%s' % (type, phost)
        return request


class AbstractAuthHandler(urllib2.BaseHandler):
    """A custom abstract authentication handler for all http authentications.

    Provides the meat to handle authentication errors and
    preventively set authentication headers after the first
    successful authentication.

    This can be used for http and proxy, as well as for basic, negotiate and
    digest authentications.

    This provides an unified interface for all authentication handlers
    (urllib2 provides far too many with different policies).

    The interaction between this handler and the urllib2
    framework is not obvious, it works as follow:

    opener.open(request) is called:

    - that may trigger http_request which will add an authentication header
      (self.build_header) if enough info is available.

    - the request is sent to the server,

    - if an authentication error is received self.auth_required is called,
      we acquire the authentication info in the error headers and call
      self.auth_match to check that we are able to try the
      authentication and complete the authentication parameters,

    - we call parent.open(request), that may trigger http_request
      and will add a header (self.build_header), but here we have
      all the required info (keep in mind that the request and
      authentication used in the recursive calls are really (and must be)
      the *same* objects).

    - if the call returns a response, the authentication have been
      successful and the request authentication parameters have been updated.
    """

    scheme = None
    """The scheme as it appears in the server header (lower cased)"""

    _max_retry = 3
    """We don't want to retry authenticating endlessly"""

    requires_username = True
    """Whether the auth mechanism requires a username."""

    # The following attributes should be defined by daughter
    # classes:
    # - auth_required_header:  the header received from the server
    # - auth_header: the header sent in the request

    def __init__(self):
        # We want to know when we enter into an try/fail cycle of
        # authentications so we initialize to None to indicate that we aren't
        # in such a cycle by default.
        self._retry_count = None

    def _parse_auth_header(self, server_header):
        """Parse the authentication header.

        :param server_header: The value of the header sent by the server
            describing the authenticaion request.

        :return: A tuple (scheme, remainder) scheme being the first word in the
            given header (lower cased), remainder may be None.
        """
        try:
            scheme, remainder = server_header.split(None, 1)
        except ValueError:
            scheme = server_header
            remainder = None
        return (scheme.lower(), remainder)

    def update_auth(self, auth, key, value):
        """Update a value in auth marking the auth as modified if needed"""
        old_value = auth.get(key, None)
        if old_value != value:
            auth[key] = value
            auth['modified'] = True

    def auth_required(self, request, headers):
        """Retry the request if the auth scheme is ours.

        :param request: The request needing authentication.
        :param headers: The headers for the authentication error response.
        :return: None or the response for the authenticated request.
        """
        # Don't try  to authenticate endlessly
        if self._retry_count is None:
            # The retry being recusrsive calls, None identify the first retry
            self._retry_count = 1
        else:
            self._retry_count += 1
            if self._retry_count > self._max_retry:
                # Let's be ready for next round
                self._retry_count = None
                return None
        server_headers = headers.getheaders(self.auth_required_header)
        if not server_headers:
            # The http error MUST have the associated
            # header. This must never happen in production code.
            raise KeyError('%s not found' % self.auth_required_header)

        auth = self.get_auth(request)
        auth['modified'] = False
        # Put some common info in auth if the caller didn't
        if auth.get('path', None) is None:
            (protocol, _, _,
             host, port, path) = urlutils.parse_url(request.get_full_url())
            self.update_auth(auth, 'protocol', protocol)
            self.update_auth(auth, 'host', host)
            self.update_auth(auth, 'port', port)
            self.update_auth(auth, 'path', path)
        # FIXME: the auth handler should be selected at a single place instead
        # of letting all handlers try to match all headers, but the current
        # design doesn't allow a simple implementation.
        for server_header in server_headers:
            # Several schemes can be proposed by the server, try to match each
            # one in turn
            matching_handler = self.auth_match(server_header, auth)
            if matching_handler:
                # auth_match may have modified auth (by adding the
                # password or changing the realm, for example)
                if (request.get_header(self.auth_header, None) is not None
                    and not auth['modified']):
                    # We already tried that, give up
                    return None

                # Only the most secure scheme proposed by the server should be
                # used, since the handlers use 'handler_order' to describe that
                # property, the first handler tried takes precedence, the
                # others should not attempt to authenticate if the best one
                # failed.
                best_scheme = auth.get('best_scheme', None)
                if best_scheme is None:
                    # At that point, if current handler should doesn't succeed
                    # the credentials are wrong (or incomplete), but we know
                    # that the associated scheme should be used.
                    best_scheme = auth['best_scheme'] = self.scheme
                if  best_scheme != self.scheme:
                    continue

                if self.requires_username and auth.get('user', None) is None:
                    # Without a known user, we can't authenticate
                    return None

                # Housekeeping
                request.connection.cleanup_pipe()
                # Retry the request with an authentication header added
                response = self.parent.open(request)
                if response:
                    self.auth_successful(request, response)
                return response
        # We are not qualified to handle the authentication.
        # Note: the authentication error handling will try all
        # available handlers. If one of them authenticates
        # successfully, a response will be returned. If none of
        # them succeeds, None will be returned and the error
        # handler will raise the 401 'Unauthorized' or the 407
        # 'Proxy Authentication Required' error.
        return None

    def add_auth_header(self, request, header):
        """Add the authentication header to the request"""
        request.add_unredirected_header(self.auth_header, header)

    def auth_match(self, header, auth):
        """Check that we are able to handle that authentication scheme.

        The request authentication parameters may need to be
        updated with info from the server. Some of these
        parameters, when combined, are considered to be the
        authentication key, if one of them change the
        authentication result may change. 'user' and 'password'
        are exampls, but some auth schemes may have others
        (digest's nonce is an example, digest's nonce_count is a
        *counter-example*). Such parameters must be updated by
        using the update_auth() method.

        :param header: The authentication header sent by the server.
        :param auth: The auth parameters already known. They may be
             updated.
        :returns: True if we can try to handle the authentication.
        """
        raise NotImplementedError(self.auth_match)

    def build_auth_header(self, auth, request):
        """Build the value of the header used to authenticate.

        :param auth: The auth parameters needed to build the header.
        :param request: The request needing authentication.

        :return: None or header.
        """
        raise NotImplementedError(self.build_auth_header)

    def auth_successful(self, request, response):
        """The authentification was successful for the request.

        Additional infos may be available in the response.

        :param request: The succesfully authenticated request.
        :param response: The server response (may contain auth info).
        """
        # It may happen that we need to reconnect later, let's be ready
        self._retry_count = None

    def get_user_password(self, auth):
        """Ask user for a password if none is already available.

        :param auth: authentication info gathered so far (from the initial url
            and then during dialog with the server).
        """
        auth_conf = config.AuthenticationConfig()
        user = auth.get('user', None)
        password = auth.get('password', None)
        realm = auth['realm']

        if user is None:
            user = auth_conf.get_user(auth['protocol'], auth['host'],
                                      port=auth['port'], path=auth['path'],
                                      realm=realm, ask=True,
                                      prompt=self.build_username_prompt(auth))
        if user is not None and password is None:
            password = auth_conf.get_password(
                auth['protocol'], auth['host'], user, port=auth['port'],
                path=auth['path'], realm=realm,
                prompt=self.build_password_prompt(auth))

        return user, password

    def _build_password_prompt(self, auth):
        """Build a prompt taking the protocol used into account.

        The AuthHandler is used by http and https, we want that information in
        the prompt, so we build the prompt from the authentication dict which
        contains all the needed parts.

        Also, http and proxy AuthHandlers present different prompts to the
        user. The daughter classes should implements a public
        build_password_prompt using this method.
        """
        prompt = '%s' % auth['protocol'].upper() + ' %(user)s@%(host)s'
        realm = auth['realm']
        if realm is not None:
            prompt += ", Realm: '%s'" % realm
        prompt += ' password'
        return prompt

    def _build_username_prompt(self, auth):
        """Build a prompt taking the protocol used into account.

        The AuthHandler is used by http and https, we want that information in
        the prompt, so we build the prompt from the authentication dict which
        contains all the needed parts.

        Also, http and proxy AuthHandlers present different prompts to the
        user. The daughter classes should implements a public
        build_username_prompt using this method.
        """
        prompt = '%s' % auth['protocol'].upper() + ' %(host)s'
        realm = auth['realm']
        if realm is not None:
            prompt += ", Realm: '%s'" % realm
        prompt += ' username'
        return prompt

    def http_request(self, request):
        """Insert an authentication header if information is available"""
        auth = self.get_auth(request)
        if self.auth_params_reusable(auth):
            self.add_auth_header(request, self.build_auth_header(auth, request))
        return request

    https_request = http_request # FIXME: Need test


class NegotiateAuthHandler(AbstractAuthHandler):
    """A authentication handler that handles WWW-Authenticate: Negotiate.

    At the moment this handler supports just Kerberos. In the future,
    NTLM support may also be added.
    """

    scheme = 'negotiate'
    handler_order = 480
    requires_username = False

    def auth_match(self, header, auth):
        scheme, raw_auth = self._parse_auth_header(header)
        if scheme != self.scheme:
            return False
        self.update_auth(auth, 'scheme', scheme)
        resp = self._auth_match_kerberos(auth)
        if resp is None:
            return False
        # Optionally should try to authenticate using NTLM here
        self.update_auth(auth, 'negotiate_response', resp)
        return True

    def _auth_match_kerberos(self, auth):
        """Try to create a GSSAPI response for authenticating against a host."""
        if not have_kerberos:
            return None
        ret, vc = kerberos.authGSSClientInit("HTTP@%(host)s" % auth)
        if ret < 1:
            trace.warning('Unable to create GSSAPI context for %s: %d',
                auth['host'], ret)
            return None
        ret = kerberos.authGSSClientStep(vc, "")
        if ret < 0:
            trace.mutter('authGSSClientStep failed: %d', ret)
            return None
        return kerberos.authGSSClientResponse(vc)

    def build_auth_header(self, auth, request):
        return "Negotiate %s" % auth['negotiate_response']

    def auth_params_reusable(self, auth):
        # If the auth scheme is known, it means a previous
        # authentication was successful, all information is
        # available, no further checks are needed.
        return (auth.get('scheme', None) == 'negotiate' and
                auth.get('negotiate_response', None) is not None)


class BasicAuthHandler(AbstractAuthHandler):
    """A custom basic authentication handler."""

    scheme = 'basic'
    handler_order = 500
    auth_regexp = re.compile('realm="([^"]*)"', re.I)

    def build_auth_header(self, auth, request):
        raw = '%s:%s' % (auth['user'], auth['password'])
        auth_header = 'Basic ' + raw.encode('base64').strip()
        return auth_header

    def extract_realm(self, header_value):
        match = self.auth_regexp.search(header_value)
        realm = None
        if match:
            realm = match.group(1)
        return match, realm

    def auth_match(self, header, auth):
        scheme, raw_auth = self._parse_auth_header(header)
        if scheme != self.scheme:
            return False

        match, realm = self.extract_realm(raw_auth)
        if match:
            # Put useful info into auth
            self.update_auth(auth, 'scheme', scheme)
            self.update_auth(auth, 'realm', realm)
            if (auth.get('user', None) is None
                or auth.get('password', None) is None):
                user, password = self.get_user_password(auth)
                self.update_auth(auth, 'user', user)
                self.update_auth(auth, 'password', password)
        return match is not None

    def auth_params_reusable(self, auth):
        # If the auth scheme is known, it means a previous
        # authentication was successful, all information is
        # available, no further checks are needed.
        return auth.get('scheme', None) == 'basic'


def get_digest_algorithm_impls(algorithm):
    H = None
    KD = None
    if algorithm == 'MD5':
        H = lambda x: osutils.md5(x).hexdigest()
    elif algorithm == 'SHA':
        H = lambda x: osutils.sha(x).hexdigest()
    if H is not None:
        KD = lambda secret, data: H("%s:%s" % (secret, data))
    return H, KD


def get_new_cnonce(nonce, nonce_count):
    raw = '%s:%d:%s:%s' % (nonce, nonce_count, time.ctime(),
                           urllib2.randombytes(8))
    return osutils.sha(raw).hexdigest()[:16]


class DigestAuthHandler(AbstractAuthHandler):
    """A custom digest authentication handler."""

    scheme = 'digest'
    # Before basic as digest is a bit more secure and should be preferred
    handler_order = 490

    def auth_params_reusable(self, auth):
        # If the auth scheme is known, it means a previous
        # authentication was successful, all information is
        # available, no further checks are needed.
        return auth.get('scheme', None) == 'digest'

    def auth_match(self, header, auth):
        scheme, raw_auth = self._parse_auth_header(header)
        if scheme != self.scheme:
            return False

        # Put the requested authentication info into a dict
        req_auth = urllib2.parse_keqv_list(urllib2.parse_http_list(raw_auth))

        # Check that we can handle that authentication
        qop = req_auth.get('qop', None)
        if qop != 'auth': # No auth-int so far
            return False

        H, KD = get_digest_algorithm_impls(req_auth.get('algorithm', 'MD5'))
        if H is None:
            return False

        realm = req_auth.get('realm', None)
        # Put useful info into auth
        self.update_auth(auth, 'scheme', scheme)
        self.update_auth(auth, 'realm', realm)
        if auth.get('user', None) is None or auth.get('password', None) is None:
            user, password = self.get_user_password(auth)
            self.update_auth(auth, 'user', user)
            self.update_auth(auth, 'password', password)

        try:
            if req_auth.get('algorithm', None) is not None:
                self.update_auth(auth, 'algorithm', req_auth.get('algorithm'))
            nonce = req_auth['nonce']
            if auth.get('nonce', None) != nonce:
                # A new nonce, never used
                self.update_auth(auth, 'nonce_count', 0)
            self.update_auth(auth, 'nonce', nonce)
            self.update_auth(auth, 'qop', qop)
            auth['opaque'] = req_auth.get('opaque', None)
        except KeyError:
            # Some required field is not there
            return False

        return True

    def build_auth_header(self, auth, request):
        url_scheme, url_selector = urllib.splittype(request.get_selector())
        sel_host, uri = urllib.splithost(url_selector)

        A1 = '%s:%s:%s' % (auth['user'], auth['realm'], auth['password'])
        A2 = '%s:%s' % (request.get_method(), uri)

        nonce = auth['nonce']
        qop = auth['qop']

        nonce_count = auth['nonce_count'] + 1
        ncvalue = '%08x' % nonce_count
        cnonce = get_new_cnonce(nonce, nonce_count)

        H, KD = get_digest_algorithm_impls(auth.get('algorithm', 'MD5'))
        nonce_data = '%s:%s:%s:%s:%s' % (nonce, ncvalue, cnonce, qop, H(A2))
        request_digest = KD(H(A1), nonce_data)

        header = 'Digest '
        header += 'username="%s", realm="%s", nonce="%s"' % (auth['user'],
                                                             auth['realm'],
                                                             nonce)
        header += ', uri="%s"' % uri
        header += ', cnonce="%s", nc=%s' % (cnonce, ncvalue)
        header += ', qop="%s"' % qop
        header += ', response="%s"' % request_digest
        # Append the optional fields
        opaque = auth.get('opaque', None)
        if opaque:
            header += ', opaque="%s"' % opaque
        if auth.get('algorithm', None):
            header += ', algorithm="%s"' % auth.get('algorithm')

        # We have used the nonce once more, update the count
        auth['nonce_count'] = nonce_count

        return header


class HTTPAuthHandler(AbstractAuthHandler):
    """Custom http authentication handler.

    Send the authentication preventively to avoid the roundtrip
    associated with the 401 error and keep the revelant info in
    the auth request attribute.
    """

    auth_required_header = 'www-authenticate'
    auth_header = 'Authorization'

    def get_auth(self, request):
        """Get the auth params from the request"""
        return request.auth

    def set_auth(self, request, auth):
        """Set the auth params for the request"""
        request.auth = auth

    def build_password_prompt(self, auth):
        return self._build_password_prompt(auth)

    def build_username_prompt(self, auth):
        return self._build_username_prompt(auth)

    def http_error_401(self, req, fp, code, msg, headers):
        return self.auth_required(req, headers)


class ProxyAuthHandler(AbstractAuthHandler):
    """Custom proxy authentication handler.

    Send the authentication preventively to avoid the roundtrip
    associated with the 407 error and keep the revelant info in
    the proxy_auth request attribute..
    """

    auth_required_header = 'proxy-authenticate'
    # FIXME: the correct capitalization is Proxy-Authorization,
    # but python-2.4 urllib2.Request insist on using capitalize()
    # instead of title().
    auth_header = 'Proxy-authorization'

    def get_auth(self, request):
        """Get the auth params from the request"""
        return request.proxy_auth

    def set_auth(self, request, auth):
        """Set the auth params for the request"""
        request.proxy_auth = auth

    def build_password_prompt(self, auth):
        prompt = self._build_password_prompt(auth)
        prompt = 'Proxy ' + prompt
        return prompt

    def build_username_prompt(self, auth):
        prompt = self._build_username_prompt(auth)
        prompt = 'Proxy ' + prompt
        return prompt

    def http_error_407(self, req, fp, code, msg, headers):
        return self.auth_required(req, headers)


class HTTPBasicAuthHandler(BasicAuthHandler, HTTPAuthHandler):
    """Custom http basic authentication handler"""


class ProxyBasicAuthHandler(BasicAuthHandler, ProxyAuthHandler):
    """Custom proxy basic authentication handler"""


class HTTPDigestAuthHandler(DigestAuthHandler, HTTPAuthHandler):
    """Custom http basic authentication handler"""


class ProxyDigestAuthHandler(DigestAuthHandler, ProxyAuthHandler):
    """Custom proxy basic authentication handler"""


class HTTPNegotiateAuthHandler(NegotiateAuthHandler, HTTPAuthHandler):
    """Custom http negotiate authentication handler"""


class ProxyNegotiateAuthHandler(NegotiateAuthHandler, ProxyAuthHandler):
    """Custom proxy negotiate authentication handler"""


class HTTPErrorProcessor(urllib2.HTTPErrorProcessor):
    """Process HTTP error responses.

    We don't really process the errors, quite the contrary
    instead, we leave our Transport handle them.
    """

    accepted_errors = [200, # Ok
                       206, # Partial content
                       404, # Not found
                       ]
    """The error codes the caller will handle.

    This can be specialized in the request on a case-by case basis, but the
    common cases are covered here.
    """

    def http_response(self, request, response):
        code, msg, hdrs = response.code, response.msg, response.info()

        accepted_errors = request.accepted_errors
        if accepted_errors is None:
            accepted_errors = self.accepted_errors

        if code not in accepted_errors:
            response = self.parent.error('http', request, response,
                                         code, msg, hdrs)
        return response

    https_response = http_response


class HTTPDefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
    """Translate common errors into bzr Exceptions"""

    def http_error_default(self, req, fp, code, msg, hdrs):
        if code == 403:
            raise errors.TransportError(
                'Server refuses to fulfill the request (403 Forbidden)'
                ' for %s' % req.get_full_url())
        else:
            raise errors.InvalidHttpResponse(req.get_full_url(),
                                             'Unable to handle http code %d: %s'
                                             % (code, msg))


class Opener(object):
    """A wrapper around urllib2.build_opener

    Daughter classes can override to build their own specific opener
    """
    # TODO: Provides hooks for daughter classes.

    def __init__(self,
                 connection=ConnectionHandler,
                 redirect=HTTPRedirectHandler,
                 error=HTTPErrorProcessor,
                 report_activity=None):
        self._opener = urllib2.build_opener(
            connection(report_activity=report_activity),
            redirect, error,
            ProxyHandler(),
            HTTPBasicAuthHandler(),
            HTTPDigestAuthHandler(),
            HTTPNegotiateAuthHandler(),
            ProxyBasicAuthHandler(),
            ProxyDigestAuthHandler(),
            ProxyNegotiateAuthHandler(),
            HTTPHandler,
            HTTPSHandler,
            HTTPDefaultErrorHandler,
            )

        self.open = self._opener.open
        if DEBUG >= 9:
            # When dealing with handler order, it's easy to mess
            # things up, the following will help understand which
            # handler is used, when and for what.
            import pprint
            pprint.pprint(self._opener.__dict__)