← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to tools/http_client.py

  • Committer: Robert Collins
  • Date: 2005-08-23 10:44:48 UTC
  • mto: (974.1.50) (1185.1.10) (1092.3.1)
  • mto: This revision was merged to the branch mainline in revision 1139.
  • Revision ID: robertc@robertcollins.net-20050823104448-fb5d448e7a5a8ee3
relace runTest with test_foo in blackbox tests

Show diffs side-by-side

added added

removed removed

Lines of Context:
tools/http_client.py
68
68
# The content is written into temporary files.  It returns a list of
69
69
# readable file objects.
70
70
 
71
 
# TODO: If we try pipelined or keepalive and the connection drop out
72
 
# then retry the request on a new connection; eventually we should perhaps
73
 
# learn that a given host or network just won't allow keepalive.
74
 
 
75
 
 
76
71
import asyncore
77
72
import socket, string, time, sys
78
73
import StringIO
79
74
import mimetools, urlparse, urllib
80
75
import logging
81
76
 
82
 
logging.basicConfig(level=logging.DEBUG,
83
 
                    format='%(asctime)s %(levelname)s %(message)s',
84
 
                    filename='/tmp/http_client.log',
85
 
                    filemode='w')
86
 
 
87
77
logger = logging.getLogger('bzr.http_client')
88
78
debug = logger.debug
89
79
info = logger.info
125
115
        self.channels = []
126
116
        self.try_pipelined = False
127
117
        self.try_keepalive = False
128
 
        self.max_channels = 5
 
118
        self.max_channels = 3
129
119
 
130
120
 
131
121
    def enqueue(self, url, consumer):
144
134
 
145
135
    def _make_channel(self):
146
136
        # proxy2 203.17.154.69
147
 
        # return HttpChannel('82.211.81.161', 80, self)         # bazaar-ng.org 
 
137
        # bazaar-ng.org 
 
138
        return HttpChannel('82.211.81.161', 80, self)
148
139
        # return HttpChannel('203.17.154.69', 8080, self)
149
 
        return HttpChannel('127.0.0.1', 8000, self)  # forwarded
 
140
        # return HttpChannel('localhost', 8000, self)
150
141
            
151
142
 
152
143
    def _wake_up_channel(self):
181
172
            choice(self.channels).take_one()
182
173
 
183
174
 
184
 
        # debug("request postponed until a channel's idle")
 
175
        debug("request left until a channel's idle")
185
176
        
186
177
 
187
178
 
403
394
                
404
395
                assert got_data
405
396
 
 
397
                debug('pass back %d bytes of %s' % (len(got_data),
 
398
                                                    self.sent_requests[0][0]))
 
399
                consumer.feed(data)
 
400
 
406
401
                self.content_remaining -= len(got_data)
407
402
 
408
 
                debug('pass back %d bytes of %s, %d remain'
409
 
                      % (len(got_data),
410
 
                         self.sent_requests[0][0],
411
 
                         self.content_remaining))
412
 
                consumer.feed(data)
413
 
 
414
403
            if self.content_remaining == 0:
415
404
                del self.sent_requests[0]
416
405
 
417
 
                debug('content complete')
418
 
                consumer.content_complete()
419
 
                
420
406
                # reset lots of things and try to get the next response header
421
407
                if self.response.connection_reply == 'close':
422
408
                    debug('server requested close')
428
414
                    self.close()
429
415
                else:
430
416
                    debug("ready for next header...")
 
417
                    consumer.content_complete()
431
418
                    self.take_one()
432
419
                self.response = None
433
420
 
441
428
        self.close()
442
429
 
443
430
 
444
 
class DummyConsumer:
445
 
    def __init__(self, url, pb):
446
 
        self.url = url
447
 
        self.outf = None
448
 
        self._pb = pb
449
 
 
450
 
    def feed(self, data):
451
 
        # print "feed", repr(data)
452
 
        # print "feed", repr(data[:20]), repr(data[-20:]), len(data)
453
 
        if not self.outf:
454
 
            base = self.url[self.url.rindex('/')+1:]
455
 
            self.outf = file('/tmp/download/' + base, 'wb')
456
 
        self.outf.write(data)
457
 
 
458
 
    def error(self, err_info):
459
 
        import traceback
460
 
        error('error reported to consumer')
461
 
        traceback.print_exception(err_info[0], err_info[1], err_info[2])
462
 
        sys.exit(1)
463
 
 
464
 
    def content_complete(self):
465
 
        info('content complete from %s' % self.url)
466
 
        self.outf.close()
467
 
        self.outf = None
468
 
        # using last_cnt is cheating
469
 
        self._pb.update('downloading inventory',
470
 
                        self._pb.last_cnt+1,
471
 
                        self._pb.last_total)
472
 
 
473
 
 
474
 
 
475
431
if __name__ == "__main__":
 
432
    class dummy_consumer:
 
433
        def __init__(self, url):
 
434
            self.url = url
 
435
 
 
436
        def feed(self, data):
 
437
            # print "feed", repr(data)
 
438
            # print "feed", repr(data[:20]), repr(data[-20:]), len(data)
 
439
            pass
 
440
            
 
441
        def error(self, err_info):
 
442
            import traceback
 
443
            traceback.print_exception(err_info[0], err_info[1], err_info[2])
 
444
 
 
445
        def content_complete(self):
 
446
            debug('content complete from %s' % self.url)
 
447
            
 
448
 
476
449
    logging.basicConfig(level=logging.DEBUG)
477
450
 
478
451
    mgr = DownloadManager()
479
452
 
480
453
    from bzrlib.branch import Branch
481
 
    from bzrlib.progress import ProgressBar
482
 
 
483
 
    pb = ProgressBar()
484
454
    revs = Branch('/home/mbp/work/bzr').revision_history()
485
 
    pb.update('downloading inventories', 0, len(revs))
486
 
 
487
 
    for rev in revs:
488
 
        url = 'http://www.bazaar-ng.org/bzr/bzr.dev/.bzr/inventory-store/' \
489
 
              + rev + '.gz'
490
 
        mgr.enqueue(url, DummyConsumer(url, pb))
491
 
 
492
 
    mgr.run()
493
 
    
494
 
 
495
 
 
 
455
 
 
456
        
496
457
    
497
458
#     for url in ['http://www.bazaar-ng.org/',
498
459
#                 'http://www.bazaar-ng.org/tutorial.html',
499
460
#                 'http://www.bazaar-ng.org/download.html',
500
461
#                 'http://www.bazaar-ng.org/bzr/bzr.dev/.bzr/revision-store/mbp@hope-20050415013653-3b3c9c3d33fae0a6.gz',
501
462
#                 ]:
 
463
 
 
464
    for rev in revs:
 
465
#        url = 'http://www.bazaar-ng.org/bzr/bzr.dev/.bzr/revision-store/' \
 
466
        url = 'http://www.bazaar-ng.org/bzr/bzr.dev/.bzr/inventory-store/' \
 
467
              + rev + '.gz'
 
468
        mgr.enqueue(url, dummy_consumer(url))
 
469
 
 
470
    mgr.run()
 
471