← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to tools/http_client.py

add a clean target

Show diffs side-by-side

added added

removed removed

Lines of Context:
tools/http_client.py
68
68
# The content is written into temporary files.  It returns a list of
69
69
# readable file objects.
70
70
 
 
71
# TODO: If we try pipelined or keepalive and the connection drop out
 
72
# then retry the request on a new connection; eventually we should perhaps
 
73
# learn that a given host or network just won't allow keepalive.
 
74
 
 
75
 
71
76
import asyncore
72
77
import socket, string, time, sys
73
78
import StringIO
74
79
import mimetools, urlparse, urllib
75
80
import logging
76
81
 
 
82
logging.basicConfig(level=logging.DEBUG,
 
83
                    format='%(asctime)s %(levelname)s %(message)s',
 
84
                    filename='/tmp/http_client.log',
 
85
                    filemode='w')
 
86
 
77
87
logger = logging.getLogger('bzr.http_client')
78
88
debug = logger.debug
79
89
info = logger.info
115
125
        self.channels = []
116
126
        self.try_pipelined = False
117
127
        self.try_keepalive = False
118
 
        self.max_channels = 3
 
128
        self.max_channels = 5
119
129
 
120
130
 
121
131
    def enqueue(self, url, consumer):
134
144
 
135
145
    def _make_channel(self):
136
146
        # proxy2 203.17.154.69
137
 
        # bazaar-ng.org 
138
 
        return HttpChannel('82.211.81.161', 80, self)
 
147
        # return HttpChannel('82.211.81.161', 80, self)         # bazaar-ng.org 
139
148
        # return HttpChannel('203.17.154.69', 8080, self)
140
 
        # return HttpChannel('localhost', 8000, self)
 
149
        return HttpChannel('127.0.0.1', 8000, self)  # forwarded
141
150
            
142
151
 
143
152
    def _wake_up_channel(self):
172
181
            choice(self.channels).take_one()
173
182
 
174
183
 
175
 
        debug("request left until a channel's idle")
 
184
        # debug("request postponed until a channel's idle")
176
185
        
177
186
 
178
187
 
394
403
                
395
404
                assert got_data
396
405
 
397
 
                debug('pass back %d bytes of %s' % (len(got_data),
398
 
                                                    self.sent_requests[0][0]))
 
406
                self.content_remaining -= len(got_data)
 
407
 
 
408
                debug('pass back %d bytes of %s, %d remain'
 
409
                      % (len(got_data),
 
410
                         self.sent_requests[0][0],
 
411
                         self.content_remaining))
399
412
                consumer.feed(data)
400
413
 
401
 
                self.content_remaining -= len(got_data)
402
 
 
403
414
            if self.content_remaining == 0:
404
415
                del self.sent_requests[0]
405
416
 
 
417
                debug('content complete')
 
418
                consumer.content_complete()
 
419
                
406
420
                # reset lots of things and try to get the next response header
407
421
                if self.response.connection_reply == 'close':
408
422
                    debug('server requested close')
414
428
                    self.close()
415
429
                else:
416
430
                    debug("ready for next header...")
417
 
                    consumer.content_complete()
418
431
                    self.take_one()
419
432
                self.response = None
420
433
 
428
441
        self.close()
429
442
 
430
443
 
 
444
class DummyConsumer:
 
445
    def __init__(self, url, pb):
 
446
        self.url = url
 
447
        self.outf = None
 
448
        self._pb = pb
 
449
 
 
450
    def feed(self, data):
 
451
        # print "feed", repr(data)
 
452
        # print "feed", repr(data[:20]), repr(data[-20:]), len(data)
 
453
        if not self.outf:
 
454
            base = self.url[self.url.rindex('/')+1:]
 
455
            self.outf = file('/tmp/download/' + base, 'wb')
 
456
        self.outf.write(data)
 
457
 
 
458
    def error(self, err_info):
 
459
        import traceback
 
460
        error('error reported to consumer')
 
461
        traceback.print_exception(err_info[0], err_info[1], err_info[2])
 
462
        sys.exit(1)
 
463
 
 
464
    def content_complete(self):
 
465
        info('content complete from %s' % self.url)
 
466
        self.outf.close()
 
467
        self.outf = None
 
468
        # using last_cnt is cheating
 
469
        self._pb.update('downloading inventory',
 
470
                        self._pb.last_cnt+1,
 
471
                        self._pb.last_total)
 
472
 
 
473
 
 
474
 
431
475
if __name__ == "__main__":
432
 
    class dummy_consumer:
433
 
        def __init__(self, url):
434
 
            self.url = url
435
 
 
436
 
        def feed(self, data):
437
 
            # print "feed", repr(data)
438
 
            # print "feed", repr(data[:20]), repr(data[-20:]), len(data)
439
 
            pass
440
 
            
441
 
        def error(self, err_info):
442
 
            import traceback
443
 
            traceback.print_exception(err_info[0], err_info[1], err_info[2])
444
 
 
445
 
        def content_complete(self):
446
 
            debug('content complete from %s' % self.url)
447
 
            
448
 
 
449
476
    logging.basicConfig(level=logging.DEBUG)
450
477
 
451
478
    mgr = DownloadManager()
452
479
 
453
480
    from bzrlib.branch import Branch
 
481
    from bzrlib.progress import ProgressBar
 
482
 
 
483
    pb = ProgressBar()
454
484
    revs = Branch('/home/mbp/work/bzr').revision_history()
455
 
 
456
 
        
 
485
    pb.update('downloading inventories', 0, len(revs))
 
486
 
 
487
    for rev in revs:
 
488
        url = 'http://www.bazaar-ng.org/bzr/bzr.dev/.bzr/inventory-store/' \
 
489
              + rev + '.gz'
 
490
        mgr.enqueue(url, DummyConsumer(url, pb))
 
491
 
 
492
    mgr.run()
 
493
    
 
494
 
 
495
 
457
496
    
458
497
#     for url in ['http://www.bazaar-ng.org/',
459
498
#                 'http://www.bazaar-ng.org/tutorial.html',
460
499
#                 'http://www.bazaar-ng.org/download.html',
461
500
#                 'http://www.bazaar-ng.org/bzr/bzr.dev/.bzr/revision-store/mbp@hope-20050415013653-3b3c9c3d33fae0a6.gz',
462
501
#                 ]:
463
 
 
464
 
    for rev in revs:
465
 
#        url = 'http://www.bazaar-ng.org/bzr/bzr.dev/.bzr/revision-store/' \
466
 
        url = 'http://www.bazaar-ng.org/bzr/bzr.dev/.bzr/inventory-store/' \
467
 
              + rev + '.gz'
468
 
        mgr.enqueue(url, dummy_consumer(url))
469
 
 
470
 
    mgr.run()
471