~bzr-pqm/bzr/bzr.dev

# $Id: http_client.py 271 2004-10-09 10:50:59Z fredrik $

# a simple asynchronous http client (based on SimpleAsyncHTTP.py from

# "Python Standard Library" by Fredrik Lundh, O'Reilly 2001)

#

# HTTP/1.1 and GZIP support added in January 2003 by Fredrik Lundh.

#

# changes:

# 2004-08-26 fl   unified http callback

# 2004-10-09 fl   factored out gzip_consumer support

#

# Copyright (c) 2001-2004 by Fredrik Lundh.  All rights reserved.

#

import asyncore

import socket, string, time, sys

import StringIO

import mimetools, urlparse, urllib

try:

    from gzip_consumer import GzipConsumer

except ImportError:

    pass

##

# Close connection.   Request handlers can raise this exception to

# indicate that the connection should be closed.

class CloseConnection(Exception):

    pass

##

# Redirect connection.  Request handlers can raise this exception to

# indicate that the a new request should be issued.

class Redirect(CloseConnection):

    def __init__(self, location):

        self.location = location

##

# Asynchronous HTTP/1.1 client.

class async_http(asyncore.dispatcher_with_send):

    # asynchronous http client

    user_agent = "http_client.py 1.2 (http://effbot.org/zone)"

    http_version = "1.1"

    proxies = urllib.getproxies()

    def __init__(self, uri, consumer, extra_headers=None):

        asyncore.dispatcher_with_send.__init__(self)

        # turn the uri into a valid request

        scheme, host, path, params, query, fragment = urlparse.urlparse(uri)

        # use origin host

        self.host = host

        # get proxy settings, if any

        proxy = self.proxies.get(scheme)

        if proxy:

            scheme, host, x, x, x, x = urlparse.urlparse(proxy)

        assert scheme == "http", "only supports HTTP requests (%s)" % scheme

        if not path:

            path = "/"

        if params:

            path = path + ";" + params

        if query:

            path = path + "?" + query

        if proxy:

            path = scheme + "://" + self.host + path

        self.path = path

        # get port number

        try:

            host, port = host.split(":", 1)

            port = int(port)

        except (TypeError, ValueError):

            port = 80 # default port

        self.consumer = consumer

        self.status = None

        self.header = None

        self.bytes_in = 0

        self.bytes_out = 0

        self.content_type = None

        self.content_length = None

        self.content_encoding = None

        self.transfer_encoding = None

        self.data = ""

        self.chunk_size = None

        self.timestamp = time.time()

        self.extra_headers = extra_headers

        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)

        try:

            self.connect((host, port))

        except socket.error:

            self.consumer.http(0, self, sys.exc_info())

    def handle_connect(self):

        # connection succeeded

        request = [

            "GET %s HTTP/%s" % (self.path, self.http_version),

            "Host: %s" % self.host,

            ]

        if GzipConsumer:

            request.append("Accept-Encoding: gzip")

        if self.extra_headers:

            request.extend(self.extra_headers)

        # make sure to include a user agent

        for header in request:

            if string.lower(header).startswith("user-agent:"):

                break

        else:

            request.append("User-Agent: %s" % self.user_agent)

        request = string.join(request, "\r\n") + "\r\n\r\n"

        self.send(request)

        self.bytes_out = self.bytes_out + len(request)

    def handle_expt(self):

        # connection failed (windows); notify consumer

        if sys.platform == "win32":

            self.close()

            self.consumer.http(0, self)

    def handle_read(self):

        # handle incoming data

        data = self.recv(2048)

        self.data = self.data + data

        self.bytes_in = self.bytes_in + len(data)

        while self.data:

            if not self.header:

                # check if we've seen a full header

                header = self.data.split("\r\n\r\n", 1)

                if len(header) <= 1:

                    return

                header, self.data = header

                # parse header

                fp = StringIO.StringIO(header)

                self.status = fp.readline().split(" ", 2)

                self.header = mimetools.Message(fp)

                # get http headers

                self.content_type = self.header.get("content-type")

                try:

                    self.content_length = int(

                        self.header.get("content-length")

                        )

                except (ValueError, TypeError):

                    self.content_length = None

                self.transfer_encoding = self.header.get("transfer-encoding")

                self.content_encoding = self.header.get("content-encoding")

                if self.content_encoding == "gzip":

                    # FIXME: report error if GzipConsumer is not available

                    self.consumer = GzipConsumer(self.consumer)

                try:

                    self.consumer.http(1, self)

                except Redirect, v:

                    # redirect

                    if v.location:

                        do_request(

                            v.location, self.consumer, self.extra_headers

                            )

                    self.close()

                    return

                except CloseConnection:

                    self.close()

                    return

            if self.transfer_encoding == "chunked" and self.chunk_size is None:

                # strip off leading whitespace

                if self.data.startswith("\r\n"):

                    self.data = self.data[2:]

                chunk_size = self.data.split("\r\n", 1)

                if len(chunk_size) <= 1:

                    return

                chunk_size, self.data = chunk_size

                try:

                    self.chunk_size = int(chunk_size, 16)

                    if self.chunk_size <= 0:

                        raise ValueError

                except ValueError:

                    return self.handle_close()

            if not self.data:

                return

            data = self.data

            self.data = ""

            chunk_size = self.chunk_size or len(data)

            if chunk_size < len(data):

                self.data = data[chunk_size:]

                data = data[:chunk_size]

                self.chunk_size = None

            else:

                self.chunk_size = chunk_size - len(data)

                if self.chunk_size <= 0:

                    self.chunk_size = None

            if data:

                self.consumer.feed(data)

            if self.content_length:

                self.content_length -= chunk_size

                if self.content_length <= 0:

                    return self.handle_close()

    def handle_close(self):

        self.consumer.close()

        self.close()

    def handle_error(self):

        self.consumer.http(0, self, sys.exc_info())

        self.close()

def do_request(uri, consumer, extra_headers=None):

    return async_http(uri, consumer, extra_headers)

if __name__ == "__main__":

    class dummy_consumer:

        def feed(self, data):

            # print "feed", repr(data)

            print "feed", repr(data[:20]), repr(data[-20:]), len(data)

        def close(self):

            print "close"

        def http(self, ok, connection, **args):

            print ok, connection, args

            print "status", connection.status

            print "header", connection.header

    try:

        url = sys.argv[1]

    except IndexError:

        url = "http://www.cnn.com/"

    do_request(url, dummy_consumer())

    asyncore.loop()