~bzr-pqm/bzr/bzr.dev : contents of bzrlib/smart/protocol.py at revision 2399.1.15

~bzr-pqm/bzr/bzr.dev : (revision 2399.1.15)

# Copyright (C) 2006, 2007 Canonical Ltd
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

"""Wire-level encoding and decoding of requests and responses for the smart
client and server.
"""


from cStringIO import StringIO

from bzrlib import errors
from bzrlib.smart import request


def _recv_tuple(from_file):
    req_line = from_file.readline()
    return _decode_tuple(req_line)


def _decode_tuple(req_line):
    if req_line == None or req_line == '':
        return None
    if req_line[-1] != '\n':
        raise errors.SmartProtocolError("request %r not terminated" % req_line)
    return tuple(req_line[:-1].split('\x01'))


def _encode_tuple(args):
    """Encode the tuple args to a bytestream."""
    return '\x01'.join(args) + '\n'


class SmartProtocolBase(object):
    """Methods common to client and server"""

    # TODO: this only actually accomodates a single block; possibly should
    # support multiple chunks?
    def _encode_bulk_data(self, body):
        """Encode body as a bulk data chunk."""
        return ''.join(('%d\n' % len(body), body, 'done\n'))

    def _serialise_offsets(self, offsets):
        """Serialise a readv offset list."""
        txt = []
        for start, length in offsets:
            txt.append('%d,%d' % (start, length))
        return '\n'.join(txt)
        

class SmartServerRequestProtocolOne(SmartProtocolBase):
    """Server-side encoding and decoding logic for smart version 1."""
    
    def __init__(self, backing_transport, write_func):
        self._backing_transport = backing_transport
        self.excess_buffer = ''
        self._finished = False
        self.in_buffer = ''
        self.has_dispatched = False
        self.request = None
        self._body_decoder = None
        self._write_func = write_func

    def accept_bytes(self, bytes):
        """Take bytes, and advance the internal state machine appropriately.
        
        :param bytes: must be a byte string
        """
        assert isinstance(bytes, str)
        self.in_buffer += bytes
        if not self.has_dispatched:
            if '\n' not in self.in_buffer:
                # no command line yet
                return
            self.has_dispatched = True
            try:
                first_line, self.in_buffer = self.in_buffer.split('\n', 1)
                first_line += '\n'
                req_args = _decode_tuple(first_line)
                self.request = request.SmartServerRequestHandler(
                    self._backing_transport, commands=request.request_handlers)
                self.request.dispatch_command(req_args[0], req_args[1:])
                if self.request.finished_reading:
                    # trivial request
                    self.excess_buffer = self.in_buffer
                    self.in_buffer = ''
                    self._send_response(self.request.response.args,
                        self.request.response.body)
            except KeyboardInterrupt:
                raise
            except Exception, exception:
                # everything else: pass to client, flush, and quit
                self._send_response(('error', str(exception)))
                return

        if self.has_dispatched:
            if self._finished:
                # nothing to do.XXX: this routine should be a single state 
                # machine too.
                self.excess_buffer += self.in_buffer
                self.in_buffer = ''
                return
            if self._body_decoder is None:
                self._body_decoder = LengthPrefixedBodyDecoder()
            self._body_decoder.accept_bytes(self.in_buffer)
            self.in_buffer = self._body_decoder.unused_data
            body_data = self._body_decoder.read_pending_data()
            self.request.accept_body(body_data)
            if self._body_decoder.finished_reading:
                self.request.end_of_body()
                assert self.request.finished_reading, \
                    "no more body, request not finished"
            if self.request.response is not None:
                self._send_response(self.request.response.args,
                    self.request.response.body)
                self.excess_buffer = self.in_buffer
                self.in_buffer = ''
            else:
                assert not self.request.finished_reading, \
                    "no response and we have finished reading."

    def _send_response(self, args, body=None):
        """Send a smart server response down the output stream."""
        assert not self._finished, 'response already sent'
        self._finished = True
        self._write_func(_encode_tuple(args))
        if body is not None:
            assert isinstance(body, str), 'body must be a str'
            bytes = self._encode_bulk_data(body)
            self._write_func(bytes)

    def next_read_size(self):
        if self._finished:
            return 0
        if self._body_decoder is None:
            return 1
        else:
            return self._body_decoder.next_read_size()


class LengthPrefixedBodyDecoder(object):
    """Decodes the length-prefixed bulk data."""
    
    def __init__(self):
        self.bytes_left = None
        self.finished_reading = False
        self.unused_data = ''
        self.state_accept = self._state_accept_expecting_length
        self.state_read = self._state_read_no_data
        self._in_buffer = ''
        self._trailer_buffer = ''
    
    def accept_bytes(self, bytes):
        """Decode as much of bytes as possible.

        If 'bytes' contains too much data it will be appended to
        self.unused_data.

        finished_reading will be set when no more data is required.  Further
        data will be appended to self.unused_data.
        """
        # accept_bytes is allowed to change the state
        current_state = self.state_accept
        self.state_accept(bytes)
        while current_state != self.state_accept:
            current_state = self.state_accept
            self.state_accept('')

    def next_read_size(self):
        if self.bytes_left is not None:
            # Ideally we want to read all the remainder of the body and the
            # trailer in one go.
            return self.bytes_left + 5
        elif self.state_accept == self._state_accept_reading_trailer:
            # Just the trailer left
            return 5 - len(self._trailer_buffer)
        elif self.state_accept == self._state_accept_expecting_length:
            # There's still at least 6 bytes left ('\n' to end the length, plus
            # 'done\n').
            return 6
        else:
            # Reading excess data.  Either way, 1 byte at a time is fine.
            return 1
        
    def read_pending_data(self):
        """Return any pending data that has been decoded."""
        return self.state_read()

    def _state_accept_expecting_length(self, bytes):
        self._in_buffer += bytes
        pos = self._in_buffer.find('\n')
        if pos == -1:
            return
        self.bytes_left = int(self._in_buffer[:pos])
        self._in_buffer = self._in_buffer[pos+1:]
        self.bytes_left -= len(self._in_buffer)
        self.state_accept = self._state_accept_reading_body
        self.state_read = self._state_read_in_buffer

    def _state_accept_reading_body(self, bytes):
        self._in_buffer += bytes
        self.bytes_left -= len(bytes)
        if self.bytes_left <= 0:
            # Finished with body
            if self.bytes_left != 0:
                self._trailer_buffer = self._in_buffer[self.bytes_left:]
                self._in_buffer = self._in_buffer[:self.bytes_left]
            self.bytes_left = None
            self.state_accept = self._state_accept_reading_trailer
        
    def _state_accept_reading_trailer(self, bytes):
        self._trailer_buffer += bytes
        # TODO: what if the trailer does not match "done\n"?  Should this raise
        # a ProtocolViolation exception?
        if self._trailer_buffer.startswith('done\n'):
            self.unused_data = self._trailer_buffer[len('done\n'):]
            self.state_accept = self._state_accept_reading_unused
            self.finished_reading = True
    
    def _state_accept_reading_unused(self, bytes):
        self.unused_data += bytes

    def _state_read_no_data(self):
        return ''

    def _state_read_in_buffer(self):
        result = self._in_buffer
        self._in_buffer = ''
        return result


class SmartClientRequestProtocolOne(SmartProtocolBase):
    """The client-side protocol for smart version 1."""

    def __init__(self, request):
        """Construct a SmartClientRequestProtocolOne.

        :param request: A SmartClientMediumRequest to serialise onto and
            deserialise from.
        """
        self._request = request
        self._body_buffer = None

    def call(self, *args):
        bytes = _encode_tuple(args)
        self._request.accept_bytes(bytes)
        self._request.finished_writing()

    def call_with_body_bytes(self, args, body):
        """Make a remote call of args with body bytes 'body'.

        After calling this, call read_response_tuple to find the result out.
        """
        bytes = _encode_tuple(args)
        self._request.accept_bytes(bytes)
        bytes = self._encode_bulk_data(body)
        self._request.accept_bytes(bytes)
        self._request.finished_writing()

    def call_with_body_readv_array(self, args, body):
        """Make a remote call with a readv array.

        The body is encoded with one line per readv offset pair. The numbers in
        each pair are separated by a comma, and no trailing \n is emitted.
        """
        bytes = _encode_tuple(args)
        self._request.accept_bytes(bytes)
        readv_bytes = self._serialise_offsets(body)
        bytes = self._encode_bulk_data(readv_bytes)
        self._request.accept_bytes(bytes)
        self._request.finished_writing()

    def cancel_read_body(self):
        """After expecting a body, a response code may indicate one otherwise.

        This method lets the domain client inform the protocol that no body
        will be transmitted. This is a terminal method: after calling it the
        protocol is not able to be used further.
        """
        self._request.finished_reading()

    def read_response_tuple(self, expect_body=False):
        """Read a response tuple from the wire.

        This should only be called once.
        """
        result = self._recv_tuple()
        if not expect_body:
            self._request.finished_reading()
        return result

    def read_body_bytes(self, count=-1):
        """Read bytes from the body, decoding into a byte stream.
        
        We read all bytes at once to ensure we've checked the trailer for 
        errors, and then feed the buffer back as read_body_bytes is called.
        """
        if self._body_buffer is not None:
            return self._body_buffer.read(count)
        _body_decoder = LengthPrefixedBodyDecoder()

        while not _body_decoder.finished_reading:
            bytes_wanted = _body_decoder.next_read_size()
            bytes = self._request.read_bytes(bytes_wanted)
            _body_decoder.accept_bytes(bytes)
        self._request.finished_reading()
        self._body_buffer = StringIO(_body_decoder.read_pending_data())
        # XXX: TODO check the trailer result.
        return self._body_buffer.read(count)

    def _recv_tuple(self):
        """Receive a tuple from the medium request."""
        line = ''
        while not line or line[-1] != '\n':
            # TODO: this is inefficient - but tuples are short.
            new_char = self._request.read_bytes(1)
            line += new_char
            assert new_char != '', "end of file reading from server."
        return _decode_tuple(line)

    def query_version(self):
        """Return protocol version number of the server."""
        self.call('hello')
        resp = self.read_response_tuple()
        if resp == ('ok', '1'):
            return 1
        else:
            raise errors.SmartProtocolError("bad response %r" % (resp,))




2400.1.5 by Andrew Bennetts Add some missing docstrings and copyright boilerplate.	1	# Copyright (C) 2006, 2007 Canonical Ltd
	2	#
	3	# This program is free software; you can redistribute it and/or modify
	4	# it under the terms of the GNU General Public License as published by
	5	# the Free Software Foundation; either version 2 of the License, or
	6	# (at your option) any later version.
	7	#
	8	# This program is distributed in the hope that it will be useful,
	9	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	10	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	11	# GNU General Public License for more details.
	12	#
	13	# You should have received a copy of the GNU General Public License
	14	# along with this program; if not, write to the Free Software
	15	# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	16
	17	"""Wire-level encoding and decoding of requests and responses for the smart
	18	client and server.
	19	"""
2400.1.3 by Andrew Bennetts Split smart transport code into several separate modules.	20
	21
	22	from cStringIO import StringIO
	23
	24	from bzrlib import errors
	25	from bzrlib.smart import request
	26
	27
	28	def _recv_tuple(from_file):
	29	req_line = from_file.readline()
	30	return _decode_tuple(req_line)
	31
	32
	33	def _decode_tuple(req_line):
	34	if req_line == None or req_line == '':
	35	return None
	36	if req_line[-1] != '\n':
	37	raise errors.SmartProtocolError("request %r not terminated" % req_line)
	38	return tuple(req_line[:-1].split('\x01'))
	39
	40
	41	def _encode_tuple(args):
	42	"""Encode the tuple args to a bytestream."""
	43	return '\x01'.join(args) + '\n'
	44
	45
	46	class SmartProtocolBase(object):
	47	"""Methods common to client and server"""
	48
	49	# TODO: this only actually accomodates a single block; possibly should
	50	# support multiple chunks?
	51	def _encode_bulk_data(self, body):
	52	"""Encode body as a bulk data chunk."""
	53	return ''.join(('%d\n' % len(body), body, 'done\n'))
	54
	55	def _serialise_offsets(self, offsets):
	56	"""Serialise a readv offset list."""
	57	txt = []
	58	for start, length in offsets:
	59	txt.append('%d,%d' % (start, length))
	60	return '\n'.join(txt)
	61
	62
	63	class SmartServerRequestProtocolOne(SmartProtocolBase):
	64	"""Server-side encoding and decoding logic for smart version 1."""
	65
	66	def __init__(self, backing_transport, write_func):
	67	self._backing_transport = backing_transport
	68	self.excess_buffer = ''
	69	self._finished = False
	70	self.in_buffer = ''
	71	self.has_dispatched = False
	72	self.request = None
	73	self._body_decoder = None
	74	self._write_func = write_func
	75
	76	def accept_bytes(self, bytes):
	77	"""Take bytes, and advance the internal state machine appropriately.
	78
	79	:param bytes: must be a byte string
	80	"""
	81	assert isinstance(bytes, str)
	82	self.in_buffer += bytes
	83	if not self.has_dispatched:
84	if '\n' not in self.in_buffer:
85	# no command line yet
86	return
87	self.has_dispatched = True
88	try:
89	first_line, self.in_buffer = self.in_buffer.split('\n', 1)
90	first_line += '\n'
91	req_args = _decode_tuple(first_line)
92	self.request = request.SmartServerRequestHandler(
2402.1.1 by Andrew Bennetts Use the Command pattern for handling smart server commands.	93	self._backing_transport, commands=request.request_handlers)
2400.1.3 by Andrew Bennetts Split smart transport code into several separate modules.	94	self.request.dispatch_command(req_args[0], req_args[1:])
	95	if self.request.finished_reading:
	96	# trivial request
	97	self.excess_buffer = self.in_buffer
	98	self.in_buffer = ''
	99	self._send_response(self.request.response.args,
	100	self.request.response.body)
	101	except KeyboardInterrupt:
	102	raise
	103	except Exception, exception:
	104	# everything else: pass to client, flush, and quit
	105	self._send_response(('error', str(exception)))
	106	return
	107
	108	if self.has_dispatched:
	109	if self._finished:
	110	# nothing to do.XXX: this routine should be a single state
	111	# machine too.
	112	self.excess_buffer += self.in_buffer
	113	self.in_buffer = ''
	114	return
	115	if self._body_decoder is None:
	116	self._body_decoder = LengthPrefixedBodyDecoder()
	117	self._body_decoder.accept_bytes(self.in_buffer)
	118	self.in_buffer = self._body_decoder.unused_data
	119	body_data = self._body_decoder.read_pending_data()
	120	self.request.accept_body(body_data)
	121	if self._body_decoder.finished_reading:
	122	self.request.end_of_body()
	123	assert self.request.finished_reading, \
	124	"no more body, request not finished"
	125	if self.request.response is not None:
	126	self._send_response(self.request.response.args,
	127	self.request.response.body)
	128	self.excess_buffer = self.in_buffer
	129	self.in_buffer = ''
	130	else:
	131	assert not self.request.finished_reading, \
	132	"no response and we have finished reading."
	133
	134	def _send_response(self, args, body=None):
	135	"""Send a smart server response down the output stream."""
	136	assert not self._finished, 'response already sent'
	137	self._finished = True
	138	self._write_func(_encode_tuple(args))
	139	if body is not None:
	140	assert isinstance(body, str), 'body must be a str'
	141	bytes = self._encode_bulk_data(body)
	142	self._write_func(bytes)
	143
	144	def next_read_size(self):
	145	if self._finished:
	146	return 0
	147	if self._body_decoder is None:
	148	return 1
	149	else:
	150	return self._body_decoder.next_read_size()
	151
	152
	153	class LengthPrefixedBodyDecoder(object):
	154	"""Decodes the length-prefixed bulk data."""
	155
	156	def __init__(self):
	157	self.bytes_left = None
158	self.finished_reading = False
159	self.unused_data = ''
160	self.state_accept = self._state_accept_expecting_length
161	self.state_read = self._state_read_no_data
162	self._in_buffer = ''
163	self._trailer_buffer = ''
164
165	def accept_bytes(self, bytes):
166	"""Decode as much of bytes as possible.
167
168	If 'bytes' contains too much data it will be appended to
169	self.unused_data.
170
171	finished_reading will be set when no more data is required. Further
172	data will be appended to self.unused_data.
173	"""
174	# accept_bytes is allowed to change the state
175	current_state = self.state_accept
176	self.state_accept(bytes)
177	while current_state != self.state_accept:
178	current_state = self.state_accept
179	self.state_accept('')
180
181	def next_read_size(self):
182	if self.bytes_left is not None:
183	# Ideally we want to read all the remainder of the body and the
184	# trailer in one go.
185	return self.bytes_left + 5
186	elif self.state_accept == self._state_accept_reading_trailer:
187	# Just the trailer left
188	return 5 - len(self._trailer_buffer)
189	elif self.state_accept == self._state_accept_expecting_length:
190	# There's still at least 6 bytes left ('\n' to end the length, plus
191	# 'done\n').
192	return 6
193	else:
194	# Reading excess data. Either way, 1 byte at a time is fine.
195	return 1
196
197	def read_pending_data(self):
198	"""Return any pending data that has been decoded."""
199	return self.state_read()
200
201	def _state_accept_expecting_length(self, bytes):
202	self._in_buffer += bytes
203	pos = self._in_buffer.find('\n')
204	if pos == -1:
205	return
206	self.bytes_left = int(self._in_buffer[:pos])
207	self._in_buffer = self._in_buffer[pos+1:]
208	self.bytes_left -= len(self._in_buffer)
209	self.state_accept = self._state_accept_reading_body
210	self.state_read = self._state_read_in_buffer
211
212	def _state_accept_reading_body(self, bytes):
213	self._in_buffer += bytes
214	self.bytes_left -= len(bytes)
215	if self.bytes_left <= 0:
216	# Finished with body
217	if self.bytes_left != 0:
218	self._trailer_buffer = self._in_buffer[self.bytes_left:]
219	self._in_buffer = self._in_buffer[:self.bytes_left]
220	self.bytes_left = None
221	self.state_accept = self._state_accept_reading_trailer
222
223	def _state_accept_reading_trailer(self, bytes):
224	self._trailer_buffer += bytes
225	# TODO: what if the trailer does not match "done\n"? Should this raise
226	# a ProtocolViolation exception?
227	if self._trailer_buffer.startswith('done\n'):
228	self.unused_data = self._trailer_buffer[len('done\n'):]
229	self.state_accept = self._state_accept_reading_unused
230	self.finished_reading = True
231
232	def _state_accept_reading_unused(self, bytes):
233	self.unused_data += bytes
234
235	def _state_read_no_data(self):
236	return ''
237
238	def _state_read_in_buffer(self):
239	result = self._in_buffer
240	self._in_buffer = ''
241	return result
242
243
244	class SmartClientRequestProtocolOne(SmartProtocolBase):
245	"""The client-side protocol for smart version 1."""
246
247	def __init__(self, request):
248	"""Construct a SmartClientRequestProtocolOne.
249
250	:param request: A SmartClientMediumRequest to serialise onto and
251	deserialise from.
252	"""
253	self._request = request
254	self._body_buffer = None
255
256	def call(self, *args):
257	bytes = _encode_tuple(args)
258	self._request.accept_bytes(bytes)
259	self._request.finished_writing()
260
261	def call_with_body_bytes(self, args, body):
262	"""Make a remote call of args with body bytes 'body'.
263
264	After calling this, call read_response_tuple to find the result out.
265	"""
266	bytes = _encode_tuple(args)
267	self._request.accept_bytes(bytes)
268	bytes = self._encode_bulk_data(body)
269	self._request.accept_bytes(bytes)
270	self._request.finished_writing()
271
272	def call_with_body_readv_array(self, args, body):
273	"""Make a remote call with a readv array.
274
275	The body is encoded with one line per readv offset pair. The numbers in
276	each pair are separated by a comma, and no trailing \n is emitted.
277	"""
278	bytes = _encode_tuple(args)
279	self._request.accept_bytes(bytes)
280	readv_bytes = self._serialise_offsets(body)
281	bytes = self._encode_bulk_data(readv_bytes)
282	self._request.accept_bytes(bytes)
283	self._request.finished_writing()
284
285	def cancel_read_body(self):
286	"""After expecting a body, a response code may indicate one otherwise.
287
288	This method lets the domain client inform the protocol that no body
289	will be transmitted. This is a terminal method: after calling it the
290	protocol is not able to be used further.
291	"""
292	self._request.finished_reading()
293
294	def read_response_tuple(self, expect_body=False):
295	"""Read a response tuple from the wire.
296
297	This should only be called once.
298	"""
299	result = self._recv_tuple()
300	if not expect_body:
301	self._request.finished_reading()
302	return result
303
304	def read_body_bytes(self, count=-1):
305	"""Read bytes from the body, decoding into a byte stream.
306
307	We read all bytes at once to ensure we've checked the trailer for
308	errors, and then feed the buffer back as read_body_bytes is called.
309	"""
310	if self._body_buffer is not None:
311	return self._body_buffer.read(count)
312	_body_decoder = LengthPrefixedBodyDecoder()
313
314	while not _body_decoder.finished_reading:
315	bytes_wanted = _body_decoder.next_read_size()
316	bytes = self._request.read_bytes(bytes_wanted)
317	_body_decoder.accept_bytes(bytes)
318	self._request.finished_reading()
319	self._body_buffer = StringIO(_body_decoder.read_pending_data())
320	# XXX: TODO check the trailer result.
321	return self._body_buffer.read(count)
322
323	def _recv_tuple(self):
324	"""Receive a tuple from the medium request."""
325	line = ''
326	while not line or line[-1] != '\n':
327	# TODO: this is inefficient - but tuples are short.
328	new_char = self._request.read_bytes(1)
329	line += new_char
330	assert new_char != '', "end of file reading from server."
331	return _decode_tuple(line)
332
333	def query_version(self):
334	"""Return protocol version number of the server."""
335	self.call('hello')
336	resp = self.read_response_tuple()
337	if resp == ('ok', '1'):
338	return 1
339	else:
340	raise errors.SmartProtocolError("bad response %r" % (resp,))
341
342
343