254
193
Version two sends the value of RESPONSE_VERSION_TWO.
256
self._write_func(self.response_marker)
258
def _send_response(self, response):
259
"""Send a smart server response down the output stream."""
261
raise AssertionError('response already sent')
262
self._finished = True
263
self._write_protocol_version()
264
self._write_success_or_failure_prefix(response)
265
self._write_func(_encode_tuple(response.args))
266
if response.body is not None:
267
if not isinstance(response.body, str):
268
raise AssertionError('body must be a str')
269
if not (response.body_stream is None):
270
raise AssertionError(
271
'body_stream and body cannot both be set')
272
bytes = self._encode_bulk_data(response.body)
273
self._write_func(bytes)
274
elif response.body_stream is not None:
275
_send_stream(response.body_stream, self._write_func)
278
def _send_stream(stream, write_func):
279
write_func('chunked\n')
280
_send_chunks(stream, write_func)
284
def _send_chunks(stream, write_func):
286
if isinstance(chunk, str):
287
bytes = "%x\n%s" % (len(chunk), chunk)
289
elif isinstance(chunk, request.FailedSmartServerResponse):
291
_send_chunks(chunk.args, write_func)
294
raise errors.BzrError(
295
'Chunks must be str or FailedSmartServerResponse, got %r'
299
class _NeedMoreBytes(Exception):
300
"""Raise this inside a _StatefulDecoder to stop decoding until more bytes
304
def __init__(self, count=None):
307
:param count: the total number of bytes needed by the current state.
308
May be None if the number of bytes needed is unknown.
313
class _StatefulDecoder(object):
314
"""Base class for writing state machines to decode byte streams.
316
Subclasses should provide a self.state_accept attribute that accepts bytes
317
and, if appropriate, updates self.state_accept to a different function.
318
accept_bytes will call state_accept as often as necessary to make sure the
319
state machine has progressed as far as possible before it returns.
321
See ProtocolThreeDecoder for an example subclass.
195
self._write_func(RESPONSE_VERSION_TWO)
198
class LengthPrefixedBodyDecoder(object):
199
"""Decodes the length-prefixed bulk data."""
324
201
def __init__(self):
202
self.bytes_left = None
325
203
self.finished_reading = False
326
self._in_buffer_list = []
327
self._in_buffer_len = 0
328
204
self.unused_data = ''
329
self.bytes_left = None
330
self._number_needed_bytes = None
332
def _get_in_buffer(self):
333
if len(self._in_buffer_list) == 1:
334
return self._in_buffer_list[0]
335
in_buffer = ''.join(self._in_buffer_list)
336
if len(in_buffer) != self._in_buffer_len:
337
raise AssertionError(
338
"Length of buffer did not match expected value: %s != %s"
339
% self._in_buffer_len, len(in_buffer))
340
self._in_buffer_list = [in_buffer]
343
def _get_in_bytes(self, count):
344
"""Grab X bytes from the input_buffer.
346
Callers should have already checked that self._in_buffer_len is >
347
count. Note, this does not consume the bytes from the buffer. The
348
caller will still need to call _get_in_buffer() and then
349
_set_in_buffer() if they actually need to consume the bytes.
351
# check if we can yield the bytes from just the first entry in our list
352
if len(self._in_buffer_list) == 0:
353
raise AssertionError('Callers must be sure we have buffered bytes'
354
' before calling _get_in_bytes')
355
if len(self._in_buffer_list[0]) > count:
356
return self._in_buffer_list[0][:count]
357
# We can't yield it from the first buffer, so collapse all buffers, and
359
in_buf = self._get_in_buffer()
360
return in_buf[:count]
362
def _set_in_buffer(self, new_buf):
363
if new_buf is not None:
364
self._in_buffer_list = [new_buf]
365
self._in_buffer_len = len(new_buf)
367
self._in_buffer_list = []
368
self._in_buffer_len = 0
205
self.state_accept = self._state_accept_expecting_length
206
self.state_read = self._state_read_no_data
208
self._trailer_buffer = ''
370
210
def accept_bytes(self, bytes):
371
211
"""Decode as much of bytes as possible.
377
217
data will be appended to self.unused_data.
379
219
# accept_bytes is allowed to change the state
380
self._number_needed_bytes = None
381
# lsprof puts a very large amount of time on this specific call for
383
self._in_buffer_list.append(bytes)
384
self._in_buffer_len += len(bytes)
386
# Run the function for the current state.
220
current_state = self.state_accept
221
self.state_accept(bytes)
222
while current_state != self.state_accept:
387
223
current_state = self.state_accept
389
while current_state != self.state_accept:
390
# The current state has changed. Run the function for the new
391
# current state, so that it can:
392
# - decode any unconsumed bytes left in a buffer, and
393
# - signal how many more bytes are expected (via raising
395
current_state = self.state_accept
397
except _NeedMoreBytes, e:
398
self._number_needed_bytes = e.count
401
class ChunkedBodyDecoder(_StatefulDecoder):
402
"""Decoder for chunked body data.
404
This is very similar the HTTP's chunked encoding. See the description of
405
streamed body data in `doc/developers/network-protocol.txt` for details.
409
_StatefulDecoder.__init__(self)
410
self.state_accept = self._state_accept_expecting_header
411
self.chunk_in_progress = None
412
self.chunks = collections.deque()
414
self.error_in_progress = None
416
def next_read_size(self):
417
# Note: the shortest possible chunk is 2 bytes: '0\n', and the
418
# end-of-body marker is 4 bytes: 'END\n'.
419
if self.state_accept == self._state_accept_reading_chunk:
420
# We're expecting more chunk content. So we're expecting at least
421
# the rest of this chunk plus an END chunk.
422
return self.bytes_left + 4
423
elif self.state_accept == self._state_accept_expecting_length:
424
if self._in_buffer_len == 0:
425
# We're expecting a chunk length. There's at least two bytes
426
# left: a digit plus '\n'.
429
# We're in the middle of reading a chunk length. So there's at
430
# least one byte left, the '\n' that terminates the length.
432
elif self.state_accept == self._state_accept_reading_unused:
434
elif self.state_accept == self._state_accept_expecting_header:
435
return max(0, len('chunked\n') - self._in_buffer_len)
437
raise AssertionError("Impossible state: %r" % (self.state_accept,))
439
def read_next_chunk(self):
441
return self.chunks.popleft()
445
def _extract_line(self):
446
in_buf = self._get_in_buffer()
447
pos = in_buf.find('\n')
449
# We haven't read a complete line yet, so request more bytes before
451
raise _NeedMoreBytes(1)
453
# Trim the prefix (including '\n' delimiter) from the _in_buffer.
454
self._set_in_buffer(in_buf[pos+1:])
458
self.unused_data = self._get_in_buffer()
459
# self._in_buffer = None
460
self._in_buffer_list = []
461
self._in_buffer_len = 0
462
self.state_accept = self._state_accept_reading_unused
464
error_args = tuple(self.error_in_progress)
465
self.chunks.append(request.FailedSmartServerResponse(error_args))
466
self.error_in_progress = None
467
self.finished_reading = True
469
def _state_accept_expecting_header(self):
470
prefix = self._extract_line()
471
if prefix == 'chunked':
472
self.state_accept = self._state_accept_expecting_length
474
raise errors.SmartProtocolError(
475
'Bad chunked body header: "%s"' % (prefix,))
477
def _state_accept_expecting_length(self):
478
prefix = self._extract_line()
481
self.error_in_progress = []
482
self._state_accept_expecting_length()
484
elif prefix == 'END':
485
# We've read the end-of-body marker.
486
# Any further bytes are unused data, including the bytes left in
491
self.bytes_left = int(prefix, 16)
492
self.chunk_in_progress = ''
493
self.state_accept = self._state_accept_reading_chunk
495
def _state_accept_reading_chunk(self):
496
in_buf = self._get_in_buffer()
497
in_buffer_len = len(in_buf)
498
self.chunk_in_progress += in_buf[:self.bytes_left]
499
self._set_in_buffer(in_buf[self.bytes_left:])
500
self.bytes_left -= in_buffer_len
501
if self.bytes_left <= 0:
502
# Finished with chunk
503
self.bytes_left = None
505
self.error_in_progress.append(self.chunk_in_progress)
507
self.chunks.append(self.chunk_in_progress)
508
self.chunk_in_progress = None
509
self.state_accept = self._state_accept_expecting_length
511
def _state_accept_reading_unused(self):
512
self.unused_data += self._get_in_buffer()
513
self._in_buffer_list = []
516
class LengthPrefixedBodyDecoder(_StatefulDecoder):
517
"""Decodes the length-prefixed bulk data."""
520
_StatefulDecoder.__init__(self)
521
self.state_accept = self._state_accept_expecting_length
522
self.state_read = self._state_read_no_data
524
self._trailer_buffer = ''
224
self.state_accept('')
526
226
def next_read_size(self):
527
227
if self.bytes_left is not None:
528
228
# Ideally we want to read all the remainder of the body and the
794
405
This prefixes the request with the value of REQUEST_VERSION_TWO.
797
response_marker = RESPONSE_VERSION_TWO
798
request_marker = REQUEST_VERSION_TWO
800
408
def read_response_tuple(self, expect_body=False):
801
409
"""Read a response tuple from the wire.
803
411
This should only be called once.
805
413
version = self._request.read_line()
806
if version != self.response_marker:
807
self._request.finished_reading()
808
raise errors.UnexpectedProtocolVersionMarker(version)
809
response_status = self._request.read_line()
810
result = SmartClientRequestProtocolOne._read_response_tuple(self)
811
self._response_is_unknown_method(result)
812
if response_status == 'success\n':
813
self.response_status = True
815
self._request.finished_reading()
817
elif response_status == 'failed\n':
818
self.response_status = False
819
self._request.finished_reading()
820
raise errors.ErrorFromSmartServer(result)
414
if version != RESPONSE_VERSION_TWO:
415
raise errors.SmartProtocolError('bad protocol marker %r' % version)
416
response_status = self._recv_line()
417
if response_status not in ('success\n', 'failed\n'):
822
418
raise errors.SmartProtocolError(
823
419
'bad protocol status %r' % response_status)
420
self.response_status = response_status == 'success\n'
421
return SmartClientRequestProtocolOne.read_response_tuple(self, expect_body)
825
423
def _write_protocol_version(self):
826
"""Write any prefixes this protocol requires.
424
r"""Write any prefixes this protocol requires.
828
426
Version two sends the value of REQUEST_VERSION_TWO.
830
self._request.accept_bytes(self.request_marker)
832
def read_streamed_body(self):
833
"""Read bytes from the body, decoding into a byte stream.
835
# Read no more than 64k at a time so that we don't risk error 10055 (no
836
# buffer space available) on Windows.
837
_body_decoder = ChunkedBodyDecoder()
838
while not _body_decoder.finished_reading:
839
bytes = self._request.read_bytes(_body_decoder.next_read_size())
841
# end of file encountered reading from server
842
raise errors.ConnectionReset(
843
"Connection lost while reading streamed body.")
844
_body_decoder.accept_bytes(bytes)
845
for body_bytes in iter(_body_decoder.read_next_chunk, None):
846
if 'hpss' in debug.debug_flags and type(body_bytes) is str:
847
mutter(' %d byte chunk read',
850
self._request.finished_reading()
853
def build_server_protocol_three(backing_transport, write_func,
855
request_handler = request.SmartServerRequestHandler(
856
backing_transport, commands=request.request_handlers,
857
root_client_path=root_client_path)
858
responder = ProtocolThreeResponder(write_func)
859
message_handler = message.ConventionalRequestHandler(request_handler, responder)
860
return ProtocolThreeDecoder(message_handler)
863
class ProtocolThreeDecoder(_StatefulDecoder):
865
response_marker = RESPONSE_VERSION_THREE
866
request_marker = REQUEST_VERSION_THREE
868
def __init__(self, message_handler, expect_version_marker=False):
869
_StatefulDecoder.__init__(self)
870
self._has_dispatched = False
872
if expect_version_marker:
873
self.state_accept = self._state_accept_expecting_protocol_version
874
# We're expecting at least the protocol version marker + some
876
self._number_needed_bytes = len(MESSAGE_VERSION_THREE) + 4
878
self.state_accept = self._state_accept_expecting_headers
879
self._number_needed_bytes = 4
880
self.decoding_failed = False
881
self.request_handler = self.message_handler = message_handler
883
def accept_bytes(self, bytes):
884
self._number_needed_bytes = None
886
_StatefulDecoder.accept_bytes(self, bytes)
887
except KeyboardInterrupt:
889
except errors.SmartMessageHandlerError, exception:
890
# We do *not* set self.decoding_failed here. The message handler
891
# has raised an error, but the decoder is still able to parse bytes
892
# and determine when this message ends.
893
log_exception_quietly()
894
self.message_handler.protocol_error(exception.exc_value)
895
# The state machine is ready to continue decoding, but the
896
# exception has interrupted the loop that runs the state machine.
897
# So we call accept_bytes again to restart it.
898
self.accept_bytes('')
899
except Exception, exception:
900
# The decoder itself has raised an exception. We cannot continue
902
self.decoding_failed = True
903
if isinstance(exception, errors.UnexpectedProtocolVersionMarker):
904
# This happens during normal operation when the client tries a
905
# protocol version the server doesn't understand, so no need to
906
# log a traceback every time.
907
# Note that this can only happen when
908
# expect_version_marker=True, which is only the case on the
912
log_exception_quietly()
913
self.message_handler.protocol_error(exception)
915
def _extract_length_prefixed_bytes(self):
916
if self._in_buffer_len < 4:
917
# A length prefix by itself is 4 bytes, and we don't even have that
919
raise _NeedMoreBytes(4)
920
(length,) = struct.unpack('!L', self._get_in_bytes(4))
921
end_of_bytes = 4 + length
922
if self._in_buffer_len < end_of_bytes:
923
# We haven't yet read as many bytes as the length-prefix says there
925
raise _NeedMoreBytes(end_of_bytes)
926
# Extract the bytes from the buffer.
927
in_buf = self._get_in_buffer()
928
bytes = in_buf[4:end_of_bytes]
929
self._set_in_buffer(in_buf[end_of_bytes:])
932
def _extract_prefixed_bencoded_data(self):
933
prefixed_bytes = self._extract_length_prefixed_bytes()
935
decoded = bdecode(prefixed_bytes)
937
raise errors.SmartProtocolError(
938
'Bytes %r not bencoded' % (prefixed_bytes,))
941
def _extract_single_byte(self):
942
if self._in_buffer_len == 0:
943
# The buffer is empty
944
raise _NeedMoreBytes(1)
945
in_buf = self._get_in_buffer()
947
self._set_in_buffer(in_buf[1:])
950
def _state_accept_expecting_protocol_version(self):
951
needed_bytes = len(MESSAGE_VERSION_THREE) - self._in_buffer_len
952
in_buf = self._get_in_buffer()
954
# We don't have enough bytes to check if the protocol version
955
# marker is right. But we can check if it is already wrong by
956
# checking that the start of MESSAGE_VERSION_THREE matches what
958
# [In fact, if the remote end isn't bzr we might never receive
959
# len(MESSAGE_VERSION_THREE) bytes. So if the bytes we have so far
960
# are wrong then we should just raise immediately rather than
962
if not MESSAGE_VERSION_THREE.startswith(in_buf):
963
# We have enough bytes to know the protocol version is wrong
964
raise errors.UnexpectedProtocolVersionMarker(in_buf)
965
raise _NeedMoreBytes(len(MESSAGE_VERSION_THREE))
966
if not in_buf.startswith(MESSAGE_VERSION_THREE):
967
raise errors.UnexpectedProtocolVersionMarker(in_buf)
968
self._set_in_buffer(in_buf[len(MESSAGE_VERSION_THREE):])
969
self.state_accept = self._state_accept_expecting_headers
971
def _state_accept_expecting_headers(self):
972
decoded = self._extract_prefixed_bencoded_data()
973
if type(decoded) is not dict:
974
raise errors.SmartProtocolError(
975
'Header object %r is not a dict' % (decoded,))
976
self.state_accept = self._state_accept_expecting_message_part
978
self.message_handler.headers_received(decoded)
980
raise errors.SmartMessageHandlerError(sys.exc_info())
982
def _state_accept_expecting_message_part(self):
983
message_part_kind = self._extract_single_byte()
984
if message_part_kind == 'o':
985
self.state_accept = self._state_accept_expecting_one_byte
986
elif message_part_kind == 's':
987
self.state_accept = self._state_accept_expecting_structure
988
elif message_part_kind == 'b':
989
self.state_accept = self._state_accept_expecting_bytes
990
elif message_part_kind == 'e':
993
raise errors.SmartProtocolError(
994
'Bad message kind byte: %r' % (message_part_kind,))
996
def _state_accept_expecting_one_byte(self):
997
byte = self._extract_single_byte()
998
self.state_accept = self._state_accept_expecting_message_part
1000
self.message_handler.byte_part_received(byte)
1002
raise errors.SmartMessageHandlerError(sys.exc_info())
1004
def _state_accept_expecting_bytes(self):
1005
# XXX: this should not buffer whole message part, but instead deliver
1006
# the bytes as they arrive.
1007
prefixed_bytes = self._extract_length_prefixed_bytes()
1008
self.state_accept = self._state_accept_expecting_message_part
1010
self.message_handler.bytes_part_received(prefixed_bytes)
1012
raise errors.SmartMessageHandlerError(sys.exc_info())
1014
def _state_accept_expecting_structure(self):
1015
structure = self._extract_prefixed_bencoded_data()
1016
self.state_accept = self._state_accept_expecting_message_part
1018
self.message_handler.structure_part_received(structure)
1020
raise errors.SmartMessageHandlerError(sys.exc_info())
1023
self.unused_data = self._get_in_buffer()
1024
self._set_in_buffer(None)
1025
self.state_accept = self._state_accept_reading_unused
1027
self.message_handler.end_received()
1029
raise errors.SmartMessageHandlerError(sys.exc_info())
1031
def _state_accept_reading_unused(self):
1032
self.unused_data = self._get_in_buffer()
1033
self._set_in_buffer(None)
1035
def next_read_size(self):
1036
if self.state_accept == self._state_accept_reading_unused:
1038
elif self.decoding_failed:
1039
# An exception occured while processing this message, probably from
1040
# self.message_handler. We're not sure that this state machine is
1041
# in a consistent state, so just signal that we're done (i.e. give
1045
if self._number_needed_bytes is not None:
1046
return self._number_needed_bytes - self._in_buffer_len
1048
raise AssertionError("don't know how many bytes are expected!")
1051
class _ProtocolThreeEncoder(object):
1053
response_marker = request_marker = MESSAGE_VERSION_THREE
1055
def __init__(self, write_func):
1057
self._real_write_func = write_func
1059
def _write_func(self, bytes):
1064
self._real_write_func(self._buf)
1067
def _serialise_offsets(self, offsets):
1068
"""Serialise a readv offset list."""
1070
for start, length in offsets:
1071
txt.append('%d,%d' % (start, length))
1072
return '\n'.join(txt)
1074
def _write_protocol_version(self):
1075
self._write_func(MESSAGE_VERSION_THREE)
1077
def _write_prefixed_bencode(self, structure):
1078
bytes = bencode(structure)
1079
self._write_func(struct.pack('!L', len(bytes)))
1080
self._write_func(bytes)
1082
def _write_headers(self, headers):
1083
self._write_prefixed_bencode(headers)
1085
def _write_structure(self, args):
1086
self._write_func('s')
1089
if type(arg) is unicode:
1090
utf8_args.append(arg.encode('utf8'))
1092
utf8_args.append(arg)
1093
self._write_prefixed_bencode(utf8_args)
1095
def _write_end(self):
1096
self._write_func('e')
1099
def _write_prefixed_body(self, bytes):
1100
self._write_func('b')
1101
self._write_func(struct.pack('!L', len(bytes)))
1102
self._write_func(bytes)
1104
def _write_error_status(self):
1105
self._write_func('oE')
1107
def _write_success_status(self):
1108
self._write_func('oS')
1111
class ProtocolThreeResponder(_ProtocolThreeEncoder):
1113
def __init__(self, write_func):
1114
_ProtocolThreeEncoder.__init__(self, write_func)
1115
self.response_sent = False
1116
self._headers = {'Software version': bzrlib.__version__}
1118
def send_error(self, exception):
1119
if self.response_sent:
1120
raise AssertionError(
1121
"send_error(%s) called, but response already sent."
1123
if isinstance(exception, errors.UnknownSmartMethod):
1124
failure = request.FailedSmartServerResponse(
1125
('UnknownMethod', exception.verb))
1126
self.send_response(failure)
1128
self.response_sent = True
1129
self._write_protocol_version()
1130
self._write_headers(self._headers)
1131
self._write_error_status()
1132
self._write_structure(('error', str(exception)))
1135
def send_response(self, response):
1136
if self.response_sent:
1137
raise AssertionError(
1138
"send_response(%r) called, but response already sent."
1140
self.response_sent = True
1141
self._write_protocol_version()
1142
self._write_headers(self._headers)
1143
if response.is_successful():
1144
self._write_success_status()
1146
self._write_error_status()
1147
self._write_structure(response.args)
1148
if response.body is not None:
1149
self._write_prefixed_body(response.body)
1150
elif response.body_stream is not None:
1151
for chunk in response.body_stream:
1152
self._write_prefixed_body(chunk)
1157
class ProtocolThreeRequester(_ProtocolThreeEncoder, Requester):
1159
def __init__(self, medium_request):
1160
_ProtocolThreeEncoder.__init__(self, medium_request.accept_bytes)
1161
self._medium_request = medium_request
1164
def set_headers(self, headers):
1165
self._headers = headers.copy()
1167
def call(self, *args):
1168
if 'hpss' in debug.debug_flags:
1169
mutter('hpss call: %s', repr(args)[1:-1])
1170
base = getattr(self._medium_request._medium, 'base', None)
1171
if base is not None:
1172
mutter(' (to %s)', base)
1173
self._request_start_time = time.time()
1174
self._write_protocol_version()
1175
self._write_headers(self._headers)
1176
self._write_structure(args)
1178
self._medium_request.finished_writing()
1180
def call_with_body_bytes(self, args, body):
1181
"""Make a remote call of args with body bytes 'body'.
1183
After calling this, call read_response_tuple to find the result out.
1185
if 'hpss' in debug.debug_flags:
1186
mutter('hpss call w/body: %s (%r...)', repr(args)[1:-1], body[:20])
1187
path = getattr(self._medium_request._medium, '_path', None)
1188
if path is not None:
1189
mutter(' (to %s)', path)
1190
mutter(' %d bytes', len(body))
1191
self._request_start_time = time.time()
1192
self._write_protocol_version()
1193
self._write_headers(self._headers)
1194
self._write_structure(args)
1195
self._write_prefixed_body(body)
1197
self._medium_request.finished_writing()
1199
def call_with_body_readv_array(self, args, body):
1200
"""Make a remote call with a readv array.
1202
The body is encoded with one line per readv offset pair. The numbers in
1203
each pair are separated by a comma, and no trailing \n is emitted.
1205
if 'hpss' in debug.debug_flags:
1206
mutter('hpss call w/readv: %s', repr(args)[1:-1])
1207
path = getattr(self._medium_request._medium, '_path', None)
1208
if path is not None:
1209
mutter(' (to %s)', path)
1210
self._request_start_time = time.time()
1211
self._write_protocol_version()
1212
self._write_headers(self._headers)
1213
self._write_structure(args)
1214
readv_bytes = self._serialise_offsets(body)
1215
if 'hpss' in debug.debug_flags:
1216
mutter(' %d bytes in readv request', len(readv_bytes))
1217
self._write_prefixed_body(readv_bytes)
1219
self._medium_request.finished_writing()
428
self._request.accept_bytes(REQUEST_VERSION_TWO)
added
removed
bzrlib/smart/protocol.py
