416
420
self._to_format = network_format_registry.get(to_network_name)
421
if self._should_fake_unknown():
422
return FailedSmartServerResponse(
423
('UnknownMethod', 'Repository.get_stream'))
417
424
return None # Signal that we want a body.
426
def _should_fake_unknown(self):
427
"""Return True if we should return UnknownMethod to the client.
429
This is a workaround for bugs in pre-1.19 clients that claim to
430
support receiving streams of CHK repositories. The pre-1.19 client
431
expects inventory records to be serialized in the format defined by
432
to_network_name, but in pre-1.19 (at least) that format definition
433
tries to use the xml5 serializer, which does not correctly handle
434
rich-roots. After 1.19 the client can also accept inventory-deltas
435
(which avoids this issue), and those clients will use the
436
Repository.get_stream_1.19 verb instead of this one.
437
So: if this repository is CHK, and the to_format doesn't match,
438
we should just fake an UnknownSmartMethod error so that the client
439
will fallback to VFS, rather than sending it a stream we know it
442
from_format = self._repository._format
443
to_format = self._to_format
444
if not from_format.supports_chks:
445
# Source not CHK: that's ok
447
if (to_format.supports_chks and
448
from_format.repository_class is to_format.repository_class and
449
from_format._serializer == to_format._serializer):
450
# Source is CHK, but target matches: that's ok
451
# (e.g. 2a->2a, or CHK2->2a)
453
# Source is CHK, and target is not CHK or incompatible CHK. We can't
454
# generate a compatible stream.
419
457
def do_body(self, body_bytes):
420
458
repository = self._repository
421
459
repository.lock_read()
451
489
repository.unlock()
492
class SmartServerRepositoryGetStream_1_19(SmartServerRepositoryGetStream):
494
def _should_fake_unknown(self):
495
"""Returns False; we don't need to workaround bugs in 1.19+ clients."""
454
499
def _stream_to_byte_stream(stream, src_format):
455
500
"""Convert a record stream to a self delimited byte stream."""
456
501
pack_writer = pack.ContainerSerialiser()
457
502
yield pack_writer.begin()
458
503
yield pack_writer.bytes_record(src_format.network_name(), '')
459
504
for substream_type, substream in stream:
505
if substream_type == 'inventory-deltas':
506
# This doesn't feel like the ideal place to issue this warning;
507
# however we don't want to do it in the Repository that's
508
# generating the stream, because that might be on the server.
509
# Instead we try to observe it as the stream goes by.
510
ui.ui_factory.warn_cross_format_fetch(src_format,
460
512
for record in substream:
461
513
if record.storage_kind in ('chunked', 'fulltext'):
462
514
serialised = record_to_fulltext_bytes(record)
515
elif record.storage_kind == 'inventory-delta':
516
serialised = record_to_inventory_delta_bytes(record)
463
517
elif record.storage_kind == 'absent':
464
518
raise ValueError("Absent factory for %s" % (record.key,))
472
526
yield pack_writer.end()
529
class _ByteStreamDecoder(object):
530
"""Helper for _byte_stream_to_stream.
532
The expected usage of this class is via the function _byte_stream_to_stream
533
which creates a _ByteStreamDecoder, pops off the stream format and then
534
yields the output of record_stream(), the main entry point to
537
Broadly this class has to unwrap two layers of iterators:
541
This is complicated by wishing to return type, iterator_for_type, but
542
getting the data for iterator_for_type when we find out type: we can't
543
simply pass a generator down to the NetworkRecordStream parser, instead
544
we have a little local state to seed each NetworkRecordStream instance,
545
and gather the type that we'll be yielding.
547
:ivar byte_stream: The byte stream being decoded.
548
:ivar stream_decoder: A pack parser used to decode the bytestream
549
:ivar current_type: The current type, used to join adjacent records of the
550
same type into a single stream.
551
:ivar first_bytes: The first bytes to give the next NetworkRecordStream.
554
def __init__(self, byte_stream):
555
"""Create a _ByteStreamDecoder."""
556
self.stream_decoder = pack.ContainerPushParser()
557
self.current_type = None
558
self.first_bytes = None
559
self.byte_stream = byte_stream
561
def iter_stream_decoder(self):
562
"""Iterate the contents of the pack from stream_decoder."""
563
# dequeue pending items
564
for record in self.stream_decoder.read_pending_records():
566
# Pull bytes of the wire, decode them to records, yield those records.
567
for bytes in self.byte_stream:
568
self.stream_decoder.accept_bytes(bytes)
569
for record in self.stream_decoder.read_pending_records():
572
def iter_substream_bytes(self):
573
if self.first_bytes is not None:
574
yield self.first_bytes
575
# If we run out of pack records, single the outer layer to stop.
576
self.first_bytes = None
577
for record in self.iter_pack_records:
578
record_names, record_bytes = record
579
record_name, = record_names
580
substream_type = record_name[0]
581
if substream_type != self.current_type:
582
# end of a substream, seed the next substream.
583
self.current_type = substream_type
584
self.first_bytes = record_bytes
588
def record_stream(self):
589
"""Yield substream_type, substream from the byte stream."""
591
# Make and consume sub generators, one per substream type:
592
while self.first_bytes is not None:
593
substream = NetworkRecordStream(self.iter_substream_bytes())
594
# after substream is fully consumed, self.current_type is set to
595
# the next type, and self.first_bytes is set to the matching bytes.
596
yield self.current_type, substream.read()
598
def seed_state(self):
599
"""Prepare the _ByteStreamDecoder to decode from the pack stream."""
600
# Set a single generator we can use to get data from the pack stream.
601
self.iter_pack_records = self.iter_stream_decoder()
602
# Seed the very first subiterator with content; after this each one
604
list(self.iter_substream_bytes())
475
607
def _byte_stream_to_stream(byte_stream):
476
608
"""Convert a byte stream into a format and a stream.
478
610
:param byte_stream: A bytes iterator, as output by _stream_to_byte_stream.
479
611
:return: (RepositoryFormat, stream_generator)
481
stream_decoder = pack.ContainerPushParser()
483
"""Closure to return the substreams."""
484
# May have fully parsed records already.
485
for record in stream_decoder.read_pending_records():
486
record_names, record_bytes = record
487
record_name, = record_names
488
substream_type = record_name[0]
489
substream = NetworkRecordStream([record_bytes])
490
yield substream_type, substream.read()
491
for bytes in byte_stream:
492
stream_decoder.accept_bytes(bytes)
493
for record in stream_decoder.read_pending_records():
494
record_names, record_bytes = record
495
record_name, = record_names
496
substream_type = record_name[0]
497
substream = NetworkRecordStream([record_bytes])
498
yield substream_type, substream.read()
613
decoder = _ByteStreamDecoder(byte_stream)
499
614
for bytes in byte_stream:
500
stream_decoder.accept_bytes(bytes)
501
for record in stream_decoder.read_pending_records(max=1):
615
decoder.stream_decoder.accept_bytes(bytes)
616
for record in decoder.stream_decoder.read_pending_records(max=1):
502
617
record_names, src_format_name = record
503
618
src_format = network_format_registry.get(src_format_name)
504
return src_format, record_stream()
619
return src_format, decoder.record_stream()
507
622
class SmartServerRepositoryUnlock(SmartServerRepositoryRequest):
650
766
return SuccessfulSmartServerResponse(('ok', ))
769
class SmartServerRepositoryInsertStream_1_19(SmartServerRepositoryInsertStreamLocked):
770
"""Insert a record stream from a RemoteSink into a repository.
772
Same as SmartServerRepositoryInsertStreamLocked, except:
773
- the lock token argument is optional
774
- servers that implement this verb accept 'inventory-delta' records in the
780
def do_repository_request(self, repository, resume_tokens, lock_token=None):
781
"""StreamSink.insert_stream for a remote repository."""
782
SmartServerRepositoryInsertStreamLocked.do_repository_request(
783
self, repository, resume_tokens, lock_token)
653
786
class SmartServerRepositoryInsertStream(SmartServerRepositoryInsertStreamLocked):
654
787
"""Insert a record stream from a RemoteSink into an unlocked repository.
added
removed
bzrlib/smart/repository.py
