77
def recreate_search(self, repository, search_bytes, discard_excess=False):
78
"""Recreate a search from its serialised form.
80
:param discard_excess: If True, and the search refers to data we don't
81
have, just silently accept that fact - the verb calling
82
recreate_search trusts that clients will look for missing things
83
they expected and get it from elsewhere.
85
lines = search_bytes.split('\n')
86
if lines[0] == 'ancestry-of':
88
search_result = graph.PendingAncestryResult(heads, repository)
89
return search_result, None
90
elif lines[0] == 'search':
91
return self.recreate_search_from_recipe(repository, lines[1:],
92
discard_excess=discard_excess)
94
return (None, FailedSmartServerResponse(('BadSearch',)))
96
def recreate_search_from_recipe(self, repository, lines,
97
discard_excess=False):
98
"""Recreate a specific revision search (vs a from-tip search).
100
:param discard_excess: If True, and the search refers to data we don't
101
have, just silently accept that fact - the verb calling
102
recreate_search trusts that clients will look for missing things
103
they expected and get it from elsewhere.
67
def recreate_search(self, repository, recipe_bytes):
68
lines = recipe_bytes.split('\n')
105
69
start_keys = set(lines[0].split(' '))
106
70
exclude_keys = set(lines[1].split(' '))
107
71
revision_count = int(lines[2])
206
158
queried_revs.update(next_revs)
207
159
parent_map = repo_graph.get_parent_map(next_revs)
208
current_revs = next_revs
209
160
next_revs = set()
210
for revision_id in current_revs:
212
parents = parent_map.get(revision_id)
213
if parents is not None:
214
# adjust for the wire
215
if parents == (_mod_revision.NULL_REVISION,):
217
# prepare the next query
218
next_revs.update(parents)
219
encoded_id = revision_id
222
encoded_id = "missing:" + revision_id
224
if (revision_id not in client_seen_revs and
225
(not missing_rev or include_missing)):
161
for revision_id, parents in parent_map.iteritems():
162
# adjust for the wire
163
if parents == (_mod_revision.NULL_REVISION,):
165
# prepare the next query
166
next_revs.update(parents)
167
if revision_id not in client_seen_revs:
226
168
# Client does not have this revision, give it to it.
227
169
# add parents to the result
228
result[encoded_id] = parents
170
result[revision_id] = parents
229
171
# Approximate the serialized cost of this revision_id.
230
size_so_far += 2 + len(encoded_id) + sum(map(len, parents))
172
size_so_far += 2 + len(revision_id) + sum(map(len, parents))
231
173
# get all the directly asked for parents, and then flesh out to
232
174
# 64K (compressed) or so. We do one level of depth at a time to
233
175
# stay in sync with the client. The 250000 magic number is
234
176
# estimated compression ratio taken from bzr.dev itself.
235
if self.no_extra_results or (
236
first_loop_done and size_so_far > 250000):
177
if first_loop_done and size_so_far > 250000:
237
178
next_revs = set()
239
180
# don't query things we've already queried
287
228
return SuccessfulSmartServerResponse(('ok', ), '\n'.join(lines))
290
class SmartServerRepositoryGetRevIdForRevno(SmartServerRepositoryReadLocked):
292
def do_readlocked_repository_request(self, repository, revno,
294
"""Find the revid for a given revno, given a known revno/revid pair.
299
found_flag, result = repository.get_rev_id_for_revno(revno, known_pair)
300
except errors.RevisionNotPresent, err:
301
if err.revision_id != known_pair[1]:
302
raise AssertionError(
303
'get_rev_id_for_revno raised RevisionNotPresent for '
304
'non-initial revision: ' + err.revision_id)
305
return FailedSmartServerResponse(
306
('nosuchrevision', err.revision_id))
308
return SuccessfulSmartServerResponse(('ok', result))
310
earliest_revno, earliest_revid = result
311
return SuccessfulSmartServerResponse(
312
('history-incomplete', earliest_revno, earliest_revid))
315
231
class SmartServerRequestHasRevision(SmartServerRepositoryRequest):
317
233
def do_repository_request(self, repository, revision_id):
408
324
return SuccessfulSmartServerResponse(('ok', token))
411
class SmartServerRepositoryGetStream(SmartServerRepositoryRequest):
413
def do_repository_request(self, repository, to_network_name):
414
"""Get a stream for inserting into a to_format repository.
416
:param repository: The repository to stream from.
417
:param to_network_name: The network name of the format of the target
420
self._to_format = network_format_registry.get(to_network_name)
421
if self._should_fake_unknown():
422
return FailedSmartServerResponse(
423
('UnknownMethod', 'Repository.get_stream'))
424
return None # Signal that we want a body.
426
def _should_fake_unknown(self):
427
"""Return True if we should return UnknownMethod to the client.
429
This is a workaround for bugs in pre-1.19 clients that claim to
430
support receiving streams of CHK repositories. The pre-1.19 client
431
expects inventory records to be serialized in the format defined by
432
to_network_name, but in pre-1.19 (at least) that format definition
433
tries to use the xml5 serializer, which does not correctly handle
434
rich-roots. After 1.19 the client can also accept inventory-deltas
435
(which avoids this issue), and those clients will use the
436
Repository.get_stream_1.19 verb instead of this one.
437
So: if this repository is CHK, and the to_format doesn't match,
438
we should just fake an UnknownSmartMethod error so that the client
439
will fallback to VFS, rather than sending it a stream we know it
442
from_format = self._repository._format
443
to_format = self._to_format
444
if not from_format.supports_chks:
445
# Source not CHK: that's ok
447
if (to_format.supports_chks and
448
from_format.repository_class is to_format.repository_class and
449
from_format._serializer == to_format._serializer):
450
# Source is CHK, but target matches: that's ok
451
# (e.g. 2a->2a, or CHK2->2a)
453
# Source is CHK, and target is not CHK or incompatible CHK. We can't
454
# generate a compatible stream.
457
def do_body(self, body_bytes):
458
repository = self._repository
459
repository.lock_read()
461
search_result, error = self.recreate_search(repository, body_bytes,
463
if error is not None:
466
source = repository._get_source(self._to_format)
467
stream = source.get_stream(search_result)
469
exc_info = sys.exc_info()
471
# On non-error, unlocking is done by the body stream handler.
474
raise exc_info[0], exc_info[1], exc_info[2]
475
return SuccessfulSmartServerResponse(('ok',),
476
body_stream=self.body_stream(stream, repository))
478
def body_stream(self, stream, repository):
479
byte_stream = _stream_to_byte_stream(stream, repository._format)
481
for bytes in byte_stream:
483
except errors.RevisionNotPresent, e:
484
# This shouldn't be able to happen, but as we don't buffer
485
# everything it can in theory happen.
487
yield FailedSmartServerResponse(('NoSuchRevision', e.revision_id))
492
class SmartServerRepositoryGetStream_1_19(SmartServerRepositoryGetStream):
494
def _should_fake_unknown(self):
495
"""Returns False; we don't need to workaround bugs in 1.19+ clients."""
499
def _stream_to_byte_stream(stream, src_format):
500
"""Convert a record stream to a self delimited byte stream."""
501
pack_writer = pack.ContainerSerialiser()
502
yield pack_writer.begin()
503
yield pack_writer.bytes_record(src_format.network_name(), '')
504
for substream_type, substream in stream:
505
for record in substream:
506
if record.storage_kind in ('chunked', 'fulltext'):
507
serialised = record_to_fulltext_bytes(record)
508
elif record.storage_kind == 'inventory-delta':
509
serialised = record_to_inventory_delta_bytes(record)
510
elif record.storage_kind == 'absent':
511
raise ValueError("Absent factory for %s" % (record.key,))
513
serialised = record.get_bytes_as(record.storage_kind)
515
# Some streams embed the whole stream into the wire
516
# representation of the first record, which means that
517
# later records have no wire representation: we skip them.
518
yield pack_writer.bytes_record(serialised, [(substream_type,)])
519
yield pack_writer.end()
522
def _byte_stream_to_stream(byte_stream):
523
"""Convert a byte stream into a format and a stream.
525
:param byte_stream: A bytes iterator, as output by _stream_to_byte_stream.
526
:return: (RepositoryFormat, stream_generator)
528
stream_decoder = pack.ContainerPushParser()
530
"""Closure to return the substreams."""
531
# May have fully parsed records already.
532
for record in stream_decoder.read_pending_records():
533
record_names, record_bytes = record
534
record_name, = record_names
535
substream_type = record_name[0]
536
substream = NetworkRecordStream([record_bytes])
537
yield substream_type, substream.read()
538
for bytes in byte_stream:
539
stream_decoder.accept_bytes(bytes)
540
for record in stream_decoder.read_pending_records():
541
record_names, record_bytes = record
542
record_name, = record_names
543
substream_type = record_name[0]
544
substream = NetworkRecordStream([record_bytes])
545
yield substream_type, substream.read()
546
for bytes in byte_stream:
547
stream_decoder.accept_bytes(bytes)
548
for record in stream_decoder.read_pending_records(max=1):
549
record_names, src_format_name = record
550
src_format = network_format_registry.get(src_format_name)
551
return src_format, record_stream()
554
327
class SmartServerRepositoryUnlock(SmartServerRepositoryRequest):
556
329
def do_repository_request(self, repository, token):
630
392
tarball.add(dirname, '.bzr') # recursive by default
635
class SmartServerRepositoryInsertStreamLocked(SmartServerRepositoryRequest):
636
"""Insert a record stream from a RemoteSink into a repository.
638
This gets bytes pushed to it by the network infrastructure and turns that
639
into a bytes iterator using a thread. That is then processed by
640
_byte_stream_to_stream.
645
def do_repository_request(self, repository, resume_tokens, lock_token):
646
"""StreamSink.insert_stream for a remote repository."""
647
repository.lock_write(token=lock_token)
648
self.do_insert_stream_request(repository, resume_tokens)
650
def do_insert_stream_request(self, repository, resume_tokens):
651
tokens = [token for token in resume_tokens.split(' ') if token]
653
self.repository = repository
654
self.queue = Queue.Queue()
655
self.insert_thread = threading.Thread(target=self._inserter_thread)
656
self.insert_thread.start()
658
def do_chunk(self, body_stream_chunk):
659
self.queue.put(body_stream_chunk)
661
def _inserter_thread(self):
663
src_format, stream = _byte_stream_to_stream(
664
self.blocking_byte_stream())
665
self.insert_result = self.repository._get_sink().insert_stream(
666
stream, src_format, self.tokens)
667
self.insert_ok = True
669
self.insert_exception = sys.exc_info()
670
self.insert_ok = False
672
def blocking_byte_stream(self):
674
bytes = self.queue.get()
675
if bytes is StopIteration:
681
self.queue.put(StopIteration)
682
if self.insert_thread is not None:
683
self.insert_thread.join()
684
if not self.insert_ok:
685
exc_info = self.insert_exception
686
raise exc_info[0], exc_info[1], exc_info[2]
687
write_group_tokens, missing_keys = self.insert_result
688
if write_group_tokens or missing_keys:
689
# bzip needed? missing keys should typically be a small set.
690
# Should this be a streaming body response ?
691
missing_keys = sorted(missing_keys)
692
bytes = bencode.bencode((write_group_tokens, missing_keys))
693
self.repository.unlock()
694
return SuccessfulSmartServerResponse(('missing-basis', bytes))
696
self.repository.unlock()
697
return SuccessfulSmartServerResponse(('ok', ))
700
class SmartServerRepositoryInsertStream_1_19(SmartServerRepositoryInsertStreamLocked):
701
"""Insert a record stream from a RemoteSink into a repository.
703
Same as SmartServerRepositoryInsertStreamLocked, except:
704
- the lock token argument is optional
705
- servers that implement this verb accept 'inventory-delta' records in the
711
def do_repository_request(self, repository, resume_tokens, lock_token=None):
712
"""StreamSink.insert_stream for a remote repository."""
713
SmartServerRepositoryInsertStreamLocked.do_repository_request(
714
self, repository, resume_tokens, lock_token)
717
class SmartServerRepositoryInsertStream(SmartServerRepositoryInsertStreamLocked):
718
"""Insert a record stream from a RemoteSink into an unlocked repository.
720
This is the same as SmartServerRepositoryInsertStreamLocked, except it
721
takes no lock_tokens; i.e. it works with an unlocked (or lock-free, e.g.
722
like pack format) repository.
727
def do_repository_request(self, repository, resume_tokens):
728
"""StreamSink.insert_stream for a remote repository."""
729
repository.lock_write()
730
self.do_insert_stream_request(repository, resume_tokens)
added
removed
bzrlib/smart/repository.py
