38
51
def do(self, path, *args):
39
52
"""Execute a repository request.
41
The repository must be at the exact path - no searching is done.
54
All Repository requests take a path to the repository as their first
55
argument. The repository must be at the exact path given by the
56
client - no searching is done.
43
58
The actual logic is delegated to self.do_repository_request.
45
:param path: The path for the repository.
46
:return: A smart server from self.do_repository_request().
60
:param client_path: The path for the repository as received from the
62
:return: A SmartServerResponse from self.do_repository_request().
48
transport = self._backing_transport.clone(path)
64
transport = self.transport_from_client_path(path)
49
65
bzrdir = BzrDir.open_from_transport(transport)
50
repository = bzrdir.open_repository()
51
return self.do_repository_request(repository, *args)
54
class SmartServerRepositoryGetRevisionGraph(SmartServerRepositoryRequest):
56
def do_repository_request(self, repository, revision_id):
66
# Save the repository for use with do_body.
67
self._repository = bzrdir.open_repository()
68
return self.do_repository_request(self._repository, *args)
70
def do_repository_request(self, repository, *args):
71
"""Override to provide an implementation for a verb."""
72
# No-op for verbs that take bodies (None as a result indicates a body
76
def recreate_search(self, repository, search_bytes, discard_excess=False):
77
"""Recreate a search from its serialised form.
79
:param discard_excess: If True, and the search refers to data we don't
80
have, just silently accept that fact - the verb calling
81
recreate_search trusts that clients will look for missing things
82
they expected and get it from elsewhere.
84
if search_bytes == 'everything':
85
return graph.EverythingResult(repository), None
86
lines = search_bytes.split('\n')
87
if lines[0] == 'ancestry-of':
89
search_result = graph.PendingAncestryResult(heads, repository)
90
return search_result, None
91
elif lines[0] == 'search':
92
return self.recreate_search_from_recipe(repository, lines[1:],
93
discard_excess=discard_excess)
95
return (None, FailedSmartServerResponse(('BadSearch',)))
97
def recreate_search_from_recipe(self, repository, lines,
98
discard_excess=False):
99
"""Recreate a specific revision search (vs a from-tip search).
101
:param discard_excess: If True, and the search refers to data we don't
102
have, just silently accept that fact - the verb calling
103
recreate_search trusts that clients will look for missing things
104
they expected and get it from elsewhere.
106
start_keys = set(lines[0].split(' '))
107
exclude_keys = set(lines[1].split(' '))
108
revision_count = int(lines[2])
109
repository.lock_read()
111
search = repository.get_graph()._make_breadth_first_searcher(
115
next_revs = search.next()
116
except StopIteration:
118
search.stop_searching_any(exclude_keys.intersection(next_revs))
119
search_result = search.get_result()
120
if (not discard_excess and
121
search_result.get_recipe()[3] != revision_count):
122
# we got back a different amount of data than expected, this
123
# gets reported as NoSuchRevision, because less revisions
124
# indicates missing revisions, and more should never happen as
125
# the excludes list considers ghosts and ensures that ghost
126
# filling races are not a problem.
127
return (None, FailedSmartServerResponse(('NoSuchRevision',)))
128
return (search_result, None)
133
class SmartServerRepositoryReadLocked(SmartServerRepositoryRequest):
134
"""Calls self.do_readlocked_repository_request."""
136
def do_repository_request(self, repository, *args):
137
"""Read lock a repository for do_readlocked_repository_request."""
138
repository.lock_read()
140
return self.do_readlocked_repository_request(repository, *args)
145
class SmartServerRepositoryGetParentMap(SmartServerRepositoryRequest):
146
"""Bzr 1.2+ - get parent data for revisions during a graph search."""
148
no_extra_results = False
150
def do_repository_request(self, repository, *revision_ids):
151
"""Get parent details for some revisions.
153
All the parents for revision_ids are returned. Additionally up to 64KB
154
of additional parent data found by performing a breadth first search
155
from revision_ids is returned. The verb takes a body containing the
156
current search state, see do_body for details.
158
If 'include-missing:' is in revision_ids, ghosts encountered in the
159
graph traversal for getting parent data are included in the result with
160
a prefix of 'missing:'.
162
:param repository: The repository to query in.
163
:param revision_ids: The utf8 encoded revision_id to answer for.
165
self._revision_ids = revision_ids
166
return None # Signal that we want a body.
168
def do_body(self, body_bytes):
169
"""Process the current search state and perform the parent lookup.
171
:return: A smart server response where the body contains an utf8
172
encoded flattened list of the parents of the revisions (the same
173
format as Repository.get_revision_graph) which has been bz2
176
repository = self._repository
177
repository.lock_read()
179
return self._do_repository_request(body_bytes)
183
def _do_repository_request(self, body_bytes):
184
repository = self._repository
185
revision_ids = set(self._revision_ids)
186
include_missing = 'include-missing:' in revision_ids
188
revision_ids.remove('include-missing:')
189
body_lines = body_bytes.split('\n')
190
search_result, error = self.recreate_search_from_recipe(
191
repository, body_lines)
192
if error is not None:
194
# TODO might be nice to start up the search again; but thats not
195
# written or tested yet.
196
client_seen_revs = set(search_result.get_keys())
197
# Always include the requested ids.
198
client_seen_revs.difference_update(revision_ids)
200
repo_graph = repository.get_graph()
204
next_revs = revision_ids
205
first_loop_done = False
207
queried_revs.update(next_revs)
208
parent_map = repo_graph.get_parent_map(next_revs)
209
current_revs = next_revs
211
for revision_id in current_revs:
213
parents = parent_map.get(revision_id)
214
if parents is not None:
215
# adjust for the wire
216
if parents == (_mod_revision.NULL_REVISION,):
218
# prepare the next query
219
next_revs.update(parents)
220
encoded_id = revision_id
223
encoded_id = "missing:" + revision_id
225
if (revision_id not in client_seen_revs and
226
(not missing_rev or include_missing)):
227
# Client does not have this revision, give it to it.
228
# add parents to the result
229
result[encoded_id] = parents
230
# Approximate the serialized cost of this revision_id.
231
size_so_far += 2 + len(encoded_id) + sum(map(len, parents))
232
# get all the directly asked for parents, and then flesh out to
233
# 64K (compressed) or so. We do one level of depth at a time to
234
# stay in sync with the client. The 250000 magic number is
235
# estimated compression ratio taken from bzr.dev itself.
236
if self.no_extra_results or (
237
first_loop_done and size_so_far > 250000):
240
# don't query things we've already queried
241
next_revs.difference_update(queried_revs)
242
first_loop_done = True
244
# sorting trivially puts lexographically similar revision ids together.
246
for revision, parents in sorted(result.items()):
247
lines.append(' '.join((revision, ) + tuple(parents)))
249
return SuccessfulSmartServerResponse(
250
('ok', ), bz2.compress('\n'.join(lines)))
253
class SmartServerRepositoryGetRevisionGraph(SmartServerRepositoryReadLocked):
255
def do_readlocked_repository_request(self, repository, revision_id):
57
256
"""Return the result of repository.get_revision_graph(revision_id).
258
Deprecated as of bzr 1.4, but supported for older clients.
59
260
:param repository: The repository to query in.
60
261
:param revision_id: The utf8 encoded revision_id to get a graph from.
61
262
:return: A smart server response where the body contains an utf8
176
409
return SuccessfulSmartServerResponse(('ok', token))
412
class SmartServerRepositoryGetStream(SmartServerRepositoryRequest):
414
def do_repository_request(self, repository, to_network_name):
415
"""Get a stream for inserting into a to_format repository.
417
The request body is 'search_bytes', a description of the revisions
420
In 2.3 this verb added support for search_bytes == 'everything'. Older
421
implementations will respond with a BadSearch error, and clients should
422
catch this and fallback appropriately.
424
:param repository: The repository to stream from.
425
:param to_network_name: The network name of the format of the target
428
self._to_format = network_format_registry.get(to_network_name)
429
if self._should_fake_unknown():
430
return FailedSmartServerResponse(
431
('UnknownMethod', 'Repository.get_stream'))
432
return None # Signal that we want a body.
434
def _should_fake_unknown(self):
435
"""Return True if we should return UnknownMethod to the client.
437
This is a workaround for bugs in pre-1.19 clients that claim to
438
support receiving streams of CHK repositories. The pre-1.19 client
439
expects inventory records to be serialized in the format defined by
440
to_network_name, but in pre-1.19 (at least) that format definition
441
tries to use the xml5 serializer, which does not correctly handle
442
rich-roots. After 1.19 the client can also accept inventory-deltas
443
(which avoids this issue), and those clients will use the
444
Repository.get_stream_1.19 verb instead of this one.
445
So: if this repository is CHK, and the to_format doesn't match,
446
we should just fake an UnknownSmartMethod error so that the client
447
will fallback to VFS, rather than sending it a stream we know it
450
from_format = self._repository._format
451
to_format = self._to_format
452
if not from_format.supports_chks:
453
# Source not CHK: that's ok
455
if (to_format.supports_chks and
456
from_format.repository_class is to_format.repository_class and
457
from_format._serializer == to_format._serializer):
458
# Source is CHK, but target matches: that's ok
459
# (e.g. 2a->2a, or CHK2->2a)
461
# Source is CHK, and target is not CHK or incompatible CHK. We can't
462
# generate a compatible stream.
465
def do_body(self, body_bytes):
466
repository = self._repository
467
repository.lock_read()
469
search_result, error = self.recreate_search(repository, body_bytes,
471
if error is not None:
474
source = repository._get_source(self._to_format)
475
stream = source.get_stream(search_result)
477
exc_info = sys.exc_info()
479
# On non-error, unlocking is done by the body stream handler.
482
raise exc_info[0], exc_info[1], exc_info[2]
483
return SuccessfulSmartServerResponse(('ok',),
484
body_stream=self.body_stream(stream, repository))
486
def body_stream(self, stream, repository):
487
byte_stream = _stream_to_byte_stream(stream, repository._format)
489
for bytes in byte_stream:
491
except errors.RevisionNotPresent, e:
492
# This shouldn't be able to happen, but as we don't buffer
493
# everything it can in theory happen.
495
yield FailedSmartServerResponse(('NoSuchRevision', e.revision_id))
500
class SmartServerRepositoryGetStream_1_19(SmartServerRepositoryGetStream):
501
"""The same as Repository.get_stream, but will return stream CHK formats to
504
See SmartServerRepositoryGetStream._should_fake_unknown.
509
def _should_fake_unknown(self):
510
"""Returns False; we don't need to workaround bugs in 1.19+ clients."""
514
def _stream_to_byte_stream(stream, src_format):
515
"""Convert a record stream to a self delimited byte stream."""
516
pack_writer = pack.ContainerSerialiser()
517
yield pack_writer.begin()
518
yield pack_writer.bytes_record(src_format.network_name(), '')
519
for substream_type, substream in stream:
520
for record in substream:
521
if record.storage_kind in ('chunked', 'fulltext'):
522
serialised = record_to_fulltext_bytes(record)
523
elif record.storage_kind == 'absent':
524
raise ValueError("Absent factory for %s" % (record.key,))
526
serialised = record.get_bytes_as(record.storage_kind)
528
# Some streams embed the whole stream into the wire
529
# representation of the first record, which means that
530
# later records have no wire representation: we skip them.
531
yield pack_writer.bytes_record(serialised, [(substream_type,)])
532
yield pack_writer.end()
535
class _ByteStreamDecoder(object):
536
"""Helper for _byte_stream_to_stream.
538
The expected usage of this class is via the function _byte_stream_to_stream
539
which creates a _ByteStreamDecoder, pops off the stream format and then
540
yields the output of record_stream(), the main entry point to
543
Broadly this class has to unwrap two layers of iterators:
547
This is complicated by wishing to return type, iterator_for_type, but
548
getting the data for iterator_for_type when we find out type: we can't
549
simply pass a generator down to the NetworkRecordStream parser, instead
550
we have a little local state to seed each NetworkRecordStream instance,
551
and gather the type that we'll be yielding.
553
:ivar byte_stream: The byte stream being decoded.
554
:ivar stream_decoder: A pack parser used to decode the bytestream
555
:ivar current_type: The current type, used to join adjacent records of the
556
same type into a single stream.
557
:ivar first_bytes: The first bytes to give the next NetworkRecordStream.
560
def __init__(self, byte_stream, record_counter):
561
"""Create a _ByteStreamDecoder."""
562
self.stream_decoder = pack.ContainerPushParser()
563
self.current_type = None
564
self.first_bytes = None
565
self.byte_stream = byte_stream
566
self._record_counter = record_counter
569
def iter_stream_decoder(self):
570
"""Iterate the contents of the pack from stream_decoder."""
571
# dequeue pending items
572
for record in self.stream_decoder.read_pending_records():
574
# Pull bytes of the wire, decode them to records, yield those records.
575
for bytes in self.byte_stream:
576
self.stream_decoder.accept_bytes(bytes)
577
for record in self.stream_decoder.read_pending_records():
580
def iter_substream_bytes(self):
581
if self.first_bytes is not None:
582
yield self.first_bytes
583
# If we run out of pack records, single the outer layer to stop.
584
self.first_bytes = None
585
for record in self.iter_pack_records:
586
record_names, record_bytes = record
587
record_name, = record_names
588
substream_type = record_name[0]
589
if substream_type != self.current_type:
590
# end of a substream, seed the next substream.
591
self.current_type = substream_type
592
self.first_bytes = record_bytes
596
def record_stream(self):
597
"""Yield substream_type, substream from the byte stream."""
598
def wrap_and_count(pb, rc, substream):
599
"""Yield records from stream while showing progress."""
602
if self.current_type != 'revisions' and self.key_count != 0:
603
# As we know the number of revisions now (in self.key_count)
604
# we can setup and use record_counter (rc).
605
if not rc.is_initialized():
606
rc.setup(self.key_count, self.key_count)
607
for record in substream.read():
609
if rc.is_initialized() and counter == rc.STEP:
610
rc.increment(counter)
611
pb.update('Estimate', rc.current, rc.max)
613
if self.current_type == 'revisions':
614
# Total records is proportional to number of revs
615
# to fetch. With remote, we used self.key_count to
616
# track the number of revs. Once we have the revs
617
# counts in self.key_count, the progress bar changes
618
# from 'Estimating..' to 'Estimate' above.
620
if counter == rc.STEP:
621
pb.update('Estimating..', self.key_count)
627
pb = ui.ui_factory.nested_progress_bar()
628
rc = self._record_counter
629
# Make and consume sub generators, one per substream type:
630
while self.first_bytes is not None:
631
substream = NetworkRecordStream(self.iter_substream_bytes())
632
# after substream is fully consumed, self.current_type is set to
633
# the next type, and self.first_bytes is set to the matching bytes.
634
yield self.current_type, wrap_and_count(pb, rc, substream)
636
pb.update('Done', rc.max, rc.max)
639
def seed_state(self):
640
"""Prepare the _ByteStreamDecoder to decode from the pack stream."""
641
# Set a single generator we can use to get data from the pack stream.
642
self.iter_pack_records = self.iter_stream_decoder()
643
# Seed the very first subiterator with content; after this each one
645
list(self.iter_substream_bytes())
648
def _byte_stream_to_stream(byte_stream, record_counter=None):
649
"""Convert a byte stream into a format and a stream.
651
:param byte_stream: A bytes iterator, as output by _stream_to_byte_stream.
652
:return: (RepositoryFormat, stream_generator)
654
decoder = _ByteStreamDecoder(byte_stream, record_counter)
655
for bytes in byte_stream:
656
decoder.stream_decoder.accept_bytes(bytes)
657
for record in decoder.stream_decoder.read_pending_records(max=1):
658
record_names, src_format_name = record
659
src_format = network_format_registry.get(src_format_name)
660
return src_format, decoder.record_stream()
179
663
class SmartServerRepositoryUnlock(SmartServerRepositoryRequest):
181
665
def do_repository_request(self, repository, token):
241
735
dirname = dirname.encode(sys.getfilesystemencoding())
242
736
# python's tarball module includes the whole path by default so
244
assert dirname.endswith('.bzr')
738
if not dirname.endswith('.bzr'):
739
raise ValueError(dirname)
245
740
tarball.add(dirname, '.bzr') # recursive by default
250
class SmartServerRepositoryStreamKnitDataForRevisions(SmartServerRepositoryRequest):
252
def do_repository_request(self, repository, *revision_ids):
253
repository.lock_read()
255
return self._do_repository_request(repository, revision_ids)
259
def _do_repository_request(self, repository, revision_ids):
260
stream = repository.get_data_stream(revision_ids)
261
filelike = StringIO()
262
pack = ContainerWriter(filelike.write)
265
for name_tuple, bytes in stream:
266
pack.add_bytes_record(bytes, [name_tuple])
267
except errors.RevisionNotPresent, e:
268
return FailedSmartServerResponse(('NoSuchRevision', e.revision_id))
270
return SuccessfulSmartServerResponse(('ok',), filelike.getvalue())
745
class SmartServerRepositoryInsertStreamLocked(SmartServerRepositoryRequest):
746
"""Insert a record stream from a RemoteSink into a repository.
748
This gets bytes pushed to it by the network infrastructure and turns that
749
into a bytes iterator using a thread. That is then processed by
750
_byte_stream_to_stream.
755
def do_repository_request(self, repository, resume_tokens, lock_token):
756
"""StreamSink.insert_stream for a remote repository."""
757
repository.lock_write(token=lock_token)
758
self.do_insert_stream_request(repository, resume_tokens)
760
def do_insert_stream_request(self, repository, resume_tokens):
761
tokens = [token for token in resume_tokens.split(' ') if token]
763
self.repository = repository
764
self.queue = Queue.Queue()
765
self.insert_thread = threading.Thread(target=self._inserter_thread)
766
self.insert_thread.start()
768
def do_chunk(self, body_stream_chunk):
769
self.queue.put(body_stream_chunk)
771
def _inserter_thread(self):
773
src_format, stream = _byte_stream_to_stream(
774
self.blocking_byte_stream())
775
self.insert_result = self.repository._get_sink().insert_stream(
776
stream, src_format, self.tokens)
777
self.insert_ok = True
779
self.insert_exception = sys.exc_info()
780
self.insert_ok = False
782
def blocking_byte_stream(self):
784
bytes = self.queue.get()
785
if bytes is StopIteration:
791
self.queue.put(StopIteration)
792
if self.insert_thread is not None:
793
self.insert_thread.join()
794
if not self.insert_ok:
795
exc_info = self.insert_exception
796
raise exc_info[0], exc_info[1], exc_info[2]
797
write_group_tokens, missing_keys = self.insert_result
798
if write_group_tokens or missing_keys:
799
# bzip needed? missing keys should typically be a small set.
800
# Should this be a streaming body response ?
801
missing_keys = sorted(missing_keys)
802
bytes = bencode.bencode((write_group_tokens, missing_keys))
803
self.repository.unlock()
804
return SuccessfulSmartServerResponse(('missing-basis', bytes))
806
self.repository.unlock()
807
return SuccessfulSmartServerResponse(('ok', ))
810
class SmartServerRepositoryInsertStream_1_19(SmartServerRepositoryInsertStreamLocked):
811
"""Insert a record stream from a RemoteSink into a repository.
813
Same as SmartServerRepositoryInsertStreamLocked, except:
814
- the lock token argument is optional
815
- servers that implement this verb accept 'inventory-delta' records in the
821
def do_repository_request(self, repository, resume_tokens, lock_token=None):
822
"""StreamSink.insert_stream for a remote repository."""
823
SmartServerRepositoryInsertStreamLocked.do_repository_request(
824
self, repository, resume_tokens, lock_token)
827
class SmartServerRepositoryInsertStream(SmartServerRepositoryInsertStreamLocked):
828
"""Insert a record stream from a RemoteSink into an unlocked repository.
830
This is the same as SmartServerRepositoryInsertStreamLocked, except it
831
takes no lock_tokens; i.e. it works with an unlocked (or lock-free, e.g.
832
like pack format) repository.
837
def do_repository_request(self, repository, resume_tokens):
838
"""StreamSink.insert_stream for a remote repository."""
839
repository.lock_write()
840
self.do_insert_stream_request(repository, resume_tokens)
added
removed
bzrlib/smart/repository.py
