~bzr-pqm/bzr/bzr.dev

Committer: Canonical.com Patch Queue Manager
Date: 2008-02-06 01:45:28 UTC
mfrom: (3211.5.4 integration)
Revision ID: pqm@pqm.ubuntu.com-20080206014528-rb8v4hl04mgxulb7

(robertc) Tune RemoteRepository.get_parent_map to not send duplicate
results, and to compress the data,
reducing round trips further. (Robert Collins)

files modified:
NEWS

bzrlib/graph.py

bzrlib/remote.py

bzrlib/smart/protocol.py

bzrlib/smart/repository.py

bzrlib/tests/test_remote.py

bzrlib/tests/test_smart.py

Show diffs side-by-side

added added

removed removed

bzrlib/remote.py

# TODO: At some point, handle upgrades by just passing the whole request

# across to run on the server.

import bz2

from cStringIO import StringIO

from bzrlib import (

272

273

self._leave_lock = False

273

274

# A cache of looked up revision parent data; reset at unlock time.

274

275

self._parents_map = None

276

if 'hpss' in debug.debug_flags:

277

self._requested_parents = None

275

278

# For tests:

276

279

# These depend on the actual remote format, so force them off for

277

280

# maximum compatibility. XXX: In future these should depend on the

476

479

self._lock_mode = 'r'

477

480

self._lock_count = 1

478

481

self._parents_map = {}

482

if 'hpss' in debug.debug_flags:

483

self._requested_parents = set()

479

484

if self._real_repository is not None:

480

485

self._real_repository.lock_read()

481

486

else:

514

519

self._lock_mode = 'w'

515

520

self._lock_count = 1

516

521

self._parents_map = {}

522

if 'hpss' in debug.debug_flags:

523

self._requested_parents = set()

517

524

elif self._lock_mode == 'r':

518

525

raise errors.ReadOnlyError(self)

519

526

else:

574

581

if self._lock_count > 0:

575

582

return

576

583

self._parents_map = None

584

if 'hpss' in debug.debug_flags:

585

self._requested_parents = None

577

586

old_mode = self._lock_mode

578

587

self._lock_mode = None

579

588

try:

776

785

len(set(self._parents_map).intersection(parent_map)),

777

786

len(parent_map))

778

787

self._parents_map.update(parent_map)

779

return dict((k, ancestry[k]) for k in keys if k in ancestry)

788

present_keys = [k for k in keys if k in ancestry]

789

if 'hpss' in debug.debug_flags:

790

self._requested_parents.update(present_keys)

791

mutter('Current RemoteRepository graph hit rate: %d%%',

792

100.0 * len(self._requested_parents) / len(self._parents_map))

793

return dict((k, ancestry[k]) for k in present_keys)

780

794

781

795

def _response_is_unknown_method(self, response, verb):

782

796

"""Return True if response is an unknonwn method response to verb.

806

820

return found_parents

807

821

else:

808

822

found_parents = {}

823

# TODO(Needs analysis): We could assume that the keys being requested

824

# from get_parent_map are in a breadth first search, so typically they

825

# will all be depth N from some common parent, and we don't have to

826

# have the server iterate from the root parent, but rather from the

827

# keys we're searching; and just tell the server the keyspace we

828

# already have; but this may be more traffic again.

829

830

# Transform self._parents_map into a search request recipe.

831

# TODO: Manage this incrementally to avoid covering the same path

832

# repeatedly. (The server will have to on each request, but the less

833

# work done the better).

834

start_set = set(self._parents_map)

835

result_parents = set()

836

for parents in self._parents_map.itervalues():

837

result_parents.update(parents)

838

stop_keys = result_parents.difference(start_set)

839

included_keys = start_set.intersection(result_parents)

840

start_set.difference_update(included_keys)

841

recipe = (start_set, stop_keys, len(self._parents_map))

842

body = self._serialise_search_recipe(recipe)

809

843

path = self.bzrdir._path_for_remote_call(self._client)

810

844

for key in keys:

811

845

assert type(key) is str

812

846

verb = 'Repository.get_parent_map'

813

response = self._client.call_expecting_body(

814

verb, path, *keys)

847

args = (path,) + tuple(keys)

848

response = self._client.call_with_body_bytes_expecting_body(

849

verb, args, self._serialise_search_recipe(recipe))

815

850

if self._response_is_unknown_method(response, verb):

816

851

# Server that does not support this method, get the whole graph.

817

852

response = self._client.call_expecting_body(

823

858

reponse[1].cancel_read_body()

824

859

raise errors.UnexpectedSmartServerResponse(response[0])

825

860

if response[0][0] == 'ok':

826

coded = response[1].read_body_bytes()

861

coded = bz2.decompress(response[1].read_body_bytes())

827

862

if coded == '':

828

863

# no revisions found

829

864

return {}

976

1011

def get_data_stream_for_search(self, search):

977

1012

REQUEST_NAME = 'Repository.stream_revisions_chunked'

978

1013

path = self.bzrdir._path_for_remote_call(self._client)

979

recipe = search.get_recipe()

980

start_keys = ' '.join(recipe[0])

981

stop_keys = ' '.join(recipe[1])

982

count = str(recipe[2])

983

body = '\n'.join((start_keys, stop_keys, count))

1014

body = self._serialise_search_recipe(search.get_recipe())

984

1015

response, protocol = self._client.call_with_body_bytes_expecting_body(

985

1016

REQUEST_NAME, (path,), body)

986

1017

1041

1072

def _make_parents_provider(self):

1042

1073

return self

1043

1074

1075

def _serialise_search_recipe(self, recipe):

1076

"""Serialise a graph search recipe.

1077

1078

:param recipe: A search recipe (start, stop, count).

1079

:return: Serialised bytes.

1080

"""

1081

start_keys = ' '.join(recipe[0])

1082

stop_keys = ' '.join(recipe[1])

1083

count = str(recipe[2])

1084

return '\n'.join((start_keys, stop_keys, count))

1085

1044

1086

1045

1087

class RemoteBranchLockableFiles(LockableFiles):

1046

1088

"""A 'LockableFiles' implementation that talks to a smart server.

Older »