« back to all changes in this revision
Viewing changes to bzrlib/repository.py
- browse files at revision 1559.1.1
- compare with another revision
- compare with revision 1534.1.7
- download diff
- download tarball
- view history from revision 1559.1.1
- Committer: Robert Collins
- Date: 2006-02-21 11:42:36 UTC
- mfrom: (1534.1.36 inter-repo-api)
- mto: This revision was merged to the branch mainline in revision 1560.
- Revision ID: robertc@robertcollins.net-20060221114236-0deb1948f0c8c956
Merge in InterRepository API support.
- files added:
- bzrlib/tests/interrepository_implementations
- files modified:
- NEWS
added
removed
bzrlib/repository.py
168
168
169
169
revision_id: only return revision ids included by revision_id.
170
170
"""
171
if self._compatible_formats(other):
172
# fast path for weave-inventory based stores.
173
# we want all revisions to satisft revision_id in other.
174
# but we dont want to stat every file here and there.
175
# we want then, all revisions other needs to satisfy revision_id
176
# checked, but not those that we have locally.
177
# so the first thing is to get a subset of the revisions to
178
# satisfy revision_id in other, and then eliminate those that
179
# we do already have.
180
# this is slow on high latency connection to self, but as as this
181
# disk format scales terribly for push anyway due to rewriting
182
# inventory.weave, this is considered acceptable.
183
# - RBC 20060209
184
if revision_id is not None:
185
other_ids = other.get_ancestry(revision_id)
186
assert other_ids.pop(0) == None
187
else:
188
other_ids = other._all_possible_ids()
189
other_ids_set = set(other_ids)
190
# other ids is the worst case to pull now.
191
# now we want to filter other_ids against what we actually
192
# have, but dont try to stat what we know we dont.
193
my_ids = set(self._all_possible_ids())
194
possibly_present_revisions = my_ids.intersection(other_ids_set)
195
actually_present_revisions = set(self._eliminate_revisions_not_present(possibly_present_revisions))
196
required_revisions = other_ids_set.difference(actually_present_revisions)
197
required_topo_revisions = [rev_id for rev_id in other_ids if rev_id in required_revisions]
198
if revision_id is not None:
199
# we used get_ancestry to determine other_ids then we are assured all
200
# revisions referenced are present as they are installed in topological order.
201
return required_topo_revisions
202
else:
203
# we only have an estimate of whats available
204
return other._eliminate_revisions_not_present(required_topo_revisions)
205
# slow code path.
206
my_ids = set(self.all_revision_ids())
207
if revision_id is not None:
208
other_ids = other.get_ancestry(revision_id)
209
assert other_ids.pop(0) == None
210
else:
211
other_ids = other.all_revision_ids()
212
result_set = set(other_ids).difference(my_ids)
213
return [rev_id for rev_id in other_ids if rev_id in result_set]
171
return InterRepository.get(other, self).missing_revision_ids(revision_id)
214
172
215
173
@staticmethod
216
174
def open(base):
222
180
control = bzrdir.BzrDir.open(base)
223
181
return control.open_repository()
224
182
225
def _compatible_formats(self, other):
226
"""Return True if the stores in self and other are 'compatible'
227
228
'compatible' means that they are both the same underlying type
229
i.e. both weave stores, or both knits and thus support fast-path
230
operations."""
231
return (isinstance(self._format, (RepositoryFormat5,
232
RepositoryFormat6,
233
RepositoryFormat7)) and
234
isinstance(other._format, (RepositoryFormat5,
235
RepositoryFormat6,
236
RepositoryFormat7)))
237
238
@needs_read_lock
239
183
def copy_content_into(self, destination, revision_id=None, basis=None):
240
184
"""Make a complete copy of the content in self into destination.
241
185
242
186
This is a destructive operation! Do not use it on existing
243
187
repositories.
244
188
"""
245
destination.lock_write()
246
try:
247
try:
248
destination.set_make_working_trees(self.make_working_trees())
249
except NotImplementedError:
250
pass
251
# optimised paths:
252
# compatible stores
253
if self._compatible_formats(destination):
254
if basis is not None:
255
# copy the basis in, then fetch remaining data.
256
basis.copy_content_into(destination, revision_id)
257
destination.fetch(self, revision_id=revision_id)
258
else:
259
# FIXME do not peek!
260
if self.control_files._transport.listable():
261
pb = bzrlib.ui.ui_factory.progress_bar()
262
copy_all(self.weave_store,
263
destination.weave_store, pb=pb)
264
pb.update('copying inventory', 0, 1)
265
destination.control_weaves.copy_multi(
266
self.control_weaves, ['inventory'])
267
copy_all(self.revision_store,
268
destination.revision_store, pb=pb)
269
else:
270
destination.fetch(self, revision_id=revision_id)
271
# compatible v4 stores
272
elif isinstance(self._format, RepositoryFormat4):
273
if not isinstance(destination._format, RepositoryFormat4):
274
raise BzrError('cannot copy v4 branches to anything other than v4 branches.')
275
store_pairs = ((self.text_store, destination.text_store),
276
(self.inventory_store, destination.inventory_store),
277
(self.revision_store, destination.revision_store))
278
try:
279
for from_store, to_store in store_pairs:
280
copy_all(from_store, to_store)
281
except UnlistableStore:
282
raise UnlistableBranch(from_store)
283
# fallback - 'fetch'
284
else:
285
destination.fetch(self, revision_id=revision_id)
286
finally:
287
destination.unlock()
189
return InterRepository.get(self, destination).copy_content(revision_id, basis)
288
190
289
@needs_write_lock
290
def fetch(self, source, revision_id=None):
191
def fetch(self, source, revision_id=None, pb=None):
291
192
"""Fetch the content required to construct revision_id from source.
292
193
293
194
If revision_id is None all content is copied.
294
195
"""
295
from bzrlib.fetch import RepoFetcher
296
mutter("Using fetch logic to copy between %s(%s) and %s(%s)",
297
source, source._format, self, self._format)
298
RepoFetcher(to_repository=self, from_repository=source, last_revision=revision_id)
196
return InterRepository.get(source, self).fetch(revision_id=revision_id,
197
pb=pb)
299
198
300
199
def unlock(self):
301
200
self.control_files.unlock()
870
769
871
770
# formats which have no format string are not discoverable
872
771
# and not independently creatable, so are not registered.
873
__default_format = RepositoryFormat7()
874
RepositoryFormat.register_format(__default_format)
875
RepositoryFormat.set_default_format(__default_format)
772
_default_format = RepositoryFormat7()
773
RepositoryFormat.register_format(_default_format)
774
RepositoryFormat.set_default_format(_default_format)
876
775
_legacy_formats = [RepositoryFormat4(),
877
776
RepositoryFormat5(),
878
777
RepositoryFormat6()]
879
778
880
779
780
class InterRepository(object):
781
"""This class represents operations taking place between two repositories.
782
783
Its instances have methods like copy_content and fetch, and contain
784
references to the source and target repositories these operations can be
785
carried out on.
786
787
Often we will provide convenience methods on 'repository' which carry out
788
operations with another repository - they will always forward to
789
InterRepository.get(other).method_name(parameters).
790
"""
791
# XXX: FIXME: FUTURE: robertc
792
# testing of these probably requires a factory in optimiser type, and
793
# then a test adapter to test each type thoroughly.
794
#
795
796
_optimisers = set()
797
"""The available optimised InterRepository types."""
798
799
def __init__(self, source, target):
800
"""Construct a default InterRepository instance. Please use 'get'.
801
802
Only subclasses of InterRepository should call
803
InterRepository.__init__ - clients should call InterRepository.get
804
instead which will create an optimised InterRepository if possible.
805
"""
806
self.source = source
807
self.target = target
808
809
@needs_write_lock
810
def copy_content(self, revision_id=None, basis=None):
811
"""Make a complete copy of the content in self into destination.
812
813
This is a destructive operation! Do not use it on existing
814
repositories.
815
816
:param revision_id: Only copy the content needed to construct
817
revision_id and its parents.
818
:param basis: Copy the needed data preferentially from basis.
819
"""
820
try:
821
self.target.set_make_working_trees(self.source.make_working_trees())
822
except NotImplementedError:
823
pass
824
# grab the basis available data
825
if basis is not None:
826
self.target.fetch(basis, revision_id=revision_id)
827
# but dont both fetching if we have the needed data now.
828
if (revision_id not in (None, NULL_REVISION) and
829
self.target.has_revision(revision_id)):
830
return
831
self.target.fetch(self.source, revision_id=revision_id)
832
833
def _double_lock(self, lock_source, lock_target):
834
"""Take out too locks, rolling back the first if the second throws."""
835
lock_source()
836
try:
837
lock_target()
838
except Exception:
839
# we want to ensure that we don't leave source locked by mistake.
840
# and any error on target should not confuse source.
841
self.source.unlock()
842
raise
843
844
@needs_write_lock
845
def fetch(self, revision_id=None, pb=None):
846
"""Fetch the content required to construct revision_id.
847
848
The content is copied from source to target.
849
850
:param revision_id: if None all content is copied, if NULL_REVISION no
851
content is copied.
852
:param pb: optional progress bar to use for progress reports. If not
853
provided a default one will be created.
854
855
Returns the copied revision count and the failed revisions in a tuple:
856
(copied, failures).
857
"""
858
from bzrlib.fetch import RepoFetcher
859
mutter("Using fetch logic to copy between %s(%s) and %s(%s)",
860
self.source, self.source._format, self.target, self.target._format)
861
f = RepoFetcher(to_repository=self.target,
862
from_repository=self.source,
863
last_revision=revision_id,
864
pb=pb)
865
return f.count_copied, f.failed_revisions
866
867
@classmethod
868
def get(klass, repository_source, repository_target):
869
"""Retrieve a InterRepository worker object for these repositories.
870
871
:param repository_source: the repository to be the 'source' member of
872
the InterRepository instance.
873
:param repository_target: the repository to be the 'target' member of
874
the InterRepository instance.
875
If an optimised InterRepository worker exists it will be used otherwise
876
a default InterRepository instance will be created.
877
"""
878
for provider in klass._optimisers:
879
if provider.is_compatible(repository_source, repository_target):
880
return provider(repository_source, repository_target)
881
return InterRepository(repository_source, repository_target)
882
883
def lock_read(self):
884
"""Take out a logical read lock.
885
886
This will lock the source branch and the target branch. The source gets
887
a read lock and the target a read lock.
888
"""
889
self._double_lock(self.source.lock_read, self.target.lock_read)
890
891
def lock_write(self):
892
"""Take out a logical write lock.
893
894
This will lock the source branch and the target branch. The source gets
895
a read lock and the target a write lock.
896
"""
897
self._double_lock(self.source.lock_read, self.target.lock_write)
898
899
@needs_read_lock
900
def missing_revision_ids(self, revision_id=None):
901
"""Return the revision ids that source has that target does not.
902
903
These are returned in topological order.
904
905
:param revision_id: only return revision ids included by this
906
revision_id.
907
"""
908
# generic, possibly worst case, slow code path.
909
target_ids = set(self.source.all_revision_ids())
910
if revision_id is not None:
911
source_ids = self.target.get_ancestry(revision_id)
912
assert source_ids.pop(0) == None
913
else:
914
source_ids = self.target.all_revision_ids()
915
result_set = set(source_ids).difference(target_ids)
916
# this may look like a no-op: its not. It preserves the ordering
917
# other_ids had while only returning the members from other_ids
918
# that we've decided we need.
919
return [rev_id for rev_id in other_ids if rev_id in result_set]
920
921
@classmethod
922
def register_optimiser(klass, optimiser):
923
"""Register an InterRepository optimiser."""
924
klass._optimisers.add(optimiser)
925
926
def unlock(self):
927
"""Release the locks on source and target."""
928
try:
929
self.target.unlock()
930
finally:
931
self.source.unlock()
932
933
@classmethod
934
def unregister_optimiser(klass, optimiser):
935
"""Unregister an InterRepository optimiser."""
936
klass._optimisers.remove(optimiser)
937
938
939
class InterWeaveRepo(InterRepository):
940
"""Optimised code paths between Weave based repositories."""
941
942
_matching_repo_format = _default_format
943
"""Repository format for testing with."""
944
945
@staticmethod
946
def is_compatible(source, target):
947
"""Be compatible with known Weave formats.
948
949
We dont test for the stores being of specific types becase that
950
could lead to confusing results, and there is no need to be
951
overly general.
952
"""
953
try:
954
return (isinstance(source._format, (RepositoryFormat5,
955
RepositoryFormat6,
956
RepositoryFormat7)) and
957
isinstance(target._format, (RepositoryFormat5,
958
RepositoryFormat6,
959
RepositoryFormat7)))
960
except AttributeError:
961
return False
962
963
@needs_write_lock
964
def copy_content(self, revision_id=None, basis=None):
965
"""See InterRepository.copy_content()."""
966
# weave specific optimised path:
967
if basis is not None:
968
# copy the basis in, then fetch remaining data.
969
basis.copy_content_into(self.target, revision_id)
970
# the basis copy_content_into could misset this.
971
try:
972
self.target.set_make_working_trees(self.source.make_working_trees())
973
except NotImplementedError:
974
pass
975
self.target.fetch(self.source, revision_id=revision_id)
976
else:
977
try:
978
self.target.set_make_working_trees(self.source.make_working_trees())
979
except NotImplementedError:
980
pass
981
# FIXME do not peek!
982
if self.source.control_files._transport.listable():
983
pb = bzrlib.ui.ui_factory.progress_bar()
984
copy_all(self.source.weave_store,
985
self.target.weave_store, pb=pb)
986
pb.update('copying inventory', 0, 1)
987
self.target.control_weaves.copy_multi(
988
self.source.control_weaves, ['inventory'])
989
copy_all(self.source.revision_store,
990
self.target.revision_store, pb=pb)
991
else:
992
self.target.fetch(self.source, revision_id=revision_id)
993
994
@needs_write_lock
995
def fetch(self, revision_id=None, pb=None):
996
"""See InterRepository.fetch()."""
997
from bzrlib.fetch import RepoFetcher
998
mutter("Using fetch logic to copy between %s(%s) and %s(%s)",
999
self.source, self.source._format, self.target, self.target._format)
1000
f = RepoFetcher(to_repository=self.target,
1001
from_repository=self.source,
1002
last_revision=revision_id,
1003
pb=pb)
1004
return f.count_copied, f.failed_revisions
1005
1006
@needs_read_lock
1007
def missing_revision_ids(self, revision_id=None):
1008
"""See InterRepository.missing_revision_ids()."""
1009
# we want all revisions to satisfy revision_id in source.
1010
# but we dont want to stat every file here and there.
1011
# we want then, all revisions other needs to satisfy revision_id
1012
# checked, but not those that we have locally.
1013
# so the first thing is to get a subset of the revisions to
1014
# satisfy revision_id in source, and then eliminate those that
1015
# we do already have.
1016
# this is slow on high latency connection to self, but as as this
1017
# disk format scales terribly for push anyway due to rewriting
1018
# inventory.weave, this is considered acceptable.
1019
# - RBC 20060209
1020
if revision_id is not None:
1021
source_ids = self.source.get_ancestry(revision_id)
1022
assert source_ids.pop(0) == None
1023
else:
1024
source_ids = self.source._all_possible_ids()
1025
source_ids_set = set(source_ids)
1026
# source_ids is the worst possible case we may need to pull.
1027
# now we want to filter source_ids against what we actually
1028
# have in target, but dont try to check for existence where we know
1029
# we do not have a revision as that would be pointless.
1030
target_ids = set(self.target._all_possible_ids())
1031
possibly_present_revisions = target_ids.intersection(source_ids_set)
1032
actually_present_revisions = set(self.target._eliminate_revisions_not_present(possibly_present_revisions))
1033
required_revisions = source_ids_set.difference(actually_present_revisions)
1034
required_topo_revisions = [rev_id for rev_id in source_ids if rev_id in required_revisions]
1035
if revision_id is not None:
1036
# we used get_ancestry to determine source_ids then we are assured all
1037
# revisions referenced are present as they are installed in topological order.
1038
# and the tip revision was validated by get_ancestry.
1039
return required_topo_revisions
1040
else:
1041
# if we just grabbed the possibly available ids, then
1042
# we only have an estimate of whats available and need to validate
1043
# that against the revision records.
1044
return self.source._eliminate_revisions_not_present(required_topo_revisions)
1045
1046
1047
InterRepository.register_optimiser(InterWeaveRepo)
1048
1049
881
1050
class RepositoryTestProviderAdapter(object):
882
1051
"""A tool to generate a suite testing multiple repository formats at once.
883
1052
906
1075
new_test.id = make_new_test_id()
907
1076
result.addTest(new_test)
908
1077
return result
1078
1079
1080
class InterRepositoryTestProviderAdapter(object):
1081
"""A tool to generate a suite testing multiple inter repository formats.
1082
1083
This is done by copying the test once for each interrepo provider and injecting
1084
the transport_server, transport_readonly_server, repository_format and
1085
repository_to_format classes into each copy.
1086
Each copy is also given a new id() to make it easy to identify.
1087
"""
1088
1089
def __init__(self, transport_server, transport_readonly_server, formats):
1090
self._transport_server = transport_server
1091
self._transport_readonly_server = transport_readonly_server
1092
self._formats = formats
1093
1094
def adapt(self, test):
1095
result = TestSuite()
1096
for interrepo_class, repository_format, repository_format_to in self._formats:
1097
new_test = deepcopy(test)
1098
new_test.transport_server = self._transport_server
1099
new_test.transport_readonly_server = self._transport_readonly_server
1100
new_test.interrepo_class = interrepo_class
1101
new_test.repository_format = repository_format
1102
new_test.repository_format_to = repository_format_to
1103
def make_new_test_id():
1104
new_id = "%s(%s)" % (new_test.id(), interrepo_class.__name__)
1105
return lambda: new_id
1106
new_test.id = make_new_test_id()
1107
result.addTest(new_test)
1108
return result
1109
1110
@staticmethod
1111
def default_test_list():
1112
"""Generate the default list of interrepo permutations to test."""
1113
result = []
1114
# test the default InterRepository between format 6 and the current
1115
# default format.
1116
# XXX: robertc 20060220 reinstate this when there are two supported
1117
# formats which do not have an optimal code path between them.
1118
#result.append((InterRepository, RepositoryFormat6(),
1119
# RepositoryFormat.get_default_format()))
1120
for optimiser in InterRepository._optimisers:
1121
result.append((optimiser,
1122
optimiser._matching_repo_format,
1123
optimiser._matching_repo_format
1124
))
1125
# if there are specific combinations we want to use, we can add them
1126
# here.
1127
return result
Loggerhead is a web-based interface for Breezy
Version: 2.0.1