32
revision as _mod_revision,
37
48
from bzrlib.decorators import needs_read_lock, needs_write_lock
38
49
from bzrlib.repository import (
52
InterSameDataRepository,
53
MetaDirVersionedFileRepository,
41
54
MetaDirRepositoryFormat,
45
58
from bzrlib.store.text import TextStore
46
from bzrlib.trace import mutter
59
from bzrlib.tuned_gzip import GzipFile, bytes_to_gzip
60
from bzrlib.versionedfile import (
62
FulltextContentFactory,
49
67
class AllInOneRepository(Repository):
50
68
"""Legacy support - the repository behaviour for all-in-one branches."""
52
_serializer = xml5.serializer_v5
54
def __init__(self, _format, a_bzrdir, _revision_store, control_store, text_store):
71
def _serializer(self):
72
return xml5.serializer_v5
74
def _escape(self, file_or_path):
75
if not isinstance(file_or_path, basestring):
76
file_or_path = '/'.join(file_or_path)
77
if file_or_path == '':
79
return urlutils.escape(osutils.safe_unicode(file_or_path))
81
def __init__(self, _format, a_bzrdir):
55
82
# we reuse one control files instance.
56
dir_mode = a_bzrdir._control_files._dir_mode
57
file_mode = a_bzrdir._control_files._file_mode
83
dir_mode = a_bzrdir._get_dir_mode()
84
file_mode = a_bzrdir._get_file_mode()
59
86
def get_store(name, compressed=True, prefixed=False):
60
87
# FIXME: This approach of assuming stores are all entirely compressed
61
# or entirely uncompressed is tidy, but breaks upgrade from
62
# some existing branches where there's a mixture; we probably
88
# or entirely uncompressed is tidy, but breaks upgrade from
89
# some existing branches where there's a mixture; we probably
63
90
# still want the option to look for both.
64
relpath = a_bzrdir._control_files._escape(name)
65
store = TextStore(a_bzrdir._control_files._transport.clone(relpath),
91
relpath = self._escape(name)
92
store = TextStore(a_bzrdir.transport.clone(relpath),
66
93
prefixed=prefixed, compressed=compressed,
68
95
file_mode=file_mode)
71
98
# not broken out yet because the controlweaves|inventory_store
72
# and text_store | weave_store bits are still different.
99
# and texts bits are still different.
73
100
if isinstance(_format, RepositoryFormat4):
74
# cannot remove these - there is still no consistent api
101
# cannot remove these - there is still no consistent api
75
102
# which allows access to this old info.
76
103
self.inventory_store = get_store('inventory-store')
77
text_store = get_store('text-store')
78
super(AllInOneRepository, self).__init__(_format, a_bzrdir, a_bzrdir._control_files, _revision_store, control_store, text_store)
104
self._text_store = get_store('text-store')
105
super(AllInOneRepository, self).__init__(_format, a_bzrdir, a_bzrdir._control_files)
81
108
def _all_possible_ids(self):
82
109
"""Return all the possible revisions that we could find."""
83
110
if 'evil' in debug.debug_flags:
84
mutter_callsite(3, "_all_possible_ids scales with size of history.")
85
return self.get_inventory_weave().versions()
111
trace.mutter_callsite(
112
3, "_all_possible_ids scales with size of history.")
113
return [key[-1] for key in self.inventories.keys()]
88
116
def _all_revision_ids(self):
89
"""Returns a list of all the revision ids in the repository.
117
"""Returns a list of all the revision ids in the repository.
91
These are in as much topological order as the underlying store can
119
These are in as much topological order as the underlying store can
92
120
present: for weaves ghosts may lead to a lack of correctness until
93
121
the reweave updates the parents list.
95
if self._revision_store.text_store.listable():
96
return self._revision_store.all_revision_ids(self.get_transaction())
97
result = self._all_possible_ids()
98
# TODO: jam 20070210 Ensure that _all_possible_ids returns non-unicode
99
# ids. (It should, since _revision_store's API should change to
100
# return utf8 revision_ids)
101
return self._eliminate_revisions_not_present(result)
103
def _check_revision_parents(self, revision, inventory):
104
"""Private to Repository and Fetch.
106
This checks the parentage of revision in an inventory weave for
107
consistency and is only applicable to inventory-weave-for-ancestry
108
using repository formats & fetchers.
110
weave_parents = inventory.get_parents(revision.revision_id)
111
weave_names = inventory.versions()
112
for parent_id in revision.parent_ids:
113
if parent_id in weave_names:
114
# this parent must not be a ghost.
115
if not parent_id in weave_parents:
117
raise errors.CorruptRepository(self)
123
return [key[-1] for key in self.revisions.keys()]
125
def _activate_new_inventory(self):
126
"""Put a replacement inventory.new into use as inventories."""
127
# Copy the content across
128
t = self.bzrdir._control_files._transport
129
t.copy('inventory.new.weave', 'inventory.weave')
130
# delete the temp inventory
131
t.delete('inventory.new.weave')
132
# Check we can parse the new weave properly as a sanity check
133
self.inventories.keys()
135
def _backup_inventory(self):
136
t = self.bzrdir._control_files._transport
137
t.copy('inventory.weave', 'inventory.backup.weave')
139
def _temp_inventories(self):
140
t = self.bzrdir._control_files._transport
141
return self._format._get_inventories(t, self, 'inventory.new')
119
143
def get_commit_builder(self, branch, parents, config, timestamp=None,
120
144
timezone=None, committer=None, revprops=None,
121
145
revision_id=None):
122
146
self._check_ascii_revisionid(revision_id, self.get_commit_builder)
123
result = WeaveCommitBuilder(self, parents, config, timestamp, timezone,
147
result = CommitBuilder(self, parents, config, timestamp, timezone,
124
148
committer, revprops, revision_id)
125
149
self.start_write_group()
206
class WeaveMetaDirRepository(MetaDirRepository):
194
class WeaveMetaDirRepository(MetaDirVersionedFileRepository):
207
195
"""A subclass of MetaDirRepository to set weave specific policy."""
209
_serializer = xml5.serializer_v5
197
def __init__(self, _format, a_bzrdir, control_files):
198
super(WeaveMetaDirRepository, self).__init__(_format, a_bzrdir, control_files)
199
self._serializer = _format._serializer
212
202
def _all_possible_ids(self):
213
203
"""Return all the possible revisions that we could find."""
214
204
if 'evil' in debug.debug_flags:
215
mutter_callsite(3, "_all_possible_ids scales with size of history.")
216
return self.get_inventory_weave().versions()
205
trace.mutter_callsite(
206
3, "_all_possible_ids scales with size of history.")
207
return [key[-1] for key in self.inventories.keys()]
219
210
def _all_revision_ids(self):
220
"""Returns a list of all the revision ids in the repository.
211
"""Returns a list of all the revision ids in the repository.
222
These are in as much topological order as the underlying store can
213
These are in as much topological order as the underlying store can
223
214
present: for weaves ghosts may lead to a lack of correctness until
224
215
the reweave updates the parents list.
226
if self._revision_store.text_store.listable():
227
return self._revision_store.all_revision_ids(self.get_transaction())
228
result = self._all_possible_ids()
229
# TODO: jam 20070210 Ensure that _all_possible_ids returns non-unicode
230
# ids. (It should, since _revision_store's API should change to
231
# return utf8 revision_ids)
232
return self._eliminate_revisions_not_present(result)
234
def _check_revision_parents(self, revision, inventory):
235
"""Private to Repository and Fetch.
237
This checks the parentage of revision in an inventory weave for
238
consistency and is only applicable to inventory-weave-for-ancestry
239
using repository formats & fetchers.
241
weave_parents = inventory.get_parents(revision.revision_id)
242
weave_names = inventory.versions()
243
for parent_id in revision.parent_ids:
244
if parent_id in weave_names:
245
# this parent must not be a ghost.
246
if not parent_id in weave_parents:
248
raise errors.CorruptRepository(self)
217
return [key[-1] for key in self.revisions.keys()]
219
def _activate_new_inventory(self):
220
"""Put a replacement inventory.new into use as inventories."""
221
# Copy the content across
223
t.copy('inventory.new.weave', 'inventory.weave')
224
# delete the temp inventory
225
t.delete('inventory.new.weave')
226
# Check we can parse the new weave properly as a sanity check
227
self.inventories.keys()
229
def _backup_inventory(self):
231
t.copy('inventory.weave', 'inventory.backup.weave')
233
def _temp_inventories(self):
235
return self._format._get_inventories(t, self, 'inventory.new')
250
237
def get_commit_builder(self, branch, parents, config, timestamp=None,
251
238
timezone=None, committer=None, revprops=None,
252
239
revision_id=None):
253
240
self._check_ascii_revisionid(revision_id, self.get_commit_builder)
254
result = WeaveCommitBuilder(self, parents, config, timestamp, timezone,
241
result = CommitBuilder(self, parents, config, timestamp, timezone,
255
242
committer, revprops, revision_id)
256
243
self.start_write_group()
260
247
def get_revision(self, revision_id):
261
248
"""Return the Revision object for a named revision"""
262
# TODO: jam 20070210 get_revision_reconcile should do this for us
263
249
r = self.get_revision_reconcile(revision_id)
264
# weave corruption can lead to absent revision markers that should be
266
# the following test is reasonably cheap (it needs a single weave read)
267
# and the weave is cached in read transactions. In write transactions
268
# it is not cached but typically we only read a small number of
269
# revisions. For knits when they are introduced we will probably want
270
# to ensure that caching write transactions are in use.
271
inv = self.get_inventory_weave()
272
self._check_revision_parents(r, inv)
276
def get_revision_graph(self, revision_id=None):
277
"""Return a dictionary containing the revision graph.
279
:param revision_id: The revision_id to get a graph from. If None, then
280
the entire revision graph is returned. This is a deprecated mode of
281
operation and will be removed in the future.
282
:return: a dictionary of revision_id->revision_parents_list.
284
if 'evil' in debug.debug_flags:
286
"get_revision_graph scales with size of history.")
287
# special case NULL_REVISION
288
if revision_id == _mod_revision.NULL_REVISION:
290
a_weave = self.get_inventory_weave()
291
all_revisions = self._eliminate_revisions_not_present(
293
entire_graph = dict([(node, tuple(a_weave.get_parents(node))) for
294
node in all_revisions])
295
if revision_id is None:
297
elif revision_id not in entire_graph:
298
raise errors.NoSuchRevision(self, revision_id)
300
# add what can be reached from revision_id
302
pending = set([revision_id])
303
while len(pending) > 0:
305
result[node] = entire_graph[node]
306
for revision_id in result[node]:
307
if revision_id not in result:
308
pending.add(revision_id)
252
def _inventory_add_lines(self, revision_id, parents, lines,
254
"""Store lines in inv_vf and return the sha1 of the inventory."""
255
present_parents = self.get_graph().get_parent_map(parents)
257
for parent in parents:
258
if parent in present_parents:
259
final_parents.append((parent,))
260
return self.inventories.add_lines((revision_id,), final_parents, lines,
261
check_content=check_content)[0]
311
263
def revision_graph_can_have_wrong_parents(self):
312
# XXX: This is an old format that we don't support full checking on, so
313
# just claim that checking for this inconsistency is not required.
452
389
_versionedfile_class = weave.WeaveFile
453
390
_matchingbzrdir = bzrdir.BzrDirFormat5()
456
super(RepositoryFormat5, self).__init__()
392
def _serializer(self):
393
return xml5.serializer_v5
458
395
def get_format_description(self):
459
396
"""See RepositoryFormat.get_format_description()."""
460
397
return "Weave repository format 5"
462
def _get_revision_store(self, repo_transport, control_files):
463
"""See RepositoryFormat._get_revision_store()."""
464
"""Return the revision store object for this a_bzrdir."""
465
return self._get_text_rev_store(repo_transport,
470
def _get_text_store(self, transport, control_files):
471
"""See RepositoryFormat._get_text_store()."""
472
return self._get_versioned_file_store('weaves', transport, control_files, prefixed=False)
399
def network_name(self):
400
"""The network name for this format is the control dirs disk label."""
401
return self._matchingbzrdir.get_format_string()
403
def _get_inventories(self, repo_transport, repo, name='inventory'):
404
mapper = versionedfile.ConstantMapper(name)
405
return versionedfile.ThunkedVersionedFiles(repo_transport,
406
weave.WeaveFile, mapper, repo.is_locked)
408
def _get_revisions(self, repo_transport, repo):
409
return RevisionTextStore(repo_transport.clone('revision-store'),
410
xml5.serializer_v5, False, versionedfile.PrefixMapper(),
411
repo.is_locked, repo.is_write_locked)
413
def _get_signatures(self, repo_transport, repo):
414
return SignatureTextStore(repo_transport.clone('revision-store'),
415
False, versionedfile.PrefixMapper(),
416
repo.is_locked, repo.is_write_locked)
418
def _get_texts(self, repo_transport, repo):
419
mapper = versionedfile.PrefixMapper()
420
base_transport = repo_transport.clone('weaves')
421
return versionedfile.ThunkedVersionedFiles(base_transport,
422
weave.WeaveFile, mapper, repo.is_locked)
475
425
class RepositoryFormat6(PreSplitOutRepositoryFormat):
562
530
weavefile.write_weave_v5(weave.Weave(), sio)
563
531
empty_weave = sio.getvalue()
565
mutter('creating repository in %s.', a_bzrdir.transport.base)
533
trace.mutter('creating repository in %s.', a_bzrdir.transport.base)
566
534
dirs = ['revision-store', 'weaves']
567
files = [('inventory.weave', StringIO(empty_weave)),
535
files = [('inventory.weave', StringIO(empty_weave)),
569
537
utf8_files = [('format', self.get_format_string())]
571
539
self._upload_blank_content(a_bzrdir, dirs, files, utf8_files, shared)
572
540
return self.open(a_bzrdir=a_bzrdir, _found=True)
574
542
def open(self, a_bzrdir, _found=False, _override_transport=None):
575
543
"""See RepositoryFormat.open().
577
545
:param _override_transport: INTERNAL USE ONLY. Allows opening the
578
546
repository at a slightly different url
579
547
than normal. I.e. during 'upgrade'.
582
550
format = RepositoryFormat.find_format(a_bzrdir)
583
assert format.__class__ == self.__class__
584
551
if _override_transport is not None:
585
552
repo_transport = _override_transport
587
554
repo_transport = a_bzrdir.get_repository_transport(None)
588
555
control_files = lockable_files.LockableFiles(repo_transport,
589
556
'lock', lockdir.LockDir)
590
text_store = self._get_text_store(repo_transport, control_files)
591
control_store = self._get_control_store(repo_transport, control_files)
592
_revision_store = self._get_revision_store(repo_transport, control_files)
593
return WeaveMetaDirRepository(_format=self,
595
control_files=control_files,
596
_revision_store=_revision_store,
597
control_store=control_store,
598
text_store=text_store)
601
class WeaveCommitBuilder(CommitBuilder):
602
"""A builder for weave based repos that don't support ghosts."""
604
def _add_text_to_weave(self, file_id, new_lines, parents, nostore_sha):
605
versionedfile = self.repository.weave_store.get_weave_or_empty(
606
file_id, self.repository.get_transaction())
607
result = versionedfile.add_lines(
608
self._new_revision_id, parents, new_lines,
609
nostore_sha=nostore_sha)[0:2]
610
versionedfile.clear_cache()
557
result = WeaveMetaDirRepository(_format=self, a_bzrdir=a_bzrdir,
558
control_files=control_files)
559
result.revisions = self._get_revisions(repo_transport, result)
560
result.signatures = self._get_signatures(repo_transport, result)
561
result.inventories = self._get_inventories(repo_transport, result)
562
result.texts = self._get_texts(repo_transport, result)
563
result.chk_bytes = None
564
result._transport = repo_transport
568
class TextVersionedFiles(VersionedFiles):
569
"""Just-a-bunch-of-files based VersionedFile stores."""
571
def __init__(self, transport, compressed, mapper, is_locked, can_write):
572
self._compressed = compressed
573
self._transport = transport
574
self._mapper = mapper
579
self._is_locked = is_locked
580
self._can_write = can_write
582
def add_lines(self, key, parents, lines):
583
"""Add a revision to the store."""
584
if not self._is_locked():
585
raise errors.ObjectNotLocked(self)
586
if not self._can_write():
587
raise errors.ReadOnlyError(self)
589
raise ValueError('bad idea to put / in %r' % (key,))
590
text = ''.join(lines)
592
text = bytes_to_gzip(text)
593
path = self._map(key)
594
self._transport.put_bytes_non_atomic(path, text, create_parent_dir=True)
596
def insert_record_stream(self, stream):
598
for record in stream:
599
# Raise an error when a record is missing.
600
if record.storage_kind == 'absent':
601
raise errors.RevisionNotPresent([record.key[0]], self)
602
# adapt to non-tuple interface
603
if record.storage_kind == 'fulltext':
604
self.add_lines(record.key, None,
605
osutils.split_lines(record.get_bytes_as('fulltext')))
607
adapter_key = record.storage_kind, 'fulltext'
609
adapter = adapters[adapter_key]
611
adapter_factory = adapter_registry.get(adapter_key)
612
adapter = adapter_factory(self)
613
adapters[adapter_key] = adapter
614
lines = osutils.split_lines(adapter.get_bytes(
615
record, record.get_bytes_as(record.storage_kind)))
617
self.add_lines(record.key, None, lines)
618
except RevisionAlreadyPresent:
621
def _load_text(self, key):
622
if not self._is_locked():
623
raise errors.ObjectNotLocked(self)
624
path = self._map(key)
626
text = self._transport.get_bytes(path)
627
compressed = self._compressed
628
except errors.NoSuchFile:
630
# try without the .gz
633
text = self._transport.get_bytes(path)
635
except errors.NoSuchFile:
640
text = GzipFile(mode='rb', fileobj=StringIO(text)).read()
644
return self._mapper.map(key) + self._ext
647
class RevisionTextStore(TextVersionedFiles):
648
"""Legacy thunk for format 4 repositories."""
650
def __init__(self, transport, serializer, compressed, mapper, is_locked,
652
"""Create a RevisionTextStore at transport with serializer."""
653
TextVersionedFiles.__init__(self, transport, compressed, mapper,
654
is_locked, can_write)
655
self._serializer = serializer
657
def _load_text_parents(self, key):
658
text = self._load_text(key)
661
parents = self._serializer.read_revision_from_string(text).parent_ids
662
return text, tuple((parent,) for parent in parents)
664
def get_parent_map(self, keys):
667
parents = self._load_text_parents(key)[1]
670
result[key] = parents
673
def get_known_graph_ancestry(self, keys):
674
"""Get a KnownGraph instance with the ancestry of keys."""
676
parent_map = self.get_parent_map(keys)
677
kg = _mod_graph.KnownGraph(parent_map)
680
def get_record_stream(self, keys, sort_order, include_delta_closure):
682
text, parents = self._load_text_parents(key)
684
yield AbsentContentFactory(key)
686
yield FulltextContentFactory(key, parents, None, text)
689
if not self._is_locked():
690
raise errors.ObjectNotLocked(self)
692
for quoted_relpath in self._transport.iter_files_recursive():
693
relpath = urllib.unquote(quoted_relpath)
694
path, ext = os.path.splitext(relpath)
697
if not relpath.endswith('.sig'):
698
relpaths.add(relpath)
699
paths = list(relpaths)
700
return set([self._mapper.unmap(path) for path in paths])
703
class SignatureTextStore(TextVersionedFiles):
704
"""Legacy thunk for format 4-7 repositories."""
706
def __init__(self, transport, compressed, mapper, is_locked, can_write):
707
TextVersionedFiles.__init__(self, transport, compressed, mapper,
708
is_locked, can_write)
709
self._ext = '.sig' + self._ext
711
def get_parent_map(self, keys):
714
text = self._load_text(key)
720
def get_record_stream(self, keys, sort_order, include_delta_closure):
722
text = self._load_text(key)
724
yield AbsentContentFactory(key)
726
yield FulltextContentFactory(key, None, None, text)
729
if not self._is_locked():
730
raise errors.ObjectNotLocked(self)
732
for quoted_relpath in self._transport.iter_files_recursive():
733
relpath = urllib.unquote(quoted_relpath)
734
path, ext = os.path.splitext(relpath)
737
if not relpath.endswith('.sig'):
739
relpaths.add(relpath[:-4])
740
paths = list(relpaths)
741
return set([self._mapper.unmap(path) for path in paths])
744
class InterWeaveRepo(InterSameDataRepository):
745
"""Optimised code paths between Weave based repositories.
747
This should be in bzrlib/repofmt/weaverepo.py but we have not yet
748
implemented lazy inter-object optimisation.
752
def _get_repo_format_to_test(self):
753
return RepositoryFormat7()
756
def is_compatible(source, target):
757
"""Be compatible with known Weave formats.
759
We don't test for the stores being of specific types because that
760
could lead to confusing results, and there is no need to be
764
return (isinstance(source._format, (RepositoryFormat5,
766
RepositoryFormat7)) and
767
isinstance(target._format, (RepositoryFormat5,
770
except AttributeError:
774
def copy_content(self, revision_id=None):
775
"""See InterRepository.copy_content()."""
776
# weave specific optimised path:
778
self.target.set_make_working_trees(self.source.make_working_trees())
779
except (errors.RepositoryUpgradeRequired, NotImplemented):
782
if self.source._transport.listable():
783
pb = ui.ui_factory.nested_progress_bar()
785
self.target.texts.insert_record_stream(
786
self.source.texts.get_record_stream(
787
self.source.texts.keys(), 'topological', False))
788
pb.update('Copying inventory', 0, 1)
789
self.target.inventories.insert_record_stream(
790
self.source.inventories.get_record_stream(
791
self.source.inventories.keys(), 'topological', False))
792
self.target.signatures.insert_record_stream(
793
self.source.signatures.get_record_stream(
794
self.source.signatures.keys(),
796
self.target.revisions.insert_record_stream(
797
self.source.revisions.get_record_stream(
798
self.source.revisions.keys(),
799
'topological', True))
803
self.target.fetch(self.source, revision_id=revision_id)
806
def search_missing_revision_ids(self, revision_id=None, find_ghosts=True):
807
"""See InterRepository.missing_revision_ids()."""
808
# we want all revisions to satisfy revision_id in source.
809
# but we don't want to stat every file here and there.
810
# we want then, all revisions other needs to satisfy revision_id
811
# checked, but not those that we have locally.
812
# so the first thing is to get a subset of the revisions to
813
# satisfy revision_id in source, and then eliminate those that
814
# we do already have.
815
# this is slow on high latency connection to self, but as this
816
# disk format scales terribly for push anyway due to rewriting
817
# inventory.weave, this is considered acceptable.
819
if revision_id is not None:
820
source_ids = self.source.get_ancestry(revision_id)
821
if source_ids[0] is not None:
822
raise AssertionError()
825
source_ids = self.source._all_possible_ids()
826
source_ids_set = set(source_ids)
827
# source_ids is the worst possible case we may need to pull.
828
# now we want to filter source_ids against what we actually
829
# have in target, but don't try to check for existence where we know
830
# we do not have a revision as that would be pointless.
831
target_ids = set(self.target._all_possible_ids())
832
possibly_present_revisions = target_ids.intersection(source_ids_set)
833
actually_present_revisions = set(
834
self.target._eliminate_revisions_not_present(possibly_present_revisions))
835
required_revisions = source_ids_set.difference(actually_present_revisions)
836
if revision_id is not None:
837
# we used get_ancestry to determine source_ids then we are assured all
838
# revisions referenced are present as they are installed in topological order.
839
# and the tip revision was validated by get_ancestry.
840
result_set = required_revisions
842
# if we just grabbed the possibly available ids, then
843
# we only have an estimate of whats available and need to validate
844
# that against the revision records.
846
self.source._eliminate_revisions_not_present(required_revisions))
847
return self.source.revision_ids_to_search_result(result_set)
614
850
_legacy_formats = [RepositoryFormat4(),
615
851
RepositoryFormat5(),
616
852
RepositoryFormat6()]
855
InterRepository.register_optimiser(InterWeaveRepo)
added
removed
bzrlib/repofmt/weaverepo.py
