163
289
if '\n' in line[:-1]:
164
290
raise errors.BzrBadParameterContainsNewline("lines")
166
def _check_write_ok(self):
167
"""Is the versioned file marked as 'finished' ? Raise if it is."""
169
raise errors.OutSideTransaction()
170
if self._access_mode != 'w':
171
raise errors.ReadOnlyObjectDirtiedError(self)
173
def clear_cache(self):
174
"""Remove any data cached in the versioned file object."""
176
def clone_text(self, new_version_id, old_version_id, parents):
177
"""Add an identical text to old_version_id as new_version_id.
179
Must raise RevisionNotPresent if the old version or any of the
180
parents are not present in file history.
182
Must raise RevisionAlreadyPresent if the new version is
183
already present in file history."""
184
self._check_write_ok()
185
return self._clone_text(new_version_id, old_version_id, parents)
187
def _clone_text(self, new_version_id, old_version_id, parents):
188
"""Helper function to do the _clone_text work."""
189
raise NotImplementedError(self.clone_text)
191
def create_empty(self, name, transport, mode=None):
192
"""Create a new versioned file of this exact type.
194
:param name: the file name
195
:param transport: the transport
196
:param mode: optional file mode.
198
raise NotImplementedError(self.create_empty)
200
def fix_parents(self, version, new_parents):
201
"""Fix the parents list for version.
203
This is done by appending a new version to the index
204
with identical data except for the parents list.
205
the parents list must be a superset of the current
208
self._check_write_ok()
209
return self._fix_parents(version, new_parents)
211
def _fix_parents(self, version, new_parents):
212
"""Helper for fix_parents."""
213
raise NotImplementedError(self.fix_parents)
215
def get_delta(self, version):
216
"""Get a delta for constructing version from some other version.
218
:return: (delta_parent, sha1, noeol, delta)
219
Where delta_parent is a version id or None to indicate no parent.
221
raise NotImplementedError(self.get_delta)
223
def get_deltas(self, versions):
224
"""Get multiple deltas at once for constructing versions.
226
:return: dict(version_id:(delta_parent, sha1, noeol, delta))
227
Where delta_parent is a version id or None to indicate no parent, and
228
version_id is the version_id created by that delta.
231
for version in versions:
232
result[version] = self.get_delta(version)
235
def get_sha1(self, version_id):
236
"""Get the stored sha1 sum for the given revision.
238
:param name: The name of the version to lookup
240
raise NotImplementedError(self.get_sha1)
242
def get_suffixes(self):
243
"""Return the file suffixes associated with this versioned file."""
244
raise NotImplementedError(self.get_suffixes)
292
def get_format_signature(self):
293
"""Get a text description of the data encoding in this file.
297
raise NotImplementedError(self.get_format_signature)
299
def make_mpdiffs(self, version_ids):
300
"""Create multiparent diffs for specified versions."""
301
knit_versions = set()
302
knit_versions.update(version_ids)
303
parent_map = self.get_parent_map(version_ids)
304
for version_id in version_ids:
306
knit_versions.update(parent_map[version_id])
308
raise errors.RevisionNotPresent(version_id, self)
309
# We need to filter out ghosts, because we can't diff against them.
310
knit_versions = set(self.get_parent_map(knit_versions).keys())
311
lines = dict(zip(knit_versions,
312
self._get_lf_split_line_list(knit_versions)))
314
for version_id in version_ids:
315
target = lines[version_id]
317
parents = [lines[p] for p in parent_map[version_id] if p in
320
# I don't know how this could ever trigger.
321
# parent_map[version_id] was already triggered in the previous
322
# for loop, and lines[p] has the 'if p in knit_versions' check,
323
# so we again won't have a KeyError.
324
raise errors.RevisionNotPresent(version_id, self)
326
left_parent_blocks = self._extract_blocks(version_id,
329
left_parent_blocks = None
330
diffs.append(multiparent.MultiParent.from_lines(target, parents,
334
def _extract_blocks(self, version_id, source, target):
337
def add_mpdiffs(self, records):
338
"""Add mpdiffs to this VersionedFile.
340
Records should be iterables of version, parents, expected_sha1,
341
mpdiff. mpdiff should be a MultiParent instance.
343
# Does this need to call self._check_write_ok()? (IanC 20070919)
345
mpvf = multiparent.MultiMemoryVersionedFile()
347
for version, parent_ids, expected_sha1, mpdiff in records:
348
versions.append(version)
349
mpvf.add_diff(mpdiff, version, parent_ids)
350
needed_parents = set()
351
for version, parent_ids, expected_sha1, mpdiff in records:
352
needed_parents.update(p for p in parent_ids
353
if not mpvf.has_version(p))
354
present_parents = set(self.get_parent_map(needed_parents).keys())
355
for parent_id, lines in zip(present_parents,
356
self._get_lf_split_line_list(present_parents)):
357
mpvf.add_version(lines, parent_id, [])
358
for (version, parent_ids, expected_sha1, mpdiff), lines in\
359
zip(records, mpvf.get_line_list(versions)):
360
if len(parent_ids) == 1:
361
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
362
mpvf.get_diff(parent_ids[0]).num_lines()))
364
left_matching_blocks = None
366
_, _, version_text = self.add_lines_with_ghosts(version,
367
parent_ids, lines, vf_parents,
368
left_matching_blocks=left_matching_blocks)
369
except NotImplementedError:
370
# The vf can't handle ghosts, so add lines normally, which will
371
# (reasonably) fail if there are ghosts in the data.
372
_, _, version_text = self.add_lines(version,
373
parent_ids, lines, vf_parents,
374
left_matching_blocks=left_matching_blocks)
375
vf_parents[version] = version_text
376
sha1s = self.get_sha1s(versions)
377
for version, parent_ids, expected_sha1, mpdiff in records:
378
if expected_sha1 != sha1s[version]:
379
raise errors.VersionedFileInvalidChecksum(version)
246
381
def get_text(self, version_id):
247
382
"""Return version contents as a text string.
447
506
raise NotImplementedError(VersionedFile.plan_merge)
449
def weave_merge(self, plan, a_marker=TextMerge.A_MARKER,
508
def weave_merge(self, plan, a_marker=TextMerge.A_MARKER,
450
509
b_marker=TextMerge.B_MARKER):
451
510
return PlanWeaveMerge(plan, a_marker, b_marker).merge_lines()[0]
513
class RecordingVersionedFilesDecorator(object):
514
"""A minimal versioned files that records calls made on it.
516
Only enough methods have been added to support tests using it to date.
518
:ivar calls: A list of the calls made; can be reset at any time by
522
def __init__(self, backing_vf):
523
"""Create a RecordingVersionedFileDsecorator decorating backing_vf.
525
:param backing_vf: The versioned file to answer all methods.
527
self._backing_vf = backing_vf
530
def add_lines(self, key, parents, lines, parent_texts=None,
531
left_matching_blocks=None, nostore_sha=None, random_id=False,
533
self.calls.append(("add_lines", key, parents, lines, parent_texts,
534
left_matching_blocks, nostore_sha, random_id, check_content))
535
return self._backing_vf.add_lines(key, parents, lines, parent_texts,
536
left_matching_blocks, nostore_sha, random_id, check_content)
539
self._backing_vf.check()
541
def get_parent_map(self, keys):
542
self.calls.append(("get_parent_map", copy(keys)))
543
return self._backing_vf.get_parent_map(keys)
545
def get_record_stream(self, keys, sort_order, include_delta_closure):
546
self.calls.append(("get_record_stream", list(keys), sort_order,
547
include_delta_closure))
548
return self._backing_vf.get_record_stream(keys, sort_order,
549
include_delta_closure)
551
def get_sha1s(self, keys):
552
self.calls.append(("get_sha1s", copy(keys)))
553
return self._backing_vf.get_sha1s(keys)
555
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
556
self.calls.append(("iter_lines_added_or_present_in_keys", copy(keys)))
557
return self._backing_vf.iter_lines_added_or_present_in_keys(keys, pb=pb)
560
self.calls.append(("keys",))
561
return self._backing_vf.keys()
564
class KeyMapper(object):
565
"""KeyMappers map between keys and underlying partitioned storage."""
568
"""Map key to an underlying storage identifier.
570
:param key: A key tuple e.g. ('file-id', 'revision-id').
571
:return: An underlying storage identifier, specific to the partitioning
574
raise NotImplementedError(self.map)
576
def unmap(self, partition_id):
577
"""Map a partitioned storage id back to a key prefix.
579
:param partition_id: The underlying partition id.
580
:return: As much of a key (or prefix) as is derivable from the partition
583
raise NotImplementedError(self.unmap)
586
class ConstantMapper(KeyMapper):
587
"""A key mapper that maps to a constant result."""
589
def __init__(self, result):
590
"""Create a ConstantMapper which will return result for all maps."""
591
self._result = result
594
"""See KeyMapper.map()."""
598
class URLEscapeMapper(KeyMapper):
599
"""Base class for use with transport backed storage.
601
This provides a map and unmap wrapper that respectively url escape and
602
unescape their outputs and inputs.
606
"""See KeyMapper.map()."""
607
return urllib.quote(self._map(key))
609
def unmap(self, partition_id):
610
"""See KeyMapper.unmap()."""
611
return self._unmap(urllib.unquote(partition_id))
614
class PrefixMapper(URLEscapeMapper):
615
"""A key mapper that extracts the first component of a key.
617
This mapper is for use with a transport based backend.
621
"""See KeyMapper.map()."""
624
def _unmap(self, partition_id):
625
"""See KeyMapper.unmap()."""
626
return (partition_id,)
629
class HashPrefixMapper(URLEscapeMapper):
630
"""A key mapper that combines the first component of a key with a hash.
632
This mapper is for use with a transport based backend.
636
"""See KeyMapper.map()."""
637
prefix = self._escape(key[0])
638
return "%02x/%s" % (adler32(prefix) & 0xff, prefix)
640
def _escape(self, prefix):
641
"""No escaping needed here."""
644
def _unmap(self, partition_id):
645
"""See KeyMapper.unmap()."""
646
return (self._unescape(osutils.basename(partition_id)),)
648
def _unescape(self, basename):
649
"""No unescaping needed for HashPrefixMapper."""
653
class HashEscapedPrefixMapper(HashPrefixMapper):
654
"""Combines the escaped first component of a key with a hash.
656
This mapper is for use with a transport based backend.
659
_safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
661
def _escape(self, prefix):
662
"""Turn a key element into a filesystem safe string.
664
This is similar to a plain urllib.quote, except
665
it uses specific safe characters, so that it doesn't
666
have to translate a lot of valid file ids.
668
# @ does not get escaped. This is because it is a valid
669
# filesystem character we use all the time, and it looks
670
# a lot better than seeing %40 all the time.
671
r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
675
def _unescape(self, basename):
676
"""Escaped names are easily unescaped by urlutils."""
677
return urllib.unquote(basename)
680
def make_versioned_files_factory(versioned_file_factory, mapper):
681
"""Create a ThunkedVersionedFiles factory.
683
This will create a callable which when called creates a
684
ThunkedVersionedFiles on a transport, using mapper to access individual
685
versioned files, and versioned_file_factory to create each individual file.
687
def factory(transport):
688
return ThunkedVersionedFiles(transport, versioned_file_factory, mapper,
693
class VersionedFiles(object):
694
"""Storage for many versioned files.
696
This object allows a single keyspace for accessing the history graph and
697
contents of named bytestrings.
699
Currently no implementation allows the graph of different key prefixes to
700
intersect, but the API does allow such implementations in the future.
702
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
703
may have a different length key-size, but that size will be constant for
704
all texts added to or retrieved from it. For instance, bzrlib uses
705
instances with a key-size of 2 for storing user files in a repository, with
706
the first element the fileid, and the second the version of that file.
708
The use of tuples allows a single code base to support several different
709
uses with only the mapping logic changing from instance to instance.
712
def add_lines(self, key, parents, lines, parent_texts=None,
713
left_matching_blocks=None, nostore_sha=None, random_id=False,
715
"""Add a text to the store.
717
:param key: The key tuple of the text to add.
718
:param parents: The parents key tuples of the text to add.
719
:param lines: A list of lines. Each line must be a bytestring. And all
720
of them except the last must be terminated with \n and contain no
721
other \n's. The last line may either contain no \n's or a single
722
terminating \n. If the lines list does meet this constraint the add
723
routine may error or may succeed - but you will be unable to read
724
the data back accurately. (Checking the lines have been split
725
correctly is expensive and extremely unlikely to catch bugs so it
726
is not done at runtime unless check_content is True.)
727
:param parent_texts: An optional dictionary containing the opaque
728
representations of some or all of the parents of version_id to
729
allow delta optimisations. VERY IMPORTANT: the texts must be those
730
returned by add_lines or data corruption can be caused.
731
:param left_matching_blocks: a hint about which areas are common
732
between the text and its left-hand-parent. The format is
733
the SequenceMatcher.get_matching_blocks format.
734
:param nostore_sha: Raise ExistingContent and do not add the lines to
735
the versioned file if the digest of the lines matches this.
736
:param random_id: If True a random id has been selected rather than
737
an id determined by some deterministic process such as a converter
738
from a foreign VCS. When True the backend may choose not to check
739
for uniqueness of the resulting key within the versioned file, so
740
this should only be done when the result is expected to be unique
742
:param check_content: If True, the lines supplied are verified to be
743
bytestrings that are correctly formed lines.
744
:return: The text sha1, the number of bytes in the text, and an opaque
745
representation of the inserted version which can be provided
746
back to future add_lines calls in the parent_texts dictionary.
748
raise NotImplementedError(self.add_lines)
750
def add_mpdiffs(self, records):
751
"""Add mpdiffs to this VersionedFile.
753
Records should be iterables of version, parents, expected_sha1,
754
mpdiff. mpdiff should be a MultiParent instance.
757
mpvf = multiparent.MultiMemoryVersionedFile()
759
for version, parent_ids, expected_sha1, mpdiff in records:
760
versions.append(version)
761
mpvf.add_diff(mpdiff, version, parent_ids)
762
needed_parents = set()
763
for version, parent_ids, expected_sha1, mpdiff in records:
764
needed_parents.update(p for p in parent_ids
765
if not mpvf.has_version(p))
766
# It seems likely that adding all the present parents as fulltexts can
767
# easily exhaust memory.
768
split_lines = osutils.split_lines
769
for record in self.get_record_stream(needed_parents, 'unordered',
771
if record.storage_kind == 'absent':
773
mpvf.add_version(split_lines(record.get_bytes_as('fulltext')),
775
for (key, parent_keys, expected_sha1, mpdiff), lines in\
776
zip(records, mpvf.get_line_list(versions)):
777
if len(parent_keys) == 1:
778
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
779
mpvf.get_diff(parent_keys[0]).num_lines()))
781
left_matching_blocks = None
782
version_sha1, _, version_text = self.add_lines(key,
783
parent_keys, lines, vf_parents,
784
left_matching_blocks=left_matching_blocks)
785
if version_sha1 != expected_sha1:
786
raise errors.VersionedFileInvalidChecksum(version)
787
vf_parents[key] = version_text
789
def annotate(self, key):
790
"""Return a list of (version-key, line) tuples for the text of key.
792
:raise RevisionNotPresent: If the key is not present.
794
raise NotImplementedError(self.annotate)
796
def check(self, progress_bar=None):
797
"""Check this object for integrity."""
798
raise NotImplementedError(self.check)
801
def check_not_reserved_id(version_id):
802
revision.check_not_reserved_id(version_id)
804
def _check_lines_not_unicode(self, lines):
805
"""Check that lines being added to a versioned file are not unicode."""
807
if line.__class__ is not str:
808
raise errors.BzrBadParameterUnicode("lines")
810
def _check_lines_are_lines(self, lines):
811
"""Check that the lines really are full lines without inline EOL."""
813
if '\n' in line[:-1]:
814
raise errors.BzrBadParameterContainsNewline("lines")
816
def get_parent_map(self, keys):
817
"""Get a map of the parents of keys.
819
:param keys: The keys to look up parents for.
820
:return: A mapping from keys to parents. Absent keys are absent from
823
raise NotImplementedError(self.get_parent_map)
825
def get_record_stream(self, keys, ordering, include_delta_closure):
826
"""Get a stream of records for keys.
828
:param keys: The keys to include.
829
:param ordering: Either 'unordered' or 'topological'. A topologically
830
sorted stream has compression parents strictly before their
832
:param include_delta_closure: If True then the closure across any
833
compression parents will be included (in the opaque data).
834
:return: An iterator of ContentFactory objects, each of which is only
835
valid until the iterator is advanced.
837
raise NotImplementedError(self.get_record_stream)
839
def get_sha1s(self, keys):
840
"""Get the sha1's of the texts for the given keys.
842
:param keys: The names of the keys to lookup
843
:return: a dict from key to sha1 digest. Keys of texts which are not
844
present in the store are not present in the returned
847
raise NotImplementedError(self.get_sha1s)
849
def insert_record_stream(self, stream):
850
"""Insert a record stream into this container.
852
:param stream: A stream of records to insert.
854
:seealso VersionedFile.get_record_stream:
856
raise NotImplementedError
858
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
859
"""Iterate over the lines in the versioned files from keys.
861
This may return lines from other keys. Each item the returned
862
iterator yields is a tuple of a line and a text version that that line
863
is present in (not introduced in).
865
Ordering of results is in whatever order is most suitable for the
866
underlying storage format.
868
If a progress bar is supplied, it may be used to indicate progress.
869
The caller is responsible for cleaning up progress bars (because this
873
* Lines are normalised by the underlying store: they will all have \n
875
* Lines are returned in arbitrary order.
877
:return: An iterator over (line, key).
879
raise NotImplementedError(self.iter_lines_added_or_present_in_keys)
882
"""Return a iterable of the keys for all the contained texts."""
883
raise NotImplementedError(self.keys)
885
def make_mpdiffs(self, keys):
886
"""Create multiparent diffs for specified keys."""
887
keys_order = tuple(keys)
888
keys = frozenset(keys)
889
knit_keys = set(keys)
890
parent_map = self.get_parent_map(keys)
891
for parent_keys in parent_map.itervalues():
893
knit_keys.update(parent_keys)
894
missing_keys = keys - set(parent_map)
896
raise errors.RevisionNotPresent(list(missing_keys)[0], self)
897
# We need to filter out ghosts, because we can't diff against them.
898
maybe_ghosts = knit_keys - keys
899
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
900
knit_keys.difference_update(ghosts)
902
split_lines = osutils.split_lines
903
for record in self.get_record_stream(knit_keys, 'topological', True):
904
lines[record.key] = split_lines(record.get_bytes_as('fulltext'))
905
# line_block_dict = {}
906
# for parent, blocks in record.extract_line_blocks():
907
# line_blocks[parent] = blocks
908
# line_blocks[record.key] = line_block_dict
910
for key in keys_order:
912
parents = parent_map[key] or []
913
# Note that filtering knit_keys can lead to a parent difference
914
# between the creation and the application of the mpdiff.
915
parent_lines = [lines[p] for p in parents if p in knit_keys]
916
if len(parent_lines) > 0:
917
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
920
left_parent_blocks = None
921
diffs.append(multiparent.MultiParent.from_lines(target,
922
parent_lines, left_parent_blocks))
925
def _extract_blocks(self, version_id, source, target):
929
class ThunkedVersionedFiles(VersionedFiles):
930
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
932
This object allows a single keyspace for accessing the history graph and
933
contents of named bytestrings.
935
Currently no implementation allows the graph of different key prefixes to
936
intersect, but the API does allow such implementations in the future.
939
def __init__(self, transport, file_factory, mapper, is_locked):
940
"""Create a ThunkedVersionedFiles."""
941
self._transport = transport
942
self._file_factory = file_factory
943
self._mapper = mapper
944
self._is_locked = is_locked
946
def add_lines(self, key, parents, lines, parent_texts=None,
947
left_matching_blocks=None, nostore_sha=None, random_id=False,
949
"""See VersionedFiles.add_lines()."""
950
path = self._mapper.map(key)
952
parents = [parent[-1] for parent in parents]
953
vf = self._get_vf(path)
956
return vf.add_lines_with_ghosts(version_id, parents, lines,
957
parent_texts=parent_texts,
958
left_matching_blocks=left_matching_blocks,
959
nostore_sha=nostore_sha, random_id=random_id,
960
check_content=check_content)
961
except NotImplementedError:
962
return vf.add_lines(version_id, parents, lines,
963
parent_texts=parent_texts,
964
left_matching_blocks=left_matching_blocks,
965
nostore_sha=nostore_sha, random_id=random_id,
966
check_content=check_content)
967
except errors.NoSuchFile:
968
# parent directory may be missing, try again.
969
self._transport.mkdir(osutils.dirname(path))
971
return vf.add_lines_with_ghosts(version_id, parents, lines,
972
parent_texts=parent_texts,
973
left_matching_blocks=left_matching_blocks,
974
nostore_sha=nostore_sha, random_id=random_id,
975
check_content=check_content)
976
except NotImplementedError:
977
return vf.add_lines(version_id, parents, lines,
978
parent_texts=parent_texts,
979
left_matching_blocks=left_matching_blocks,
980
nostore_sha=nostore_sha, random_id=random_id,
981
check_content=check_content)
983
def annotate(self, key):
984
"""Return a list of (version-key, line) tuples for the text of key.
986
:raise RevisionNotPresent: If the key is not present.
989
path = self._mapper.map(prefix)
990
vf = self._get_vf(path)
991
origins = vf.annotate(key[-1])
993
for origin, line in origins:
994
result.append((prefix + (origin,), line))
997
def check(self, progress_bar=None):
998
"""See VersionedFiles.check()."""
999
for prefix, vf in self._iter_all_components():
1002
def get_parent_map(self, keys):
1003
"""Get a map of the parents of keys.
1005
:param keys: The keys to look up parents for.
1006
:return: A mapping from keys to parents. Absent keys are absent from
1009
prefixes = self._partition_keys(keys)
1011
for prefix, suffixes in prefixes.items():
1012
path = self._mapper.map(prefix)
1013
vf = self._get_vf(path)
1014
parent_map = vf.get_parent_map(suffixes)
1015
for key, parents in parent_map.items():
1016
result[prefix + (key,)] = tuple(
1017
prefix + (parent,) for parent in parents)
1020
def _get_vf(self, path):
1021
if not self._is_locked():
1022
raise errors.ObjectNotLocked(self)
1023
return self._file_factory(path, self._transport, create=True,
1024
get_scope=lambda:None)
1026
def _partition_keys(self, keys):
1027
"""Turn keys into a dict of prefix:suffix_list."""
1030
prefix_keys = result.setdefault(key[:-1], [])
1031
prefix_keys.append(key[-1])
1034
def _get_all_prefixes(self):
1035
# Identify all key prefixes.
1036
# XXX: A bit hacky, needs polish.
1037
if type(self._mapper) == ConstantMapper:
1038
paths = [self._mapper.map(())]
1042
for quoted_relpath in self._transport.iter_files_recursive():
1043
path, ext = os.path.splitext(quoted_relpath)
1045
paths = list(relpaths)
1046
prefixes = [self._mapper.unmap(path) for path in paths]
1047
return zip(paths, prefixes)
1049
def get_record_stream(self, keys, ordering, include_delta_closure):
1050
"""See VersionedFiles.get_record_stream()."""
1051
# Ordering will be taken care of by each partitioned store; group keys
1054
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1055
suffixes = [(suffix,) for suffix in suffixes]
1056
for record in vf.get_record_stream(suffixes, ordering,
1057
include_delta_closure):
1058
if record.parents is not None:
1059
record.parents = tuple(
1060
prefix + parent for parent in record.parents)
1061
record.key = prefix + record.key
1064
def _iter_keys_vf(self, keys):
1065
prefixes = self._partition_keys(keys)
1067
for prefix, suffixes in prefixes.items():
1068
path = self._mapper.map(prefix)
1069
vf = self._get_vf(path)
1070
yield prefix, suffixes, vf
1072
def get_sha1s(self, keys):
1073
"""See VersionedFiles.get_sha1s()."""
1075
for prefix,suffixes, vf in self._iter_keys_vf(keys):
1076
vf_sha1s = vf.get_sha1s(suffixes)
1077
for suffix, sha1 in vf_sha1s.iteritems():
1078
sha1s[prefix + (suffix,)] = sha1
1081
def insert_record_stream(self, stream):
1082
"""Insert a record stream into this container.
1084
:param stream: A stream of records to insert.
1086
:seealso VersionedFile.get_record_stream:
1088
for record in stream:
1089
prefix = record.key[:-1]
1090
key = record.key[-1:]
1091
if record.parents is not None:
1092
parents = [parent[-1:] for parent in record.parents]
1095
thunk_record = AdapterFactory(key, parents, record)
1096
path = self._mapper.map(prefix)
1097
# Note that this parses the file many times; we can do better but
1098
# as this only impacts weaves in terms of performance, it is
1100
vf = self._get_vf(path)
1101
vf.insert_record_stream([thunk_record])
1103
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1104
"""Iterate over the lines in the versioned files from keys.
1106
This may return lines from other keys. Each item the returned
1107
iterator yields is a tuple of a line and a text version that that line
1108
is present in (not introduced in).
1110
Ordering of results is in whatever order is most suitable for the
1111
underlying storage format.
1113
If a progress bar is supplied, it may be used to indicate progress.
1114
The caller is responsible for cleaning up progress bars (because this
1118
* Lines are normalised by the underlying store: they will all have \n
1120
* Lines are returned in arbitrary order.
1122
:return: An iterator over (line, key).
1124
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1125
for line, version in vf.iter_lines_added_or_present_in_versions(suffixes):
1126
yield line, prefix + (version,)
1128
def _iter_all_components(self):
1129
for path, prefix in self._get_all_prefixes():
1130
yield prefix, self._get_vf(path)
1133
"""See VersionedFiles.keys()."""
1135
for prefix, vf in self._iter_all_components():
1136
for suffix in vf.versions():
1137
result.add(prefix + (suffix,))
1141
class _PlanMergeVersionedFile(VersionedFiles):
1142
"""A VersionedFile for uncommitted and committed texts.
1144
It is intended to allow merges to be planned with working tree texts.
1145
It implements only the small part of the VersionedFiles interface used by
1146
PlanMerge. It falls back to multiple versionedfiles for data not stored in
1147
_PlanMergeVersionedFile itself.
1149
:ivar: fallback_versionedfiles a list of VersionedFiles objects that can be
1150
queried for missing texts.
1153
def __init__(self, file_id):
1154
"""Create a _PlanMergeVersionedFile.
1156
:param file_id: Used with _PlanMerge code which is not yet fully
1157
tuple-keyspace aware.
1159
self._file_id = file_id
1160
# fallback locations
1161
self.fallback_versionedfiles = []
1162
# Parents for locally held keys.
1164
# line data for locally held keys.
1166
# key lookup providers
1167
self._providers = [DictParentsProvider(self._parents)]
1169
def plan_merge(self, ver_a, ver_b, base=None):
1170
"""See VersionedFile.plan_merge"""
1171
from bzrlib.merge import _PlanMerge
1173
return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1174
old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1175
new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
1176
return _PlanMerge._subtract_plans(old_plan, new_plan)
1178
def plan_lca_merge(self, ver_a, ver_b, base=None):
1179
from bzrlib.merge import _PlanLCAMerge
1181
new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
1184
old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
1185
return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
1187
def add_lines(self, key, parents, lines):
1188
"""See VersionedFiles.add_lines
1190
Lines are added locally, not to fallback versionedfiles. Also, ghosts
1191
are permitted. Only reserved ids are permitted.
1193
if type(key) is not tuple:
1194
raise TypeError(key)
1195
if not revision.is_reserved_id(key[-1]):
1196
raise ValueError('Only reserved ids may be used')
1198
raise ValueError('Parents may not be None')
1200
raise ValueError('Lines may not be None')
1201
self._parents[key] = tuple(parents)
1202
self._lines[key] = lines
1204
def get_record_stream(self, keys, ordering, include_delta_closure):
1207
if key in self._lines:
1208
lines = self._lines[key]
1209
parents = self._parents[key]
1211
yield FulltextContentFactory(key, parents, None,
1213
for versionedfile in self.fallback_versionedfiles:
1214
for record in versionedfile.get_record_stream(
1215
pending, 'unordered', True):
1216
if record.storage_kind == 'absent':
1219
pending.remove(record.key)
1223
# report absent entries
1225
yield AbsentContentFactory(key)
1227
def get_parent_map(self, keys):
1228
"""See VersionedFiles.get_parent_map"""
1229
# We create a new provider because a fallback may have been added.
1230
# If we make fallbacks private we can update a stack list and avoid
1231
# object creation thrashing.
1234
if revision.NULL_REVISION in keys:
1235
keys.remove(revision.NULL_REVISION)
1236
result[revision.NULL_REVISION] = ()
1237
self._providers = self._providers[:1] + self.fallback_versionedfiles
1239
_StackedParentsProvider(self._providers).get_parent_map(keys))
1240
for key, parents in result.iteritems():
1242
result[key] = (revision.NULL_REVISION,)
454
1246
class PlanWeaveMerge(TextMerge):
455
1247
"""Weave merge that takes a plan as its input.
524
1323
PlanWeaveMerge.__init__(self, plan, a_marker, b_marker)
527
class InterVersionedFile(InterObject):
528
"""This class represents operations taking place between two versionedfiles..
530
Its instances have methods like join, and contain
531
references to the source and target versionedfiles these operations can be
534
Often we will provide convenience methods on 'versionedfile' which carry out
535
operations with another versionedfile - they will always forward to
536
InterVersionedFile.get(other).method_name(parameters).
1326
class VirtualVersionedFiles(VersionedFiles):
1327
"""Dummy implementation for VersionedFiles that uses other functions for
1328
obtaining fulltexts and parent maps.
1330
This is always on the bottom of the stack and uses string keys
1331
(rather than tuples) internally.
540
"""The available optimised InterVersionedFile types."""
542
def join(self, pb=None, msg=None, version_ids=None, ignore_missing=False):
543
"""Integrate versions from self.source into self.target.
545
If version_ids is None all versions from source should be
546
incorporated into this versioned file.
548
Must raise RevisionNotPresent if any of the specified versions
549
are not present in the other files history unless ignore_missing is
550
supplied when they are silently skipped.
553
# - if the target is empty, just add all the versions from
554
# source to target, otherwise:
555
# - make a temporary versioned file of type target
556
# - insert the source content into it one at a time
558
if not self.target.versions():
561
# Make a new target-format versioned file.
562
temp_source = self.target.create_empty("temp", MemoryTransport())
564
version_ids = self._get_source_version_ids(version_ids, ignore_missing)
565
graph = self.source.get_graph(version_ids)
566
order = topo_sort(graph.items())
567
pb = ui.ui_factory.nested_progress_bar()
570
# TODO for incremental cross-format work:
571
# make a versioned file with the following content:
572
# all revisions we have been asked to join
573
# all their ancestors that are *not* in target already.
574
# the immediate parents of the above two sets, with
575
# empty parent lists - these versions are in target already
576
# and the incorrect version data will be ignored.
577
# TODO: for all ancestors that are present in target already,
578
# check them for consistent data, this requires moving sha1 from
580
# TODO: remove parent texts when they are not relevant any more for
581
# memory pressure reduction. RBC 20060313
582
# pb.update('Converting versioned data', 0, len(order))
583
# deltas = self.source.get_deltas(order)
584
for index, version in enumerate(order):
585
pb.update('Converting versioned data', index, len(order))
586
parent_text = target.add_lines(version,
587
self.source.get_parents(version),
588
self.source.get_lines(version),
589
parent_texts=parent_texts)
590
parent_texts[version] = parent_text
591
#delta_parent, sha1, noeol, delta = deltas[version]
592
#target.add_delta(version,
593
# self.source.get_parents(version),
598
#target.get_lines(version)
600
# this should hit the native code path for target
601
if target is not self.target:
602
return self.target.join(temp_source,
610
def _get_source_version_ids(self, version_ids, ignore_missing):
611
"""Determine the version ids to be used from self.source.
613
:param version_ids: The caller-supplied version ids to check. (None
614
for all). If None is in version_ids, it is stripped.
615
:param ignore_missing: if True, remove missing ids from the version
616
list. If False, raise RevisionNotPresent on
617
a missing version id.
618
:return: A set of version ids.
620
if version_ids is None:
621
# None cannot be in source.versions
622
return set(self.source.versions())
625
return set(self.source.versions()).intersection(set(version_ids))
1334
def __init__(self, get_parent_map, get_lines):
1335
"""Create a VirtualVersionedFiles.
1337
:param get_parent_map: Same signature as Repository.get_parent_map.
1338
:param get_lines: Should return lines for specified key or None if
1341
super(VirtualVersionedFiles, self).__init__()
1342
self._get_parent_map = get_parent_map
1343
self._get_lines = get_lines
1345
def check(self, progressbar=None):
1346
"""See VersionedFiles.check.
1348
:note: Always returns True for VirtualVersionedFiles.
1352
def add_mpdiffs(self, records):
1353
"""See VersionedFiles.mpdiffs.
1355
:note: Not implemented for VirtualVersionedFiles.
1357
raise NotImplementedError(self.add_mpdiffs)
1359
def get_parent_map(self, keys):
1360
"""See VersionedFiles.get_parent_map."""
1361
return dict([((k,), tuple([(p,) for p in v]))
1362
for k,v in self._get_parent_map([k for (k,) in keys]).iteritems()])
1364
def get_sha1s(self, keys):
1365
"""See VersionedFiles.get_sha1s."""
1368
lines = self._get_lines(k)
1369
if lines is not None:
1370
if not isinstance(lines, list):
1371
raise AssertionError
1372
ret[(k,)] = osutils.sha_strings(lines)
1375
def get_record_stream(self, keys, ordering, include_delta_closure):
1376
"""See VersionedFiles.get_record_stream."""
1377
for (k,) in list(keys):
1378
lines = self._get_lines(k)
1379
if lines is not None:
1380
if not isinstance(lines, list):
1381
raise AssertionError
1382
yield FulltextContentFactory((k,), None,
1383
sha1=osutils.sha_strings(lines),
1384
text=''.join(lines))
627
new_version_ids = set()
628
for version in version_ids:
631
if not self.source.has_version(version):
632
raise errors.RevisionNotPresent(version, str(self.source))
634
new_version_ids.add(version)
635
return new_version_ids
638
class InterVersionedFileTestProviderAdapter(object):
639
"""A tool to generate a suite testing multiple inter versioned-file classes.
641
This is done by copying the test once for each InterVersionedFile provider
642
and injecting the transport_server, transport_readonly_server,
643
versionedfile_factory and versionedfile_factory_to classes into each copy.
644
Each copy is also given a new id() to make it easy to identify.
647
def __init__(self, transport_server, transport_readonly_server, formats):
648
self._transport_server = transport_server
649
self._transport_readonly_server = transport_readonly_server
650
self._formats = formats
652
def adapt(self, test):
654
for (interversionedfile_class,
655
versionedfile_factory,
656
versionedfile_factory_to) in self._formats:
657
new_test = deepcopy(test)
658
new_test.transport_server = self._transport_server
659
new_test.transport_readonly_server = self._transport_readonly_server
660
new_test.interversionedfile_class = interversionedfile_class
661
new_test.versionedfile_factory = versionedfile_factory
662
new_test.versionedfile_factory_to = versionedfile_factory_to
663
def make_new_test_id():
664
new_id = "%s(%s)" % (new_test.id(), interversionedfile_class.__name__)
665
return lambda: new_id
666
new_test.id = make_new_test_id()
667
result.addTest(new_test)
671
def default_test_list():
672
"""Generate the default list of interversionedfile permutations to test."""
673
from bzrlib.weave import WeaveFile
674
from bzrlib.knit import KnitVersionedFile
676
# test the fallback InterVersionedFile from annotated knits to weave
677
result.append((InterVersionedFile,
680
for optimiser in InterVersionedFile._optimisers:
681
result.append((optimiser,
682
optimiser._matching_file_from_factory,
683
optimiser._matching_file_to_factory
685
# if there are specific combinations we want to use, we can add them
1386
yield AbsentContentFactory((k,))
added
removed
bzrlib/versionedfile.py
