« back to all changes in this revision
Viewing changes to bzrlib/versionedfile.py
- browse files at revision 3510
- compare with another revision
- compare with revision 1713
- download diff
- download tarball
- view history from revision 3510
- Committer: Canonical.com Patch Queue Manager
- Date: 2008-06-25 10:36:36 UTC
- mfrom: (3350.6.12 versionedfiles)
- Revision ID: pqm@pqm.ubuntu.com-20080625103636-6kxh4e1gmyn82f50
(mbp for robertc) VersionedFiles refactoring
- files removed:
- bzrlib/store/revision/__init__.py
- bzrlib/tests/interversionedfile_implementations
- bzrlib/tests/revisionstore_implementations
- files modified:
- NEWS
added
removed
bzrlib/versionedfile.py
19
19
20
20
"""Versioned text file storage api."""
21
21
22
from cStringIO import StringIO
23
import os
24
import urllib
25
from zlib import adler32
26
22
27
from bzrlib.lazy_import import lazy_import
23
28
lazy_import(globals(), """
24
29
30
35
revision,
31
36
ui,
32
37
)
33
from bzrlib.graph import Graph
38
from bzrlib.graph import DictParentsProvider, Graph, _StackedParentsProvider
34
39
from bzrlib.transport.memory import MemoryTransport
35
40
""")
36
37
from cStringIO import StringIO
38
39
41
from bzrlib.inter import InterObject
40
42
from bzrlib.registry import Registry
41
43
from bzrlib.symbol_versioning import *
70
72
:ivar parents: A tuple of parent keys for self.key. If the object has
71
73
no parent information, None (as opposed to () for an empty list of
72
74
parents).
73
"""
75
"""
74
76
75
77
def __init__(self):
76
78
"""Create a ContentFactory."""
80
82
self.parents = None
81
83
82
84
83
class AbsentContentFactory(object):
85
class FulltextContentFactory(ContentFactory):
86
"""Static data content factory.
87
88
This takes a fulltext when created and just returns that during
89
get_bytes_as('fulltext').
90
91
:ivar sha1: None, or the sha1 of the content fulltext.
92
:ivar storage_kind: The native storage kind of this factory. Always
93
'fulltext'.
94
:ivar key: The key of this content. Each key is a tuple with a single
95
string in it.
96
:ivar parents: A tuple of parent keys for self.key. If the object has
97
no parent information, None (as opposed to () for an empty list of
98
parents).
99
"""
100
101
def __init__(self, key, parents, sha1, text):
102
"""Create a ContentFactory."""
103
self.sha1 = sha1
104
self.storage_kind = 'fulltext'
105
self.key = key
106
self.parents = parents
107
self._text = text
108
109
def get_bytes_as(self, storage_kind):
110
if storage_kind == self.storage_kind:
111
return self._text
112
raise errors.UnavailableRepresentation(self.key, storage_kind,
113
self.storage_kind)
114
115
116
class AbsentContentFactory(ContentFactory):
84
117
"""A placeholder content factory for unavailable texts.
85
118
86
119
:ivar sha1: None.
98
131
self.parents = None
99
132
100
133
134
class AdapterFactory(ContentFactory):
135
"""A content factory to adapt between key prefix's."""
136
137
def __init__(self, key, parents, adapted):
138
"""Create an adapter factory instance."""
139
self.key = key
140
self.parents = parents
141
self._adapted = adapted
142
143
def __getattr__(self, attr):
144
"""Return a member from the adapted object."""
145
if attr in ('key', 'parents'):
146
return self.__dict__[attr]
147
else:
148
return getattr(self._adapted, attr)
149
150
101
151
def filter_absent(record_stream):
102
152
"""Adapt a record stream to remove absent records."""
103
153
for record in record_stream:
418
468
"""
419
469
raise NotImplementedError(self.annotate)
420
470
421
@deprecated_method(one_five)
422
def join(self, other, pb=None, msg=None, version_ids=None,
423
ignore_missing=False):
424
"""Integrate versions from other into this versioned file.
425
426
If version_ids is None all versions from other should be
427
incorporated into this versioned file.
428
429
Must raise RevisionNotPresent if any of the specified versions
430
are not present in the other file's history unless ignore_missing
431
is supplied in which case they are silently skipped.
432
"""
433
self._check_write_ok()
434
return InterVersionedFile.get(other, self).join(
435
pb,
436
msg,
437
version_ids,
438
ignore_missing)
439
440
471
def iter_lines_added_or_present_in_versions(self, version_ids=None,
441
472
pb=None):
442
473
"""Iterate over the lines in the versioned file from version_ids.
486
517
return PlanWeaveMerge(plan, a_marker, b_marker).merge_lines()[0]
487
518
488
519
489
class RecordingVersionedFileDecorator(object):
490
"""A minimal versioned file that records calls made on it.
520
class RecordingVersionedFilesDecorator(object):
521
"""A minimal versioned files that records calls made on it.
491
522
492
523
Only enough methods have been added to support tests using it to date.
493
524
496
527
"""
497
528
498
529
def __init__(self, backing_vf):
499
"""Create a RecordingVersionedFileDecorator decorating backing_vf.
530
"""Create a RecordingVersionedFileDsecorator decorating backing_vf.
500
531
501
532
:param backing_vf: The versioned file to answer all methods.
502
533
"""
503
534
self._backing_vf = backing_vf
504
535
self.calls = []
505
536
506
def get_lines(self, version_ids):
507
self.calls.append(("get_lines", version_ids))
508
return self._backing_vf.get_lines(version_ids)
509
510
511
class _PlanMergeVersionedFile(object):
537
def get_record_stream(self, keys, sort_order, include_delta_closure):
538
self.calls.append(("get_record_stream", keys, sort_order,
539
include_delta_closure))
540
return self._backing_vf.get_record_stream(keys, sort_order,
541
include_delta_closure)
542
543
544
class KeyMapper(object):
545
"""KeyMappers map between keys and underlying partitioned storage."""
546
547
def map(self, key):
548
"""Map key to an underlying storage identifier.
549
550
:param key: A key tuple e.g. ('file-id', 'revision-id').
551
:return: An underlying storage identifier, specific to the partitioning
552
mechanism.
553
"""
554
raise NotImplementedError(self.map)
555
556
def unmap(self, partition_id):
557
"""Map a partitioned storage id back to a key prefix.
558
559
:param partition_id: The underlying partition id.
560
:return: As much of a key (or prefix) as is derivable from the partition
561
id.
562
"""
563
raise NotImplementedError(self.unmap)
564
565
566
class ConstantMapper(KeyMapper):
567
"""A key mapper that maps to a constant result."""
568
569
def __init__(self, result):
570
"""Create a ConstantMapper which will return result for all maps."""
571
self._result = result
572
573
def map(self, key):
574
"""See KeyMapper.map()."""
575
return self._result
576
577
578
class URLEscapeMapper(KeyMapper):
579
"""Base class for use with transport backed storage.
580
581
This provides a map and unmap wrapper that respectively url escape and
582
unescape their outputs and inputs.
583
"""
584
585
def map(self, key):
586
"""See KeyMapper.map()."""
587
return urllib.quote(self._map(key))
588
589
def unmap(self, partition_id):
590
"""See KeyMapper.unmap()."""
591
return self._unmap(urllib.unquote(partition_id))
592
593
594
class PrefixMapper(URLEscapeMapper):
595
"""A key mapper that extracts the first component of a key.
596
597
This mapper is for use with a transport based backend.
598
"""
599
600
def _map(self, key):
601
"""See KeyMapper.map()."""
602
return key[0]
603
604
def _unmap(self, partition_id):
605
"""See KeyMapper.unmap()."""
606
return (partition_id,)
607
608
609
class HashPrefixMapper(URLEscapeMapper):
610
"""A key mapper that combines the first component of a key with a hash.
611
612
This mapper is for use with a transport based backend.
613
"""
614
615
def _map(self, key):
616
"""See KeyMapper.map()."""
617
prefix = self._escape(key[0])
618
return "%02x/%s" % (adler32(prefix) & 0xff, prefix)
619
620
def _escape(self, prefix):
621
"""No escaping needed here."""
622
return prefix
623
624
def _unmap(self, partition_id):
625
"""See KeyMapper.unmap()."""
626
return (self._unescape(osutils.basename(partition_id)),)
627
628
def _unescape(self, basename):
629
"""No unescaping needed for HashPrefixMapper."""
630
return basename
631
632
633
class HashEscapedPrefixMapper(HashPrefixMapper):
634
"""Combines the escaped first component of a key with a hash.
635
636
This mapper is for use with a transport based backend.
637
"""
638
639
_safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
640
641
def _escape(self, prefix):
642
"""Turn a key element into a filesystem safe string.
643
644
This is similar to a plain urllib.quote, except
645
it uses specific safe characters, so that it doesn't
646
have to translate a lot of valid file ids.
647
"""
648
# @ does not get escaped. This is because it is a valid
649
# filesystem character we use all the time, and it looks
650
# a lot better than seeing %40 all the time.
651
r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
652
for c in prefix]
653
return ''.join(r)
654
655
def _unescape(self, basename):
656
"""Escaped names are easily unescaped by urlutils."""
657
return urllib.unquote(basename)
658
659
660
def make_versioned_files_factory(versioned_file_factory, mapper):
661
"""Create a ThunkedVersionedFiles factory.
662
663
This will create a callable which when called creates a
664
ThunkedVersionedFiles on a transport, using mapper to access individual
665
versioned files, and versioned_file_factory to create each individual file.
666
"""
667
def factory(transport):
668
return ThunkedVersionedFiles(transport, versioned_file_factory, mapper,
669
lambda:True)
670
return factory
671
672
673
class VersionedFiles(object):
674
"""Storage for many versioned files.
675
676
This object allows a single keyspace for accessing the history graph and
677
contents of named bytestrings.
678
679
Currently no implementation allows the graph of different key prefixes to
680
intersect, but the API does allow such implementations in the future.
681
682
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
683
may have a different length key-size, but that size will be constant for
684
all texts added to or retrieved from it. For instance, bzrlib uses
685
instances with a key-size of 2 for storing user files in a repository, with
686
the first element the fileid, and the second the version of that file.
687
688
The use of tuples allows a single code base to support several different
689
uses with only the mapping logic changing from instance to instance.
690
"""
691
692
def add_lines(self, key, parents, lines, parent_texts=None,
693
left_matching_blocks=None, nostore_sha=None, random_id=False,
694
check_content=True):
695
"""Add a text to the store.
696
697
:param key: The key tuple of the text to add.
698
:param parents: The parents key tuples of the text to add.
699
:param lines: A list of lines. Each line must be a bytestring. And all
700
of them except the last must be terminated with \n and contain no
701
other \n's. The last line may either contain no \n's or a single
702
terminating \n. If the lines list does meet this constraint the add
703
routine may error or may succeed - but you will be unable to read
704
the data back accurately. (Checking the lines have been split
705
correctly is expensive and extremely unlikely to catch bugs so it
706
is not done at runtime unless check_content is True.)
707
:param parent_texts: An optional dictionary containing the opaque
708
representations of some or all of the parents of version_id to
709
allow delta optimisations. VERY IMPORTANT: the texts must be those
710
returned by add_lines or data corruption can be caused.
711
:param left_matching_blocks: a hint about which areas are common
712
between the text and its left-hand-parent. The format is
713
the SequenceMatcher.get_matching_blocks format.
714
:param nostore_sha: Raise ExistingContent and do not add the lines to
715
the versioned file if the digest of the lines matches this.
716
:param random_id: If True a random id has been selected rather than
717
an id determined by some deterministic process such as a converter
718
from a foreign VCS. When True the backend may choose not to check
719
for uniqueness of the resulting key within the versioned file, so
720
this should only be done when the result is expected to be unique
721
anyway.
722
:param check_content: If True, the lines supplied are verified to be
723
bytestrings that are correctly formed lines.
724
:return: The text sha1, the number of bytes in the text, and an opaque
725
representation of the inserted version which can be provided
726
back to future add_lines calls in the parent_texts dictionary.
727
"""
728
raise NotImplementedError(self.add_lines)
729
730
def add_mpdiffs(self, records):
731
"""Add mpdiffs to this VersionedFile.
732
733
Records should be iterables of version, parents, expected_sha1,
734
mpdiff. mpdiff should be a MultiParent instance.
735
"""
736
vf_parents = {}
737
mpvf = multiparent.MultiMemoryVersionedFile()
738
versions = []
739
for version, parent_ids, expected_sha1, mpdiff in records:
740
versions.append(version)
741
mpvf.add_diff(mpdiff, version, parent_ids)
742
needed_parents = set()
743
for version, parent_ids, expected_sha1, mpdiff in records:
744
needed_parents.update(p for p in parent_ids
745
if not mpvf.has_version(p))
746
# It seems likely that adding all the present parents as fulltexts can
747
# easily exhaust memory.
748
present_parents = set(self.get_parent_map(needed_parents).keys())
749
split_lines = osutils.split_lines
750
for record in self.get_record_stream(present_parents, 'unordered',
751
True):
752
mpvf.add_version(split_lines(record.get_bytes_as('fulltext')),
753
record.key, [])
754
for (key, parent_keys, expected_sha1, mpdiff), lines in\
755
zip(records, mpvf.get_line_list(versions)):
756
if len(parent_keys) == 1:
757
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
758
mpvf.get_diff(parent_keys[0]).num_lines()))
759
else:
760
left_matching_blocks = None
761
version_sha1, _, version_text = self.add_lines(key,
762
parent_keys, lines, vf_parents,
763
left_matching_blocks=left_matching_blocks)
764
if version_sha1 != expected_sha1:
765
raise errors.VersionedFileInvalidChecksum(version)
766
vf_parents[key] = version_text
767
768
def annotate(self, key):
769
"""Return a list of (version-key, line) tuples for the text of key.
770
771
:raise RevisionNotPresent: If the key is not present.
772
"""
773
raise NotImplementedError(self.annotate)
774
775
def check(self, progress_bar=None):
776
"""Check this object for integrity."""
777
raise NotImplementedError(self.check)
778
779
@staticmethod
780
def check_not_reserved_id(version_id):
781
revision.check_not_reserved_id(version_id)
782
783
def _check_lines_not_unicode(self, lines):
784
"""Check that lines being added to a versioned file are not unicode."""
785
for line in lines:
786
if line.__class__ is not str:
787
raise errors.BzrBadParameterUnicode("lines")
788
789
def _check_lines_are_lines(self, lines):
790
"""Check that the lines really are full lines without inline EOL."""
791
for line in lines:
792
if '\n' in line[:-1]:
793
raise errors.BzrBadParameterContainsNewline("lines")
794
795
def get_parent_map(self, keys):
796
"""Get a map of the parents of keys.
797
798
:param keys: The keys to look up parents for.
799
:return: A mapping from keys to parents. Absent keys are absent from
800
the mapping.
801
"""
802
raise NotImplementedError(self.get_parent_map)
803
804
def get_record_stream(self, keys, ordering, include_delta_closure):
805
"""Get a stream of records for keys.
806
807
:param keys: The keys to include.
808
:param ordering: Either 'unordered' or 'topological'. A topologically
809
sorted stream has compression parents strictly before their
810
children.
811
:param include_delta_closure: If True then the closure across any
812
compression parents will be included (in the opaque data).
813
:return: An iterator of ContentFactory objects, each of which is only
814
valid until the iterator is advanced.
815
"""
816
raise NotImplementedError(self.get_record_stream)
817
818
def get_sha1s(self, keys):
819
"""Get the sha1's of the texts for the given keys.
820
821
:param keys: The names of the keys to lookup
822
:return: a list of sha1s matching keys.
823
"""
824
raise NotImplementedError(self.get_sha1s)
825
826
def insert_record_stream(self, stream):
827
"""Insert a record stream into this container.
828
829
:param stream: A stream of records to insert.
830
:return: None
831
:seealso VersionedFile.get_record_stream:
832
"""
833
raise NotImplementedError
834
835
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
836
"""Iterate over the lines in the versioned files from keys.
837
838
This may return lines from other keys. Each item the returned
839
iterator yields is a tuple of a line and a text version that that line
840
is present in (not introduced in).
841
842
Ordering of results is in whatever order is most suitable for the
843
underlying storage format.
844
845
If a progress bar is supplied, it may be used to indicate progress.
846
The caller is responsible for cleaning up progress bars (because this
847
is an iterator).
848
849
NOTES:
850
* Lines are normalised by the underlying store: they will all have \n
851
terminators.
852
* Lines are returned in arbitrary order.
853
854
:return: An iterator over (line, key).
855
"""
856
raise NotImplementedError(self.iter_lines_added_or_present_in_keys)
857
858
def keys(self):
859
"""Return a iterable of the keys for all the contained texts."""
860
raise NotImplementedError(self.keys)
861
862
def make_mpdiffs(self, keys):
863
"""Create multiparent diffs for specified keys."""
864
keys_order = tuple(keys)
865
keys = frozenset(keys)
866
knit_keys = set(keys)
867
parent_map = self.get_parent_map(keys)
868
for parent_keys in parent_map.itervalues():
869
if parent_keys:
870
knit_keys.update(parent_keys)
871
missing_keys = keys - set(parent_map)
872
if missing_keys:
873
raise errors.RevisionNotPresent(missing_keys.pop(), self)
874
# We need to filter out ghosts, because we can't diff against them.
875
maybe_ghosts = knit_keys - keys
876
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
877
knit_keys.difference_update(ghosts)
878
lines = {}
879
split_lines = osutils.split_lines
880
for record in self.get_record_stream(knit_keys, 'topological', True):
881
lines[record.key] = split_lines(record.get_bytes_as('fulltext'))
882
# line_block_dict = {}
883
# for parent, blocks in record.extract_line_blocks():
884
# line_blocks[parent] = blocks
885
# line_blocks[record.key] = line_block_dict
886
diffs = []
887
for key in keys_order:
888
target = lines[key]
889
parents = parent_map[key] or []
890
# Note that filtering knit_keys can lead to a parent difference
891
# between the creation and the application of the mpdiff.
892
parent_lines = [lines[p] for p in parents if p in knit_keys]
893
if len(parent_lines) > 0:
894
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
895
target)
896
else:
897
left_parent_blocks = None
898
diffs.append(multiparent.MultiParent.from_lines(target,
899
parent_lines, left_parent_blocks))
900
return diffs
901
902
def _extract_blocks(self, version_id, source, target):
903
return None
904
905
906
class ThunkedVersionedFiles(VersionedFiles):
907
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
908
909
This object allows a single keyspace for accessing the history graph and
910
contents of named bytestrings.
911
912
Currently no implementation allows the graph of different key prefixes to
913
intersect, but the API does allow such implementations in the future.
914
"""
915
916
def __init__(self, transport, file_factory, mapper, is_locked):
917
"""Create a ThunkedVersionedFiles."""
918
self._transport = transport
919
self._file_factory = file_factory
920
self._mapper = mapper
921
self._is_locked = is_locked
922
923
def add_lines(self, key, parents, lines, parent_texts=None,
924
left_matching_blocks=None, nostore_sha=None, random_id=False,
925
check_content=True):
926
"""See VersionedFiles.add_lines()."""
927
path = self._mapper.map(key)
928
version_id = key[-1]
929
parents = [parent[-1] for parent in parents]
930
vf = self._get_vf(path)
931
try:
932
try:
933
return vf.add_lines_with_ghosts(version_id, parents, lines,
934
parent_texts=parent_texts,
935
left_matching_blocks=left_matching_blocks,
936
nostore_sha=nostore_sha, random_id=random_id,
937
check_content=check_content)
938
except NotImplementedError:
939
return vf.add_lines(version_id, parents, lines,
940
parent_texts=parent_texts,
941
left_matching_blocks=left_matching_blocks,
942
nostore_sha=nostore_sha, random_id=random_id,
943
check_content=check_content)
944
except errors.NoSuchFile:
945
# parent directory may be missing, try again.
946
self._transport.mkdir(osutils.dirname(path))
947
try:
948
return vf.add_lines_with_ghosts(version_id, parents, lines,
949
parent_texts=parent_texts,
950
left_matching_blocks=left_matching_blocks,
951
nostore_sha=nostore_sha, random_id=random_id,
952
check_content=check_content)
953
except NotImplementedError:
954
return vf.add_lines(version_id, parents, lines,
955
parent_texts=parent_texts,
956
left_matching_blocks=left_matching_blocks,
957
nostore_sha=nostore_sha, random_id=random_id,
958
check_content=check_content)
959
960
def annotate(self, key):
961
"""Return a list of (version-key, line) tuples for the text of key.
962
963
:raise RevisionNotPresent: If the key is not present.
964
"""
965
prefix = key[:-1]
966
path = self._mapper.map(prefix)
967
vf = self._get_vf(path)
968
origins = vf.annotate(key[-1])
969
result = []
970
for origin, line in origins:
971
result.append((prefix + (origin,), line))
972
return result
973
974
def check(self, progress_bar=None):
975
"""See VersionedFiles.check()."""
976
for prefix, vf in self._iter_all_components():
977
vf.check()
978
979
def get_parent_map(self, keys):
980
"""Get a map of the parents of keys.
981
982
:param keys: The keys to look up parents for.
983
:return: A mapping from keys to parents. Absent keys are absent from
984
the mapping.
985
"""
986
prefixes = self._partition_keys(keys)
987
result = {}
988
for prefix, suffixes in prefixes.items():
989
path = self._mapper.map(prefix)
990
vf = self._get_vf(path)
991
parent_map = vf.get_parent_map(suffixes)
992
for key, parents in parent_map.items():
993
result[prefix + (key,)] = tuple(
994
prefix + (parent,) for parent in parents)
995
return result
996
997
def _get_vf(self, path):
998
if not self._is_locked():
999
raise errors.ObjectNotLocked(self)
1000
return self._file_factory(path, self._transport, create=True,
1001
get_scope=lambda:None)
1002
1003
def _partition_keys(self, keys):
1004
"""Turn keys into a dict of prefix:suffix_list."""
1005
result = {}
1006
for key in keys:
1007
prefix_keys = result.setdefault(key[:-1], [])
1008
prefix_keys.append(key[-1])
1009
return result
1010
1011
def _get_all_prefixes(self):
1012
# Identify all key prefixes.
1013
# XXX: A bit hacky, needs polish.
1014
if type(self._mapper) == ConstantMapper:
1015
paths = [self._mapper.map(())]
1016
prefixes = [()]
1017
else:
1018
relpaths = set()
1019
for quoted_relpath in self._transport.iter_files_recursive():
1020
path, ext = os.path.splitext(quoted_relpath)
1021
relpaths.add(path)
1022
paths = list(relpaths)
1023
prefixes = [self._mapper.unmap(path) for path in paths]
1024
return zip(paths, prefixes)
1025
1026
def get_record_stream(self, keys, ordering, include_delta_closure):
1027
"""See VersionedFiles.get_record_stream()."""
1028
# Ordering will be taken care of by each partitioned store; group keys
1029
# by partition.
1030
keys = sorted(keys)
1031
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1032
suffixes = [(suffix,) for suffix in suffixes]
1033
for record in vf.get_record_stream(suffixes, ordering,
1034
include_delta_closure):
1035
if record.parents is not None:
1036
record.parents = tuple(
1037
prefix + parent for parent in record.parents)
1038
record.key = prefix + record.key
1039
yield record
1040
1041
def _iter_keys_vf(self, keys):
1042
prefixes = self._partition_keys(keys)
1043
sha1s = {}
1044
for prefix, suffixes in prefixes.items():
1045
path = self._mapper.map(prefix)
1046
vf = self._get_vf(path)
1047
yield prefix, suffixes, vf
1048
1049
def get_sha1s(self, keys):
1050
"""See VersionedFiles.get_sha1s()."""
1051
sha1s = {}
1052
for prefix,suffixes, vf in self._iter_keys_vf(keys):
1053
vf_sha1s = vf.get_sha1s(suffixes)
1054
for suffix, sha1 in zip(suffixes, vf.get_sha1s(suffixes)):
1055
sha1s[prefix + (suffix,)] = sha1
1056
return [sha1s[key] for key in keys]
1057
1058
def insert_record_stream(self, stream):
1059
"""Insert a record stream into this container.
1060
1061
:param stream: A stream of records to insert.
1062
:return: None
1063
:seealso VersionedFile.get_record_stream:
1064
"""
1065
for record in stream:
1066
prefix = record.key[:-1]
1067
key = record.key[-1:]
1068
if record.parents is not None:
1069
parents = [parent[-1:] for parent in record.parents]
1070
else:
1071
parents = None
1072
thunk_record = AdapterFactory(key, parents, record)
1073
path = self._mapper.map(prefix)
1074
# Note that this parses the file many times; we can do better but
1075
# as this only impacts weaves in terms of performance, it is
1076
# tolerable.
1077
vf = self._get_vf(path)
1078
vf.insert_record_stream([thunk_record])
1079
1080
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1081
"""Iterate over the lines in the versioned files from keys.
1082
1083
This may return lines from other keys. Each item the returned
1084
iterator yields is a tuple of a line and a text version that that line
1085
is present in (not introduced in).
1086
1087
Ordering of results is in whatever order is most suitable for the
1088
underlying storage format.
1089
1090
If a progress bar is supplied, it may be used to indicate progress.
1091
The caller is responsible for cleaning up progress bars (because this
1092
is an iterator).
1093
1094
NOTES:
1095
* Lines are normalised by the underlying store: they will all have \n
1096
terminators.
1097
* Lines are returned in arbitrary order.
1098
1099
:return: An iterator over (line, key).
1100
"""
1101
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1102
for line, version in vf.iter_lines_added_or_present_in_versions(suffixes):
1103
yield line, prefix + (version,)
1104
1105
def _iter_all_components(self):
1106
for path, prefix in self._get_all_prefixes():
1107
yield prefix, self._get_vf(path)
1108
1109
def keys(self):
1110
"""See VersionedFiles.keys()."""
1111
result = set()
1112
for prefix, vf in self._iter_all_components():
1113
for suffix in vf.versions():
1114
result.add(prefix + (suffix,))
1115
return result
1116
1117
1118
class _PlanMergeVersionedFile(VersionedFiles):
512
1119
"""A VersionedFile for uncommitted and committed texts.
513
1120
514
1121
It is intended to allow merges to be planned with working tree texts.
515
It implements only the small part of the VersionedFile interface used by
1122
It implements only the small part of the VersionedFiles interface used by
516
1123
PlanMerge. It falls back to multiple versionedfiles for data not stored in
517
1124
_PlanMergeVersionedFile itself.
1125
1126
:ivar: fallback_versionedfiles a list of VersionedFiles objects that can be
1127
queried for missing texts.
518
1128
"""
519
1129
520
def __init__(self, file_id, fallback_versionedfiles=None):
521
"""Constuctor
1130
def __init__(self, file_id):
1131
"""Create a _PlanMergeVersionedFile.
522
1132
523
:param file_id: Used when raising exceptions.
524
:param fallback_versionedfiles: If supplied, the set of fallbacks to
525
use. Otherwise, _PlanMergeVersionedFile.fallback_versionedfiles
526
can be appended to later.
1133
:param file_id: Used with _PlanMerge code which is not yet fully
1134
tuple-keyspace aware.
527
1135
"""
528
1136
self._file_id = file_id
529
if fallback_versionedfiles is None:
530
self.fallback_versionedfiles = []
531
else:
532
self.fallback_versionedfiles = fallback_versionedfiles
1137
# fallback locations
1138
self.fallback_versionedfiles = []
1139
# Parents for locally held keys.
533
1140
self._parents = {}
1141
# line data for locally held keys.
534
1142
self._lines = {}
1143
# key lookup providers
1144
self._providers = [DictParentsProvider(self._parents)]
535
1145
536
1146
def plan_merge(self, ver_a, ver_b, base=None):
537
1147
"""See VersionedFile.plan_merge"""
538
1148
from bzrlib.merge import _PlanMerge
539
1149
if base is None:
540
return _PlanMerge(ver_a, ver_b, self).plan_merge()
541
old_plan = list(_PlanMerge(ver_a, base, self).plan_merge())
542
new_plan = list(_PlanMerge(ver_a, ver_b, self).plan_merge())
1150
return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1151
old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1152
new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
543
1153
return _PlanMerge._subtract_plans(old_plan, new_plan)
544
1154
545
1155
def plan_lca_merge(self, ver_a, ver_b, base=None):
546
1156
from bzrlib.merge import _PlanLCAMerge
547
graph = self._get_graph()
548
new_plan = _PlanLCAMerge(ver_a, ver_b, self, graph).plan_merge()
1157
graph = Graph(self)
1158
new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
549
1159
if base is None:
550
1160
return new_plan
551
old_plan = _PlanLCAMerge(ver_a, base, self, graph).plan_merge()
1161
old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
552
1162
return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
553
1163
554
def add_lines(self, version_id, parents, lines):
555
"""See VersionedFile.add_lines
1164
def add_lines(self, key, parents, lines):
1165
"""See VersionedFiles.add_lines
556
1166
557
Lines are added locally, not fallback versionedfiles. Also, ghosts are
558
permitted. Only reserved ids are permitted.
1167
Lines are added locally, not to fallback versionedfiles. Also, ghosts
1168
are permitted. Only reserved ids are permitted.
559
1169
"""
560
if not revision.is_reserved_id(version_id):
1170
if type(key) is not tuple:
1171
raise TypeError(key)
1172
if not revision.is_reserved_id(key[-1]):
561
1173
raise ValueError('Only reserved ids may be used')
562
1174
if parents is None:
563
1175
raise ValueError('Parents may not be None')
564
1176
if lines is None:
565
1177
raise ValueError('Lines may not be None')
566
self._parents[version_id] = tuple(parents)
567
self._lines[version_id] = lines
1178
self._parents[key] = tuple(parents)
1179
self._lines[key] = lines
568
1180
569
def get_lines(self, version_id):
570
"""See VersionedFile.get_ancestry"""
571
lines = self._lines.get(version_id)
572
if lines is not None:
573
return lines
1181
def get_record_stream(self, keys, ordering, include_delta_closure):
1182
pending = set(keys)
1183
for key in keys:
1184
if key in self._lines:
1185
lines = self._lines[key]
1186
parents = self._parents[key]
1187
pending.remove(key)
1188
yield FulltextContentFactory(key, parents, None,
1189
''.join(lines))
574
1190
for versionedfile in self.fallback_versionedfiles:
575
try:
576
return versionedfile.get_lines(version_id)
577
except errors.RevisionNotPresent:
578
continue
579
else:
580
raise errors.RevisionNotPresent(version_id, self._file_id)
581
582
def get_ancestry(self, version_id, topo_sorted=False):
583
"""See VersionedFile.get_ancestry.
584
585
Note that this implementation assumes that if a VersionedFile can
586
answer get_ancestry at all, it can give an authoritative answer. In
587
fact, ghosts can invalidate this assumption. But it's good enough
588
99% of the time, and far cheaper/simpler.
589
590
Also note that the results of this version are never topologically
591
sorted, and are a set.
592
"""
593
if topo_sorted:
594
raise ValueError('This implementation does not provide sorting')
595
parents = self._parents.get(version_id)
596
if parents is None:
597
for vf in self.fallback_versionedfiles:
598
try:
599
return vf.get_ancestry(version_id, topo_sorted=False)
600
except errors.RevisionNotPresent:
1191
for record in versionedfile.get_record_stream(
1192
pending, 'unordered', True):
1193
if record.storage_kind == 'absent':
601
1194
continue
602
else:
603
raise errors.RevisionNotPresent(version_id, self._file_id)
604
ancestry = set([version_id])
605
for parent in parents:
606
ancestry.update(self.get_ancestry(parent, topo_sorted=False))
607
return ancestry
608
609
def get_parent_map(self, version_ids):
610
"""See VersionedFile.get_parent_map"""
611
result = {}
612
pending = set(version_ids)
613
for key in version_ids:
614
try:
615
result[key] = self._parents[key]
616
except KeyError:
617
pass
618
pending = pending - set(result.keys())
619
for versionedfile in self.fallback_versionedfiles:
620
parents = versionedfile.get_parent_map(pending)
621
result.update(parents)
622
pending = pending - set(parents.keys())
1195
else:
1196
pending.remove(record.key)
1197
yield record
623
1198
if not pending:
624
return result
1199
return
1200
# report absent entries
1201
for key in pending:
1202
yield AbsentContentFactory(key)
1203
1204
def get_parent_map(self, keys):
1205
"""See VersionedFiles.get_parent_map"""
1206
# We create a new provider because a fallback may have been added.
1207
# If we make fallbacks private we can update a stack list and avoid
1208
# object creation thrashing.
1209
keys = set(keys)
1210
result = {}
1211
if revision.NULL_REVISION in keys:
1212
keys.remove(revision.NULL_REVISION)
1213
result[revision.NULL_REVISION] = ()
1214
self._providers = self._providers[:1] + self.fallback_versionedfiles
1215
result.update(
1216
_StackedParentsProvider(self._providers).get_parent_map(keys))
1217
for key, parents in result.iteritems():
1218
if parents == ():
1219
result[key] = (revision.NULL_REVISION,)
625
1220
return result
626
1221
627
def _get_graph(self):
628
from bzrlib.graph import (
629
DictParentsProvider,
630
Graph,
631
_StackedParentsProvider,
632
)
633
from bzrlib.repofmt.knitrepo import _KnitParentsProvider
634
parent_providers = [DictParentsProvider(self._parents)]
635
for vf in self.fallback_versionedfiles:
636
parent_providers.append(_KnitParentsProvider(vf))
637
return Graph(_StackedParentsProvider(parent_providers))
638
639
1222
640
1223
class PlanWeaveMerge(TextMerge):
641
1224
"""Weave merge that takes a plan as its input.
717
1300
PlanWeaveMerge.__init__(self, plan, a_marker, b_marker)
718
1301
719
1302
720
class InterVersionedFile(InterObject):
721
"""This class represents operations taking place between two VersionedFiles.
722
723
Its instances have methods like join, and contain
724
references to the source and target versionedfiles these operations can be
725
carried out on.
726
727
Often we will provide convenience methods on 'versionedfile' which carry out
728
operations with another versionedfile - they will always forward to
729
InterVersionedFile.get(other).method_name(parameters).
730
"""
731
732
_optimisers = []
733
"""The available optimised InterVersionedFile types."""
734
735
def join(self, pb=None, msg=None, version_ids=None, ignore_missing=False):
736
"""Integrate versions from self.source into self.target.
737
738
If version_ids is None all versions from source should be
739
incorporated into this versioned file.
740
741
Must raise RevisionNotPresent if any of the specified versions
742
are not present in the other file's history unless ignore_missing is
743
supplied in which case they are silently skipped.
744
"""
745
target = self.target
746
version_ids = self._get_source_version_ids(version_ids, ignore_missing)
747
graph = Graph(self.source)
748
search = graph._make_breadth_first_searcher(version_ids)
749
transitive_ids = set()
750
map(transitive_ids.update, list(search))
751
parent_map = self.source.get_parent_map(transitive_ids)
752
order = tsort.topo_sort(parent_map.items())
753
pb = ui.ui_factory.nested_progress_bar()
754
parent_texts = {}
755
try:
756
# TODO for incremental cross-format work:
757
# make a versioned file with the following content:
758
# all revisions we have been asked to join
759
# all their ancestors that are *not* in target already.
760
# the immediate parents of the above two sets, with
761
# empty parent lists - these versions are in target already
762
# and the incorrect version data will be ignored.
763
# TODO: for all ancestors that are present in target already,
764
# check them for consistent data, this requires moving sha1 from
765
#
766
# TODO: remove parent texts when they are not relevant any more for
767
# memory pressure reduction. RBC 20060313
768
# pb.update('Converting versioned data', 0, len(order))
769
total = len(order)
770
for index, version in enumerate(order):
771
pb.update('Converting versioned data', index, total)
772
if version in target:
773
continue
774
_, _, parent_text = target.add_lines(version,
775
parent_map[version],
776
self.source.get_lines(version),
777
parent_texts=parent_texts)
778
parent_texts[version] = parent_text
779
return total
780
finally:
781
pb.finished()
782
783
def _get_source_version_ids(self, version_ids, ignore_missing):
784
"""Determine the version ids to be used from self.source.
785
786
:param version_ids: The caller-supplied version ids to check. (None
787
for all). If None is in version_ids, it is stripped.
788
:param ignore_missing: if True, remove missing ids from the version
789
list. If False, raise RevisionNotPresent on
790
a missing version id.
791
:return: A set of version ids.
792
"""
793
if version_ids is None:
794
# None cannot be in source.versions
795
return set(self.source.versions())
796
else:
797
if ignore_missing:
798
return set(self.source.versions()).intersection(set(version_ids))
799
else:
800
new_version_ids = set()
801
for version in version_ids:
802
if version is None:
803
continue
804
if not self.source.has_version(version):
805
raise errors.RevisionNotPresent(version, str(self.source))
806
else:
807
new_version_ids.add(version)
808
return new_version_ids
Loggerhead is a web-based interface for Breezy
Version: 2.0.1