« back to all changes in this revision
Viewing changes to bzrlib/versionedfile.py
- browse files at revision 3363.8.1
- compare with another revision
- compare with revision 3363.2.35
- download diff
- download tarball
- view history from revision 3363.8.1
- Committer: Aaron Bentley
- Date: 2008-07-02 13:26:59 UTC
- mfrom: (3363.7.1 annotate)
- mto: (3363.2.38 is_executable)
- mto: This revision was merged to the branch mainline in revision 3517.
- Revision ID: aaron@aaronbentley.com-20080702132659-ofq4dotjwmhachh1
Merge with annotate
- files removed:
- bzrlib/store/revision/__init__.py
- bzrlib/tests/interversionedfile_implementations
- bzrlib/tests/revisionstore_implementations
- files modified:
- NEWS
added
removed
bzrlib/versionedfile.py
19
19
20
20
"""Versioned text file storage api."""
21
21
22
from copy import copy
23
from cStringIO import StringIO
24
import os
25
import urllib
26
from zlib import adler32
27
22
28
from bzrlib.lazy_import import lazy_import
23
29
lazy_import(globals(), """
24
30
30
36
revision,
31
37
ui,
32
38
)
33
from bzrlib.graph import Graph
39
from bzrlib.graph import DictParentsProvider, Graph, _StackedParentsProvider
34
40
from bzrlib.transport.memory import MemoryTransport
35
41
""")
36
37
from cStringIO import StringIO
38
39
42
from bzrlib.inter import InterObject
40
43
from bzrlib.registry import Registry
41
44
from bzrlib.symbol_versioning import *
70
73
:ivar parents: A tuple of parent keys for self.key. If the object has
71
74
no parent information, None (as opposed to () for an empty list of
72
75
parents).
73
"""
76
"""
74
77
75
78
def __init__(self):
76
79
"""Create a ContentFactory."""
80
83
self.parents = None
81
84
82
85
83
class AbsentContentFactory(object):
86
class FulltextContentFactory(ContentFactory):
87
"""Static data content factory.
88
89
This takes a fulltext when created and just returns that during
90
get_bytes_as('fulltext').
91
92
:ivar sha1: None, or the sha1 of the content fulltext.
93
:ivar storage_kind: The native storage kind of this factory. Always
94
'fulltext'.
95
:ivar key: The key of this content. Each key is a tuple with a single
96
string in it.
97
:ivar parents: A tuple of parent keys for self.key. If the object has
98
no parent information, None (as opposed to () for an empty list of
99
parents).
100
"""
101
102
def __init__(self, key, parents, sha1, text):
103
"""Create a ContentFactory."""
104
self.sha1 = sha1
105
self.storage_kind = 'fulltext'
106
self.key = key
107
self.parents = parents
108
self._text = text
109
110
def get_bytes_as(self, storage_kind):
111
if storage_kind == self.storage_kind:
112
return self._text
113
raise errors.UnavailableRepresentation(self.key, storage_kind,
114
self.storage_kind)
115
116
117
class AbsentContentFactory(ContentFactory):
84
118
"""A placeholder content factory for unavailable texts.
85
119
86
120
:ivar sha1: None.
98
132
self.parents = None
99
133
100
134
135
class AdapterFactory(ContentFactory):
136
"""A content factory to adapt between key prefix's."""
137
138
def __init__(self, key, parents, adapted):
139
"""Create an adapter factory instance."""
140
self.key = key
141
self.parents = parents
142
self._adapted = adapted
143
144
def __getattr__(self, attr):
145
"""Return a member from the adapted object."""
146
if attr in ('key', 'parents'):
147
return self.__dict__[attr]
148
else:
149
return getattr(self._adapted, attr)
150
151
101
152
def filter_absent(record_stream):
102
153
"""Adapt a record stream to remove absent records."""
103
154
for record in record_stream:
322
373
parent_ids, lines, vf_parents,
323
374
left_matching_blocks=left_matching_blocks)
324
375
vf_parents[version] = version_text
325
for (version, parent_ids, expected_sha1, mpdiff), sha1 in\
326
zip(records, self.get_sha1s(versions)):
327
if expected_sha1 != sha1:
376
sha1s = self.get_sha1s(versions)
377
for version, parent_ids, expected_sha1, mpdiff in records:
378
if expected_sha1 != sha1s[version]:
328
379
raise errors.VersionedFileInvalidChecksum(version)
329
380
330
def get_sha1s(self, version_ids):
331
"""Get the stored sha1 sums for the given revisions.
332
333
:param version_ids: The names of the versions to lookup
334
:return: a list of sha1s in order according to the version_ids
335
"""
336
raise NotImplementedError(self.get_sha1s)
337
338
381
def get_text(self, version_id):
339
382
"""Return version contents as a text string.
340
383
418
461
"""
419
462
raise NotImplementedError(self.annotate)
420
463
421
@deprecated_method(one_five)
422
def join(self, other, pb=None, msg=None, version_ids=None,
423
ignore_missing=False):
424
"""Integrate versions from other into this versioned file.
425
426
If version_ids is None all versions from other should be
427
incorporated into this versioned file.
428
429
Must raise RevisionNotPresent if any of the specified versions
430
are not present in the other file's history unless ignore_missing
431
is supplied in which case they are silently skipped.
432
"""
433
self._check_write_ok()
434
return InterVersionedFile.get(other, self).join(
435
pb,
436
msg,
437
version_ids,
438
ignore_missing)
439
440
464
def iter_lines_added_or_present_in_versions(self, version_ids=None,
441
465
pb=None):
442
466
"""Iterate over the lines in the versioned file from version_ids.
486
510
return PlanWeaveMerge(plan, a_marker, b_marker).merge_lines()[0]
487
511
488
512
489
class RecordingVersionedFileDecorator(object):
490
"""A minimal versioned file that records calls made on it.
513
class RecordingVersionedFilesDecorator(object):
514
"""A minimal versioned files that records calls made on it.
491
515
492
516
Only enough methods have been added to support tests using it to date.
493
517
496
520
"""
497
521
498
522
def __init__(self, backing_vf):
499
"""Create a RecordingVersionedFileDecorator decorating backing_vf.
523
"""Create a RecordingVersionedFileDsecorator decorating backing_vf.
500
524
501
525
:param backing_vf: The versioned file to answer all methods.
502
526
"""
503
527
self._backing_vf = backing_vf
504
528
self.calls = []
505
529
506
def get_lines(self, version_ids):
507
self.calls.append(("get_lines", version_ids))
508
return self._backing_vf.get_lines(version_ids)
509
510
511
class _PlanMergeVersionedFile(object):
530
def add_lines(self, key, parents, lines, parent_texts=None,
531
left_matching_blocks=None, nostore_sha=None, random_id=False,
532
check_content=True):
533
self.calls.append(("add_lines", key, parents, lines, parent_texts,
534
left_matching_blocks, nostore_sha, random_id, check_content))
535
return self._backing_vf.add_lines(key, parents, lines, parent_texts,
536
left_matching_blocks, nostore_sha, random_id, check_content)
537
538
def get_parent_map(self, keys):
539
self.calls.append(("get_parent_map", copy(keys)))
540
return self._backing_vf.get_parent_map(keys)
541
542
def get_record_stream(self, keys, sort_order, include_delta_closure):
543
self.calls.append(("get_record_stream", list(keys), sort_order,
544
include_delta_closure))
545
return self._backing_vf.get_record_stream(keys, sort_order,
546
include_delta_closure)
547
548
def get_sha1s(self, keys):
549
self.calls.append(("get_sha1s", copy(keys)))
550
return self._backing_vf.get_sha1s(keys)
551
552
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
553
self.calls.append(("iter_lines_added_or_present_in_keys", copy(keys)))
554
return self._backing_vf.iter_lines_added_or_present_in_keys(keys, pb=pb)
555
556
def keys(self):
557
self.calls.append(("keys",))
558
return self._backing_vf.keys()
559
560
561
class KeyMapper(object):
562
"""KeyMappers map between keys and underlying partitioned storage."""
563
564
def map(self, key):
565
"""Map key to an underlying storage identifier.
566
567
:param key: A key tuple e.g. ('file-id', 'revision-id').
568
:return: An underlying storage identifier, specific to the partitioning
569
mechanism.
570
"""
571
raise NotImplementedError(self.map)
572
573
def unmap(self, partition_id):
574
"""Map a partitioned storage id back to a key prefix.
575
576
:param partition_id: The underlying partition id.
577
:return: As much of a key (or prefix) as is derivable from the partition
578
id.
579
"""
580
raise NotImplementedError(self.unmap)
581
582
583
class ConstantMapper(KeyMapper):
584
"""A key mapper that maps to a constant result."""
585
586
def __init__(self, result):
587
"""Create a ConstantMapper which will return result for all maps."""
588
self._result = result
589
590
def map(self, key):
591
"""See KeyMapper.map()."""
592
return self._result
593
594
595
class URLEscapeMapper(KeyMapper):
596
"""Base class for use with transport backed storage.
597
598
This provides a map and unmap wrapper that respectively url escape and
599
unescape their outputs and inputs.
600
"""
601
602
def map(self, key):
603
"""See KeyMapper.map()."""
604
return urllib.quote(self._map(key))
605
606
def unmap(self, partition_id):
607
"""See KeyMapper.unmap()."""
608
return self._unmap(urllib.unquote(partition_id))
609
610
611
class PrefixMapper(URLEscapeMapper):
612
"""A key mapper that extracts the first component of a key.
613
614
This mapper is for use with a transport based backend.
615
"""
616
617
def _map(self, key):
618
"""See KeyMapper.map()."""
619
return key[0]
620
621
def _unmap(self, partition_id):
622
"""See KeyMapper.unmap()."""
623
return (partition_id,)
624
625
626
class HashPrefixMapper(URLEscapeMapper):
627
"""A key mapper that combines the first component of a key with a hash.
628
629
This mapper is for use with a transport based backend.
630
"""
631
632
def _map(self, key):
633
"""See KeyMapper.map()."""
634
prefix = self._escape(key[0])
635
return "%02x/%s" % (adler32(prefix) & 0xff, prefix)
636
637
def _escape(self, prefix):
638
"""No escaping needed here."""
639
return prefix
640
641
def _unmap(self, partition_id):
642
"""See KeyMapper.unmap()."""
643
return (self._unescape(osutils.basename(partition_id)),)
644
645
def _unescape(self, basename):
646
"""No unescaping needed for HashPrefixMapper."""
647
return basename
648
649
650
class HashEscapedPrefixMapper(HashPrefixMapper):
651
"""Combines the escaped first component of a key with a hash.
652
653
This mapper is for use with a transport based backend.
654
"""
655
656
_safe = "abcdefghijklmnopqrstuvwxyz0123456789-_@,."
657
658
def _escape(self, prefix):
659
"""Turn a key element into a filesystem safe string.
660
661
This is similar to a plain urllib.quote, except
662
it uses specific safe characters, so that it doesn't
663
have to translate a lot of valid file ids.
664
"""
665
# @ does not get escaped. This is because it is a valid
666
# filesystem character we use all the time, and it looks
667
# a lot better than seeing %40 all the time.
668
r = [((c in self._safe) and c or ('%%%02x' % ord(c)))
669
for c in prefix]
670
return ''.join(r)
671
672
def _unescape(self, basename):
673
"""Escaped names are easily unescaped by urlutils."""
674
return urllib.unquote(basename)
675
676
677
def make_versioned_files_factory(versioned_file_factory, mapper):
678
"""Create a ThunkedVersionedFiles factory.
679
680
This will create a callable which when called creates a
681
ThunkedVersionedFiles on a transport, using mapper to access individual
682
versioned files, and versioned_file_factory to create each individual file.
683
"""
684
def factory(transport):
685
return ThunkedVersionedFiles(transport, versioned_file_factory, mapper,
686
lambda:True)
687
return factory
688
689
690
class VersionedFiles(object):
691
"""Storage for many versioned files.
692
693
This object allows a single keyspace for accessing the history graph and
694
contents of named bytestrings.
695
696
Currently no implementation allows the graph of different key prefixes to
697
intersect, but the API does allow such implementations in the future.
698
699
The keyspace is expressed via simple tuples. Any instance of VersionedFiles
700
may have a different length key-size, but that size will be constant for
701
all texts added to or retrieved from it. For instance, bzrlib uses
702
instances with a key-size of 2 for storing user files in a repository, with
703
the first element the fileid, and the second the version of that file.
704
705
The use of tuples allows a single code base to support several different
706
uses with only the mapping logic changing from instance to instance.
707
"""
708
709
def add_lines(self, key, parents, lines, parent_texts=None,
710
left_matching_blocks=None, nostore_sha=None, random_id=False,
711
check_content=True):
712
"""Add a text to the store.
713
714
:param key: The key tuple of the text to add.
715
:param parents: The parents key tuples of the text to add.
716
:param lines: A list of lines. Each line must be a bytestring. And all
717
of them except the last must be terminated with \n and contain no
718
other \n's. The last line may either contain no \n's or a single
719
terminating \n. If the lines list does meet this constraint the add
720
routine may error or may succeed - but you will be unable to read
721
the data back accurately. (Checking the lines have been split
722
correctly is expensive and extremely unlikely to catch bugs so it
723
is not done at runtime unless check_content is True.)
724
:param parent_texts: An optional dictionary containing the opaque
725
representations of some or all of the parents of version_id to
726
allow delta optimisations. VERY IMPORTANT: the texts must be those
727
returned by add_lines or data corruption can be caused.
728
:param left_matching_blocks: a hint about which areas are common
729
between the text and its left-hand-parent. The format is
730
the SequenceMatcher.get_matching_blocks format.
731
:param nostore_sha: Raise ExistingContent and do not add the lines to
732
the versioned file if the digest of the lines matches this.
733
:param random_id: If True a random id has been selected rather than
734
an id determined by some deterministic process such as a converter
735
from a foreign VCS. When True the backend may choose not to check
736
for uniqueness of the resulting key within the versioned file, so
737
this should only be done when the result is expected to be unique
738
anyway.
739
:param check_content: If True, the lines supplied are verified to be
740
bytestrings that are correctly formed lines.
741
:return: The text sha1, the number of bytes in the text, and an opaque
742
representation of the inserted version which can be provided
743
back to future add_lines calls in the parent_texts dictionary.
744
"""
745
raise NotImplementedError(self.add_lines)
746
747
def add_mpdiffs(self, records):
748
"""Add mpdiffs to this VersionedFile.
749
750
Records should be iterables of version, parents, expected_sha1,
751
mpdiff. mpdiff should be a MultiParent instance.
752
"""
753
vf_parents = {}
754
mpvf = multiparent.MultiMemoryVersionedFile()
755
versions = []
756
for version, parent_ids, expected_sha1, mpdiff in records:
757
versions.append(version)
758
mpvf.add_diff(mpdiff, version, parent_ids)
759
needed_parents = set()
760
for version, parent_ids, expected_sha1, mpdiff in records:
761
needed_parents.update(p for p in parent_ids
762
if not mpvf.has_version(p))
763
# It seems likely that adding all the present parents as fulltexts can
764
# easily exhaust memory.
765
split_lines = osutils.split_lines
766
for record in self.get_record_stream(needed_parents, 'unordered',
767
True):
768
if record.storage_kind == 'absent':
769
continue
770
mpvf.add_version(split_lines(record.get_bytes_as('fulltext')),
771
record.key, [])
772
for (key, parent_keys, expected_sha1, mpdiff), lines in\
773
zip(records, mpvf.get_line_list(versions)):
774
if len(parent_keys) == 1:
775
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
776
mpvf.get_diff(parent_keys[0]).num_lines()))
777
else:
778
left_matching_blocks = None
779
version_sha1, _, version_text = self.add_lines(key,
780
parent_keys, lines, vf_parents,
781
left_matching_blocks=left_matching_blocks)
782
if version_sha1 != expected_sha1:
783
raise errors.VersionedFileInvalidChecksum(version)
784
vf_parents[key] = version_text
785
786
def annotate(self, key):
787
"""Return a list of (version-key, line) tuples for the text of key.
788
789
:raise RevisionNotPresent: If the key is not present.
790
"""
791
raise NotImplementedError(self.annotate)
792
793
def check(self, progress_bar=None):
794
"""Check this object for integrity."""
795
raise NotImplementedError(self.check)
796
797
@staticmethod
798
def check_not_reserved_id(version_id):
799
revision.check_not_reserved_id(version_id)
800
801
def _check_lines_not_unicode(self, lines):
802
"""Check that lines being added to a versioned file are not unicode."""
803
for line in lines:
804
if line.__class__ is not str:
805
raise errors.BzrBadParameterUnicode("lines")
806
807
def _check_lines_are_lines(self, lines):
808
"""Check that the lines really are full lines without inline EOL."""
809
for line in lines:
810
if '\n' in line[:-1]:
811
raise errors.BzrBadParameterContainsNewline("lines")
812
813
def get_parent_map(self, keys):
814
"""Get a map of the parents of keys.
815
816
:param keys: The keys to look up parents for.
817
:return: A mapping from keys to parents. Absent keys are absent from
818
the mapping.
819
"""
820
raise NotImplementedError(self.get_parent_map)
821
822
def get_record_stream(self, keys, ordering, include_delta_closure):
823
"""Get a stream of records for keys.
824
825
:param keys: The keys to include.
826
:param ordering: Either 'unordered' or 'topological'. A topologically
827
sorted stream has compression parents strictly before their
828
children.
829
:param include_delta_closure: If True then the closure across any
830
compression parents will be included (in the opaque data).
831
:return: An iterator of ContentFactory objects, each of which is only
832
valid until the iterator is advanced.
833
"""
834
raise NotImplementedError(self.get_record_stream)
835
836
def get_sha1s(self, keys):
837
"""Get the sha1's of the texts for the given keys.
838
839
:param keys: The names of the keys to lookup
840
:return: a dict from key to sha1 digest. Keys of texts which are not
841
present in the store are not present in the returned
842
dictionary.
843
"""
844
raise NotImplementedError(self.get_sha1s)
845
846
def insert_record_stream(self, stream):
847
"""Insert a record stream into this container.
848
849
:param stream: A stream of records to insert.
850
:return: None
851
:seealso VersionedFile.get_record_stream:
852
"""
853
raise NotImplementedError
854
855
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
856
"""Iterate over the lines in the versioned files from keys.
857
858
This may return lines from other keys. Each item the returned
859
iterator yields is a tuple of a line and a text version that that line
860
is present in (not introduced in).
861
862
Ordering of results is in whatever order is most suitable for the
863
underlying storage format.
864
865
If a progress bar is supplied, it may be used to indicate progress.
866
The caller is responsible for cleaning up progress bars (because this
867
is an iterator).
868
869
NOTES:
870
* Lines are normalised by the underlying store: they will all have \n
871
terminators.
872
* Lines are returned in arbitrary order.
873
874
:return: An iterator over (line, key).
875
"""
876
raise NotImplementedError(self.iter_lines_added_or_present_in_keys)
877
878
def keys(self):
879
"""Return a iterable of the keys for all the contained texts."""
880
raise NotImplementedError(self.keys)
881
882
def make_mpdiffs(self, keys):
883
"""Create multiparent diffs for specified keys."""
884
keys_order = tuple(keys)
885
keys = frozenset(keys)
886
knit_keys = set(keys)
887
parent_map = self.get_parent_map(keys)
888
for parent_keys in parent_map.itervalues():
889
if parent_keys:
890
knit_keys.update(parent_keys)
891
missing_keys = keys - set(parent_map)
892
if missing_keys:
893
raise errors.RevisionNotPresent(missing_keys.pop(), self)
894
# We need to filter out ghosts, because we can't diff against them.
895
maybe_ghosts = knit_keys - keys
896
ghosts = maybe_ghosts - set(self.get_parent_map(maybe_ghosts))
897
knit_keys.difference_update(ghosts)
898
lines = {}
899
split_lines = osutils.split_lines
900
for record in self.get_record_stream(knit_keys, 'topological', True):
901
lines[record.key] = split_lines(record.get_bytes_as('fulltext'))
902
# line_block_dict = {}
903
# for parent, blocks in record.extract_line_blocks():
904
# line_blocks[parent] = blocks
905
# line_blocks[record.key] = line_block_dict
906
diffs = []
907
for key in keys_order:
908
target = lines[key]
909
parents = parent_map[key] or []
910
# Note that filtering knit_keys can lead to a parent difference
911
# between the creation and the application of the mpdiff.
912
parent_lines = [lines[p] for p in parents if p in knit_keys]
913
if len(parent_lines) > 0:
914
left_parent_blocks = self._extract_blocks(key, parent_lines[0],
915
target)
916
else:
917
left_parent_blocks = None
918
diffs.append(multiparent.MultiParent.from_lines(target,
919
parent_lines, left_parent_blocks))
920
return diffs
921
922
def _extract_blocks(self, version_id, source, target):
923
return None
924
925
926
class ThunkedVersionedFiles(VersionedFiles):
927
"""Storage for many versioned files thunked onto a 'VersionedFile' class.
928
929
This object allows a single keyspace for accessing the history graph and
930
contents of named bytestrings.
931
932
Currently no implementation allows the graph of different key prefixes to
933
intersect, but the API does allow such implementations in the future.
934
"""
935
936
def __init__(self, transport, file_factory, mapper, is_locked):
937
"""Create a ThunkedVersionedFiles."""
938
self._transport = transport
939
self._file_factory = file_factory
940
self._mapper = mapper
941
self._is_locked = is_locked
942
943
def add_lines(self, key, parents, lines, parent_texts=None,
944
left_matching_blocks=None, nostore_sha=None, random_id=False,
945
check_content=True):
946
"""See VersionedFiles.add_lines()."""
947
path = self._mapper.map(key)
948
version_id = key[-1]
949
parents = [parent[-1] for parent in parents]
950
vf = self._get_vf(path)
951
try:
952
try:
953
return vf.add_lines_with_ghosts(version_id, parents, lines,
954
parent_texts=parent_texts,
955
left_matching_blocks=left_matching_blocks,
956
nostore_sha=nostore_sha, random_id=random_id,
957
check_content=check_content)
958
except NotImplementedError:
959
return vf.add_lines(version_id, parents, lines,
960
parent_texts=parent_texts,
961
left_matching_blocks=left_matching_blocks,
962
nostore_sha=nostore_sha, random_id=random_id,
963
check_content=check_content)
964
except errors.NoSuchFile:
965
# parent directory may be missing, try again.
966
self._transport.mkdir(osutils.dirname(path))
967
try:
968
return vf.add_lines_with_ghosts(version_id, parents, lines,
969
parent_texts=parent_texts,
970
left_matching_blocks=left_matching_blocks,
971
nostore_sha=nostore_sha, random_id=random_id,
972
check_content=check_content)
973
except NotImplementedError:
974
return vf.add_lines(version_id, parents, lines,
975
parent_texts=parent_texts,
976
left_matching_blocks=left_matching_blocks,
977
nostore_sha=nostore_sha, random_id=random_id,
978
check_content=check_content)
979
980
def annotate(self, key):
981
"""Return a list of (version-key, line) tuples for the text of key.
982
983
:raise RevisionNotPresent: If the key is not present.
984
"""
985
prefix = key[:-1]
986
path = self._mapper.map(prefix)
987
vf = self._get_vf(path)
988
origins = vf.annotate(key[-1])
989
result = []
990
for origin, line in origins:
991
result.append((prefix + (origin,), line))
992
return result
993
994
def check(self, progress_bar=None):
995
"""See VersionedFiles.check()."""
996
for prefix, vf in self._iter_all_components():
997
vf.check()
998
999
def get_parent_map(self, keys):
1000
"""Get a map of the parents of keys.
1001
1002
:param keys: The keys to look up parents for.
1003
:return: A mapping from keys to parents. Absent keys are absent from
1004
the mapping.
1005
"""
1006
prefixes = self._partition_keys(keys)
1007
result = {}
1008
for prefix, suffixes in prefixes.items():
1009
path = self._mapper.map(prefix)
1010
vf = self._get_vf(path)
1011
parent_map = vf.get_parent_map(suffixes)
1012
for key, parents in parent_map.items():
1013
result[prefix + (key,)] = tuple(
1014
prefix + (parent,) for parent in parents)
1015
return result
1016
1017
def _get_vf(self, path):
1018
if not self._is_locked():
1019
raise errors.ObjectNotLocked(self)
1020
return self._file_factory(path, self._transport, create=True,
1021
get_scope=lambda:None)
1022
1023
def _partition_keys(self, keys):
1024
"""Turn keys into a dict of prefix:suffix_list."""
1025
result = {}
1026
for key in keys:
1027
prefix_keys = result.setdefault(key[:-1], [])
1028
prefix_keys.append(key[-1])
1029
return result
1030
1031
def _get_all_prefixes(self):
1032
# Identify all key prefixes.
1033
# XXX: A bit hacky, needs polish.
1034
if type(self._mapper) == ConstantMapper:
1035
paths = [self._mapper.map(())]
1036
prefixes = [()]
1037
else:
1038
relpaths = set()
1039
for quoted_relpath in self._transport.iter_files_recursive():
1040
path, ext = os.path.splitext(quoted_relpath)
1041
relpaths.add(path)
1042
paths = list(relpaths)
1043
prefixes = [self._mapper.unmap(path) for path in paths]
1044
return zip(paths, prefixes)
1045
1046
def get_record_stream(self, keys, ordering, include_delta_closure):
1047
"""See VersionedFiles.get_record_stream()."""
1048
# Ordering will be taken care of by each partitioned store; group keys
1049
# by partition.
1050
keys = sorted(keys)
1051
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1052
suffixes = [(suffix,) for suffix in suffixes]
1053
for record in vf.get_record_stream(suffixes, ordering,
1054
include_delta_closure):
1055
if record.parents is not None:
1056
record.parents = tuple(
1057
prefix + parent for parent in record.parents)
1058
record.key = prefix + record.key
1059
yield record
1060
1061
def _iter_keys_vf(self, keys):
1062
prefixes = self._partition_keys(keys)
1063
sha1s = {}
1064
for prefix, suffixes in prefixes.items():
1065
path = self._mapper.map(prefix)
1066
vf = self._get_vf(path)
1067
yield prefix, suffixes, vf
1068
1069
def get_sha1s(self, keys):
1070
"""See VersionedFiles.get_sha1s()."""
1071
sha1s = {}
1072
for prefix,suffixes, vf in self._iter_keys_vf(keys):
1073
vf_sha1s = vf.get_sha1s(suffixes)
1074
for suffix, sha1 in vf_sha1s.iteritems():
1075
sha1s[prefix + (suffix,)] = sha1
1076
return sha1s
1077
1078
def insert_record_stream(self, stream):
1079
"""Insert a record stream into this container.
1080
1081
:param stream: A stream of records to insert.
1082
:return: None
1083
:seealso VersionedFile.get_record_stream:
1084
"""
1085
for record in stream:
1086
prefix = record.key[:-1]
1087
key = record.key[-1:]
1088
if record.parents is not None:
1089
parents = [parent[-1:] for parent in record.parents]
1090
else:
1091
parents = None
1092
thunk_record = AdapterFactory(key, parents, record)
1093
path = self._mapper.map(prefix)
1094
# Note that this parses the file many times; we can do better but
1095
# as this only impacts weaves in terms of performance, it is
1096
# tolerable.
1097
vf = self._get_vf(path)
1098
vf.insert_record_stream([thunk_record])
1099
1100
def iter_lines_added_or_present_in_keys(self, keys, pb=None):
1101
"""Iterate over the lines in the versioned files from keys.
1102
1103
This may return lines from other keys. Each item the returned
1104
iterator yields is a tuple of a line and a text version that that line
1105
is present in (not introduced in).
1106
1107
Ordering of results is in whatever order is most suitable for the
1108
underlying storage format.
1109
1110
If a progress bar is supplied, it may be used to indicate progress.
1111
The caller is responsible for cleaning up progress bars (because this
1112
is an iterator).
1113
1114
NOTES:
1115
* Lines are normalised by the underlying store: they will all have \n
1116
terminators.
1117
* Lines are returned in arbitrary order.
1118
1119
:return: An iterator over (line, key).
1120
"""
1121
for prefix, suffixes, vf in self._iter_keys_vf(keys):
1122
for line, version in vf.iter_lines_added_or_present_in_versions(suffixes):
1123
yield line, prefix + (version,)
1124
1125
def _iter_all_components(self):
1126
for path, prefix in self._get_all_prefixes():
1127
yield prefix, self._get_vf(path)
1128
1129
def keys(self):
1130
"""See VersionedFiles.keys()."""
1131
result = set()
1132
for prefix, vf in self._iter_all_components():
1133
for suffix in vf.versions():
1134
result.add(prefix + (suffix,))
1135
return result
1136
1137
1138
class _PlanMergeVersionedFile(VersionedFiles):
512
1139
"""A VersionedFile for uncommitted and committed texts.
513
1140
514
1141
It is intended to allow merges to be planned with working tree texts.
515
It implements only the small part of the VersionedFile interface used by
1142
It implements only the small part of the VersionedFiles interface used by
516
1143
PlanMerge. It falls back to multiple versionedfiles for data not stored in
517
1144
_PlanMergeVersionedFile itself.
1145
1146
:ivar: fallback_versionedfiles a list of VersionedFiles objects that can be
1147
queried for missing texts.
518
1148
"""
519
1149
520
def __init__(self, file_id, fallback_versionedfiles=None):
521
"""Constuctor
1150
def __init__(self, file_id):
1151
"""Create a _PlanMergeVersionedFile.
522
1152
523
:param file_id: Used when raising exceptions.
524
:param fallback_versionedfiles: If supplied, the set of fallbacks to
525
use. Otherwise, _PlanMergeVersionedFile.fallback_versionedfiles
526
can be appended to later.
1153
:param file_id: Used with _PlanMerge code which is not yet fully
1154
tuple-keyspace aware.
527
1155
"""
528
1156
self._file_id = file_id
529
if fallback_versionedfiles is None:
530
self.fallback_versionedfiles = []
531
else:
532
self.fallback_versionedfiles = fallback_versionedfiles
1157
# fallback locations
1158
self.fallback_versionedfiles = []
1159
# Parents for locally held keys.
533
1160
self._parents = {}
1161
# line data for locally held keys.
534
1162
self._lines = {}
1163
# key lookup providers
1164
self._providers = [DictParentsProvider(self._parents)]
535
1165
536
1166
def plan_merge(self, ver_a, ver_b, base=None):
537
1167
"""See VersionedFile.plan_merge"""
538
1168
from bzrlib.merge import _PlanMerge
539
1169
if base is None:
540
return _PlanMerge(ver_a, ver_b, self).plan_merge()
541
old_plan = list(_PlanMerge(ver_a, base, self).plan_merge())
542
new_plan = list(_PlanMerge(ver_a, ver_b, self).plan_merge())
1170
return _PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge()
1171
old_plan = list(_PlanMerge(ver_a, base, self, (self._file_id,)).plan_merge())
1172
new_plan = list(_PlanMerge(ver_a, ver_b, self, (self._file_id,)).plan_merge())
543
1173
return _PlanMerge._subtract_plans(old_plan, new_plan)
544
1174
545
1175
def plan_lca_merge(self, ver_a, ver_b, base=None):
546
1176
from bzrlib.merge import _PlanLCAMerge
547
graph = self._get_graph()
548
new_plan = _PlanLCAMerge(ver_a, ver_b, self, graph).plan_merge()
1177
graph = Graph(self)
1178
new_plan = _PlanLCAMerge(ver_a, ver_b, self, (self._file_id,), graph).plan_merge()
549
1179
if base is None:
550
1180
return new_plan
551
old_plan = _PlanLCAMerge(ver_a, base, self, graph).plan_merge()
1181
old_plan = _PlanLCAMerge(ver_a, base, self, (self._file_id,), graph).plan_merge()
552
1182
return _PlanLCAMerge._subtract_plans(list(old_plan), list(new_plan))
553
1183
554
def add_lines(self, version_id, parents, lines):
555
"""See VersionedFile.add_lines
1184
def add_lines(self, key, parents, lines):
1185
"""See VersionedFiles.add_lines
556
1186
557
Lines are added locally, not fallback versionedfiles. Also, ghosts are
558
permitted. Only reserved ids are permitted.
1187
Lines are added locally, not to fallback versionedfiles. Also, ghosts
1188
are permitted. Only reserved ids are permitted.
559
1189
"""
560
if not revision.is_reserved_id(version_id):
1190
if type(key) is not tuple:
1191
raise TypeError(key)
1192
if not revision.is_reserved_id(key[-1]):
561
1193
raise ValueError('Only reserved ids may be used')
562
1194
if parents is None:
563
1195
raise ValueError('Parents may not be None')
564
1196
if lines is None:
565
1197
raise ValueError('Lines may not be None')
566
self._parents[version_id] = tuple(parents)
567
self._lines[version_id] = lines
1198
self._parents[key] = tuple(parents)
1199
self._lines[key] = lines
568
1200
569
def get_lines(self, version_id):
570
"""See VersionedFile.get_ancestry"""
571
lines = self._lines.get(version_id)
572
if lines is not None:
573
return lines
1201
def get_record_stream(self, keys, ordering, include_delta_closure):
1202
pending = set(keys)
1203
for key in keys:
1204
if key in self._lines:
1205
lines = self._lines[key]
1206
parents = self._parents[key]
1207
pending.remove(key)
1208
yield FulltextContentFactory(key, parents, None,
1209
''.join(lines))
574
1210
for versionedfile in self.fallback_versionedfiles:
575
try:
576
return versionedfile.get_lines(version_id)
577
except errors.RevisionNotPresent:
578
continue
579
else:
580
raise errors.RevisionNotPresent(version_id, self._file_id)
581
582
def get_ancestry(self, version_id, topo_sorted=False):
583
"""See VersionedFile.get_ancestry.
584
585
Note that this implementation assumes that if a VersionedFile can
586
answer get_ancestry at all, it can give an authoritative answer. In
587
fact, ghosts can invalidate this assumption. But it's good enough
588
99% of the time, and far cheaper/simpler.
589
590
Also note that the results of this version are never topologically
591
sorted, and are a set.
592
"""
593
if topo_sorted:
594
raise ValueError('This implementation does not provide sorting')
595
parents = self._parents.get(version_id)
596
if parents is None:
597
for vf in self.fallback_versionedfiles:
598
try:
599
return vf.get_ancestry(version_id, topo_sorted=False)
600
except errors.RevisionNotPresent:
1211
for record in versionedfile.get_record_stream(
1212
pending, 'unordered', True):
1213
if record.storage_kind == 'absent':
601
1214
continue
602
else:
603
raise errors.RevisionNotPresent(version_id, self._file_id)
604
ancestry = set([version_id])
605
for parent in parents:
606
ancestry.update(self.get_ancestry(parent, topo_sorted=False))
607
return ancestry
608
609
def get_parent_map(self, version_ids):
610
"""See VersionedFile.get_parent_map"""
611
result = {}
612
pending = set(version_ids)
613
for key in version_ids:
614
try:
615
result[key] = self._parents[key]
616
except KeyError:
617
pass
618
pending = pending - set(result.keys())
619
for versionedfile in self.fallback_versionedfiles:
620
parents = versionedfile.get_parent_map(pending)
621
result.update(parents)
622
pending = pending - set(parents.keys())
1215
else:
1216
pending.remove(record.key)
1217
yield record
623
1218
if not pending:
624
return result
1219
return
1220
# report absent entries
1221
for key in pending:
1222
yield AbsentContentFactory(key)
1223
1224
def get_parent_map(self, keys):
1225
"""See VersionedFiles.get_parent_map"""
1226
# We create a new provider because a fallback may have been added.
1227
# If we make fallbacks private we can update a stack list and avoid
1228
# object creation thrashing.
1229
keys = set(keys)
1230
result = {}
1231
if revision.NULL_REVISION in keys:
1232
keys.remove(revision.NULL_REVISION)
1233
result[revision.NULL_REVISION] = ()
1234
self._providers = self._providers[:1] + self.fallback_versionedfiles
1235
result.update(
1236
_StackedParentsProvider(self._providers).get_parent_map(keys))
1237
for key, parents in result.iteritems():
1238
if parents == ():
1239
result[key] = (revision.NULL_REVISION,)
625
1240
return result
626
1241
627
def _get_graph(self):
628
from bzrlib.graph import (
629
DictParentsProvider,
630
Graph,
631
_StackedParentsProvider,
632
)
633
from bzrlib.repofmt.knitrepo import _KnitParentsProvider
634
parent_providers = [DictParentsProvider(self._parents)]
635
for vf in self.fallback_versionedfiles:
636
parent_providers.append(_KnitParentsProvider(vf))
637
return Graph(_StackedParentsProvider(parent_providers))
638
639
1242
640
1243
class PlanWeaveMerge(TextMerge):
641
1244
"""Weave merge that takes a plan as its input.
717
1320
PlanWeaveMerge.__init__(self, plan, a_marker, b_marker)
718
1321
719
1322
720
class InterVersionedFile(InterObject):
721
"""This class represents operations taking place between two VersionedFiles.
722
723
Its instances have methods like join, and contain
724
references to the source and target versionedfiles these operations can be
725
carried out on.
726
727
Often we will provide convenience methods on 'versionedfile' which carry out
728
operations with another versionedfile - they will always forward to
729
InterVersionedFile.get(other).method_name(parameters).
730
"""
731
732
_optimisers = []
733
"""The available optimised InterVersionedFile types."""
734
735
def join(self, pb=None, msg=None, version_ids=None, ignore_missing=False):
736
"""Integrate versions from self.source into self.target.
737
738
If version_ids is None all versions from source should be
739
incorporated into this versioned file.
740
741
Must raise RevisionNotPresent if any of the specified versions
742
are not present in the other file's history unless ignore_missing is
743
supplied in which case they are silently skipped.
744
"""
745
target = self.target
746
version_ids = self._get_source_version_ids(version_ids, ignore_missing)
747
graph = Graph(self.source)
748
search = graph._make_breadth_first_searcher(version_ids)
749
transitive_ids = set()
750
map(transitive_ids.update, list(search))
751
parent_map = self.source.get_parent_map(transitive_ids)
752
order = tsort.topo_sort(parent_map.items())
753
pb = ui.ui_factory.nested_progress_bar()
754
parent_texts = {}
755
try:
756
# TODO for incremental cross-format work:
757
# make a versioned file with the following content:
758
# all revisions we have been asked to join
759
# all their ancestors that are *not* in target already.
760
# the immediate parents of the above two sets, with
761
# empty parent lists - these versions are in target already
762
# and the incorrect version data will be ignored.
763
# TODO: for all ancestors that are present in target already,
764
# check them for consistent data, this requires moving sha1 from
765
#
766
# TODO: remove parent texts when they are not relevant any more for
767
# memory pressure reduction. RBC 20060313
768
# pb.update('Converting versioned data', 0, len(order))
769
total = len(order)
770
for index, version in enumerate(order):
771
pb.update('Converting versioned data', index, total)
772
if version in target:
773
continue
774
_, _, parent_text = target.add_lines(version,
775
parent_map[version],
776
self.source.get_lines(version),
777
parent_texts=parent_texts)
778
parent_texts[version] = parent_text
779
return total
780
finally:
781
pb.finished()
782
783
def _get_source_version_ids(self, version_ids, ignore_missing):
784
"""Determine the version ids to be used from self.source.
785
786
:param version_ids: The caller-supplied version ids to check. (None
787
for all). If None is in version_ids, it is stripped.
788
:param ignore_missing: if True, remove missing ids from the version
789
list. If False, raise RevisionNotPresent on
790
a missing version id.
791
:return: A set of version ids.
792
"""
793
if version_ids is None:
794
# None cannot be in source.versions
795
return set(self.source.versions())
796
else:
797
if ignore_missing:
798
return set(self.source.versions()).intersection(set(version_ids))
799
else:
800
new_version_ids = set()
801
for version in version_ids:
802
if version is None:
803
continue
804
if not self.source.has_version(version):
805
raise errors.RevisionNotPresent(version, str(self.source))
806
else:
807
new_version_ids.add(version)
808
return new_version_ids
Loggerhead is a web-based interface for Breezy
Version: 2.0.1