22
22
"""Weave - storage of related text file versions"""
24
# before intset (r923) 2000 versions in 41.5s
25
# with intset (r926) 2000 versions in 93s !!!
26
# better to just use plain sets.
28
# making _extract build and return a list, rather than being a generator
31
# with python -O, r923 does 2000 versions in 36.87s
33
# with optimizations to avoid mutating lists - 35.75! I guess copying
34
# all the elements every time costs more than the small manipulations.
35
# a surprisingly small change.
37
# r931, which avoids using a generator for extract, does 36.98s
39
# with memoized inclusions, takes 41.49s; not very good
41
# with slots, takes 37.35s; without takes 39.16, a bit surprising
43
# with the delta calculation mixed in with the add method, rather than
44
# separated, takes 36.78s
46
# with delta folded in and mutation of the list, 36.13s
48
# with all this and simplification of add code, 33s
54
# TODO: Perhaps have copy method for Weave instances?
25
56
# XXX: If we do weaves this way, will a merge still behave the same
26
57
# way if it's done in a different order? That's a pretty desirable
44
75
# TODO: Parallel-extract that passes back each line along with a
45
76
# description of which revisions include it. Nice for checking all
46
# shas or calculating stats in parallel.
48
79
# TODO: Using a single _extract routine and then processing the output
49
80
# is probably inefficient. It's simple enough that we can afford to
50
81
# have slight specializations for different ways its used: annotate,
51
82
# basis for add, get, etc.
53
# TODO: Probably the API should work only in names to hide the integer
54
# indexes from the user.
56
# TODO: Is there any potential performance win by having an add()
57
# variant that is passed a pre-cooked version of the single basis
60
# TODO: Reweave can possibly be made faster by remembering diffs
61
# where the basis and destination are unchanged.
63
# FIXME: Sometimes we will be given a parents list for a revision
64
# that includes some redundant parents (i.e. already a parent of
65
# something in the list.) We should eliminate them. This can
66
# be done fairly efficiently because the sequence numbers constrain
67
# the possible relationships.
84
# TODO: Perhaps the API should work only in names to hide the integer
85
# indexes from the user?
72
from difflib import SequenceMatcher
74
from bzrlib.trace import mutter
75
from bzrlib.errors import (WeaveError, WeaveFormatError, WeaveParentMismatch,
76
WeaveRevisionNotPresent, WeaveRevisionAlreadyPresent)
77
import bzrlib.errors as errors
78
from bzrlib.tsort import topo_sort
91
from cStringIO import StringIO
93
from bzrlib.osutils import sha_strings
96
class WeaveError(Exception):
97
"""Exception in processing weave"""
100
class WeaveFormatError(WeaveError):
101
"""Weave invariant violated"""
81
104
class Weave(object):
82
105
"""weave - versioned text file storage.
223
228
return self._name_map[name]
225
raise WeaveRevisionNotPresent(name, self)
228
return self._names[:]
230
def iter_names(self):
231
"""Yield a list of all names in this weave."""
232
return iter(self._names)
230
raise WeaveError("name %r not present in weave %r" %
231
(name, self._weave_name))
234
234
def idx_to_name(self, version):
235
235
return self._names[version]
237
def _check_repeated_add(self, name, parents, text, sha1):
238
def _check_repeated_add(self, name, parents, text):
238
239
"""Check that a duplicated add is OK.
240
241
If it is, return the (old) index; otherwise raise an exception.
242
243
idx = self.lookup(name)
243
if sorted(self._parents[idx]) != sorted(parents) \
244
or sha1 != self._sha1s[idx]:
245
raise WeaveRevisionAlreadyPresent(name, self)
244
if sorted(self._parents[idx]) != sorted(parents):
245
raise WeaveError("name \"%s\" already present in weave "
246
"with different parents" % name)
247
new_sha1 = sha_strings(text)
248
if new_sha1 != self._sha1s[idx]:
249
raise WeaveError("name \"%s\" already present in weave "
250
"with different text" % name)
248
def add(self, name, parents, text, sha1=None):
255
def add(self, name, parents, text):
249
256
"""Add a single text on top of the weave.
251
258
Returns the index number of the newly added version.
371
373
return new_version
373
def add_identical(self, old_rev_id, new_rev_id, parents):
374
"""Add an identical text to old_rev_id as new_rev_id."""
375
old_lines = self.get(self.lookup(old_rev_id))
376
self.add(new_rev_id, parents, old_lines)
378
376
def inclusions(self, versions):
379
377
"""Return set of all ancestors of given version(s)."""
380
378
i = set(versions)
381
for v in xrange(max(versions), 0, -1):
383
# include all its parents
384
i.update(self._parents[v])
386
## except IndexError:
387
## raise ValueError("version %d not present in weave" % v)
383
# include all its parents
384
i.update(self._parents[v])
388
raise ValueError("version %d not present in weave" % v)
390
391
def parents(self, version):
391
392
return self._parents[version]
394
def parent_names(self, version):
395
"""Return version names for parents of a version."""
396
return map(self.idx_to_name, self._parents[self.lookup(version)])
399
395
def minimal_parents(self, version):
400
396
"""Find the minimal set of parents for the version."""
401
397
included = self._parents[version]
544
537
result.append((istack[-1], lineno, l))
547
raise WeaveFormatError("unclosed insertion blocks "
548
"at end of weave: %s" % istack)
541
raise WFE("unclosed insertion blocks at end of weave",
550
raise WeaveFormatError("unclosed deletion blocks at end of weave: %s"
544
raise WFE("unclosed deletion blocks at end of weave",
555
551
def get_iter(self, name_or_index):
556
552
"""Yield lines for the specified version."""
557
553
incls = [self.maybe_lookup(name_or_index)]
562
# We don't have sha1 sums for multiple entries
564
554
for origin, lineno, line in self._extract(incls):
569
expected_sha1 = self._sha1s[index]
570
measured_sha1 = cur_sha.hexdigest()
571
if measured_sha1 != expected_sha1:
572
raise errors.WeaveInvalidChecksum(
573
'file %s, revision %s, expected: %s, measured %s'
574
% (self._weave_name, self._names[index],
575
expected_sha1, measured_sha1))
578
def get_text(self, name_or_index):
579
return ''.join(self.get_iter(name_or_index))
558
def get_text(self, version):
580
559
assert isinstance(version, int)
583
def get_lines(self, name_or_index):
561
s.writelines(self.get_iter(version))
565
def get(self, name_or_index):
584
566
return list(self.get_iter(name_or_index))
590
def get_sha1(self, name):
591
"""Get the stored sha1 sum for the given revision.
593
:param name: The name of the version to lookup
595
return self._sha1s[self.lookup(name)]
597
569
def mash_iter(self, included):
598
570
"""Return composed version of multiple included versions."""
599
included = map(self.maybe_lookup, included)
600
571
for origin, lineno, text in self._extract(included):
629
601
raise WeaveFormatError("invalid included version %d for index %d"
630
602
% (inclusions[-1], version))
632
# try extracting all versions; parallel extraction is used
604
# try extracting all versions; this is a bit slow and parallel
605
# extraction could be used
633
606
nv = self.numversions()
634
sha1s = [sha.new() for i in range(nv)]
635
texts = [[] for i in range(nv)]
638
# For creating the ancestry, IntSet is much faster (3.7s vs 0.17s)
639
# The problem is that set membership is much more expensive
641
for p in self._parents[i]:
642
new_inc.update(inclusions[p])
644
#assert set(new_inc) == self.inclusions([i]), 'failed %s != %s' % (new_inc, self.inclusions([i]))
645
inclusions.append(new_inc)
647
nlines = len(self._weave)
649
update_text = 'checking weave'
651
short_name = os.path.basename(self._weave_name)
652
update_text = 'checking %s' % (short_name,)
653
update_text = update_text[:25]
655
for lineno, insert, deleteset, line in self._walk():
607
for version in range(nv):
657
progress_bar.update(update_text, lineno, nlines)
659
for j, j_inc in enumerate(inclusions):
660
# The active inclusion must be an ancestor,
661
# and no ancestors must have deleted this line,
662
# because we don't support resurrection.
663
if (insert in j_inc) and not (deleteset & j_inc):
664
sha1s[j].update(line)
666
for version in range(nv):
667
hd = sha1s[version].hexdigest()
609
progress_bar.update('checking text', version, nv)
611
for l in self.get_iter(version):
668
614
expected = self._sha1s[version]
669
615
if hd != expected:
670
raise errors.WeaveInvalidChecksum(
671
"mismatched sha1 for version %s: "
672
"got %s, expected %s"
673
% (self._names[version], hd, expected))
616
raise WeaveError("mismatched sha1 for version %d; "
617
"got %s, expected %s"
618
% (version, hd, expected))
675
620
# TODO: check insertions are properly nested, that there are
676
621
# no lines outside of insertion blocks, that deletions are
677
622
# properly paired, etc.
626
def merge(self, merge_versions):
627
"""Automerge and mark conflicts between versions.
629
This returns a sequence, each entry describing alternatives
630
for a chunk of the file. Each of the alternatives is given as
633
If there is a chunk of the file where there's no diagreement,
634
only one alternative is given.
637
# approach: find the included versions common to all the
639
raise NotImplementedError()
679
643
def _delta(self, included, lines):
680
644
"""Return changes from basis to new revision.
798
758
'killed-both'), \
802
def join(self, other, pb=None, msg=None):
804
"""Integrate versions from other into this weave.
806
The resulting weave contains all the history of both weaves;
807
any version you could retrieve from either self or other can be
808
retrieved from self after this call.
810
It is illegal for the two weaves to contain different values
811
or different parents for any version. See also reweave().
813
:param other: The other weave to pull into this one
814
:param pb: An optional progress bar
815
:param msg: An optional message to display for progress
817
if other.numversions() == 0:
818
return # nothing to update, easy
819
# two loops so that we do not change ourselves before verifying it
821
# work through in index order to make sure we get all dependencies
822
for other_idx, name in enumerate(other._names):
823
self._check_version_consistent(other, other_idx, name)
831
for other_idx, name in enumerate(other._names):
832
# TODO: If all the parents of the other version are already
833
# present then we can avoid some work by just taking the delta
834
# and adjusting the offsets.
835
new_parents = self._imported_parents(other, other_idx)
836
sha1 = other._sha1s[other_idx]
840
if name in self._name_map:
841
idx = self.lookup(name)
842
n1 = set(map(other.idx_to_name, other._parents[other_idx]))
843
n2 = set(map(self.idx_to_name, self._parents[idx]))
844
if sha1 == self._sha1s[idx] and n1 == n2:
848
pb.update(msg, other_idx, len(other._names))
851
lines = other.get_lines(other_idx)
852
self.add(name, new_parents, lines, sha1)
854
mutter("merged = %d, processed = %d, file_id=%s; deltat=%d"%(
855
merged,processed,self._weave_name, time.time( )-time0))
860
def _imported_parents(self, other, other_idx):
861
"""Return list of parents in self corresponding to indexes in other."""
863
for parent_idx in other._parents[other_idx]:
864
parent_name = other._names[parent_idx]
865
if parent_name not in self._names:
866
# should not be possible
867
raise WeaveError("missing parent {%s} of {%s} in %r"
868
% (parent_name, other._name_map[other_idx], self))
869
new_parents.append(self._name_map[parent_name])
872
def _check_version_consistent(self, other, other_idx, name):
873
"""Check if a version in consistent in this and other.
875
To be consistent it must have:
878
* the same direct parents (by name, not index, and disregarding
881
If present & correct return True;
882
if not present in self return False;
883
if inconsistent raise error."""
884
this_idx = self._name_map.get(name, -1)
886
if self._sha1s[this_idx] != other._sha1s[other_idx]:
887
raise WeaveError("inconsistent texts for version {%s} "
888
"when joining weaves"
890
self_parents = self._parents[this_idx]
891
other_parents = other._parents[other_idx]
892
n1 = set([self._names[i] for i in self_parents])
893
n2 = set([other._names[i] for i in other_parents])
895
raise WeaveParentMismatch("inconsistent parents "
896
"for version {%s}: %s vs %s" % (name, n1, n2))
902
def reweave(self, other, pb=None, msg=None):
903
"""Reweave self with other.
905
:param other: The other weave to merge
906
:param pb: An optional progress bar, indicating how far done we are
907
:param msg: An optional message for the progress
909
new_weave = reweave(self, other, pb=pb, msg=msg)
910
for attr in self.__slots__:
911
setattr(self, attr, getattr(new_weave, attr))
914
def reweave(wa, wb, pb=None, msg=None):
915
"""Combine two weaves and return the result.
917
This works even if a revision R has different parents in
918
wa and wb. In the resulting weave all the parents are given.
920
This is done by just building up a new weave, maintaining ordering
921
of the versions in the two inputs. More efficient approaches
922
might be possible but it should only be necessary to do
923
this operation rarely, when a new previously ghost version is
926
:param pb: An optional progress bar, indicating how far done we are
927
:param msg: An optional message for the progress
931
queue_a = range(wa.numversions())
932
queue_b = range(wb.numversions())
933
# first determine combined parents of all versions
934
# map from version name -> all parent names
935
combined_parents = _reweave_parent_graphs(wa, wb)
936
mutter("combined parents: %r", combined_parents)
937
order = topo_sort(combined_parents.iteritems())
938
mutter("order to reweave: %r", order)
943
for idx, name in enumerate(order):
945
pb.update(msg, idx, len(order))
946
if name in wa._name_map:
947
lines = wa.get_lines(name)
948
if name in wb._name_map:
949
lines_b = wb.get_lines(name)
951
mutter('Weaves differ on content. rev_id {%s}', name)
952
mutter('weaves: %s, %s', wa._weave_name, wb._weave_name)
954
lines = list(difflib.unified_diff(lines, lines_b,
955
wa._weave_name, wb._weave_name))
956
mutter('lines:\n%s', ''.join(lines))
957
raise errors.WeaveTextDiffers(name, wa, wb)
959
lines = wb.get_lines(name)
960
wr.add(name, combined_parents[name], lines)
964
def _reweave_parent_graphs(wa, wb):
965
"""Return combined parent ancestry for two weaves.
967
Returned as a list of (version_name, set(parent_names))"""
969
for weave in [wa, wb]:
970
for idx, name in enumerate(weave._names):
971
p = combined.setdefault(name, set())
972
p.update(map(weave.idx_to_name, weave._parents[idx]))
976
767
def weave_toc(w):
added
removed
bzrlib/weave.py
