22
22
"""Weave - storage of related text file versions"""
24
# before intset (r923) 2000 versions in 41.5s
25
# with intset (r926) 2000 versions in 93s !!!
26
# better to just use plain sets.
28
# making _extract build and return a list, rather than being a generator
31
# with python -O, r923 does 2000 versions in 36.87s
33
# with optimizations to avoid mutating lists - 35.75! I guess copying
34
# all the elements every time costs more than the small manipulations.
35
# a surprisingly small change.
37
# r931, which avoids using a generator for extract, does 36.98s
39
# with memoized inclusions, takes 41.49s; not very good
41
# with slots, takes 37.35s; without takes 39.16, a bit surprising
43
# with the delta calculation mixed in with the add method, rather than
44
# separated, takes 36.78s
46
# with delta folded in and mutation of the list, 36.13s
48
# with all this and simplification of add code, 33s
54
# TODO: Perhaps have copy method for Weave instances?
56
25
# XXX: If we do weaves this way, will a merge still behave the same
57
26
# way if it's done in a different order? That's a pretty desirable
75
44
# TODO: Parallel-extract that passes back each line along with a
76
45
# description of which revisions include it. Nice for checking all
46
# shas or calculating stats in parallel.
79
48
# TODO: Using a single _extract routine and then processing the output
80
49
# is probably inefficient. It's simple enough that we can afford to
81
50
# have slight specializations for different ways its used: annotate,
82
51
# basis for add, get, etc.
84
# TODO: Perhaps the API should work only in names to hide the integer
85
# indexes from the user?
53
# TODO: Probably the API should work only in names to hide the integer
54
# indexes from the user.
56
# TODO: Is there any potential performance win by having an add()
57
# variant that is passed a pre-cooked version of the single basis
60
# TODO: Reweave can possibly be made faster by remembering diffs
61
# where the basis and destination are unchanged.
63
# FIXME: Sometimes we will be given a parents list for a revision
64
# that includes some redundant parents (i.e. already a parent of
65
# something in the list.) We should eliminate them. This can
66
# be done fairly efficiently because the sequence numbers constrain
67
# the possible relationships.
93
class WeaveError(Exception):
94
"""Exception in processing weave"""
97
class WeaveFormatError(WeaveError):
98
"""Weave invariant violated"""
71
from difflib import SequenceMatcher
73
from bzrlib.trace import mutter
74
from bzrlib.errors import (WeaveError, WeaveFormatError, WeaveParentMismatch,
75
WeaveRevisionNotPresent, WeaveRevisionAlreadyPresent)
76
import bzrlib.errors as errors
77
from bzrlib.tsort import topo_sort
101
80
class Weave(object):
102
81
"""weave - versioned text file storage.
183
162
For each name, the version number.
165
Descriptive name of this weave; typically the filename if known.
186
__slots__ = ['_weave', '_parents', '_sha1s', '_names', '_name_map']
169
__slots__ = ['_weave', '_parents', '_sha1s', '_names', '_name_map',
172
def __init__(self, weave_name=None):
190
174
self._parents = []
193
177
self._name_map = {}
178
self._weave_name = weave_name
181
return "Weave(%r)" % self._weave_name
185
"""Return a deep copy of self.
187
The copy can be modified without affecting the original weave."""
189
other._weave = self._weave[:]
190
other._parents = self._parents[:]
191
other._sha1s = self._sha1s[:]
192
other._names = self._names[:]
193
other._name_map = self._name_map.copy()
194
other._weave_name = self._weave_name
196
197
def __eq__(self, other):
197
198
if not isinstance(other, Weave):
204
205
def __ne__(self, other):
205
206
return not self.__eq__(other)
208
def __contains__(self, name):
209
return self._name_map.has_key(name)
211
def maybe_lookup(self, name_or_index):
212
"""Convert possible symbolic name to index, or pass through indexes."""
213
if isinstance(name_or_index, (int, long)):
216
return self.lookup(name_or_index)
208
219
def lookup(self, name):
220
"""Convert symbolic version name to index."""
210
222
return self._name_map[name]
212
raise WeaveError("name %s not present in weave" % name)
224
raise WeaveRevisionNotPresent(name, self)
227
return self._names[:]
229
def iter_names(self):
230
"""Yield a list of all names in this weave."""
231
return iter(self._names)
233
def idx_to_name(self, version):
234
return self._names[version]
236
def _check_repeated_add(self, name, parents, text, sha1):
237
"""Check that a duplicated add is OK.
239
If it is, return the (old) index; otherwise raise an exception.
241
idx = self.lookup(name)
242
if sorted(self._parents[idx]) != sorted(parents) \
243
or sha1 != self._sha1s[idx]:
244
raise WeaveRevisionAlreadyPresent(name, self)
215
def add(self, name, parents, text):
247
def add(self, name, parents, text, sha1=None):
216
248
"""Add a single text on top of the weave.
218
250
Returns the index number of the newly added version.
225
257
List or set of direct parent version numbers.
228
Sequence of lines to be added in the new version."""
260
Sequence of lines to be added in the new version.
262
sha -- SHA-1 of the file, if known. This is trusted to be
265
from bzrlib.osutils import sha_strings
230
267
assert isinstance(name, basestring)
269
sha1 = sha_strings(text)
231
270
if name in self._name_map:
232
raise WeaveError("name %r already present in weave" % name)
271
return self._check_repeated_add(name, parents, text, sha1)
273
parents = map(self.maybe_lookup, parents)
234
274
self._check_versions(parents)
235
275
## self._check_lines(text)
236
276
new_version = len(self._parents)
243
279
# if we abort after here the (in-memory) weave will be corrupt because only
244
280
# some fields are updated
335
370
return new_version
372
def add_identical(self, old_rev_id, new_rev_id, parents):
373
"""Add an identical text to old_rev_id as new_rev_id."""
374
old_lines = self.get(self.lookup(old_rev_id))
375
self.add(new_rev_id, parents, old_lines)
338
377
def inclusions(self, versions):
339
378
"""Return set of all ancestors of given version(s)."""
340
379
i = set(versions)
345
# include all its parents
346
i.update(self._parents[v])
350
raise ValueError("version %d not present in weave" % v)
380
for v in xrange(max(versions), 0, -1):
382
# include all its parents
383
i.update(self._parents[v])
385
## except IndexError:
386
## raise ValueError("version %d not present in weave" % v)
389
def parents(self, version):
390
return self._parents[version]
393
def parent_names(self, version):
394
"""Return version names for parents of a version."""
395
return map(self.idx_to_name, self._parents[self.lookup(version)])
353
398
def minimal_parents(self, version):
490
538
result.append((istack[-1], lineno, l))
494
raise WFE("unclosed insertion blocks at end of weave",
541
raise WeaveFormatError("unclosed insertion blocks "
542
"at end of weave: %s" % istack)
497
raise WFE("unclosed deletion blocks at end of weave",
544
raise WeaveFormatError("unclosed deletion blocks at end of weave: %s"
504
def get_iter(self, version):
549
def get_iter(self, name_or_index):
505
550
"""Yield lines for the specified version."""
506
for origin, lineno, line in self._extract([version]):
551
incls = [self.maybe_lookup(name_or_index)]
556
# We don't have sha1 sums for multiple entries
558
for origin, lineno, line in self._extract(incls):
510
def get(self, index):
511
return list(self.get_iter(index))
563
expected_sha1 = self._sha1s[index]
564
measured_sha1 = s.hexdigest()
565
if measured_sha1 != expected_sha1:
566
raise errors.WeaveInvalidChecksum(
567
'file %s, revision %s, expected: %s, measured %s'
568
% (self._weave_name, self._names[index],
569
expected_sha1, measured_sha1))
572
def get_text(self, name_or_index):
573
return ''.join(self.get_iter(name_or_index))
574
assert isinstance(version, int)
577
def get_lines(self, name_or_index):
578
return list(self.get_iter(name_or_index))
514
584
def mash_iter(self, included):
515
585
"""Return composed version of multiple included versions."""
586
included = map(self.maybe_lookup, included)
516
587
for origin, lineno, text in self._extract(included):
703
760
'killed-both'), \
764
def join(self, other):
766
"""Integrate versions from other into this weave.
768
The resulting weave contains all the history of both weaves;
769
any version you could retrieve from either self or other can be
770
retrieved from self after this call.
772
It is illegal for the two weaves to contain different values
773
or different parents for any version. See also reweave().
775
if other.numversions() == 0:
776
return # nothing to update, easy
777
# two loops so that we do not change ourselves before verifying it
779
# work through in index order to make sure we get all dependencies
780
for other_idx, name in enumerate(other._names):
781
self._check_version_consistent(other, other_idx, name)
786
for other_idx, name in enumerate(other._names):
787
# TODO: If all the parents of the other version are already
788
# present then we can avoid some work by just taking the delta
789
# and adjusting the offsets.
790
new_parents = self._imported_parents(other, other_idx)
791
sha1 = other._sha1s[other_idx]
795
if name in self._names:
796
idx = self.lookup(name)
797
n1 = map(other.idx_to_name, other._parents[other_idx] )
798
n2 = map(self.idx_to_name, self._parents[other_idx] )
799
if sha1 == self._sha1s[idx] and n1 == n2:
803
lines = other.get_lines(other_idx)
804
self.add(name, new_parents, lines, sha1)
806
mutter("merged = %d, processed = %d, file_id=%s; deltat=%d"%(
807
merged,processed,self._weave_name, time.time( )-time0))
812
def _imported_parents(self, other, other_idx):
813
"""Return list of parents in self corresponding to indexes in other."""
815
for parent_idx in other._parents[other_idx]:
816
parent_name = other._names[parent_idx]
817
if parent_name not in self._names:
818
# should not be possible
819
raise WeaveError("missing parent {%s} of {%s} in %r"
820
% (parent_name, other._name_map[other_idx], self))
821
new_parents.append(self._name_map[parent_name])
824
def _check_version_consistent(self, other, other_idx, name):
825
"""Check if a version in consistent in this and other.
827
To be consistent it must have:
830
* the same direct parents (by name, not index, and disregarding
833
If present & correct return True;
834
if not present in self return False;
835
if inconsistent raise error."""
836
this_idx = self._name_map.get(name, -1)
838
if self._sha1s[this_idx] != other._sha1s[other_idx]:
839
raise WeaveError("inconsistent texts for version {%s} "
840
"when joining weaves"
842
self_parents = self._parents[this_idx]
843
other_parents = other._parents[other_idx]
844
n1 = [self._names[i] for i in self_parents]
845
n2 = [other._names[i] for i in other_parents]
849
raise WeaveParentMismatch("inconsistent parents "
850
"for version {%s}: %s vs %s" % (name, n1, n2))
856
def reweave(self, other):
857
"""Reweave self with other."""
858
new_weave = reweave(self, other)
859
for attr in self.__slots__:
860
setattr(self, attr, getattr(new_weave, attr))
864
"""Combine two weaves and return the result.
866
This works even if a revision R has different parents in
867
wa and wb. In the resulting weave all the parents are given.
869
This is done by just building up a new weave, maintaining ordering
870
of the versions in the two inputs. More efficient approaches
871
might be possible but it should only be necessary to do
872
this operation rarely, when a new previously ghost version is
877
queue_a = range(wa.numversions())
878
queue_b = range(wb.numversions())
879
# first determine combined parents of all versions
880
# map from version name -> all parent names
881
combined_parents = _reweave_parent_graphs(wa, wb)
882
mutter("combined parents: %r", combined_parents)
883
order = topo_sort(combined_parents.iteritems())
884
mutter("order to reweave: %r", order)
886
if name in wa._name_map:
887
lines = wa.get_lines(name)
888
if name in wb._name_map:
889
assert lines == wb.get_lines(name)
891
lines = wb.get_lines(name)
892
wr.add(name, combined_parents[name], lines)
896
def _reweave_parent_graphs(wa, wb):
897
"""Return combined parent ancestry for two weaves.
899
Returned as a list of (version_name, set(parent_names))"""
901
for weave in [wa, wb]:
902
for idx, name in enumerate(weave._names):
903
p = combined.setdefault(name, set())
904
p.update(map(weave.idx_to_name, weave._parents[idx]))
712
908
def weave_toc(w):
added
removed
bzrlib/weave.py
