22
22
"""Weave - storage of related text file versions"""
24
# before intset (r923) 2000 versions in 41.5s
25
# with intset (r926) 2000 versions in 93s !!!
26
# better to just use plain sets.
28
# making _extract build and return a list, rather than being a generator
31
# with python -O, r923 does 2000 versions in 36.87s
33
# with optimizations to avoid mutating lists - 35.75! I guess copying
34
# all the elements every time costs more than the small manipulations.
35
# a surprisingly small change.
37
# r931, which avoids using a generator for extract, does 36.98s
39
# with memoized inclusions, takes 41.49s; not very good
41
# with slots, takes 37.35s; without takes 39.16, a bit surprising
43
# with the delta calculation mixed in with the add method, rather than
44
# separated, takes 36.78s
46
# with delta folded in and mutation of the list, 36.13s
48
# with all this and simplification of add code, 33s
54
# TODO: Perhaps have copy method for Weave instances?
56
25
# XXX: If we do weaves this way, will a merge still behave the same
57
26
# way if it's done in a different order? That's a pretty desirable
75
44
# TODO: Parallel-extract that passes back each line along with a
76
45
# description of which revisions include it. Nice for checking all
46
# shas or calculating stats in parallel.
79
48
# TODO: Using a single _extract routine and then processing the output
80
49
# is probably inefficient. It's simple enough that we can afford to
81
50
# have slight specializations for different ways its used: annotate,
82
51
# basis for add, get, etc.
84
# TODO: Perhaps the API should work only in names to hide the integer
85
# indexes from the user?
53
# TODO: Probably the API should work only in names to hide the integer
54
# indexes from the user.
56
# TODO: Is there any potential performance win by having an add()
57
# variant that is passed a pre-cooked version of the single basis
60
# TODO: Reweave can possibly be made faster by remembering diffs
61
# where the basis and destination are unchanged.
63
# FIXME: Sometimes we will be given a parents list for a revision
64
# that includes some redundant parents (i.e. already a parent of
65
# something in the list.) We should eliminate them. This can
66
# be done fairly efficiently because the sequence numbers constrain
67
# the possible relationships.
90
from cStringIO import StringIO
93
class WeaveError(Exception):
94
"""Exception in processing weave"""
97
class WeaveFormatError(WeaveError):
98
"""Weave invariant violated"""
71
from difflib import SequenceMatcher
73
from bzrlib.trace import mutter
74
from bzrlib.errors import WeaveError, WeaveFormatError, WeaveParentMismatch, \
75
WeaveRevisionNotPresent, WeaveRevisionAlreadyPresent
76
from bzrlib.tsort import topo_sort
101
79
class Weave(object):
102
80
"""weave - versioned text file storage.
210
204
def __ne__(self, other):
211
205
return not self.__eq__(other)
207
def __contains__(self, name):
208
return self._name_map.has_key(name)
210
def maybe_lookup(self, name_or_index):
211
"""Convert possible symbolic name to index, or pass through indexes."""
212
if isinstance(name_or_index, (int, long)):
215
return self.lookup(name_or_index)
214
218
def lookup(self, name):
219
"""Convert symbolic version name to index."""
216
221
return self._name_map[name]
218
raise WeaveError("name %s not present in weave %s" %
219
(name, self._weave_name))
223
raise WeaveRevisionNotPresent(name, self)
226
return self._names[:]
228
def iter_names(self):
229
"""Yield a list of all names in this weave."""
230
return iter(self._names)
232
def idx_to_name(self, version):
233
return self._names[version]
235
def _check_repeated_add(self, name, parents, text, sha1):
236
"""Check that a duplicated add is OK.
238
If it is, return the (old) index; otherwise raise an exception.
240
idx = self.lookup(name)
241
if sorted(self._parents[idx]) != sorted(parents) \
242
or sha1 != self._sha1s[idx]:
243
raise WeaveRevisionAlreadyPresent(name, self)
222
def add(self, name, parents, text):
246
def add(self, name, parents, text, sha1=None):
223
247
"""Add a single text on top of the weave.
225
249
Returns the index number of the newly added version.
232
256
List or set of direct parent version numbers.
235
Sequence of lines to be added in the new version."""
259
Sequence of lines to be added in the new version.
261
sha -- SHA-1 of the file, if known. This is trusted to be
264
from bzrlib.osutils import sha_strings
237
266
assert isinstance(name, basestring)
268
sha1 = sha_strings(text)
238
269
if name in self._name_map:
239
raise WeaveError("name %r already present in weave" % name)
270
return self._check_repeated_add(name, parents, text, sha1)
272
parents = map(self.maybe_lookup, parents)
241
273
self._check_versions(parents)
242
274
## self._check_lines(text)
243
275
new_version = len(self._parents)
250
278
# if we abort after here the (in-memory) weave will be corrupt because only
251
279
# some fields are updated
342
369
return new_version
371
def add_identical(self, old_rev_id, new_rev_id, parents):
372
"""Add an identical text to old_rev_id as new_rev_id."""
373
old_lines = self.get(self.lookup(old_rev_id))
374
self.add(new_rev_id, parents, old_lines)
345
376
def inclusions(self, versions):
346
377
"""Return set of all ancestors of given version(s)."""
347
378
i = set(versions)
352
# include all its parents
353
i.update(self._parents[v])
357
raise ValueError("version %d not present in weave" % v)
379
for v in xrange(max(versions), 0, -1):
381
# include all its parents
382
i.update(self._parents[v])
384
## except IndexError:
385
## raise ValueError("version %d not present in weave" % v)
388
def parents(self, version):
389
return self._parents[version]
392
def parent_names(self, version):
393
"""Return version names for parents of a version."""
394
return map(self.idx_to_name, self._parents[self.lookup(version)])
360
397
def minimal_parents(self, version):
400
437
raise IndexError("invalid version number %r" % i)
403
def annotate(self, index):
404
return list(self.annotate_iter(index))
407
def annotate_iter(self, version):
440
def annotate(self, name_or_index):
441
return list(self.annotate_iter(name_or_index))
444
def annotate_iter(self, name_or_index):
408
445
"""Yield list of (index-id, line) pairs for the specified version.
410
447
The index indicates when the line originated in the weave."""
411
for origin, lineno, text in self._extract([version]):
448
incls = [self.maybe_lookup(name_or_index)]
449
for origin, lineno, text in self._extract(incls):
412
450
yield origin, text
416
453
"""Walk the weave.
501
537
result.append((istack[-1], lineno, l))
505
raise WFE("unclosed insertion blocks at end of weave",
540
raise WeaveFormatError("unclosed insertion blocks "
541
"at end of weave: %s" % istack)
508
raise WFE("unclosed deletion blocks at end of weave",
543
raise WeaveFormatError("unclosed deletion blocks at end of weave: %s"
515
def get_iter(self, version):
548
def get_iter(self, name_or_index):
516
549
"""Yield lines for the specified version."""
517
for origin, lineno, line in self._extract([version]):
550
incls = [self.maybe_lookup(name_or_index)]
551
for origin, lineno, line in self._extract(incls):
521
def get_text(self, version):
555
def get_text(self, name_or_index):
556
return ''.join(self.get_iter(name_or_index))
522
557
assert isinstance(version, int)
524
s.writelines(self.get_iter(version))
528
def get(self, index):
529
return list(self.get_iter(index))
560
def get_lines(self, name_or_index):
561
return list(self.get_iter(name_or_index))
532
567
def mash_iter(self, included):
533
568
"""Return composed version of multiple included versions."""
569
included = map(self.maybe_lookup, included)
534
570
for origin, lineno, text in self._extract(included):
743
def join(self, other):
744
"""Integrate versions from other into this weave.
746
The resulting weave contains all the history of both weaves;
747
any version you could retrieve from either self or other can be
748
retrieved from self after this call.
750
It is illegal for the two weaves to contain different values
751
or different parents for any version. See also reweave().
753
if other.numversions() == 0:
754
return # nothing to update, easy
755
# two loops so that we do not change ourselves before verifying it
757
# work through in index order to make sure we get all dependencies
758
for other_idx, name in enumerate(other._names):
759
if self._check_version_consistent(other, other_idx, name):
761
for other_idx, name in enumerate(other._names):
762
# TODO: If all the parents of the other version are already
763
# present then we can avoid some work by just taking the delta
764
# and adjusting the offsets.
765
new_parents = self._imported_parents(other, other_idx)
766
lines = other.get_lines(other_idx)
767
sha1 = other._sha1s[other_idx]
768
self.add(name, new_parents, lines, sha1)
771
def _imported_parents(self, other, other_idx):
772
"""Return list of parents in self corresponding to indexes in other."""
774
for parent_idx in other._parents[other_idx]:
775
parent_name = other._names[parent_idx]
776
if parent_name not in self._names:
777
# should not be possible
778
raise WeaveError("missing parent {%s} of {%s} in %r"
779
% (parent_name, other._name_map[other_idx], self))
780
new_parents.append(self._name_map[parent_name])
783
def _check_version_consistent(self, other, other_idx, name):
784
"""Check if a version in consistent in this and other.
786
To be consistent it must have:
789
* the same direct parents (by name, not index, and disregarding
792
If present & correct return True;
793
if not present in self return False;
794
if inconsistent raise error."""
795
this_idx = self._name_map.get(name, -1)
797
if self._sha1s[this_idx] != other._sha1s[other_idx]:
798
raise WeaveError("inconsistent texts for version {%s} "
799
"when joining weaves"
801
self_parents = self._parents[this_idx]
802
other_parents = other._parents[other_idx]
803
n1 = [self._names[i] for i in self_parents]
804
n2 = [other._names[i] for i in other_parents]
808
raise WeaveParentMismatch("inconsistent parents "
809
"for version {%s}: %s vs %s" % (name, n1, n2))
815
def reweave(self, other):
816
"""Reweave self with other."""
817
new_weave = reweave(self, other)
818
for attr in self.__slots__:
819
setattr(self, attr, getattr(new_weave, attr))
823
"""Combine two weaves and return the result.
825
This works even if a revision R has different parents in
826
wa and wb. In the resulting weave all the parents are given.
828
This is done by just building up a new weave, maintaining ordering
829
of the versions in the two inputs. More efficient approaches
830
might be possible but it should only be necessary to do
831
this operation rarely, when a new previously ghost version is
836
queue_a = range(wa.numversions())
837
queue_b = range(wb.numversions())
838
# first determine combined parents of all versions
839
# map from version name -> all parent names
840
combined_parents = _reweave_parent_graphs(wa, wb)
841
mutter("combined parents: %r", combined_parents)
842
order = topo_sort(combined_parents.iteritems())
843
mutter("order to reweave: %r", order)
845
if name in wa._name_map:
846
lines = wa.get_lines(name)
847
if name in wb._name_map:
848
assert lines == wb.get_lines(name)
850
lines = wb.get_lines(name)
851
wr.add(name, combined_parents[name], lines)
855
def _reweave_parent_graphs(wa, wb):
856
"""Return combined parent ancestry for two weaves.
858
Returned as a list of (version_name, set(parent_names))"""
860
for weave in [wa, wb]:
861
for idx, name in enumerate(weave._names):
862
p = combined.setdefault(name, set())
863
p.update(map(weave.idx_to_name, weave._parents[idx]))
730
867
def weave_toc(w):
added
removed
bzrlib/weave.py
