22
22
"""Weave - storage of related text file versions"""
24
# before intset (r923) 2000 versions in 41.5s
25
# with intset (r926) 2000 versions in 93s !!!
26
# better to just use plain sets.
28
# making _extract build and return a list, rather than being a generator
31
# with python -O, r923 does 2000 versions in 36.87s
33
# with optimizations to avoid mutating lists - 35.75! I guess copying
34
# all the elements every time costs more than the small manipulations.
35
# a surprisingly small change.
37
# r931, which avoids using a generator for extract, does 36.98s
39
# with memoized inclusions, takes 41.49s; not very good
41
# with slots, takes 37.35s; without takes 39.16, a bit surprising
43
# with the delta calculation mixed in with the add method, rather than
44
# separated, takes 36.78s
46
# with delta folded in and mutation of the list, 36.13s
48
# with all this and simplification of add code, 33s
51
# TODO: Perhaps have copy method for Weave instances?
25
53
# XXX: If we do weaves this way, will a merge still behave the same
26
54
# way if it's done in a different order? That's a pretty desirable
44
74
# TODO: Parallel-extract that passes back each line along with a
45
75
# description of which revisions include it. Nice for checking all
46
# shas or calculating stats in parallel.
48
# TODO: Using a single _extract routine and then processing the output
49
# is probably inefficient. It's simple enough that we can afford to
50
# have slight specializations for different ways its used: annotate,
51
# basis for add, get, etc.
53
# TODO: Probably the API should work only in names to hide the integer
54
# indexes from the user.
56
# TODO: Is there any potential performance win by having an add()
57
# variant that is passed a pre-cooked version of the single basis
60
# TODO: Reweave can possibly be made faster by remembering diffs
61
# where the basis and destination are unchanged.
63
# FIXME: Sometimes we will be given a parents list for a revision
64
# that includes some redundant parents (i.e. already a parent of
65
# something in the list.) We should eliminate them. This can
66
# be done fairly efficiently because the sequence numbers constrain
67
# the possible relationships.
71
from difflib import SequenceMatcher
73
from bzrlib.trace import mutter
74
from bzrlib.errors import WeaveError, WeaveFormatError, WeaveParentMismatch, \
75
WeaveRevisionNotPresent, WeaveRevisionAlreadyPresent
76
from bzrlib.tsort import topo_sort
81
class WeaveError(Exception):
82
"""Exception in processing weave"""
85
class WeaveFormatError(WeaveError):
86
"""Weave invariant violated"""
79
89
class Weave(object):
80
90
"""weave - versioned text file storage.
152
162
each version; the parent's parents are implied.
155
List of hex SHA-1 of each version.
158
List of symbolic names for each version. Each should be unique.
161
For each name, the version number.
164
Descriptive name of this weave; typically the filename if known.
165
List of hex SHA-1 of each version, or None if not recorded.
168
__slots__ = ['_weave', '_parents', '_sha1s', '_names', '_name_map',
168
__slots__ = ['_weave', '_parents', '_sha1s']
171
def __init__(self, weave_name=None):
173
172
self._parents = []
177
self._weave_name = weave_name
181
"""Return a deep copy of self.
183
The copy can be modified without affecting the original weave."""
185
other._weave = self._weave[:]
186
other._parents = self._parents[:]
187
other._sha1s = self._sha1s[:]
188
other._names = self._names[:]
189
other._name_map = self._name_map.copy()
190
other._weave_name = self._weave_name
193
176
def __eq__(self, other):
194
177
if not isinstance(other, Weave):
196
179
return self._parents == other._parents \
197
and self._weave == other._weave \
198
and self._sha1s == other._sha1s
180
and self._weave == other._weave
201
183
def __ne__(self, other):
202
184
return not self.__eq__(other)
204
def __contains__(self, name):
205
return self._name_map.has_key(name)
207
def maybe_lookup(self, name_or_index):
208
"""Convert possible symbolic name to index, or pass through indexes."""
209
if isinstance(name_or_index, (int, long)):
212
return self.lookup(name_or_index)
215
def lookup(self, name):
216
"""Convert symbolic version name to index."""
218
return self._name_map[name]
220
raise WeaveRevisionNotPresent(name, self)
223
return self._names[:]
225
def iter_names(self):
226
"""Yield a list of all names in this weave."""
227
return iter(self._names)
229
def idx_to_name(self, version):
230
return self._names[version]
232
def _check_repeated_add(self, name, parents, text, sha1):
233
"""Check that a duplicated add is OK.
235
If it is, return the (old) index; otherwise raise an exception.
237
idx = self.lookup(name)
238
if sorted(self._parents[idx]) != sorted(parents) \
239
or sha1 != self._sha1s[idx]:
240
raise WeaveRevisionAlreadyPresent(name, self)
243
def add(self, name, parents, text, sha1=None):
187
def add(self, parents, text):
244
188
"""Add a single text on top of the weave.
246
190
Returns the index number of the newly added version.
249
Symbolic name for this version.
250
(Typically the revision-id of the revision that added it.)
253
193
List or set of direct parent version numbers.
256
Sequence of lines to be added in the new version.
258
sha -- SHA-1 of the file, if known. This is trusted to be
261
from bzrlib.osutils import sha_strings
263
assert isinstance(name, basestring)
265
sha1 = sha_strings(text)
266
if name in self._name_map:
267
return self._check_repeated_add(name, parents, text, sha1)
269
parents = map(self.maybe_lookup, parents)
196
Sequence of lines to be added in the new version."""
270
198
self._check_versions(parents)
271
199
## self._check_lines(text)
272
200
new_version = len(self._parents)
275
# if we abort after here the (in-memory) weave will be corrupt because only
276
# some fields are updated
277
self._parents.append(parents[:])
208
# if we abort after here the weave will be corrupt
209
self._parents.append(frozenset(parents))
278
210
self._sha1s.append(sha1)
279
self._names.append(name)
280
self._name_map[name] = new_version
366
297
return new_version
368
def add_identical(self, old_rev_id, new_rev_id, parents):
369
"""Add an identical text to old_rev_id as new_rev_id."""
370
old_lines = self.get(self.lookup(old_rev_id))
371
self.add(new_rev_id, parents, old_lines)
373
300
def inclusions(self, versions):
374
301
"""Return set of all ancestors of given version(s)."""
375
302
i = set(versions)
376
for v in xrange(max(versions), 0, -1):
378
# include all its parents
379
i.update(self._parents[v])
381
## except IndexError:
382
## raise ValueError("version %d not present in weave" % v)
385
def parents(self, version):
386
return self._parents[version]
389
def parent_names(self, version):
390
"""Return version names for parents of a version."""
391
return map(self.idx_to_name, self._parents[self.lookup(version)])
307
# include all its parents
308
i.update(self._parents[v])
312
raise ValueError("version %d not present in weave" % v)
394
315
def minimal_parents(self, version):
534
452
result.append((istack[-1], lineno, l))
537
raise WeaveFormatError("unclosed insertion blocks "
538
"at end of weave: %s" % istack)
456
raise WFE("unclosed insertion blocks at end of weave",
540
raise WeaveFormatError("unclosed deletion blocks at end of weave: %s"
459
raise WFE("unclosed deletion blocks at end of weave",
545
def get_iter(self, name_or_index):
466
def get_iter(self, version):
546
467
"""Yield lines for the specified version."""
547
incls = [self.maybe_lookup(name_or_index)]
548
for origin, lineno, line in self._extract(incls):
468
for origin, lineno, line in self._extract([version]):
552
def get_text(self, name_or_index):
553
return ''.join(self.get_iter(name_or_index))
554
assert isinstance(version, int)
557
def get_lines(self, name_or_index):
558
return list(self.get_iter(name_or_index))
472
def get(self, index):
473
return list(self.get_iter(index))
564
476
def mash_iter(self, included):
565
477
"""Return composed version of multiple included versions."""
566
included = map(self.maybe_lookup, included)
567
478
for origin, lineno, text in self._extract(included):
757
def join(self, other):
758
"""Integrate versions from other into this weave.
760
The resulting weave contains all the history of both weaves;
761
any version you could retrieve from either self or other can be
762
retrieved from self after this call.
764
It is illegal for the two weaves to contain different values
765
or different parents for any version. See also reweave().
767
if other.numversions() == 0:
768
return # nothing to update, easy
769
# two loops so that we do not change ourselves before verifying it
771
# work through in index order to make sure we get all dependencies
772
for other_idx, name in enumerate(other._names):
773
if self._check_version_consistent(other, other_idx, name):
775
for other_idx, name in enumerate(other._names):
776
# TODO: If all the parents of the other version are already
777
# present then we can avoid some work by just taking the delta
778
# and adjusting the offsets.
779
new_parents = self._imported_parents(other, other_idx)
780
lines = other.get_lines(other_idx)
781
sha1 = other._sha1s[other_idx]
782
self.add(name, new_parents, lines, sha1)
785
def _imported_parents(self, other, other_idx):
786
"""Return list of parents in self corresponding to indexes in other."""
788
for parent_idx in other._parents[other_idx]:
789
parent_name = other._names[parent_idx]
790
if parent_name not in self._names:
791
# should not be possible
792
raise WeaveError("missing parent {%s} of {%s} in %r"
793
% (parent_name, other._name_map[other_idx], self))
794
new_parents.append(self._name_map[parent_name])
797
def _check_version_consistent(self, other, other_idx, name):
798
"""Check if a version in consistent in this and other.
800
To be consistent it must have:
803
* the same direct parents (by name, not index, and disregarding
806
If present & correct return True;
807
if not present in self return False;
808
if inconsistent raise error."""
809
this_idx = self._name_map.get(name, -1)
811
if self._sha1s[this_idx] != other._sha1s[other_idx]:
812
raise WeaveError("inconsistent texts for version {%s} "
813
"when joining weaves"
815
self_parents = self._parents[this_idx]
816
other_parents = other._parents[other_idx]
817
n1 = [self._names[i] for i in self_parents]
818
n2 = [other._names[i] for i in other_parents]
822
raise WeaveParentMismatch("inconsistent parents "
823
"for version {%s}: %s vs %s" % (name, n1, n2))
829
def reweave(self, other):
830
"""Reweave self with other."""
831
new_weave = reweave(self, other)
832
for attr in self.__slots__:
833
setattr(self, attr, getattr(new_weave, attr))
837
"""Combine two weaves and return the result.
839
This works even if a revision R has different parents in
840
wa and wb. In the resulting weave all the parents are given.
842
This is done by just building up a new weave, maintaining ordering
843
of the versions in the two inputs. More efficient approaches
844
might be possible but it should only be necessary to do
845
this operation rarely, when a new previously ghost version is
850
queue_a = range(wa.numversions())
851
queue_b = range(wb.numversions())
852
# first determine combined parents of all versions
853
# map from version name -> all parent names
854
combined_parents = _reweave_parent_graphs(wa, wb)
855
mutter("combined parents: %r", combined_parents)
856
order = topo_sort(combined_parents.iteritems())
857
mutter("order to reweave: %r", order)
859
if name in wa._name_map:
860
lines = wa.get_lines(name)
861
if name in wb._name_map:
862
assert lines == wb.get_lines(name)
864
lines = wb.get_lines(name)
865
wr.add(name, combined_parents[name], lines)
869
def _reweave_parent_graphs(wa, wb):
870
"""Return combined parent ancestry for two weaves.
872
Returned as a list of (version_name, set(parent_names))"""
874
for weave in [wa, wb]:
875
for idx, name in enumerate(weave._names):
876
p = combined.setdefault(name, set())
877
p.update(map(weave.idx_to_name, weave._parents[idx]))
882
"""Show the weave's table-of-contents"""
883
print '%6s %50s %10s %10s' % ('ver', 'name', 'sha1', 'parents')
884
for i in (6, 50, 10, 10):
676
"""Show some text information about the weave."""
677
print '%6s %40s %20s' % ('ver', 'sha1', 'parents')
678
for i in (6, 40, 20):
887
681
for i in range(w.numversions()):
888
682
sha1 = w._sha1s[i]
890
parent_str = ' '.join(map(str, w._parents[i]))
891
print '%6d %-50.50s %10.10s %s' % (i, name, sha1, parent_str)
895
def weave_stats(weave_file, pb):
683
print '%6d %40s %s' % (i, sha1, ' '.join(map(str, w._parents[i])))
687
def weave_stats(weave_file):
688
from bzrlib.progress import ProgressBar
896
689
from bzrlib.weavefile import read_weave
898
693
wf = file(weave_file, 'rb')
899
694
w = read_weave(wf)
900
695
# FIXME: doesn't work on pipes
added
removed
bzrlib/weave.py
