22
22
"""Weave - storage of related text file versions"""
24
# before intset (r923) 2000 versions in 41.5s
25
# with intset (r926) 2000 versions in 93s !!!
26
# better to just use plain sets.
28
# making _extract build and return a list, rather than being a generator
31
# with python -O, r923 does 2000 versions in 36.87s
33
# with optimizations to avoid mutating lists - 35.75! I guess copying
34
# all the elements every time costs more than the small manipulations.
35
# a surprisingly small change.
37
# r931, which avoids using a generator for extract, does 36.98s
39
# with memoized inclusions, takes 41.49s; not very good
41
# with slots, takes 37.35s; without takes 39.16, a bit surprising
43
# with the delta calculation mixed in with the add method, rather than
44
# separated, takes 36.78s
46
# with delta folded in and mutation of the list, 36.13s
48
# with all this and simplification of add code, 33s
51
# TODO: Perhaps have copy method for Weave instances?
25
53
# XXX: If we do weaves this way, will a merge still behave the same
26
54
# way if it's done in a different order? That's a pretty desirable
44
74
# TODO: Parallel-extract that passes back each line along with a
45
75
# description of which revisions include it. Nice for checking all
46
# shas or calculating stats in parallel.
48
# TODO: Using a single _extract routine and then processing the output
49
# is probably inefficient. It's simple enough that we can afford to
50
# have slight specializations for different ways its used: annotate,
51
# basis for add, get, etc.
53
# TODO: Probably the API should work only in names to hide the integer
54
# indexes from the user.
56
# TODO: Is there any potential performance win by having an add()
57
# variant that is passed a pre-cooked version of the single basis
60
# TODO: Reweave can possibly be made faster by remembering diffs
61
# where the basis and destination are unchanged.
63
# FIXME: Sometimes we will be given a parents list for a revision
64
# that includes some redundant parents (i.e. already a parent of
65
# something in the list.) We should eliminate them. This can
66
# be done fairly efficiently because the sequence numbers constrain
67
# the possible relationships.
72
from difflib import SequenceMatcher
74
from bzrlib.trace import mutter
75
from bzrlib.errors import (WeaveError, WeaveFormatError, WeaveParentMismatch,
76
WeaveRevisionNotPresent, WeaveRevisionAlreadyPresent)
77
import bzrlib.errors as errors
78
from bzrlib.tsort import topo_sort
81
class WeaveError(Exception):
82
"""Exception in processing weave"""
85
class WeaveFormatError(WeaveError):
86
"""Weave invariant violated"""
81
89
class Weave(object):
82
90
"""weave - versioned text file storage.
154
160
each version; the parent's parents are implied.
157
List of hex SHA-1 of each version.
160
List of symbolic names for each version. Each should be unique.
163
For each name, the version number.
166
Descriptive name of this weave; typically the filename if known.
163
List of hex SHA-1 of each version, or None if not recorded.
170
__slots__ = ['_weave', '_parents', '_sha1s', '_names', '_name_map',
166
__slots__ = ['_weave', '_parents', '_sha1s']
173
def __init__(self, weave_name=None):
175
170
self._parents = []
179
self._weave_name = weave_name
182
return "Weave(%r)" % self._weave_name
186
"""Return a deep copy of self.
188
The copy can be modified without affecting the original weave."""
190
other._weave = self._weave[:]
191
other._parents = self._parents[:]
192
other._sha1s = self._sha1s[:]
193
other._names = self._names[:]
194
other._name_map = self._name_map.copy()
195
other._weave_name = self._weave_name
198
174
def __eq__(self, other):
199
175
if not isinstance(other, Weave):
201
177
return self._parents == other._parents \
202
and self._weave == other._weave \
203
and self._sha1s == other._sha1s
178
and self._weave == other._weave
206
181
def __ne__(self, other):
207
182
return not self.__eq__(other)
209
def __contains__(self, name):
210
return self._name_map.has_key(name)
212
def maybe_lookup(self, name_or_index):
213
"""Convert possible symbolic name to index, or pass through indexes."""
214
if isinstance(name_or_index, (int, long)):
217
return self.lookup(name_or_index)
220
def lookup(self, name):
221
"""Convert symbolic version name to index."""
223
return self._name_map[name]
225
raise WeaveRevisionNotPresent(name, self)
228
return self._names[:]
230
def iter_names(self):
231
"""Yield a list of all names in this weave."""
232
return iter(self._names)
234
def idx_to_name(self, version):
235
return self._names[version]
237
def _check_repeated_add(self, name, parents, text, sha1):
238
"""Check that a duplicated add is OK.
240
If it is, return the (old) index; otherwise raise an exception.
242
idx = self.lookup(name)
243
if sorted(self._parents[idx]) != sorted(parents) \
244
or sha1 != self._sha1s[idx]:
245
raise WeaveRevisionAlreadyPresent(name, self)
248
def add(self, name, parents, text, sha1=None):
185
def add(self, parents, text):
249
186
"""Add a single text on top of the weave.
251
188
Returns the index number of the newly added version.
254
Symbolic name for this version.
255
(Typically the revision-id of the revision that added it.)
258
191
List or set of direct parent version numbers.
261
Sequence of lines to be added in the new version.
263
sha -- SHA-1 of the file, if known. This is trusted to be
266
from bzrlib.osutils import sha_strings
268
assert isinstance(name, basestring)
270
sha1 = sha_strings(text)
271
if name in self._name_map:
272
return self._check_repeated_add(name, parents, text, sha1)
274
parents = map(self.maybe_lookup, parents)
194
Sequence of lines to be added in the new version."""
275
196
self._check_versions(parents)
276
197
## self._check_lines(text)
277
198
new_version = len(self._parents)
280
# if we abort after here the (in-memory) weave will be corrupt because only
281
# some fields are updated
282
self._parents.append(parents[:])
206
# if we abort after here the weave will be corrupt
207
self._parents.append(frozenset(parents))
283
208
self._sha1s.append(sha1)
284
self._names.append(name)
285
self._name_map[name] = new_version
364
288
# we don't destroy ourselves
366
290
self._weave[i:i] = ([('{', new_version)]
292
+ [('}', new_version)])
369
293
offset += 2 + (j2 - j1)
371
295
return new_version
373
def add_identical(self, old_rev_id, new_rev_id, parents):
374
"""Add an identical text to old_rev_id as new_rev_id."""
375
old_lines = self.get(self.lookup(old_rev_id))
376
self.add(new_rev_id, parents, old_lines)
378
298
def inclusions(self, versions):
379
299
"""Return set of all ancestors of given version(s)."""
380
300
i = set(versions)
381
for v in xrange(max(versions), 0, -1):
383
# include all its parents
384
i.update(self._parents[v])
386
## except IndexError:
387
## raise ValueError("version %d not present in weave" % v)
390
def parents(self, version):
391
return self._parents[version]
394
def parent_names(self, version):
395
"""Return version names for parents of a version."""
396
return map(self.idx_to_name, self._parents[self.lookup(version)])
305
# include all its parents
306
i.update(self._parents[v])
310
raise ValueError("version %d not present in weave" % v)
399
313
def minimal_parents(self, version):
544
451
result.append((istack[-1], lineno, l))
547
raise WeaveFormatError("unclosed insertion blocks "
548
"at end of weave: %s" % istack)
455
raise WFE("unclosed insertion blocks at end of weave",
550
raise WeaveFormatError("unclosed deletion blocks at end of weave: %s"
458
raise WFE("unclosed deletion blocks at end of weave",
555
def get_iter(self, name_or_index):
465
def get_iter(self, version):
556
466
"""Yield lines for the specified version."""
557
incls = [self.maybe_lookup(name_or_index)]
562
# We don't have sha1 sums for multiple entries
564
for origin, lineno, line in self._extract(incls):
467
for origin, lineno, line in self._extract([version]):
569
expected_sha1 = self._sha1s[index]
570
measured_sha1 = cur_sha.hexdigest()
571
if measured_sha1 != expected_sha1:
572
raise errors.WeaveInvalidChecksum(
573
'file %s, revision %s, expected: %s, measured %s'
574
% (self._weave_name, self._names[index],
575
expected_sha1, measured_sha1))
578
def get_text(self, name_or_index):
579
return ''.join(self.get_iter(name_or_index))
580
assert isinstance(version, int)
583
def get_lines(self, name_or_index):
584
return list(self.get_iter(name_or_index))
590
def get_sha1(self, name):
591
"""Get the stored sha1 sum for the given revision.
593
:param name: The name of the version to lookup
595
return self._sha1s[self.lookup(name)]
471
def get(self, index):
472
return list(self.get_iter(index))
597
475
def mash_iter(self, included):
598
476
"""Return composed version of multiple included versions."""
599
included = map(self.maybe_lookup, included)
600
477
for origin, lineno, text in self._extract(included):
629
507
raise WeaveFormatError("invalid included version %d for index %d"
630
508
% (inclusions[-1], version))
632
# try extracting all versions; parallel extraction is used
510
# try extracting all versions; this is a bit slow and parallel
511
# extraction could be used
633
513
nv = self.numversions()
634
sha1s = [sha.new() for i in range(nv)]
635
texts = [[] for i in range(nv)]
638
# For creating the ancestry, IntSet is much faster (3.7s vs 0.17s)
639
# The problem is that set membership is much more expensive
641
for p in self._parents[i]:
642
new_inc.update(inclusions[p])
644
#assert set(new_inc) == self.inclusions([i]), 'failed %s != %s' % (new_inc, self.inclusions([i]))
645
inclusions.append(new_inc)
647
nlines = len(self._weave)
649
update_text = 'checking weave'
651
short_name = os.path.basename(self._weave_name)
652
update_text = 'checking %s' % (short_name,)
653
update_text = update_text[:25]
655
for lineno, insert, deleteset, line in self._walk():
514
for version in range(nv):
657
progress_bar.update(update_text, lineno, nlines)
659
for j, j_inc in enumerate(inclusions):
660
# The active inclusion must be an ancestor,
661
# and no ancestors must have deleted this line,
662
# because we don't support resurrection.
663
if (insert in j_inc) and not (deleteset & j_inc):
664
sha1s[j].update(line)
666
for version in range(nv):
667
hd = sha1s[version].hexdigest()
516
progress_bar.update('checking text', version, nv)
518
for l in self.get_iter(version):
668
521
expected = self._sha1s[version]
669
522
if hd != expected:
670
raise errors.WeaveInvalidChecksum(
671
"mismatched sha1 for version %s: "
672
"got %s, expected %s"
673
% (self._names[version], hd, expected))
523
raise WeaveError("mismatched sha1 for version %d; "
524
"got %s, expected %s"
525
% (version, hd, expected))
675
527
# TODO: check insertions are properly nested, that there are
676
528
# no lines outside of insertion blocks, that deletions are
677
529
# properly paired, etc.
533
def merge(self, merge_versions):
534
"""Automerge and mark conflicts between versions.
536
This returns a sequence, each entry describing alternatives
537
for a chunk of the file. Each of the alternatives is given as
540
If there is a chunk of the file where there's no diagreement,
541
only one alternative is given.
544
# approach: find the included versions common to all the
546
raise NotImplementedError()
679
550
def _delta(self, included, lines):
680
551
"""Return changes from basis to new revision.
798
665
'killed-both'), \
802
def join(self, other):
804
"""Integrate versions from other into this weave.
806
The resulting weave contains all the history of both weaves;
807
any version you could retrieve from either self or other can be
808
retrieved from self after this call.
810
It is illegal for the two weaves to contain different values
811
or different parents for any version. See also reweave().
813
if other.numversions() == 0:
814
return # nothing to update, easy
815
# two loops so that we do not change ourselves before verifying it
817
# work through in index order to make sure we get all dependencies
818
for other_idx, name in enumerate(other._names):
819
self._check_version_consistent(other, other_idx, name)
824
for other_idx, name in enumerate(other._names):
825
# TODO: If all the parents of the other version are already
826
# present then we can avoid some work by just taking the delta
827
# and adjusting the offsets.
828
new_parents = self._imported_parents(other, other_idx)
829
sha1 = other._sha1s[other_idx]
833
if name in self._names:
834
idx = self.lookup(name)
835
n1 = map(other.idx_to_name, other._parents[other_idx] )
836
n2 = map(self.idx_to_name, self._parents[other_idx] )
837
if sha1 == self._sha1s[idx] and n1 == n2:
841
lines = other.get_lines(other_idx)
842
self.add(name, new_parents, lines, sha1)
844
mutter("merged = %d, processed = %d, file_id=%s; deltat=%d"%(
845
merged,processed,self._weave_name, time.time( )-time0))
850
def _imported_parents(self, other, other_idx):
851
"""Return list of parents in self corresponding to indexes in other."""
853
for parent_idx in other._parents[other_idx]:
854
parent_name = other._names[parent_idx]
855
if parent_name not in self._names:
856
# should not be possible
857
raise WeaveError("missing parent {%s} of {%s} in %r"
858
% (parent_name, other._name_map[other_idx], self))
859
new_parents.append(self._name_map[parent_name])
862
def _check_version_consistent(self, other, other_idx, name):
863
"""Check if a version in consistent in this and other.
865
To be consistent it must have:
868
* the same direct parents (by name, not index, and disregarding
871
If present & correct return True;
872
if not present in self return False;
873
if inconsistent raise error."""
874
this_idx = self._name_map.get(name, -1)
876
if self._sha1s[this_idx] != other._sha1s[other_idx]:
877
raise WeaveError("inconsistent texts for version {%s} "
878
"when joining weaves"
880
self_parents = self._parents[this_idx]
881
other_parents = other._parents[other_idx]
882
n1 = [self._names[i] for i in self_parents]
883
n2 = [other._names[i] for i in other_parents]
887
raise WeaveParentMismatch("inconsistent parents "
888
"for version {%s}: %s vs %s" % (name, n1, n2))
894
def reweave(self, other):
895
"""Reweave self with other."""
896
new_weave = reweave(self, other)
897
for attr in self.__slots__:
898
setattr(self, attr, getattr(new_weave, attr))
902
"""Combine two weaves and return the result.
904
This works even if a revision R has different parents in
905
wa and wb. In the resulting weave all the parents are given.
907
This is done by just building up a new weave, maintaining ordering
908
of the versions in the two inputs. More efficient approaches
909
might be possible but it should only be necessary to do
910
this operation rarely, when a new previously ghost version is
915
queue_a = range(wa.numversions())
916
queue_b = range(wb.numversions())
917
# first determine combined parents of all versions
918
# map from version name -> all parent names
919
combined_parents = _reweave_parent_graphs(wa, wb)
920
mutter("combined parents: %r", combined_parents)
921
order = topo_sort(combined_parents.iteritems())
922
mutter("order to reweave: %r", order)
924
if name in wa._name_map:
925
lines = wa.get_lines(name)
926
if name in wb._name_map:
927
assert lines == wb.get_lines(name)
929
lines = wb.get_lines(name)
930
wr.add(name, combined_parents[name], lines)
934
def _reweave_parent_graphs(wa, wb):
935
"""Return combined parent ancestry for two weaves.
937
Returned as a list of (version_name, set(parent_names))"""
939
for weave in [wa, wb]:
940
for idx, name in enumerate(weave._names):
941
p = combined.setdefault(name, set())
942
p.update(map(weave.idx_to_name, weave._parents[idx]))
947
"""Show the weave's table-of-contents"""
948
print '%6s %50s %10s %10s' % ('ver', 'name', 'sha1', 'parents')
949
for i in (6, 50, 10, 10):
675
"""Show some text information about the weave."""
676
print '%6s %40s %20s' % ('ver', 'sha1', 'parents')
677
for i in (6, 40, 20):
952
680
for i in range(w.numversions()):
953
681
sha1 = w._sha1s[i]
955
parent_str = ' '.join(map(str, w._parents[i]))
956
print '%6d %-50.50s %10.10s %s' % (i, name, sha1, parent_str)
960
def weave_stats(weave_file, pb):
682
print '%6d %40s %s' % (i, sha1, ' '.join(map(str, w._parents[i])))
686
def weave_stats(weave_file):
687
from bzrlib.progress import ProgressBar
961
688
from bzrlib.weavefile import read_weave
963
692
wf = file(weave_file, 'rb')
964
693
w = read_weave(wf)
965
694
# FIXME: doesn't work on pipes
added
removed
bzrlib/weave.py
