22
22
"""Weave - storage of related text file versions"""
24
# before intset (r923) 2000 versions in 41.5s
25
# with intset (r926) 2000 versions in 93s !!!
26
# better to just use plain sets.
28
# making _extract build and return a list, rather than being a generator
31
# with python -O, r923 does 2000 versions in 36.87s
33
# with optimizations to avoid mutating lists - 35.75! I guess copying
34
# all the elements every time costs more than the small manipulations.
35
# a surprisingly small change.
37
# r931, which avoids using a generator for extract, does 36.98s
39
# with memoized inclusions, takes 41.49s; not very good
41
# with slots, takes 37.35s; without takes 39.16, a bit surprising
43
# with the delta calculation mixed in with the add method, rather than
44
# separated, takes 36.78s
46
# with delta folded in and mutation of the list, 36.13s
48
# with all this and simplification of add code, 33s
51
# TODO: Perhaps have copy method for Weave instances?
53
25
# XXX: If we do weaves this way, will a merge still behave the same
54
26
# way if it's done in a different order? That's a pretty desirable
74
44
# TODO: Parallel-extract that passes back each line along with a
75
45
# description of which revisions include it. Nice for checking all
46
# shas or calculating stats in parallel.
48
# TODO: Using a single _extract routine and then processing the output
49
# is probably inefficient. It's simple enough that we can afford to
50
# have slight specializations for different ways its used: annotate,
51
# basis for add, get, etc.
53
# TODO: Perhaps the API should work only in names to hide the integer
54
# indexes from the user?
56
# TODO: Is there any potential performance win by having an add()
57
# variant that is passed a pre-cooked version of the single basis
60
# TODO: Have a way to go back and insert a revision V1 that is a parent
61
# of an already-stored revision V2. This means some lines previously
62
# counted as new in V2 will be discovered to have actually come from V1.
63
# It is probably necessary to insert V1, then compute a whole new diff
64
# from the mashed ancestors to V2. This must be repeated for every
65
# direct child of V1. The deltas from V2 to its descendents won't change,
66
# although their location within the weave may change. It may be possible
67
# to just adjust the location of those instructions rather than
68
# re-weaving the whole thing. This is expected to be a fairly rare
69
# operation, only used when inserting data that was previously a ghost.
75
from difflib import SequenceMatcher
77
from bzrlib.trace import mutter
81
80
class WeaveError(Exception):
160
165
each version; the parent's parents are implied.
163
List of hex SHA-1 of each version, or None if not recorded.
168
List of hex SHA-1 of each version.
171
List of symbolic names for each version. Each should be unique.
174
For each name, the version number.
177
Descriptive name of this weave; typically the filename if known.
166
__slots__ = ['_weave', '_parents', '_sha1s']
181
__slots__ = ['_weave', '_parents', '_sha1s', '_names', '_name_map',
184
def __init__(self, weave_name=None):
170
186
self._parents = []
190
self._weave_name = weave_name
194
"""Return a deep copy of self.
196
The copy can be modified without affecting the original weave."""
198
other._weave = self._weave[:]
199
other._parents = self._parents[:]
200
other._sha1s = self._sha1s[:]
201
other._names = self._names[:]
202
other._name_map = self._name_map.copy()
203
other._weave_name = self._weave_name
174
206
def __eq__(self, other):
175
207
if not isinstance(other, Weave):
177
209
return self._parents == other._parents \
178
and self._weave == other._weave
210
and self._weave == other._weave \
211
and self._sha1s == other._sha1s
181
214
def __ne__(self, other):
182
215
return not self.__eq__(other)
185
def add(self, parents, text):
218
def maybe_lookup(self, name_or_index):
219
"""Convert possible symbolic name to index, or pass through indexes."""
220
if isinstance(name_or_index, (int, long)):
223
return self.lookup(name_or_index)
226
def lookup(self, name):
227
"""Convert symbolic version name to index."""
229
return self._name_map[name]
231
raise WeaveError("name %r not present in weave %r" %
232
(name, self._weave_name))
235
def iter_names(self):
236
"""Yield a list of all names in this weave."""
237
return iter(self._names)
239
def idx_to_name(self, version):
240
return self._names[version]
243
def _check_repeated_add(self, name, parents, text, sha1):
244
"""Check that a duplicated add is OK.
246
If it is, return the (old) index; otherwise raise an exception.
248
idx = self.lookup(name)
249
if sorted(self._parents[idx]) != sorted(parents):
250
raise WeaveError("name \"%s\" already present in weave "
251
"with different parents" % name)
252
if sha1 != self._sha1s[idx]:
253
raise WeaveError("name \"%s\" already present in weave "
254
"with different text" % name)
259
def add(self, name, parents, text, sha1=None):
186
260
"""Add a single text on top of the weave.
188
262
Returns the index number of the newly added version.
265
Symbolic name for this version.
266
(Typically the revision-id of the revision that added it.)
191
269
List or set of direct parent version numbers.
194
Sequence of lines to be added in the new version."""
272
Sequence of lines to be added in the new version.
274
sha -- SHA-1 of the file, if known. This is trusted to be
277
from bzrlib.osutils import sha_strings
279
assert isinstance(name, basestring)
281
sha1 = sha_strings(text)
282
if name in self._name_map:
283
return self._check_repeated_add(name, parents, text, sha1)
285
parents = map(self.maybe_lookup, parents)
196
286
self._check_versions(parents)
197
287
## self._check_lines(text)
198
288
new_version = len(self._parents)
206
# if we abort after here the weave will be corrupt
207
self._parents.append(frozenset(parents))
291
# if we abort after here the (in-memory) weave will be corrupt because only
292
# some fields are updated
293
self._parents.append(parents[:])
208
294
self._sha1s.append(sha1)
295
self._names.append(name)
296
self._name_map[name] = new_version
287
375
# we don't destroy ourselves
289
377
self._weave[i:i] = ([('{', new_version)]
291
+ [('}', new_version)])
292
380
offset += 2 + (j2 - j1)
294
382
return new_version
384
def add_identical(self, old_rev_id, new_rev_id, parents):
385
"""Add an identical text to old_rev_id as new_rev_id."""
386
old_lines = self.get(self.lookup(old_rev_id))
387
self.add(new_rev_id, parents, old_lines)
297
389
def inclusions(self, versions):
298
390
"""Return set of all ancestors of given version(s)."""
299
391
i = set(versions)
304
# include all its parents
305
i.update(self._parents[v])
309
raise ValueError("version %d not present in weave" % v)
392
for v in xrange(max(versions), 0, -1):
394
# include all its parents
395
i.update(self._parents[v])
397
## except IndexError:
398
## raise ValueError("version %d not present in weave" % v)
401
def parents(self, version):
402
return self._parents[version]
405
def parent_names(self, version):
406
"""Return version names for parents of a version."""
407
return map(self.idx_to_name, self._parents[self.lookup(version)])
312
410
def minimal_parents(self, version):
464
def get_iter(self, version):
566
def get_iter(self, name_or_index):
465
567
"""Yield lines for the specified version."""
466
for origin, lineno, line in self._extract([version]):
568
incls = [self.maybe_lookup(name_or_index)]
569
for origin, lineno, line in self._extract(incls):
470
def get(self, index):
471
return list(self.get_iter(index))
573
def get_text(self, name_or_index):
574
return ''.join(self.get_iter(name_or_index))
575
assert isinstance(version, int)
578
def get_lines(self, name_or_index):
579
return list(self.get_iter(name_or_index))
474
585
def mash_iter(self, included):
475
586
"""Return composed version of multiple included versions."""
587
included = map(self.maybe_lookup, included)
476
588
for origin, lineno, text in self._extract(included):
674
"""Show some text information about the weave."""
675
print '%6s %40s %20s' % ('ver', 'sha1', 'parents')
676
for i in (6, 40, 20):
778
def join(self, other):
779
"""Integrate versions from other into this weave.
781
The resulting weave contains all the history of both weaves;
782
any version you could retrieve from either self or other can be
783
retrieved from self after this call.
785
It is illegal for the two weaves to contain different values
786
or different parents for any version. See also reweave().
788
if other.numversions() == 0:
789
return # nothing to update, easy
790
# two loops so that we do not change ourselves before verifying it
792
# work through in index order to make sure we get all dependencies
793
for other_idx, name in enumerate(other._names):
794
if self._check_version_consistent(other, other_idx, name):
796
for other_idx, name in enumerate(other._names):
797
# TODO: If all the parents of the other version are already
798
# present then we can avoid some work by just taking the delta
799
# and adjusting the offsets.
800
new_parents = self._imported_parents(other, other_idx)
801
lines = other.get_lines(other_idx)
802
sha1 = other._sha1s[other_idx]
803
self.add(name, new_parents, lines, sha1)
806
def _imported_parents(self, other, other_idx):
807
"""Return list of parents in self corresponding to indexes in other."""
809
for parent_idx in other._parents[other_idx]:
810
parent_name = other._names[parent_idx]
811
if parent_name not in self._names:
812
# should not be possible
813
raise WeaveError("missing parent {%s} of {%s} in %r"
814
% (parent_name, other._name_map[other_idx], self))
815
new_parents.append(self._name_map[parent_name])
818
def _check_version_consistent(self, other, other_idx, name):
819
"""Check if a version in consistent in this and other.
821
To be consistent it must have:
824
* the same direct parents (by name, not index, and disregarding
827
If present & correct return True;
828
if not present in self return False;
829
if inconsistent raise error."""
830
this_idx = self._name_map.get(name, -1)
832
if self._sha1s[this_idx] != other._sha1s[other_idx]:
833
raise WeaveError("inconsistent texts for version {%s} "
834
"when joining weaves"
836
self_parents = self._parents[this_idx]
837
other_parents = other._parents[other_idx]
838
n1 = [self._names[i] for i in self_parents]
839
n2 = [other._names[i] for i in other_parents]
843
raise WeaveParentMismatch("inconsistent parents "
844
"for version {%s}: %s vs %s" % (name, n1, n2))
850
def reweave(self, other):
851
"""Reweave self with other."""
852
new_weave = reweave(self, other)
853
for attr in self.__slots__:
854
setattr(self, attr, getattr(new_weave, attr))
858
"""Combine two weaves and return the result.
860
This works even if a revision R has different parents in
861
wa and wb. In the resulting weave all the parents are given.
863
This is done by just building up a new weave, maintaining ordering
864
of the versions in the two inputs. More efficient approaches
865
might be possible but it should only be necessary to do
866
this operation rarely, when a new previously ghost version is
871
queue_a = range(wa.numversions())
872
queue_b = range(wb.numversions())
873
# first determine combined parents of all versions
874
# map from version name -> all parent names
875
combined_parents = _reweave_parent_graphs(wa, wb)
876
mutter("combined parents: %r", combined_parents)
877
order = _make_reweave_order(wa._names, wb._names, combined_parents)
878
mutter("order to reweave: %r", order)
880
if name in wa._name_map:
881
lines = wa.get_lines(name)
882
if name in wb._name_map:
883
assert lines == wb.get_lines(name)
885
lines = wb.get_lines(name)
886
wr.add(name, combined_parents[name], lines)
890
def _reweave_parent_graphs(wa, wb):
891
"""Return combined parent ancestry for two weaves.
893
Returned as a list of (version_name, set(parent_names))"""
895
for weave in [wa, wb]:
896
for idx, name in enumerate(weave._names):
897
p = combined.setdefault(name, set())
898
p.update(map(weave.idx_to_name, weave._parents[idx]))
902
def _make_reweave_order(wa_order, wb_order, combined_parents):
903
"""Return an order to reweave versions respecting parents."""
907
next_a = next_b = None
908
len_a = len(wa_order)
909
len_b = len(wb_order)
910
while ia < len(wa_order) or ib < len(wb_order):
912
next_a = wa_order[ia]
916
if combined_parents[next_a].issubset(done):
918
result.append(next_a)
922
next_b = wb_order[ib]
926
elif combined_parents[next_b].issubset(done):
928
result.append(next_b)
931
raise WeaveError("don't know how to reweave at {%s} and {%s}"
937
"""Show the weave's table-of-contents"""
938
print '%6s %50s %10s %10s' % ('ver', 'name', 'sha1', 'parents')
939
for i in (6, 50, 10, 10):
679
942
for i in range(w.numversions()):
680
943
sha1 = w._sha1s[i]
681
print '%6d %40s %s' % (i, sha1, ' '.join(map(str, w._parents[i])))
685
def weave_stats(weave_file):
686
from bzrlib.progress import ProgressBar
945
parent_str = ' '.join(map(str, w._parents[i]))
946
print '%6d %-50.50s %10.10s %s' % (i, name, sha1, parent_str)
950
def weave_stats(weave_file, pb):
687
951
from bzrlib.weavefile import read_weave
691
953
wf = file(weave_file, 'rb')
692
954
w = read_weave(wf)
693
955
# FIXME: doesn't work on pipes