199
213
return self._parents == other._parents \
200
214
and self._weave == other._weave \
201
215
and self._sha1s == other._sha1s
204
217
def __ne__(self, other):
205
218
return not self.__eq__(other)
220
@deprecated_method(zero_eight)
221
def idx_to_name(self, index):
222
"""Old public interface, the public interface is all names now."""
225
def _idx_to_name(self, version):
226
return self._names[version]
228
@deprecated_method(zero_eight)
208
229
def lookup(self, name):
230
"""Backwards compatability thunk:
232
Return name, as name is valid in the api now, and spew deprecation
237
def _lookup(self, name):
238
"""Convert symbolic version name to index."""
210
240
return self._name_map[name]
212
raise WeaveError("name %s not present in weave" % name)
215
def add(self, name, parents, text):
242
raise RevisionNotPresent(name, self._weave_name)
244
@deprecated_method(zero_eight)
245
def iter_names(self):
246
"""Deprecated convenience function, please see VersionedFile.names()."""
247
return iter(self.names())
249
@deprecated_method(zero_eight)
251
"""See Weave.versions for the current api."""
252
return self.versions()
255
"""See VersionedFile.versions."""
256
return self._names[:]
258
def has_version(self, version_id):
259
"""See VersionedFile.has_version."""
260
return self._name_map.has_key(version_id)
262
__contains__ = has_version
264
def get_delta(self, version_id):
265
"""See VersionedFile.get_delta."""
266
return self.get_deltas([version_id])[version_id]
268
def get_deltas(self, version_ids):
269
"""See VersionedFile.get_deltas."""
270
version_ids = self.get_ancestry(version_ids)
271
for version_id in version_ids:
272
if not self.has_version(version_id):
273
raise RevisionNotPresent(version_id, self)
274
# try extracting all versions; parallel extraction is used
275
nv = self.num_versions()
281
last_parent_lines = {}
283
parent_inclusions = {}
288
# its simplest to generate a full set of prepared variables.
290
name = self._names[i]
291
sha1s[name] = self.get_sha1(name)
292
parents_list = self.get_parents(name)
294
parent = parents_list[0]
295
parents[name] = parent
296
parent_inclusions[name] = inclusions[parent]
299
parent_inclusions[name] = set()
300
# we want to emit start, finish, replacement_length, replacement_lines tuples.
301
diff_hunks[name] = []
302
current_hunks[name] = [0, 0, 0, []] # #start, finish, repl_length, repl_tuples
303
parent_linenums[name] = 0
305
parent_noeols[name] = False
306
last_parent_lines[name] = None
307
new_inc = set([name])
308
for p in self._parents[i]:
309
new_inc.update(inclusions[self._idx_to_name(p)])
310
# debug only, known good so far.
311
#assert set(new_inc) == set(self.get_ancestry(name)), \
312
# 'failed %s != %s' % (set(new_inc), set(self.get_ancestry(name)))
313
inclusions[name] = new_inc
315
nlines = len(self._weave)
317
for lineno, inserted, deletes, line in self._walk_internal():
318
# a line is active in a version if:
319
# insert is in the versions inclusions
321
# deleteset & the versions inclusions is an empty set.
322
# so - if we have a included by mapping - version is included by
323
# children, we get a list of children to examine for deletes affect
324
# ing them, which is less than the entire set of children.
325
for version_id in version_ids:
326
# The active inclusion must be an ancestor,
327
# and no ancestors must have deleted this line,
328
# because we don't support resurrection.
329
parent_inclusion = parent_inclusions[version_id]
330
inclusion = inclusions[version_id]
331
parent_active = inserted in parent_inclusion and not (deletes & parent_inclusion)
332
version_active = inserted in inclusion and not (deletes & inclusion)
333
if not parent_active and not version_active:
334
# unrelated line of ancestry
336
elif parent_active and version_active:
338
parent_linenum = parent_linenums[version_id]
339
if current_hunks[version_id] != [parent_linenum, parent_linenum, 0, []]:
340
diff_hunks[version_id].append(tuple(current_hunks[version_id]))
342
current_hunks[version_id] = [parent_linenum, parent_linenum, 0, []]
343
parent_linenums[version_id] = parent_linenum
346
noeols[version_id] = True
349
elif parent_active and not version_active:
351
current_hunks[version_id][1] += 1
352
parent_linenums[version_id] += 1
353
last_parent_lines[version_id] = line
354
elif not parent_active and version_active:
356
# noeol only occurs at the end of a file because we
357
# diff linewise. We want to show noeol changes as a
358
# empty diff unless the actual eol-less content changed.
361
if last_parent_lines[version_id][-1] != '\n':
362
parent_noeols[version_id] = True
363
except (TypeError, IndexError):
366
if theline[-1] != '\n':
367
noeols[version_id] = True
371
parent_should_go = False
373
if parent_noeols[version_id] == noeols[version_id]:
374
# no noeol toggle, so trust the weaves statement
375
# that this line is changed.
377
if parent_noeols[version_id]:
378
theline = theline + '\n'
379
elif parent_noeols[version_id]:
380
# parent has no eol, we do:
381
# our line is new, report as such..
383
elif noeols[version_id]:
384
# append a eol so that it looks like
386
theline = theline + '\n'
387
if parents[version_id] is not None:
388
#if last_parent_lines[version_id] is not None:
389
parent_should_go = True
390
if last_parent_lines[version_id] != theline:
393
#parent_should_go = False
395
current_hunks[version_id][2] += 1
396
current_hunks[version_id][3].append((inserted, theline))
398
# last hunk last parent line is not eaten
399
current_hunks[version_id][1] -= 1
400
if current_hunks[version_id][1] < 0:
401
current_hunks[version_id][1] = 0
402
# import pdb;pdb.set_trace()
403
# assert current_hunks[version_id][1] >= 0
407
version = self._idx_to_name(i)
408
if current_hunks[version] != [0, 0, 0, []]:
409
diff_hunks[version].append(tuple(current_hunks[version]))
411
for version_id in version_ids:
412
result[version_id] = (
416
diff_hunks[version_id],
420
def get_parents(self, version_id):
421
"""See VersionedFile.get_parent."""
422
return map(self._idx_to_name, self._parents[self._lookup(version_id)])
424
def _check_repeated_add(self, name, parents, text, sha1):
425
"""Check that a duplicated add is OK.
427
If it is, return the (old) index; otherwise raise an exception.
429
idx = self._lookup(name)
430
if sorted(self._parents[idx]) != sorted(parents) \
431
or sha1 != self._sha1s[idx]:
432
raise RevisionAlreadyPresent(name, self._weave_name)
435
@deprecated_method(zero_eight)
436
def add_identical(self, old_rev_id, new_rev_id, parents):
437
"""Please use Weave.clone_text now."""
438
return self.clone_text(new_rev_id, old_rev_id, parents)
440
def _add_lines(self, version_id, parents, lines, parent_texts):
441
"""See VersionedFile.add_lines."""
442
return self._add(version_id, lines, map(self._lookup, parents))
444
@deprecated_method(zero_eight)
445
def add(self, name, parents, text, sha1=None):
446
"""See VersionedFile.add_lines for the non deprecated api."""
447
return self._add(name, text, map(self._maybe_lookup, parents), sha1)
449
def _add(self, version_id, lines, parents, sha1=None):
216
450
"""Add a single text on top of the weave.
218
452
Returns the index number of the newly added version.
221
455
Symbolic name for this version.
222
456
(Typically the revision-id of the revision that added it.)
225
459
List or set of direct parent version numbers.
228
Sequence of lines to be added in the new version."""
230
assert isinstance(name, basestring)
231
if name in self._name_map:
232
raise WeaveError("name %r already present in weave" % name)
462
Sequence of lines to be added in the new version.
465
assert isinstance(version_id, basestring)
467
sha1 = sha_strings(lines)
468
if version_id in self._name_map:
469
return self._check_repeated_add(version_id, parents, lines, sha1)
234
471
self._check_versions(parents)
235
## self._check_lines(text)
472
## self._check_lines(lines)
236
473
new_version = len(self._parents)
243
475
# if we abort after here the (in-memory) weave will be corrupt because only
244
476
# some fields are updated
477
# XXX: FIXME implement a succeed-or-fail of the rest of this routine.
478
# - Robert Collins 20060226
245
479
self._parents.append(parents[:])
246
480
self._sha1s.append(sha1)
247
self._names.append(name)
248
self._name_map[name] = new_version
481
self._names.append(version_id)
482
self._name_map[version_id] = new_version
486
assert isinstance(l, basestring)
834
assert l.__class__ in (str, unicode)
487
835
if isactive is None:
488
836
isactive = (not dset) and istack and (istack[-1] in included)
490
838
result.append((istack[-1], lineno, l))
494
raise WFE("unclosed insertion blocks at end of weave",
841
raise WeaveFormatError("unclosed insertion blocks "
842
"at end of weave: %s" % istack)
497
raise WFE("unclosed deletion blocks at end of weave",
504
def get_iter(self, version):
505
"""Yield lines for the specified version."""
506
for origin, lineno, line in self._extract([version]):
510
def get(self, index):
511
return list(self.get_iter(index))
514
def mash_iter(self, included):
515
"""Return composed version of multiple included versions."""
516
for origin, lineno, text in self._extract(included):
520
def dump(self, to_file):
521
from pprint import pprint
522
print >>to_file, "Weave._weave = ",
523
pprint(self._weave, to_file)
524
print >>to_file, "Weave._parents = ",
525
pprint(self._parents, to_file)
844
raise WeaveFormatError("unclosed deletion blocks at end of weave: %s"
848
@deprecated_method(zero_eight)
849
def get_iter(self, name_or_index):
850
"""Deprecated, please do not use. Lookups are not not needed.
852
Please use get_lines now.
854
return iter(self.get_lines(self._maybe_lookup(name_or_index)))
856
@deprecated_method(zero_eight)
857
def maybe_lookup(self, name_or_index):
858
"""Deprecated, please do not use. Lookups are not not needed."""
859
return self._maybe_lookup(name_or_index)
861
def _maybe_lookup(self, name_or_index):
862
"""Convert possible symbolic name to index, or pass through indexes.
866
if isinstance(name_or_index, (int, long)):
869
return self._lookup(name_or_index)
871
@deprecated_method(zero_eight)
872
def get(self, version_id):
873
"""Please use either Weave.get_text or Weave.get_lines as desired."""
874
return self.get_lines(version_id)
876
def get_lines(self, version_id):
877
"""See VersionedFile.get_lines()."""
878
int_index = self._maybe_lookup(version_id)
879
result = [line for (origin, lineno, line) in self._extract([int_index])]
880
expected_sha1 = self._sha1s[int_index]
881
measured_sha1 = sha_strings(result)
882
if measured_sha1 != expected_sha1:
883
raise errors.WeaveInvalidChecksum(
884
'file %s, revision %s, expected: %s, measured %s'
885
% (self._weave_name, version_id,
886
expected_sha1, measured_sha1))
889
def get_sha1(self, name):
890
"""Get the stored sha1 sum for the given revision.
892
:param name: The name of the version to lookup
894
return self._sha1s[self._lookup(name)]
896
@deprecated_method(zero_eight)
529
897
def numversions(self):
898
"""How many versions are in this weave?
900
Deprecated in favour of num_versions.
902
return self.num_versions()
904
def num_versions(self):
905
"""How many versions are in this weave?"""
530
906
l = len(self._parents)
531
907
assert l == len(self._sha1s)
536
return self.numversions()
910
__len__ = num_versions
539
912
def check(self, progress_bar=None):
540
# check no circular inclusions
541
for version in range(self.numversions()):
913
# TODO evaluate performance hit of using string sets in this routine.
914
# TODO: check no circular inclusions
915
# TODO: create a nested progress bar
916
for version in range(self.num_versions()):
542
917
inclusions = list(self._parents[version])
544
919
inclusions.sort()
546
921
raise WeaveFormatError("invalid included version %d for index %d"
547
922
% (inclusions[-1], version))
549
# try extracting all versions; this is a bit slow and parallel
550
# extraction could be used
551
nv = self.numversions()
552
for version in range(nv):
924
# try extracting all versions; parallel extraction is used
925
nv = self.num_versions()
930
# For creating the ancestry, IntSet is much faster (3.7s vs 0.17s)
931
# The problem is that set membership is much more expensive
932
name = self._idx_to_name(i)
933
sha1s[name] = sha.new()
935
new_inc = set([name])
936
for p in self._parents[i]:
937
new_inc.update(inclusions[self._idx_to_name(p)])
939
assert set(new_inc) == set(self.get_ancestry(name)), \
940
'failed %s != %s' % (set(new_inc), set(self.get_ancestry(name)))
941
inclusions[name] = new_inc
943
nlines = len(self._weave)
945
update_text = 'checking weave'
947
short_name = os.path.basename(self._weave_name)
948
update_text = 'checking %s' % (short_name,)
949
update_text = update_text[:25]
951
for lineno, insert, deleteset, line in self._walk_internal():
554
progress_bar.update('checking text', version, nv)
556
for l in self.get_iter(version):
559
expected = self._sha1s[version]
953
progress_bar.update(update_text, lineno, nlines)
955
for name, name_inclusions in inclusions.items():
956
# The active inclusion must be an ancestor,
957
# and no ancestors must have deleted this line,
958
# because we don't support resurrection.
959
if (insert in name_inclusions) and not (deleteset & name_inclusions):
960
sha1s[name].update(line)
963
version = self._idx_to_name(i)
964
hd = sha1s[version].hexdigest()
965
expected = self._sha1s[i]
560
966
if hd != expected:
561
raise WeaveError("mismatched sha1 for version %d; "
562
"got %s, expected %s"
563
% (version, hd, expected))
967
raise errors.WeaveInvalidChecksum(
968
"mismatched sha1 for version %s: "
969
"got %s, expected %s"
970
% (version, hd, expected))
565
972
# TODO: check insertions are properly nested, that there are
566
973
# no lines outside of insertion blocks, that deletions are
567
974
# properly paired, etc.
571
def merge(self, merge_versions):
572
"""Automerge and mark conflicts between versions.
574
This returns a sequence, each entry describing alternatives
575
for a chunk of the file. Each of the alternatives is given as
578
If there is a chunk of the file where there's no diagreement,
579
only one alternative is given.
582
# approach: find the included versions common to all the
584
raise NotImplementedError()
588
def _delta(self, included, lines):
589
"""Return changes from basis to new revision.
591
The old text for comparison is the union of included revisions.
593
This is used in inserting a new text.
595
Delta is returned as a sequence of
596
(weave1, weave2, newlines).
598
This indicates that weave1:weave2 of the old weave should be
599
replaced by the sequence of lines in newlines. Note that
600
these line numbers are positions in the total weave and don't
601
correspond to the lines in any extracted version, or even the
602
extracted union of included versions.
604
If line1=line2, this is a pure insert; if newlines=[] this is a
605
pure delete. (Similar to difflib.)
610
def plan_merge(self, ver_a, ver_b):
611
"""Return pseudo-annotation indicating how the two versions merge.
613
This is computed between versions a and b and their common
616
Weave lines present in none of them are skipped entirely.
618
inc_a = self.inclusions([ver_a])
619
inc_b = self.inclusions([ver_b])
620
inc_c = inc_a & inc_b
622
for lineno, insert, deleteset, line in self._walk():
623
if deleteset & inc_c:
624
# killed in parent; can't be in either a or b
625
# not relevant to our work
626
yield 'killed-base', line
627
elif insert in inc_c:
628
# was inserted in base
629
killed_a = bool(deleteset & inc_a)
630
killed_b = bool(deleteset & inc_b)
631
if killed_a and killed_b:
632
yield 'killed-both', line
634
yield 'killed-a', line
636
yield 'killed-b', line
638
yield 'unchanged', line
639
elif insert in inc_a:
640
if deleteset & inc_a:
641
yield 'ghost-a', line
645
elif insert in inc_b:
646
if deleteset & inc_b:
647
yield 'ghost-b', line
651
# not in either revision
652
yield 'irrelevant', line
654
yield 'unchanged', '' # terminator
658
def weave_merge(self, plan):
663
for state, line in plan:
664
if state == 'unchanged' or state == 'killed-both':
665
# resync and flush queued conflicts changes if any
666
if not lines_a and not lines_b:
668
elif ch_a and not ch_b:
670
for l in lines_a: yield l
671
elif ch_b and not ch_a:
672
for l in lines_b: yield l
673
elif lines_a == lines_b:
674
for l in lines_a: yield l
677
for l in lines_a: yield l
679
for l in lines_b: yield l
686
if state == 'unchanged':
689
elif state == 'killed-a':
692
elif state == 'killed-b':
695
elif state == 'new-a':
698
elif state == 'new-b':
702
assert state in ('irrelevant', 'ghost-a', 'ghost-b', 'killed-base',
976
def _join(self, other, pb, msg, version_ids, ignore_missing):
977
"""Worker routine for join()."""
978
if not other.versions():
979
return # nothing to update, easy
982
for version_id in version_ids:
983
if not other.has_version(version_id) and not ignore_missing:
984
raise RevisionNotPresent(version_id, self._weave_name)
986
version_ids = other.versions()
988
# two loops so that we do not change ourselves before verifying it
990
# work through in index order to make sure we get all dependencies
993
# get the selected versions only that are in other.versions.
994
version_ids = set(other.versions()).intersection(set(version_ids))
995
# pull in the referenced graph.
996
version_ids = other.get_ancestry(version_ids)
997
pending_graph = [(version, other.get_parents(version)) for
998
version in version_ids]
999
for name in topo_sort(pending_graph):
1000
other_idx = other._name_map[name]
1001
# returns True if we have it, False if we need it.
1002
if not self._check_version_consistent(other, other_idx, name):
1003
names_to_join.append((other_idx, name))
1012
for other_idx, name in names_to_join:
1013
# TODO: If all the parents of the other version are already
1014
# present then we can avoid some work by just taking the delta
1015
# and adjusting the offsets.
1016
new_parents = self._imported_parents(other, other_idx)
1017
sha1 = other._sha1s[other_idx]
1022
pb.update(msg, merged, len(names_to_join))
1024
lines = other.get_lines(other_idx)
1025
self._add(name, lines, new_parents, sha1)
1027
mutter("merged = %d, processed = %d, file_id=%s; deltat=%d"%(
1028
merged, processed, self._weave_name, time.time()-time0))
1030
def _imported_parents(self, other, other_idx):
1031
"""Return list of parents in self corresponding to indexes in other."""
1033
for parent_idx in other._parents[other_idx]:
1034
parent_name = other._names[parent_idx]
1035
if parent_name not in self._name_map:
1036
# should not be possible
1037
raise WeaveError("missing parent {%s} of {%s} in %r"
1038
% (parent_name, other._name_map[other_idx], self))
1039
new_parents.append(self._name_map[parent_name])
1042
def _check_version_consistent(self, other, other_idx, name):
1043
"""Check if a version in consistent in this and other.
1045
To be consistent it must have:
1048
* the same direct parents (by name, not index, and disregarding
1051
If present & correct return True;
1052
if not present in self return False;
1053
if inconsistent raise error."""
1054
this_idx = self._name_map.get(name, -1)
1056
if self._sha1s[this_idx] != other._sha1s[other_idx]:
1057
raise errors.WeaveTextDiffers(name, self, other)
1058
self_parents = self._parents[this_idx]
1059
other_parents = other._parents[other_idx]
1060
n1 = set([self._names[i] for i in self_parents])
1061
n2 = set([other._names[i] for i in other_parents])
1062
if not self._compatible_parents(n1, n2):
1063
raise WeaveParentMismatch("inconsistent parents "
1064
"for version {%s}: %s vs %s" % (name, n1, n2))
1070
@deprecated_method(zero_eight)
1071
def reweave(self, other, pb=None, msg=None):
1072
"""reweave has been superceded by plain use of join."""
1073
return self.join(other, pb, msg)
1075
def _reweave(self, other, pb, msg):
1076
"""Reweave self with other - internal helper for join().
1078
:param other: The other weave to merge
1079
:param pb: An optional progress bar, indicating how far done we are
1080
:param msg: An optional message for the progress
1082
new_weave = _reweave(self, other, pb=pb, msg=msg)
1083
self._copy_weave_content(new_weave)
1085
def _copy_weave_content(self, otherweave):
1086
"""adsorb the content from otherweave."""
1087
for attr in self.__slots__:
1088
if attr != '_weave_name':
1089
setattr(self, attr, copy(getattr(otherweave, attr)))
1092
class WeaveFile(Weave):
1093
"""A WeaveFile represents a Weave on disk and writes on change."""
1095
WEAVE_SUFFIX = '.weave'
1097
def __init__(self, name, transport, filemode=None, create=False, access_mode='w'):
1098
"""Create a WeaveFile.
1100
:param create: If not True, only open an existing knit.
1102
super(WeaveFile, self).__init__(name, access_mode)
1103
self._transport = transport
1104
self._filemode = filemode
1106
_read_weave_v5(self._transport.get(name + WeaveFile.WEAVE_SUFFIX), self)
1107
except errors.NoSuchFile:
1113
def _add_lines(self, version_id, parents, lines, parent_texts):
1114
"""Add a version and save the weave."""
1115
result = super(WeaveFile, self)._add_lines(version_id, parents, lines,
1120
def _clone_text(self, new_version_id, old_version_id, parents):
1121
"""See VersionedFile.clone_text."""
1122
super(WeaveFile, self)._clone_text(new_version_id, old_version_id, parents)
1125
def copy_to(self, name, transport):
1126
"""See VersionedFile.copy_to()."""
1127
# as we are all in memory always, just serialise to the new place.
1129
write_weave_v5(self, sio)
1131
transport.put(name + WeaveFile.WEAVE_SUFFIX, sio, self._filemode)
1133
def create_empty(self, name, transport, filemode=None):
1134
return WeaveFile(name, transport, filemode, create=True)
1137
"""Save the weave."""
1138
self._check_write_ok()
1140
write_weave_v5(self, sio)
1142
self._transport.put(self._weave_name + WeaveFile.WEAVE_SUFFIX,
1148
"""See VersionedFile.get_suffixes()."""
1149
return [WeaveFile.WEAVE_SUFFIX]
1151
def join(self, other, pb=None, msg=None, version_ids=None,
1152
ignore_missing=False):
1153
"""Join other into self and save."""
1154
super(WeaveFile, self).join(other, pb, msg, version_ids, ignore_missing)
1158
@deprecated_function(zero_eight)
1159
def reweave(wa, wb, pb=None, msg=None):
1160
"""reweaving is deprecation, please just use weave.join()."""
1161
_reweave(wa, wb, pb, msg)
1163
def _reweave(wa, wb, pb=None, msg=None):
1164
"""Combine two weaves and return the result.
1166
This works even if a revision R has different parents in
1167
wa and wb. In the resulting weave all the parents are given.
1169
This is done by just building up a new weave, maintaining ordering
1170
of the versions in the two inputs. More efficient approaches
1171
might be possible but it should only be necessary to do
1172
this operation rarely, when a new previously ghost version is
1175
:param pb: An optional progress bar, indicating how far done we are
1176
:param msg: An optional message for the progress
1180
queue_a = range(wa.num_versions())
1181
queue_b = range(wb.num_versions())
1182
# first determine combined parents of all versions
1183
# map from version name -> all parent names
1184
combined_parents = _reweave_parent_graphs(wa, wb)
1185
mutter("combined parents: %r", combined_parents)
1186
order = topo_sort(combined_parents.iteritems())
1187
mutter("order to reweave: %r", order)
1192
for idx, name in enumerate(order):
1194
pb.update(msg, idx, len(order))
1195
if name in wa._name_map:
1196
lines = wa.get_lines(name)
1197
if name in wb._name_map:
1198
lines_b = wb.get_lines(name)
1199
if lines != lines_b:
1200
mutter('Weaves differ on content. rev_id {%s}', name)
1201
mutter('weaves: %s, %s', wa._weave_name, wb._weave_name)
1203
lines = list(difflib.unified_diff(lines, lines_b,
1204
wa._weave_name, wb._weave_name))
1205
mutter('lines:\n%s', ''.join(lines))
1206
raise errors.WeaveTextDiffers(name, wa, wb)
1208
lines = wb.get_lines(name)
1209
wr._add(name, lines, [wr._lookup(i) for i in combined_parents[name]])
1212
def _reweave_parent_graphs(wa, wb):
1213
"""Return combined parent ancestry for two weaves.
1215
Returned as a list of (version_name, set(parent_names))"""
1217
for weave in [wa, wb]:
1218
for idx, name in enumerate(weave._names):
1219
p = combined.setdefault(name, set())
1220
p.update(map(weave._idx_to_name, weave._parents[idx]))
712
1224
def weave_toc(w):
added
removed
bzrlib/weave.py
