71
71
from copy import copy
72
72
from cStringIO import StringIO
77
from bzrlib.lazy_import import lazy_import
78
lazy_import(globals(), """
79
from bzrlib import tsort
78
81
from bzrlib import (
81
from bzrlib.trace import mutter
82
86
from bzrlib.errors import (WeaveError, WeaveFormatError, WeaveParentMismatch,
83
87
RevisionAlreadyPresent,
84
88
RevisionNotPresent,
89
UnavailableRepresentation,
85
90
WeaveRevisionAlreadyPresent,
86
91
WeaveRevisionNotPresent,
88
import bzrlib.errors as errors
89
from bzrlib.osutils import sha_strings
93
from bzrlib.osutils import dirname, sha, sha_strings, split_lines
90
94
import bzrlib.patiencediff
91
from bzrlib.symbol_versioning import (deprecated_method,
95
from bzrlib.tsort import topo_sort
96
from bzrlib.versionedfile import VersionedFile, InterVersionedFile
95
from bzrlib.revision import NULL_REVISION
96
from bzrlib.symbol_versioning import *
97
from bzrlib.trace import mutter
98
from bzrlib.versionedfile import (
97
105
from bzrlib.weavefile import _read_weave_v5, write_weave_v5
108
class WeaveContentFactory(ContentFactory):
109
"""Content factory for streaming from weaves.
111
:seealso ContentFactory:
114
def __init__(self, version, weave):
115
"""Create a WeaveContentFactory for version from weave."""
116
ContentFactory.__init__(self)
117
self.sha1 = weave.get_sha1s([version])[version]
118
self.key = (version,)
119
parents = weave.get_parent_map([version])[version]
120
self.parents = tuple((parent,) for parent in parents)
121
self.storage_kind = 'fulltext'
124
def get_bytes_as(self, storage_kind):
125
if storage_kind == 'fulltext':
126
return self._weave.get_text(self.key[-1])
127
elif storage_kind == 'chunked':
128
return self._weave.get_lines(self.key[-1])
130
raise UnavailableRepresentation(self.key, storage_kind, 'fulltext')
100
133
class Weave(VersionedFile):
101
134
"""weave - versioned text file storage.
103
136
A Weave manages versions of line-based text files, keeping track
104
137
of the originating version for each line.
274
304
__contains__ = has_version
276
def get_delta(self, version_id):
277
"""See VersionedFile.get_delta."""
278
return self.get_deltas([version_id])[version_id]
280
def get_deltas(self, version_ids):
281
"""See VersionedFile.get_deltas."""
282
version_ids = self.get_ancestry(version_ids)
306
def get_record_stream(self, versions, ordering, include_delta_closure):
307
"""Get a stream of records for versions.
309
:param versions: The versions to include. Each version is a tuple
311
:param ordering: Either 'unordered' or 'topological'. A topologically
312
sorted stream has compression parents strictly before their
314
:param include_delta_closure: If True then the closure across any
315
compression parents will be included (in the opaque data).
316
:return: An iterator of ContentFactory objects, each of which is only
317
valid until the iterator is advanced.
319
versions = [version[-1] for version in versions]
320
if ordering == 'topological':
321
parents = self.get_parent_map(versions)
322
new_versions = tsort.topo_sort(parents)
323
new_versions.extend(set(versions).difference(set(parents)))
324
versions = new_versions
325
elif ordering == 'groupcompress':
326
parents = self.get_parent_map(versions)
327
new_versions = sort_groupcompress(parents)
328
new_versions.extend(set(versions).difference(set(parents)))
329
versions = new_versions
330
for version in versions:
332
yield WeaveContentFactory(version, self)
334
yield AbsentContentFactory((version,))
336
def get_parent_map(self, version_ids):
337
"""See VersionedFile.get_parent_map."""
283
339
for version_id in version_ids:
284
if not self.has_version(version_id):
285
raise RevisionNotPresent(version_id, self)
286
# try extracting all versions; parallel extraction is used
287
nv = self.num_versions()
293
last_parent_lines = {}
295
parent_inclusions = {}
300
# its simplest to generate a full set of prepared variables.
302
name = self._names[i]
303
sha1s[name] = self.get_sha1(name)
304
parents_list = self.get_parents(name)
306
parent = parents_list[0]
307
parents[name] = parent
308
parent_inclusions[name] = inclusions[parent]
311
parent_inclusions[name] = set()
312
# we want to emit start, finish, replacement_length, replacement_lines tuples.
313
diff_hunks[name] = []
314
current_hunks[name] = [0, 0, 0, []] # #start, finish, repl_length, repl_tuples
315
parent_linenums[name] = 0
317
parent_noeols[name] = False
318
last_parent_lines[name] = None
319
new_inc = set([name])
320
for p in self._parents[i]:
321
new_inc.update(inclusions[self._idx_to_name(p)])
322
# debug only, known good so far.
323
#assert set(new_inc) == set(self.get_ancestry(name)), \
324
# 'failed %s != %s' % (set(new_inc), set(self.get_ancestry(name)))
325
inclusions[name] = new_inc
327
nlines = len(self._weave)
329
for lineno, inserted, deletes, line in self._walk_internal():
330
# a line is active in a version if:
331
# insert is in the versions inclusions
333
# deleteset & the versions inclusions is an empty set.
334
# so - if we have a included by mapping - version is included by
335
# children, we get a list of children to examine for deletes affect
336
# ing them, which is less than the entire set of children.
337
for version_id in version_ids:
338
# The active inclusion must be an ancestor,
339
# and no ancestors must have deleted this line,
340
# because we don't support resurrection.
341
parent_inclusion = parent_inclusions[version_id]
342
inclusion = inclusions[version_id]
343
parent_active = inserted in parent_inclusion and not (deletes & parent_inclusion)
344
version_active = inserted in inclusion and not (deletes & inclusion)
345
if not parent_active and not version_active:
346
# unrelated line of ancestry
340
if version_id == NULL_REVISION:
345
map(self._idx_to_name,
346
self._parents[self._lookup(version_id)]))
347
except RevisionNotPresent:
348
elif parent_active and version_active:
350
parent_linenum = parent_linenums[version_id]
351
if current_hunks[version_id] != [parent_linenum, parent_linenum, 0, []]:
352
diff_hunks[version_id].append(tuple(current_hunks[version_id]))
354
current_hunks[version_id] = [parent_linenum, parent_linenum, 0, []]
355
parent_linenums[version_id] = parent_linenum
358
noeols[version_id] = True
361
elif parent_active and not version_active:
363
current_hunks[version_id][1] += 1
364
parent_linenums[version_id] += 1
365
last_parent_lines[version_id] = line
366
elif not parent_active and version_active:
368
# noeol only occurs at the end of a file because we
369
# diff linewise. We want to show noeol changes as a
370
# empty diff unless the actual eol-less content changed.
373
if last_parent_lines[version_id][-1] != '\n':
374
parent_noeols[version_id] = True
375
except (TypeError, IndexError):
378
if theline[-1] != '\n':
379
noeols[version_id] = True
383
parent_should_go = False
385
if parent_noeols[version_id] == noeols[version_id]:
386
# no noeol toggle, so trust the weaves statement
387
# that this line is changed.
389
if parent_noeols[version_id]:
390
theline = theline + '\n'
391
elif parent_noeols[version_id]:
392
# parent has no eol, we do:
393
# our line is new, report as such..
395
elif noeols[version_id]:
396
# append a eol so that it looks like
398
theline = theline + '\n'
399
if parents[version_id] is not None:
400
#if last_parent_lines[version_id] is not None:
401
parent_should_go = True
402
if last_parent_lines[version_id] != theline:
405
#parent_should_go = False
407
current_hunks[version_id][2] += 1
408
current_hunks[version_id][3].append((inserted, theline))
410
# last hunk last parent line is not eaten
411
current_hunks[version_id][1] -= 1
412
if current_hunks[version_id][1] < 0:
413
current_hunks[version_id][1] = 0
414
# import pdb;pdb.set_trace()
415
# assert current_hunks[version_id][1] >= 0
419
version = self._idx_to_name(i)
420
if current_hunks[version] != [0, 0, 0, []]:
421
diff_hunks[version].append(tuple(current_hunks[version]))
423
for version_id in version_ids:
424
result[version_id] = (
428
diff_hunks[version_id],
349
result[version_id] = parents
432
def get_parents(self, version_id):
433
"""See VersionedFile.get_parent."""
434
return map(self._idx_to_name, self._parents[self._lookup(version_id)])
352
def get_parents_with_ghosts(self, version_id):
353
raise NotImplementedError(self.get_parents_with_ghosts)
355
def insert_record_stream(self, stream):
356
"""Insert a record stream into this versioned file.
358
:param stream: A stream of records to insert.
360
:seealso VersionedFile.get_record_stream:
363
for record in stream:
364
# Raise an error when a record is missing.
365
if record.storage_kind == 'absent':
366
raise RevisionNotPresent([record.key[0]], self)
367
# adapt to non-tuple interface
368
parents = [parent[0] for parent in record.parents]
369
if (record.storage_kind == 'fulltext'
370
or record.storage_kind == 'chunked'):
371
self.add_lines(record.key[0], parents,
372
osutils.chunks_to_lines(record.get_bytes_as('chunked')))
374
adapter_key = record.storage_kind, 'fulltext'
376
adapter = adapters[adapter_key]
378
adapter_factory = adapter_registry.get(adapter_key)
379
adapter = adapter_factory(self)
380
adapters[adapter_key] = adapter
381
lines = split_lines(adapter.get_bytes(record))
383
self.add_lines(record.key[0], parents, lines)
384
except RevisionAlreadyPresent:
436
387
def _check_repeated_add(self, name, parents, text, sha1):
437
388
"""Check that a duplicated add is OK.
444
395
raise RevisionAlreadyPresent(name, self._weave_name)
447
@deprecated_method(zero_eight)
448
def add_identical(self, old_rev_id, new_rev_id, parents):
449
"""Please use Weave.clone_text now."""
450
return self.clone_text(new_rev_id, old_rev_id, parents)
452
def _add_lines(self, version_id, parents, lines, parent_texts):
398
def _add_lines(self, version_id, parents, lines, parent_texts,
399
left_matching_blocks, nostore_sha, random_id, check_content):
453
400
"""See VersionedFile.add_lines."""
454
return self._add(version_id, lines, map(self._lookup, parents))
456
@deprecated_method(zero_eight)
457
def add(self, name, parents, text, sha1=None):
458
"""See VersionedFile.add_lines for the non deprecated api."""
459
return self._add(name, text, map(self._maybe_lookup, parents), sha1)
461
def _add(self, version_id, lines, parents, sha1=None):
401
idx = self._add(version_id, lines, map(self._lookup, parents),
402
nostore_sha=nostore_sha)
403
return sha_strings(lines), sum(map(len, lines)), idx
405
def _add(self, version_id, lines, parents, sha1=None, nostore_sha=None):
462
406
"""Add a single text on top of the weave.
464
408
Returns the index number of the newly added version.
467
411
Symbolic name for this version.
468
412
(Typically the revision-id of the revision that added it.)
413
If None, a name will be allocated based on the hash. (sha1:SHAHASH)
471
416
List or set of direct parent version numbers.
474
419
Sequence of lines to be added in the new version.
421
:param nostore_sha: See VersionedFile.add_lines.
477
assert isinstance(version_id, basestring)
478
423
self._check_lines_not_unicode(lines)
479
424
self._check_lines_are_lines(lines)
481
426
sha1 = sha_strings(lines)
427
if sha1 == nostore_sha:
428
raise errors.ExistingContent
429
if version_id is None:
430
version_id = "sha1:" + sha1
482
431
if version_id in self._name_map:
483
432
return self._check_repeated_add(version_id, parents, lines, sha1)
971
855
# no lines outside of insertion blocks, that deletions are
972
856
# properly paired, etc.
974
def _join(self, other, pb, msg, version_ids, ignore_missing):
975
"""Worker routine for join()."""
976
if not other.versions():
977
return # nothing to update, easy
980
# versions is never none, InterWeave checks this.
983
# two loops so that we do not change ourselves before verifying it
985
# work through in index order to make sure we get all dependencies
988
# get the selected versions only that are in other.versions.
989
version_ids = set(other.versions()).intersection(set(version_ids))
990
# pull in the referenced graph.
991
version_ids = other.get_ancestry(version_ids)
992
pending_graph = [(version, other.get_parents(version)) for
993
version in version_ids]
994
for name in topo_sort(pending_graph):
995
other_idx = other._name_map[name]
996
# returns True if we have it, False if we need it.
997
if not self._check_version_consistent(other, other_idx, name):
998
names_to_join.append((other_idx, name))
1007
for other_idx, name in names_to_join:
1008
# TODO: If all the parents of the other version are already
1009
# present then we can avoid some work by just taking the delta
1010
# and adjusting the offsets.
1011
new_parents = self._imported_parents(other, other_idx)
1012
sha1 = other._sha1s[other_idx]
1017
pb.update(msg, merged, len(names_to_join))
1019
lines = other.get_lines(other_idx)
1020
self._add(name, lines, new_parents, sha1)
1022
mutter("merged = %d, processed = %d, file_id=%s; deltat=%d"%(
1023
merged, processed, self._weave_name, time.time()-time0))
1025
858
def _imported_parents(self, other, other_idx):
1026
859
"""Return list of parents in self corresponding to indexes in other."""
1027
860
new_parents = []
1127
953
transport.put_file(name + WeaveFile.WEAVE_SUFFIX, sio, self._filemode)
1129
def create_empty(self, name, transport, filemode=None):
1130
return WeaveFile(name, transport, filemode, create=True)
1132
955
def _save(self):
1133
956
"""Save the weave."""
1134
957
self._check_write_ok()
1135
958
sio = StringIO()
1136
959
write_weave_v5(self, sio)
1138
self._transport.put_file(self._weave_name + WeaveFile.WEAVE_SUFFIX,
961
bytes = sio.getvalue()
962
path = self._weave_name + WeaveFile.WEAVE_SUFFIX
964
self._transport.put_bytes(path, bytes, self._filemode)
965
except errors.NoSuchFile:
966
self._transport.mkdir(dirname(path))
967
self._transport.put_bytes(path, bytes, self._filemode)
1143
970
def get_suffixes():
1144
971
"""See VersionedFile.get_suffixes()."""
1145
972
return [WeaveFile.WEAVE_SUFFIX]
1147
def join(self, other, pb=None, msg=None, version_ids=None,
1148
ignore_missing=False):
1149
"""Join other into self and save."""
1150
super(WeaveFile, self).join(other, pb, msg, version_ids, ignore_missing)
974
def insert_record_stream(self, stream):
975
super(WeaveFile, self).insert_record_stream(stream)
1154
@deprecated_function(zero_eight)
1155
def reweave(wa, wb, pb=None, msg=None):
1156
"""reweaving is deprecation, please just use weave.join()."""
1157
_reweave(wa, wb, pb, msg)
1159
979
def _reweave(wa, wb, pb=None, msg=None):
1160
980
"""Combine two weaves and return the result.
1162
This works even if a revision R has different parents in
982
This works even if a revision R has different parents in
1163
983
wa and wb. In the resulting weave all the parents are given.
1165
This is done by just building up a new weave, maintaining ordering
985
This is done by just building up a new weave, maintaining ordering
1166
986
of the versions in the two inputs. More efficient approaches
1167
might be possible but it should only be necessary to do
1168
this operation rarely, when a new previously ghost version is
987
might be possible but it should only be necessary to do
988
this operation rarely, when a new previously ghost version is
1171
991
:param pb: An optional progress bar, indicating how far done we are
added
removed
bzrlib/weave.py
