71
71
from copy import copy
72
72
from cStringIO import StringIO
77
from bzrlib.lazy_import import lazy_import
78
lazy_import(globals(), """
79
from bzrlib import tsort
81
78
from bzrlib import (
81
from bzrlib.trace import mutter
86
82
from bzrlib.errors import (WeaveError, WeaveFormatError, WeaveParentMismatch,
87
83
RevisionAlreadyPresent,
88
84
RevisionNotPresent,
89
UnavailableRepresentation,
90
85
WeaveRevisionAlreadyPresent,
91
86
WeaveRevisionNotPresent,
93
from bzrlib.osutils import dirname, sha, sha_strings, split_lines
88
import bzrlib.errors as errors
89
from bzrlib.osutils import sha_strings
94
90
import bzrlib.patiencediff
95
from bzrlib.revision import NULL_REVISION
96
from bzrlib.symbol_versioning import *
97
from bzrlib.trace import mutter
98
from bzrlib.versionedfile import (
91
from bzrlib.symbol_versioning import (deprecated_method,
95
from bzrlib.tsort import topo_sort
96
from bzrlib.versionedfile import VersionedFile, InterVersionedFile
105
97
from bzrlib.weavefile import _read_weave_v5, write_weave_v5
108
class WeaveContentFactory(ContentFactory):
109
"""Content factory for streaming from weaves.
111
:seealso ContentFactory:
114
def __init__(self, version, weave):
115
"""Create a WeaveContentFactory for version from weave."""
116
ContentFactory.__init__(self)
117
self.sha1 = weave.get_sha1s([version])[version]
118
self.key = (version,)
119
parents = weave.get_parent_map([version])[version]
120
self.parents = tuple((parent,) for parent in parents)
121
self.storage_kind = 'fulltext'
124
def get_bytes_as(self, storage_kind):
125
if storage_kind == 'fulltext':
126
return self._weave.get_text(self.key[-1])
127
elif storage_kind == 'chunked':
128
return self._weave.get_lines(self.key[-1])
130
raise UnavailableRepresentation(self.key, storage_kind, 'fulltext')
133
100
class Weave(VersionedFile):
134
101
"""weave - versioned text file storage.
136
103
A Weave manages versions of line-based text files, keeping track
137
104
of the originating version for each line.
304
274
__contains__ = has_version
306
def get_record_stream(self, versions, ordering, include_delta_closure):
307
"""Get a stream of records for versions.
309
:param versions: The versions to include. Each version is a tuple
311
:param ordering: Either 'unordered' or 'topological'. A topologically
312
sorted stream has compression parents strictly before their
314
:param include_delta_closure: If True then the closure across any
315
compression parents will be included (in the opaque data).
316
:return: An iterator of ContentFactory objects, each of which is only
317
valid until the iterator is advanced.
319
versions = [version[-1] for version in versions]
320
if ordering == 'topological':
321
parents = self.get_parent_map(versions)
322
new_versions = tsort.topo_sort(parents)
323
new_versions.extend(set(versions).difference(set(parents)))
324
versions = new_versions
325
elif ordering == 'groupcompress':
326
parents = self.get_parent_map(versions)
327
new_versions = sort_groupcompress(parents)
328
new_versions.extend(set(versions).difference(set(parents)))
329
versions = new_versions
330
for version in versions:
332
yield WeaveContentFactory(version, self)
334
yield AbsentContentFactory((version,))
336
def get_parent_map(self, version_ids):
337
"""See VersionedFile.get_parent_map."""
276
def get_delta(self, version_id):
277
"""See VersionedFile.get_delta."""
278
return self.get_deltas([version_id])[version_id]
280
def get_deltas(self, version_ids):
281
"""See VersionedFile.get_deltas."""
282
version_ids = self.get_ancestry(version_ids)
339
283
for version_id in version_ids:
340
if version_id == NULL_REVISION:
345
map(self._idx_to_name,
346
self._parents[self._lookup(version_id)]))
347
except RevisionNotPresent:
284
if not self.has_version(version_id):
285
raise RevisionNotPresent(version_id, self)
286
# try extracting all versions; parallel extraction is used
287
nv = self.num_versions()
293
last_parent_lines = {}
295
parent_inclusions = {}
300
# its simplest to generate a full set of prepared variables.
302
name = self._names[i]
303
sha1s[name] = self.get_sha1(name)
304
parents_list = self.get_parents(name)
306
parent = parents_list[0]
307
parents[name] = parent
308
parent_inclusions[name] = inclusions[parent]
311
parent_inclusions[name] = set()
312
# we want to emit start, finish, replacement_length, replacement_lines tuples.
313
diff_hunks[name] = []
314
current_hunks[name] = [0, 0, 0, []] # #start, finish, repl_length, repl_tuples
315
parent_linenums[name] = 0
317
parent_noeols[name] = False
318
last_parent_lines[name] = None
319
new_inc = set([name])
320
for p in self._parents[i]:
321
new_inc.update(inclusions[self._idx_to_name(p)])
322
# debug only, known good so far.
323
#assert set(new_inc) == set(self.get_ancestry(name)), \
324
# 'failed %s != %s' % (set(new_inc), set(self.get_ancestry(name)))
325
inclusions[name] = new_inc
327
nlines = len(self._weave)
329
for lineno, inserted, deletes, line in self._walk_internal():
330
# a line is active in a version if:
331
# insert is in the versions inclusions
333
# deleteset & the versions inclusions is an empty set.
334
# so - if we have a included by mapping - version is included by
335
# children, we get a list of children to examine for deletes affect
336
# ing them, which is less than the entire set of children.
337
for version_id in version_ids:
338
# The active inclusion must be an ancestor,
339
# and no ancestors must have deleted this line,
340
# because we don't support resurrection.
341
parent_inclusion = parent_inclusions[version_id]
342
inclusion = inclusions[version_id]
343
parent_active = inserted in parent_inclusion and not (deletes & parent_inclusion)
344
version_active = inserted in inclusion and not (deletes & inclusion)
345
if not parent_active and not version_active:
346
# unrelated line of ancestry
349
result[version_id] = parents
348
elif parent_active and version_active:
350
parent_linenum = parent_linenums[version_id]
351
if current_hunks[version_id] != [parent_linenum, parent_linenum, 0, []]:
352
diff_hunks[version_id].append(tuple(current_hunks[version_id]))
354
current_hunks[version_id] = [parent_linenum, parent_linenum, 0, []]
355
parent_linenums[version_id] = parent_linenum
358
noeols[version_id] = True
361
elif parent_active and not version_active:
363
current_hunks[version_id][1] += 1
364
parent_linenums[version_id] += 1
365
last_parent_lines[version_id] = line
366
elif not parent_active and version_active:
368
# noeol only occurs at the end of a file because we
369
# diff linewise. We want to show noeol changes as a
370
# empty diff unless the actual eol-less content changed.
373
if last_parent_lines[version_id][-1] != '\n':
374
parent_noeols[version_id] = True
375
except (TypeError, IndexError):
378
if theline[-1] != '\n':
379
noeols[version_id] = True
383
parent_should_go = False
385
if parent_noeols[version_id] == noeols[version_id]:
386
# no noeol toggle, so trust the weaves statement
387
# that this line is changed.
389
if parent_noeols[version_id]:
390
theline = theline + '\n'
391
elif parent_noeols[version_id]:
392
# parent has no eol, we do:
393
# our line is new, report as such..
395
elif noeols[version_id]:
396
# append a eol so that it looks like
398
theline = theline + '\n'
399
if parents[version_id] is not None:
400
#if last_parent_lines[version_id] is not None:
401
parent_should_go = True
402
if last_parent_lines[version_id] != theline:
405
#parent_should_go = False
407
current_hunks[version_id][2] += 1
408
current_hunks[version_id][3].append((inserted, theline))
410
# last hunk last parent line is not eaten
411
current_hunks[version_id][1] -= 1
412
if current_hunks[version_id][1] < 0:
413
current_hunks[version_id][1] = 0
414
# import pdb;pdb.set_trace()
415
# assert current_hunks[version_id][1] >= 0
419
version = self._idx_to_name(i)
420
if current_hunks[version] != [0, 0, 0, []]:
421
diff_hunks[version].append(tuple(current_hunks[version]))
423
for version_id in version_ids:
424
result[version_id] = (
428
diff_hunks[version_id],
352
def get_parents_with_ghosts(self, version_id):
353
raise NotImplementedError(self.get_parents_with_ghosts)
355
def insert_record_stream(self, stream):
356
"""Insert a record stream into this versioned file.
358
:param stream: A stream of records to insert.
360
:seealso VersionedFile.get_record_stream:
363
for record in stream:
364
# Raise an error when a record is missing.
365
if record.storage_kind == 'absent':
366
raise RevisionNotPresent([record.key[0]], self)
367
# adapt to non-tuple interface
368
parents = [parent[0] for parent in record.parents]
369
if (record.storage_kind == 'fulltext'
370
or record.storage_kind == 'chunked'):
371
self.add_lines(record.key[0], parents,
372
osutils.chunks_to_lines(record.get_bytes_as('chunked')))
374
adapter_key = record.storage_kind, 'fulltext'
376
adapter = adapters[adapter_key]
378
adapter_factory = adapter_registry.get(adapter_key)
379
adapter = adapter_factory(self)
380
adapters[adapter_key] = adapter
381
lines = split_lines(adapter.get_bytes(record))
383
self.add_lines(record.key[0], parents, lines)
384
except RevisionAlreadyPresent:
432
def get_parents(self, version_id):
433
"""See VersionedFile.get_parent."""
434
return map(self._idx_to_name, self._parents[self._lookup(version_id)])
387
436
def _check_repeated_add(self, name, parents, text, sha1):
388
437
"""Check that a duplicated add is OK.
395
444
raise RevisionAlreadyPresent(name, self._weave_name)
398
def _add_lines(self, version_id, parents, lines, parent_texts,
399
left_matching_blocks, nostore_sha, random_id, check_content):
447
@deprecated_method(zero_eight)
448
def add_identical(self, old_rev_id, new_rev_id, parents):
449
"""Please use Weave.clone_text now."""
450
return self.clone_text(new_rev_id, old_rev_id, parents)
452
def _add_lines(self, version_id, parents, lines, parent_texts):
400
453
"""See VersionedFile.add_lines."""
401
idx = self._add(version_id, lines, map(self._lookup, parents),
402
nostore_sha=nostore_sha)
403
return sha_strings(lines), sum(map(len, lines)), idx
405
def _add(self, version_id, lines, parents, sha1=None, nostore_sha=None):
454
return self._add(version_id, lines, map(self._lookup, parents))
456
@deprecated_method(zero_eight)
457
def add(self, name, parents, text, sha1=None):
458
"""See VersionedFile.add_lines for the non deprecated api."""
459
return self._add(name, text, map(self._maybe_lookup, parents), sha1)
461
def _add(self, version_id, lines, parents, sha1=None):
406
462
"""Add a single text on top of the weave.
408
464
Returns the index number of the newly added version.
411
467
Symbolic name for this version.
412
468
(Typically the revision-id of the revision that added it.)
413
If None, a name will be allocated based on the hash. (sha1:SHAHASH)
416
471
List or set of direct parent version numbers.
419
474
Sequence of lines to be added in the new version.
421
:param nostore_sha: See VersionedFile.add_lines.
477
assert isinstance(version_id, basestring)
423
478
self._check_lines_not_unicode(lines)
424
479
self._check_lines_are_lines(lines)
426
481
sha1 = sha_strings(lines)
427
if sha1 == nostore_sha:
428
raise errors.ExistingContent
429
if version_id is None:
430
version_id = "sha1:" + sha1
431
482
if version_id in self._name_map:
432
483
return self._check_repeated_add(version_id, parents, lines, sha1)
855
975
# no lines outside of insertion blocks, that deletions are
856
976
# properly paired, etc.
978
def _join(self, other, pb, msg, version_ids, ignore_missing):
979
"""Worker routine for join()."""
980
if not other.versions():
981
return # nothing to update, easy
984
# versions is never none, InterWeave checks this.
987
# two loops so that we do not change ourselves before verifying it
989
# work through in index order to make sure we get all dependencies
992
# get the selected versions only that are in other.versions.
993
version_ids = set(other.versions()).intersection(set(version_ids))
994
# pull in the referenced graph.
995
version_ids = other.get_ancestry(version_ids)
996
pending_graph = [(version, other.get_parents(version)) for
997
version in version_ids]
998
for name in topo_sort(pending_graph):
999
other_idx = other._name_map[name]
1000
# returns True if we have it, False if we need it.
1001
if not self._check_version_consistent(other, other_idx, name):
1002
names_to_join.append((other_idx, name))
1011
for other_idx, name in names_to_join:
1012
# TODO: If all the parents of the other version are already
1013
# present then we can avoid some work by just taking the delta
1014
# and adjusting the offsets.
1015
new_parents = self._imported_parents(other, other_idx)
1016
sha1 = other._sha1s[other_idx]
1021
pb.update(msg, merged, len(names_to_join))
1023
lines = other.get_lines(other_idx)
1024
self._add(name, lines, new_parents, sha1)
1026
mutter("merged = %d, processed = %d, file_id=%s; deltat=%d"%(
1027
merged, processed, self._weave_name, time.time()-time0))
858
1029
def _imported_parents(self, other, other_idx):
859
1030
"""Return list of parents in self corresponding to indexes in other."""
860
1031
new_parents = []
953
1131
transport.put_file(name + WeaveFile.WEAVE_SUFFIX, sio, self._filemode)
1133
def create_empty(self, name, transport, filemode=None):
1134
return WeaveFile(name, transport, filemode, create=True)
955
1136
def _save(self):
956
1137
"""Save the weave."""
957
1138
self._check_write_ok()
958
1139
sio = StringIO()
959
1140
write_weave_v5(self, sio)
961
bytes = sio.getvalue()
962
path = self._weave_name + WeaveFile.WEAVE_SUFFIX
964
self._transport.put_bytes(path, bytes, self._filemode)
965
except errors.NoSuchFile:
966
self._transport.mkdir(dirname(path))
967
self._transport.put_bytes(path, bytes, self._filemode)
1142
self._transport.put_file(self._weave_name + WeaveFile.WEAVE_SUFFIX,
970
1147
def get_suffixes():
971
1148
"""See VersionedFile.get_suffixes()."""
972
1149
return [WeaveFile.WEAVE_SUFFIX]
974
def insert_record_stream(self, stream):
975
super(WeaveFile, self).insert_record_stream(stream)
1151
def join(self, other, pb=None, msg=None, version_ids=None,
1152
ignore_missing=False):
1153
"""Join other into self and save."""
1154
super(WeaveFile, self).join(other, pb, msg, version_ids, ignore_missing)
1158
@deprecated_function(zero_eight)
1159
def reweave(wa, wb, pb=None, msg=None):
1160
"""reweaving is deprecation, please just use weave.join()."""
1161
_reweave(wa, wb, pb, msg)
979
1163
def _reweave(wa, wb, pb=None, msg=None):
980
1164
"""Combine two weaves and return the result.
982
This works even if a revision R has different parents in
1166
This works even if a revision R has different parents in
983
1167
wa and wb. In the resulting weave all the parents are given.
985
This is done by just building up a new weave, maintaining ordering
1169
This is done by just building up a new weave, maintaining ordering
986
1170
of the versions in the two inputs. More efficient approaches
987
might be possible but it should only be necessary to do
988
this operation rarely, when a new previously ghost version is
1171
might be possible but it should only be necessary to do
1172
this operation rarely, when a new previously ghost version is
991
1175
:param pb: An optional progress bar, indicating how far done we are
added
removed
bzrlib/weave.py
