7
7
# it under the terms of the GNU General Public License as published by
8
8
# the Free Software Foundation; either version 2 of the License, or
9
9
# (at your option) any later version.
11
11
# This program is distributed in the hope that it will be useful,
12
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
14
# GNU General Public License for more details.
16
16
# You should have received a copy of the GNU General Public License
17
17
# along with this program; if not, write to the Free Software
18
18
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20
20
"""Versioned text file storage api."""
23
from copy import deepcopy
24
from unittest import TestSuite
27
import bzrlib.errors as errors
22
from bzrlib.lazy_import import lazy_import
23
lazy_import(globals(), """
33
from bzrlib.transport.memory import MemoryTransport
36
from cStringIO import StringIO
28
38
from bzrlib.inter import InterObject
29
from bzrlib.symbol_versioning import *
30
39
from bzrlib.textmerge import TextMerge
31
from bzrlib.transport.memory import MemoryTransport
32
from bzrlib.tsort import topo_sort
36
42
class VersionedFile(object):
75
77
"""Returns whether version is present."""
76
78
raise NotImplementedError(self.has_version)
78
def add_delta(self, version_id, parents, delta_parent, sha1, noeol, delta):
79
"""Add a text to the versioned file via a pregenerated delta.
81
:param version_id: The version id being added.
82
:param parents: The parents of the version_id.
83
:param delta_parent: The parent this delta was created against.
84
:param sha1: The sha1 of the full text.
85
:param delta: The delta instructions. See get_delta for details.
87
self._check_write_ok()
88
if self.has_version(version_id):
89
raise errors.RevisionAlreadyPresent(version_id, self)
90
return self._add_delta(version_id, parents, delta_parent, sha1, noeol, delta)
92
def _add_delta(self, version_id, parents, delta_parent, sha1, noeol, delta):
93
"""Class specific routine to add a delta.
95
This generic version simply applies the delta to the delta_parent and
98
# strip annotation from delta
100
for start, stop, delta_len, delta_lines in delta:
101
new_delta.append((start, stop, delta_len, [text for origin, text in delta_lines]))
102
if delta_parent is not None:
103
parent_full = self.get_lines(delta_parent)
106
new_full = self._apply_delta(parent_full, new_delta)
107
# its impossible to have noeol on an empty file
108
if noeol and new_full[-1][-1] == '\n':
109
new_full[-1] = new_full[-1][:-1]
110
self.add_lines(version_id, parents, new_full)
112
def add_lines(self, version_id, parents, lines, parent_texts=None):
80
def add_lines(self, version_id, parents, lines, parent_texts=None,
81
left_matching_blocks=None, nostore_sha=None, random_id=False,
113
83
"""Add a single text on top of the versioned file.
115
85
Must raise RevisionAlreadyPresent if the new version is
118
88
Must raise RevisionNotPresent if any of the given parents are
119
89
not present in file history.
91
:param lines: A list of lines. Each line must be a bytestring. And all
92
of them except the last must be terminated with \n and contain no
93
other \n's. The last line may either contain no \n's or a single
94
terminated \n. If the lines list does meet this constraint the add
95
routine may error or may succeed - but you will be unable to read
96
the data back accurately. (Checking the lines have been split
97
correctly is expensive and extremely unlikely to catch bugs so it
98
is not done at runtime unless check_content is True.)
120
99
:param parent_texts: An optional dictionary containing the opaque
121
representations of some or all of the parents of
122
version_id to allow delta optimisations.
123
VERY IMPORTANT: the texts must be those returned
124
by add_lines or data corruption can be caused.
125
:return: An opaque representation of the inserted version which can be
126
provided back to future add_lines calls in the parent_texts
100
representations of some or all of the parents of version_id to
101
allow delta optimisations. VERY IMPORTANT: the texts must be those
102
returned by add_lines or data corruption can be caused.
103
:param left_matching_blocks: a hint about which areas are common
104
between the text and its left-hand-parent. The format is
105
the SequenceMatcher.get_matching_blocks format.
106
:param nostore_sha: Raise ExistingContent and do not add the lines to
107
the versioned file if the digest of the lines matches this.
108
:param random_id: If True a random id has been selected rather than
109
an id determined by some deterministic process such as a converter
110
from a foreign VCS. When True the backend may choose not to check
111
for uniqueness of the resulting key within the versioned file, so
112
this should only be done when the result is expected to be unique
114
:param check_content: If True, the lines supplied are verified to be
115
bytestrings that are correctly formed lines.
116
:return: The text sha1, the number of bytes in the text, and an opaque
117
representation of the inserted version which can be provided
118
back to future add_lines calls in the parent_texts dictionary.
120
version_id = osutils.safe_revision_id(version_id)
121
parents = [osutils.safe_revision_id(v) for v in parents]
129
122
self._check_write_ok()
130
return self._add_lines(version_id, parents, lines, parent_texts)
123
return self._add_lines(version_id, parents, lines, parent_texts,
124
left_matching_blocks, nostore_sha, random_id, check_content)
132
def _add_lines(self, version_id, parents, lines, parent_texts):
126
def _add_lines(self, version_id, parents, lines, parent_texts,
127
left_matching_blocks, nostore_sha, random_id, check_content):
133
128
"""Helper to do the class specific add_lines."""
134
129
raise NotImplementedError(self.add_lines)
136
131
def add_lines_with_ghosts(self, version_id, parents, lines,
132
parent_texts=None, nostore_sha=None, random_id=False,
138
134
"""Add lines to the versioned file, allowing ghosts to be present.
140
This takes the same parameters as add_lines.
136
This takes the same parameters as add_lines and returns the same.
138
version_id = osutils.safe_revision_id(version_id)
139
parents = [osutils.safe_revision_id(v) for v in parents]
142
140
self._check_write_ok()
143
141
return self._add_lines_with_ghosts(version_id, parents, lines,
142
parent_texts, nostore_sha, random_id, check_content)
146
def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts):
144
def _add_lines_with_ghosts(self, version_id, parents, lines, parent_texts,
145
nostore_sha, random_id, check_content):
147
146
"""Helper to do class specific add_lines_with_ghosts."""
148
147
raise NotImplementedError(self.add_lines_with_ghosts)
198
210
raise NotImplementedError(self.create_empty)
200
def fix_parents(self, version, new_parents):
201
"""Fix the parents list for version.
203
This is done by appending a new version to the index
204
with identical data except for the parents list.
205
the parents list must be a superset of the current
208
self._check_write_ok()
209
return self._fix_parents(version, new_parents)
211
def _fix_parents(self, version, new_parents):
212
"""Helper for fix_parents."""
213
raise NotImplementedError(self.fix_parents)
215
def get_delta(self, version):
216
"""Get a delta for constructing version from some other version.
218
:return: (delta_parent, sha1, noeol, delta)
219
Where delta_parent is a version id or None to indicate no parent.
221
raise NotImplementedError(self.get_delta)
223
def get_deltas(self, versions):
224
"""Get multiple deltas at once for constructing versions.
226
:return: dict(version_id:(delta_parent, sha1, noeol, delta))
227
Where delta_parent is a version id or None to indicate no parent, and
228
version_id is the version_id created by that delta.
231
for version in versions:
232
result[version] = self.get_delta(version)
212
def get_format_signature(self):
213
"""Get a text description of the data encoding in this file.
217
raise NotImplementedError(self.get_format_signature)
219
def make_mpdiffs(self, version_ids):
220
"""Create multiparent diffs for specified versions"""
221
knit_versions = set()
222
for version_id in version_ids:
223
knit_versions.add(version_id)
224
knit_versions.update(self.get_parents(version_id))
225
lines = dict(zip(knit_versions,
226
self._get_lf_split_line_list(knit_versions)))
228
for version_id in version_ids:
229
target = lines[version_id]
230
parents = [lines[p] for p in self.get_parents(version_id)]
232
left_parent_blocks = self._extract_blocks(version_id,
235
left_parent_blocks = None
236
diffs.append(multiparent.MultiParent.from_lines(target, parents,
240
def _extract_blocks(self, version_id, source, target):
243
def add_mpdiffs(self, records):
244
"""Add mpdiffs to this versionedfile
246
Records should be iterables of version, parents, expected_sha1,
247
mpdiff. mpdiff should be a MultiParent instance.
250
mpvf = multiparent.MultiMemoryVersionedFile()
252
for version, parent_ids, expected_sha1, mpdiff in records:
253
versions.append(version)
254
mpvf.add_diff(mpdiff, version, parent_ids)
255
needed_parents = set()
256
for version, parent_ids, expected_sha1, mpdiff in records:
257
needed_parents.update(p for p in parent_ids
258
if not mpvf.has_version(p))
259
for parent_id, lines in zip(needed_parents,
260
self._get_lf_split_line_list(needed_parents)):
261
mpvf.add_version(lines, parent_id, [])
262
for (version, parent_ids, expected_sha1, mpdiff), lines in\
263
zip(records, mpvf.get_line_list(versions)):
264
if len(parent_ids) == 1:
265
left_matching_blocks = list(mpdiff.get_matching_blocks(0,
266
mpvf.get_diff(parent_ids[0]).num_lines()))
268
left_matching_blocks = None
269
_, _, version_text = self.add_lines(version, parent_ids, lines,
270
vf_parents, left_matching_blocks=left_matching_blocks)
271
vf_parents[version] = version_text
272
for (version, parent_ids, expected_sha1, mpdiff), sha1 in\
273
zip(records, self.get_sha1s(versions)):
274
if expected_sha1 != sha1:
275
raise errors.VersionedFileInvalidChecksum(version)
235
277
def get_sha1(self, version_id):
236
278
"""Get the stored sha1 sum for the given revision.
384
def iter_lines_added_or_present_in_versions(self, version_ids=None):
437
def iter_lines_added_or_present_in_versions(self, version_ids=None,
385
439
"""Iterate over the lines in the versioned file from version_ids.
387
441
This may return lines from other versions, and does not return the
388
442
specific version marker at this point. The api may be changed
389
443
during development to include the version that the versioned file
390
444
thinks is relevant, but given that such hints are just guesses,
391
its better not to have it if we dont need it.
445
its better not to have it if we don't need it.
447
If a progress bar is supplied, it may be used to indicate progress.
448
The caller is responsible for cleaning up progress bars (because this
393
451
NOTES: Lines are normalised: they will all have \n terminators.
394
452
Lines are returned in arbitrary order.
396
454
raise NotImplementedError(self.iter_lines_added_or_present_in_versions)
456
def iter_parents(self, version_ids):
457
"""Iterate through the parents for many version ids.
459
:param version_ids: An iterable yielding version_ids.
460
:return: An iterator that yields (version_id, parents). Requested
461
version_ids not present in the versioned file are simply skipped.
462
The order is undefined, allowing for different optimisations in
463
the underlying implementation.
465
for version_id in version_ids:
467
yield version_id, tuple(self.get_parents(version_id))
468
except errors.RevisionNotPresent:
398
471
def transaction_finished(self):
399
472
"""The transaction that this file was opened in has finished.
404
477
self.finished = True
406
@deprecated_method(zero_eight)
407
def walk(self, version_ids=None):
408
"""Walk the versioned file as a weave-like structure, for
409
versions relative to version_ids. Yields sequence of (lineno,
410
insert, deletes, text) for each relevant line.
412
Must raise RevisionNotPresent if any of the specified versions
413
are not present in the file history.
415
:param version_ids: the version_ids to walk with respect to. If not
416
supplied the entire weave-like structure is walked.
418
walk is deprecated in favour of iter_lines_added_or_present_in_versions
420
raise NotImplementedError(self.walk)
422
@deprecated_method(zero_eight)
423
def iter_names(self):
424
"""Walk the names list."""
425
return iter(self.versions())
427
479
def plan_merge(self, ver_a, ver_b):
428
480
"""Return pseudo-annotation indicating how the two versions merge.
580
632
# TODO: remove parent texts when they are not relevant any more for
581
633
# memory pressure reduction. RBC 20060313
582
634
# pb.update('Converting versioned data', 0, len(order))
583
# deltas = self.source.get_deltas(order)
584
635
for index, version in enumerate(order):
585
636
pb.update('Converting versioned data', index, len(order))
586
parent_text = target.add_lines(version,
637
_, _, parent_text = target.add_lines(version,
587
638
self.source.get_parents(version),
588
639
self.source.get_lines(version),
589
640
parent_texts=parent_texts)
590
641
parent_texts[version] = parent_text
591
#delta_parent, sha1, noeol, delta = deltas[version]
592
#target.add_delta(version,
593
# self.source.get_parents(version),
598
#target.get_lines(version)
600
643
# this should hit the native code path for target
601
644
if target is not self.target:
634
678
new_version_ids.add(version)
635
679
return new_version_ids
638
class InterVersionedFileTestProviderAdapter(object):
639
"""A tool to generate a suite testing multiple inter versioned-file classes.
641
This is done by copying the test once for each interversionedfile provider
642
and injecting the transport_server, transport_readonly_server,
643
versionedfile_factory and versionedfile_factory_to classes into each copy.
644
Each copy is also given a new id() to make it easy to identify.
647
def __init__(self, transport_server, transport_readonly_server, formats):
648
self._transport_server = transport_server
649
self._transport_readonly_server = transport_readonly_server
650
self._formats = formats
652
def adapt(self, test):
654
for (interversionedfile_class,
655
versionedfile_factory,
656
versionedfile_factory_to) in self._formats:
657
new_test = deepcopy(test)
658
new_test.transport_server = self._transport_server
659
new_test.transport_readonly_server = self._transport_readonly_server
660
new_test.interversionedfile_class = interversionedfile_class
661
new_test.versionedfile_factory = versionedfile_factory
662
new_test.versionedfile_factory_to = versionedfile_factory_to
663
def make_new_test_id():
664
new_id = "%s(%s)" % (new_test.id(), interversionedfile_class.__name__)
665
return lambda: new_id
666
new_test.id = make_new_test_id()
667
result.addTest(new_test)
671
def default_test_list():
672
"""Generate the default list of interversionedfile permutations to test."""
673
from bzrlib.weave import WeaveFile
674
from bzrlib.knit import KnitVersionedFile
676
# test the fallback InterVersionedFile from annotated knits to weave
677
result.append((InterVersionedFile,
680
for optimiser in InterVersionedFile._optimisers:
681
result.append((optimiser,
682
optimiser._matching_file_from_factory,
683
optimiser._matching_file_to_factory
685
# if there are specific combinations we want to use, we can add them
added
removed
bzrlib/versionedfile.py
