1
# Copyright (C) 2005 Canonical Ltd
4
# Johan Rydberg <jrydberg@gnu.org>
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
11
# This program is distributed in the hope that it will be useful,
12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
# GNU General Public License for more details.
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21
# TODO: might be nice to create a versionedfile with some type of corruption
22
# considered typical and check that it can be detected/corrected.
24
from itertools import chain, izip
25
from StringIO import StringIO
33
from bzrlib.errors import (
35
RevisionAlreadyPresent,
38
from bzrlib import knit as _mod_knit
39
from bzrlib.knit import (
46
from bzrlib.symbol_versioning import one_four, one_five
47
from bzrlib.tests import (
49
TestCaseWithMemoryTransport,
54
split_suite_by_condition,
57
from bzrlib.tests.http_utils import TestCaseWithWebserver
58
from bzrlib.trace import mutter
59
from bzrlib.transport import get_transport
60
from bzrlib.transport.memory import MemoryTransport
61
from bzrlib.tsort import topo_sort
62
from bzrlib.tuned_gzip import GzipFile
63
import bzrlib.versionedfile as versionedfile
64
from bzrlib.versionedfile import (
66
HashEscapedPrefixMapper,
68
VirtualVersionedFiles,
69
make_versioned_files_factory,
71
from bzrlib.weave import WeaveFile
72
from bzrlib.weavefile import read_weave, write_weave
75
def load_tests(standard_tests, module, loader):
76
"""Parameterize VersionedFiles tests for different implementations."""
77
to_adapt, result = split_suite_by_condition(
78
standard_tests, condition_isinstance(TestVersionedFiles))
79
len_one_adapter = TestScenarioApplier()
80
len_two_adapter = TestScenarioApplier()
81
# We want to be sure of behaviour for:
82
# weaves prefix layout (weave texts)
83
# individually named weaves (weave inventories)
84
# annotated knits - prefix|hash|hash-escape layout, we test the third only
85
# as it is the most complex mapper.
86
# individually named knits
87
# individual no-graph knits in packs (signatures)
88
# individual graph knits in packs (inventories)
89
# individual graph nocompression knits in packs (revisions)
90
# plain text knits in packs (texts)
91
len_one_adapter.scenarios = [
94
'factory':make_versioned_files_factory(WeaveFile,
95
ConstantMapper('inventory')),
98
'support_partial_insertion': False,
102
'factory':make_file_factory(False, ConstantMapper('revisions')),
105
'support_partial_insertion': False,
107
('named-nograph-nodelta-knit-pack', {
108
'cleanup':cleanup_pack_knit,
109
'factory':make_pack_factory(False, False, 1),
112
'support_partial_insertion': False,
114
('named-graph-knit-pack', {
115
'cleanup':cleanup_pack_knit,
116
'factory':make_pack_factory(True, True, 1),
119
'support_partial_insertion': True,
121
('named-graph-nodelta-knit-pack', {
122
'cleanup':cleanup_pack_knit,
123
'factory':make_pack_factory(True, False, 1),
126
'support_partial_insertion': False,
129
len_two_adapter.scenarios = [
132
'factory':make_versioned_files_factory(WeaveFile,
136
'support_partial_insertion': False,
138
('annotated-knit-escape', {
140
'factory':make_file_factory(True, HashEscapedPrefixMapper()),
143
'support_partial_insertion': False,
145
('plain-knit-pack', {
146
'cleanup':cleanup_pack_knit,
147
'factory':make_pack_factory(True, True, 2),
150
'support_partial_insertion': True,
153
for test in iter_suite_tests(to_adapt):
154
result.addTests(len_one_adapter.adapt(test))
155
result.addTests(len_two_adapter.adapt(test))
159
def get_diamond_vf(f, trailing_eol=True, left_only=False):
160
"""Get a diamond graph to exercise deltas and merges.
162
:param trailing_eol: If True end the last line with \n.
166
'base': (('origin',),),
167
'left': (('base',),),
168
'right': (('base',),),
169
'merged': (('left',), ('right',)),
171
# insert a diamond graph to exercise deltas and merges.
176
f.add_lines('origin', [], ['origin' + last_char])
177
f.add_lines('base', ['origin'], ['base' + last_char])
178
f.add_lines('left', ['base'], ['base\n', 'left' + last_char])
180
f.add_lines('right', ['base'],
181
['base\n', 'right' + last_char])
182
f.add_lines('merged', ['left', 'right'],
183
['base\n', 'left\n', 'right\n', 'merged' + last_char])
187
def get_diamond_files(files, key_length, trailing_eol=True, left_only=False,
189
"""Get a diamond graph to exercise deltas and merges.
191
This creates a 5-node graph in files. If files supports 2-length keys two
192
graphs are made to exercise the support for multiple ids.
194
:param trailing_eol: If True end the last line with \n.
195
:param key_length: The length of keys in files. Currently supports length 1
197
:param left_only: If True do not add the right and merged nodes.
198
:param nograph: If True, do not provide parents to the add_lines calls;
199
this is useful for tests that need inserted data but have graphless
201
:return: The results of the add_lines calls.
206
prefixes = [('FileA',), ('FileB',)]
207
# insert a diamond graph to exercise deltas and merges.
213
def get_parents(suffix_list):
217
result = [prefix + suffix for suffix in suffix_list]
219
# we loop over each key because that spreads the inserts across prefixes,
220
# which is how commit operates.
221
for prefix in prefixes:
222
result.append(files.add_lines(prefix + ('origin',), (),
223
['origin' + last_char]))
224
for prefix in prefixes:
225
result.append(files.add_lines(prefix + ('base',),
226
get_parents([('origin',)]), ['base' + last_char]))
227
for prefix in prefixes:
228
result.append(files.add_lines(prefix + ('left',),
229
get_parents([('base',)]),
230
['base\n', 'left' + last_char]))
232
for prefix in prefixes:
233
result.append(files.add_lines(prefix + ('right',),
234
get_parents([('base',)]),
235
['base\n', 'right' + last_char]))
236
for prefix in prefixes:
237
result.append(files.add_lines(prefix + ('merged',),
238
get_parents([('left',), ('right',)]),
239
['base\n', 'left\n', 'right\n', 'merged' + last_char]))
243
class VersionedFileTestMixIn(object):
244
"""A mixin test class for testing VersionedFiles.
246
This is not an adaptor-style test at this point because
247
theres no dynamic substitution of versioned file implementations,
248
they are strictly controlled by their owning repositories.
251
def get_transaction(self):
252
if not hasattr(self, '_transaction'):
253
self._transaction = None
254
return self._transaction
258
f.add_lines('r0', [], ['a\n', 'b\n'])
259
f.add_lines('r1', ['r0'], ['b\n', 'c\n'])
261
versions = f.versions()
262
self.assertTrue('r0' in versions)
263
self.assertTrue('r1' in versions)
264
self.assertEquals(f.get_lines('r0'), ['a\n', 'b\n'])
265
self.assertEquals(f.get_text('r0'), 'a\nb\n')
266
self.assertEquals(f.get_lines('r1'), ['b\n', 'c\n'])
267
self.assertEqual(2, len(f))
268
self.assertEqual(2, f.num_versions())
270
self.assertRaises(RevisionNotPresent,
271
f.add_lines, 'r2', ['foo'], [])
272
self.assertRaises(RevisionAlreadyPresent,
273
f.add_lines, 'r1', [], [])
275
# this checks that reopen with create=True does not break anything.
276
f = self.reopen_file(create=True)
279
def test_adds_with_parent_texts(self):
282
_, _, parent_texts['r0'] = f.add_lines('r0', [], ['a\n', 'b\n'])
284
_, _, parent_texts['r1'] = f.add_lines_with_ghosts('r1',
285
['r0', 'ghost'], ['b\n', 'c\n'], parent_texts=parent_texts)
286
except NotImplementedError:
287
# if the format doesn't support ghosts, just add normally.
288
_, _, parent_texts['r1'] = f.add_lines('r1',
289
['r0'], ['b\n', 'c\n'], parent_texts=parent_texts)
290
f.add_lines('r2', ['r1'], ['c\n', 'd\n'], parent_texts=parent_texts)
291
self.assertNotEqual(None, parent_texts['r0'])
292
self.assertNotEqual(None, parent_texts['r1'])
294
versions = f.versions()
295
self.assertTrue('r0' in versions)
296
self.assertTrue('r1' in versions)
297
self.assertTrue('r2' in versions)
298
self.assertEquals(f.get_lines('r0'), ['a\n', 'b\n'])
299
self.assertEquals(f.get_lines('r1'), ['b\n', 'c\n'])
300
self.assertEquals(f.get_lines('r2'), ['c\n', 'd\n'])
301
self.assertEqual(3, f.num_versions())
302
origins = f.annotate('r1')
303
self.assertEquals(origins[0][0], 'r0')
304
self.assertEquals(origins[1][0], 'r1')
305
origins = f.annotate('r2')
306
self.assertEquals(origins[0][0], 'r1')
307
self.assertEquals(origins[1][0], 'r2')
310
f = self.reopen_file()
313
def test_add_unicode_content(self):
314
# unicode content is not permitted in versioned files.
315
# versioned files version sequences of bytes only.
317
self.assertRaises(errors.BzrBadParameterUnicode,
318
vf.add_lines, 'a', [], ['a\n', u'b\n', 'c\n'])
320
(errors.BzrBadParameterUnicode, NotImplementedError),
321
vf.add_lines_with_ghosts, 'a', [], ['a\n', u'b\n', 'c\n'])
323
def test_add_follows_left_matching_blocks(self):
324
"""If we change left_matching_blocks, delta changes
326
Note: There are multiple correct deltas in this case, because
327
we start with 1 "a" and we get 3.
330
if isinstance(vf, WeaveFile):
331
raise TestSkipped("WeaveFile ignores left_matching_blocks")
332
vf.add_lines('1', [], ['a\n'])
333
vf.add_lines('2', ['1'], ['a\n', 'a\n', 'a\n'],
334
left_matching_blocks=[(0, 0, 1), (1, 3, 0)])
335
self.assertEqual(['a\n', 'a\n', 'a\n'], vf.get_lines('2'))
336
vf.add_lines('3', ['1'], ['a\n', 'a\n', 'a\n'],
337
left_matching_blocks=[(0, 2, 1), (1, 3, 0)])
338
self.assertEqual(['a\n', 'a\n', 'a\n'], vf.get_lines('3'))
340
def test_inline_newline_throws(self):
341
# \r characters are not permitted in lines being added
343
self.assertRaises(errors.BzrBadParameterContainsNewline,
344
vf.add_lines, 'a', [], ['a\n\n'])
346
(errors.BzrBadParameterContainsNewline, NotImplementedError),
347
vf.add_lines_with_ghosts, 'a', [], ['a\n\n'])
348
# but inline CR's are allowed
349
vf.add_lines('a', [], ['a\r\n'])
351
vf.add_lines_with_ghosts('b', [], ['a\r\n'])
352
except NotImplementedError:
355
def test_add_reserved(self):
357
self.assertRaises(errors.ReservedId,
358
vf.add_lines, 'a:', [], ['a\n', 'b\n', 'c\n'])
360
def test_add_lines_nostoresha(self):
361
"""When nostore_sha is supplied using old content raises."""
363
empty_text = ('a', [])
364
sample_text_nl = ('b', ["foo\n", "bar\n"])
365
sample_text_no_nl = ('c', ["foo\n", "bar"])
367
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
368
sha, _, _ = vf.add_lines(version, [], lines)
370
# we now have a copy of all the lines in the vf.
371
for sha, (version, lines) in zip(
372
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
373
self.assertRaises(errors.ExistingContent,
374
vf.add_lines, version + "2", [], lines,
376
# and no new version should have been added.
377
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
380
def test_add_lines_with_ghosts_nostoresha(self):
381
"""When nostore_sha is supplied using old content raises."""
383
empty_text = ('a', [])
384
sample_text_nl = ('b', ["foo\n", "bar\n"])
385
sample_text_no_nl = ('c', ["foo\n", "bar"])
387
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
388
sha, _, _ = vf.add_lines(version, [], lines)
390
# we now have a copy of all the lines in the vf.
391
# is the test applicable to this vf implementation?
393
vf.add_lines_with_ghosts('d', [], [])
394
except NotImplementedError:
395
raise TestSkipped("add_lines_with_ghosts is optional")
396
for sha, (version, lines) in zip(
397
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
398
self.assertRaises(errors.ExistingContent,
399
vf.add_lines_with_ghosts, version + "2", [], lines,
401
# and no new version should have been added.
402
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
405
def test_add_lines_return_value(self):
406
# add_lines should return the sha1 and the text size.
408
empty_text = ('a', [])
409
sample_text_nl = ('b', ["foo\n", "bar\n"])
410
sample_text_no_nl = ('c', ["foo\n", "bar"])
411
# check results for the three cases:
412
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
413
# the first two elements are the same for all versioned files:
414
# - the digest and the size of the text. For some versioned files
415
# additional data is returned in additional tuple elements.
416
result = vf.add_lines(version, [], lines)
417
self.assertEqual(3, len(result))
418
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
420
# parents should not affect the result:
421
lines = sample_text_nl[1]
422
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
423
vf.add_lines('d', ['b', 'c'], lines)[0:2])
425
def test_get_reserved(self):
427
self.assertRaises(errors.ReservedId, vf.get_texts, ['b:'])
428
self.assertRaises(errors.ReservedId, vf.get_lines, 'b:')
429
self.assertRaises(errors.ReservedId, vf.get_text, 'b:')
431
def test_add_unchanged_last_line_noeol_snapshot(self):
432
"""Add a text with an unchanged last line with no eol should work."""
433
# Test adding this in a number of chain lengths; because the interface
434
# for VersionedFile does not allow forcing a specific chain length, we
435
# just use a small base to get the first snapshot, then a much longer
436
# first line for the next add (which will make the third add snapshot)
437
# and so on. 20 has been chosen as an aribtrary figure - knits use 200
438
# as a capped delta length, but ideally we would have some way of
439
# tuning the test to the store (e.g. keep going until a snapshot
441
for length in range(20):
443
vf = self.get_file('case-%d' % length)
446
for step in range(length):
447
version = prefix % step
448
lines = (['prelude \n'] * step) + ['line']
449
vf.add_lines(version, parents, lines)
450
version_lines[version] = lines
452
vf.add_lines('no-eol', parents, ['line'])
453
vf.get_texts(version_lines.keys())
454
self.assertEqualDiff('line', vf.get_text('no-eol'))
456
def test_get_texts_eol_variation(self):
457
# similar to the failure in <http://bugs.launchpad.net/234748>
459
sample_text_nl = ["line\n"]
460
sample_text_no_nl = ["line"]
467
lines = sample_text_nl
469
lines = sample_text_no_nl
470
# left_matching blocks is an internal api; it operates on the
471
# *internal* representation for a knit, which is with *all* lines
472
# being normalised to end with \n - even the final line in a no_nl
473
# file. Using it here ensures that a broken internal implementation
474
# (which is what this test tests) will generate a correct line
475
# delta (which is to say, an empty delta).
476
vf.add_lines(version, parents, lines,
477
left_matching_blocks=[(0, 0, 1)])
479
versions.append(version)
480
version_lines[version] = lines
482
vf.get_texts(versions)
483
vf.get_texts(reversed(versions))
485
def test_add_lines_with_matching_blocks_noeol_last_line(self):
486
"""Add a text with an unchanged last line with no eol should work."""
487
from bzrlib import multiparent
488
# Hand verified sha1 of the text we're adding.
489
sha1 = '6a1d115ec7b60afb664dc14890b5af5ce3c827a4'
490
# Create a mpdiff which adds a new line before the trailing line, and
491
# reuse the last line unaltered (which can cause annotation reuse).
492
# Test adding this in two situations:
493
# On top of a new insertion
494
vf = self.get_file('fulltext')
495
vf.add_lines('noeol', [], ['line'])
496
vf.add_lines('noeol2', ['noeol'], ['newline\n', 'line'],
497
left_matching_blocks=[(0, 1, 1)])
498
self.assertEqualDiff('newline\nline', vf.get_text('noeol2'))
500
vf = self.get_file('delta')
501
vf.add_lines('base', [], ['line'])
502
vf.add_lines('noeol', ['base'], ['prelude\n', 'line'])
503
vf.add_lines('noeol2', ['noeol'], ['newline\n', 'line'],
504
left_matching_blocks=[(1, 1, 1)])
505
self.assertEqualDiff('newline\nline', vf.get_text('noeol2'))
507
def test_make_mpdiffs(self):
508
from bzrlib import multiparent
509
vf = self.get_file('foo')
510
sha1s = self._setup_for_deltas(vf)
511
new_vf = self.get_file('bar')
512
for version in multiparent.topo_iter(vf):
513
mpdiff = vf.make_mpdiffs([version])[0]
514
new_vf.add_mpdiffs([(version, vf.get_parent_map([version])[version],
515
vf.get_sha1s([version])[version], mpdiff)])
516
self.assertEqualDiff(vf.get_text(version),
517
new_vf.get_text(version))
519
def test_make_mpdiffs_with_ghosts(self):
520
vf = self.get_file('foo')
522
vf.add_lines_with_ghosts('text', ['ghost'], ['line\n'])
523
except NotImplementedError:
524
# old Weave formats do not allow ghosts
526
self.assertRaises(errors.RevisionNotPresent, vf.make_mpdiffs, ['ghost'])
528
def _setup_for_deltas(self, f):
529
self.assertFalse(f.has_version('base'))
530
# add texts that should trip the knit maximum delta chain threshold
531
# as well as doing parallel chains of data in knits.
532
# this is done by two chains of 25 insertions
533
f.add_lines('base', [], ['line\n'])
534
f.add_lines('noeol', ['base'], ['line'])
535
# detailed eol tests:
536
# shared last line with parent no-eol
537
f.add_lines('noeolsecond', ['noeol'], ['line\n', 'line'])
538
# differing last line with parent, both no-eol
539
f.add_lines('noeolnotshared', ['noeolsecond'], ['line\n', 'phone'])
540
# add eol following a noneol parent, change content
541
f.add_lines('eol', ['noeol'], ['phone\n'])
542
# add eol following a noneol parent, no change content
543
f.add_lines('eolline', ['noeol'], ['line\n'])
544
# noeol with no parents:
545
f.add_lines('noeolbase', [], ['line'])
546
# noeol preceeding its leftmost parent in the output:
547
# this is done by making it a merge of two parents with no common
548
# anestry: noeolbase and noeol with the
549
# later-inserted parent the leftmost.
550
f.add_lines('eolbeforefirstparent', ['noeolbase', 'noeol'], ['line'])
551
# two identical eol texts
552
f.add_lines('noeoldup', ['noeol'], ['line'])
554
text_name = 'chain1-'
556
sha1s = {0 :'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
557
1 :'45e21ea146a81ea44a821737acdb4f9791c8abe7',
558
2 :'e1f11570edf3e2a070052366c582837a4fe4e9fa',
559
3 :'26b4b8626da827088c514b8f9bbe4ebf181edda1',
560
4 :'e28a5510be25ba84d31121cff00956f9970ae6f6',
561
5 :'d63ec0ce22e11dcf65a931b69255d3ac747a318d',
562
6 :'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
563
7 :'95c14da9cafbf828e3e74a6f016d87926ba234ab',
564
8 :'779e9a0b28f9f832528d4b21e17e168c67697272',
565
9 :'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
566
10:'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
567
11:'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
568
12:'31a2286267f24d8bedaa43355f8ad7129509ea85',
569
13:'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
570
14:'2c4b1736566b8ca6051e668de68650686a3922f2',
571
15:'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
572
16:'b0d2e18d3559a00580f6b49804c23fea500feab3',
573
17:'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
574
18:'5cf64a3459ae28efa60239e44b20312d25b253f3',
575
19:'1ebed371807ba5935958ad0884595126e8c4e823',
576
20:'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
577
21:'01edc447978004f6e4e962b417a4ae1955b6fe5d',
578
22:'d8d8dc49c4bf0bab401e0298bb5ad827768618bb',
579
23:'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
580
24:'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
581
25:'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
583
for depth in range(26):
584
new_version = text_name + '%s' % depth
585
text = text + ['line\n']
586
f.add_lines(new_version, [next_parent], text)
587
next_parent = new_version
589
text_name = 'chain2-'
591
for depth in range(26):
592
new_version = text_name + '%s' % depth
593
text = text + ['line\n']
594
f.add_lines(new_version, [next_parent], text)
595
next_parent = new_version
598
def test_ancestry(self):
600
self.assertEqual([], f.get_ancestry([]))
601
f.add_lines('r0', [], ['a\n', 'b\n'])
602
f.add_lines('r1', ['r0'], ['b\n', 'c\n'])
603
f.add_lines('r2', ['r0'], ['b\n', 'c\n'])
604
f.add_lines('r3', ['r2'], ['b\n', 'c\n'])
605
f.add_lines('rM', ['r1', 'r2'], ['b\n', 'c\n'])
606
self.assertEqual([], f.get_ancestry([]))
607
versions = f.get_ancestry(['rM'])
608
# there are some possibilities:
612
# so we check indexes
613
r0 = versions.index('r0')
614
r1 = versions.index('r1')
615
r2 = versions.index('r2')
616
self.assertFalse('r3' in versions)
617
rM = versions.index('rM')
618
self.assertTrue(r0 < r1)
619
self.assertTrue(r0 < r2)
620
self.assertTrue(r1 < rM)
621
self.assertTrue(r2 < rM)
623
self.assertRaises(RevisionNotPresent,
624
f.get_ancestry, ['rM', 'rX'])
626
self.assertEqual(set(f.get_ancestry('rM')),
627
set(f.get_ancestry('rM', topo_sorted=False)))
629
def test_mutate_after_finish(self):
630
self._transaction = 'before'
632
self._transaction = 'after'
633
self.assertRaises(errors.OutSideTransaction, f.add_lines, '', [], [])
634
self.assertRaises(errors.OutSideTransaction, f.add_lines_with_ghosts, '', [], [])
636
def test_copy_to(self):
638
f.add_lines('0', [], ['a\n'])
639
t = MemoryTransport()
641
for suffix in self.get_factory().get_suffixes():
642
self.assertTrue(t.has('foo' + suffix))
644
def test_get_suffixes(self):
646
# and should be a list
647
self.assertTrue(isinstance(self.get_factory().get_suffixes(), list))
649
def test_get_parent_map(self):
651
f.add_lines('r0', [], ['a\n', 'b\n'])
653
{'r0':()}, f.get_parent_map(['r0']))
654
f.add_lines('r1', ['r0'], ['a\n', 'b\n'])
656
{'r1':('r0',)}, f.get_parent_map(['r1']))
660
f.get_parent_map(['r0', 'r1']))
661
f.add_lines('r2', [], ['a\n', 'b\n'])
662
f.add_lines('r3', [], ['a\n', 'b\n'])
663
f.add_lines('m', ['r0', 'r1', 'r2', 'r3'], ['a\n', 'b\n'])
665
{'m':('r0', 'r1', 'r2', 'r3')}, f.get_parent_map(['m']))
666
self.assertEqual({}, f.get_parent_map('y'))
670
f.get_parent_map(['r0', 'y', 'r1']))
672
def test_annotate(self):
674
f.add_lines('r0', [], ['a\n', 'b\n'])
675
f.add_lines('r1', ['r0'], ['c\n', 'b\n'])
676
origins = f.annotate('r1')
677
self.assertEquals(origins[0][0], 'r1')
678
self.assertEquals(origins[1][0], 'r0')
680
self.assertRaises(RevisionNotPresent,
683
def test_detection(self):
684
# Test weaves detect corruption.
686
# Weaves contain a checksum of their texts.
687
# When a text is extracted, this checksum should be
690
w = self.get_file_corrupted_text()
692
self.assertEqual('hello\n', w.get_text('v1'))
693
self.assertRaises(errors.WeaveInvalidChecksum, w.get_text, 'v2')
694
self.assertRaises(errors.WeaveInvalidChecksum, w.get_lines, 'v2')
695
self.assertRaises(errors.WeaveInvalidChecksum, w.check)
697
w = self.get_file_corrupted_checksum()
699
self.assertEqual('hello\n', w.get_text('v1'))
700
self.assertRaises(errors.WeaveInvalidChecksum, w.get_text, 'v2')
701
self.assertRaises(errors.WeaveInvalidChecksum, w.get_lines, 'v2')
702
self.assertRaises(errors.WeaveInvalidChecksum, w.check)
704
def get_file_corrupted_text(self):
705
"""Return a versioned file with corrupt text but valid metadata."""
706
raise NotImplementedError(self.get_file_corrupted_text)
708
def reopen_file(self, name='foo'):
709
"""Open the versioned file from disk again."""
710
raise NotImplementedError(self.reopen_file)
712
def test_iter_lines_added_or_present_in_versions(self):
713
# test that we get at least an equalset of the lines added by
714
# versions in the weave
715
# the ordering here is to make a tree so that dumb searches have
716
# more changes to muck up.
718
class InstrumentedProgress(progress.DummyProgress):
722
progress.DummyProgress.__init__(self)
725
def update(self, msg=None, current=None, total=None):
726
self.updates.append((msg, current, total))
729
# add a base to get included
730
vf.add_lines('base', [], ['base\n'])
731
# add a ancestor to be included on one side
732
vf.add_lines('lancestor', [], ['lancestor\n'])
733
# add a ancestor to be included on the other side
734
vf.add_lines('rancestor', ['base'], ['rancestor\n'])
735
# add a child of rancestor with no eofile-nl
736
vf.add_lines('child', ['rancestor'], ['base\n', 'child\n'])
737
# add a child of lancestor and base to join the two roots
738
vf.add_lines('otherchild',
739
['lancestor', 'base'],
740
['base\n', 'lancestor\n', 'otherchild\n'])
741
def iter_with_versions(versions, expected):
742
# now we need to see what lines are returned, and how often.
744
progress = InstrumentedProgress()
745
# iterate over the lines
746
for line in vf.iter_lines_added_or_present_in_versions(versions,
748
lines.setdefault(line, 0)
750
if []!= progress.updates:
751
self.assertEqual(expected, progress.updates)
753
lines = iter_with_versions(['child', 'otherchild'],
754
[('Walking content.', 0, 2),
755
('Walking content.', 1, 2),
756
('Walking content.', 2, 2)])
757
# we must see child and otherchild
758
self.assertTrue(lines[('child\n', 'child')] > 0)
759
self.assertTrue(lines[('otherchild\n', 'otherchild')] > 0)
760
# we dont care if we got more than that.
763
lines = iter_with_versions(None, [('Walking content.', 0, 5),
764
('Walking content.', 1, 5),
765
('Walking content.', 2, 5),
766
('Walking content.', 3, 5),
767
('Walking content.', 4, 5),
768
('Walking content.', 5, 5)])
769
# all lines must be seen at least once
770
self.assertTrue(lines[('base\n', 'base')] > 0)
771
self.assertTrue(lines[('lancestor\n', 'lancestor')] > 0)
772
self.assertTrue(lines[('rancestor\n', 'rancestor')] > 0)
773
self.assertTrue(lines[('child\n', 'child')] > 0)
774
self.assertTrue(lines[('otherchild\n', 'otherchild')] > 0)
776
def test_add_lines_with_ghosts(self):
777
# some versioned file formats allow lines to be added with parent
778
# information that is > than that in the format. Formats that do
779
# not support this need to raise NotImplementedError on the
780
# add_lines_with_ghosts api.
782
# add a revision with ghost parents
783
# The preferred form is utf8, but we should translate when needed
784
parent_id_unicode = u'b\xbfse'
785
parent_id_utf8 = parent_id_unicode.encode('utf8')
787
vf.add_lines_with_ghosts('notbxbfse', [parent_id_utf8], [])
788
except NotImplementedError:
789
# check the other ghost apis are also not implemented
790
self.assertRaises(NotImplementedError, vf.get_ancestry_with_ghosts, ['foo'])
791
self.assertRaises(NotImplementedError, vf.get_parents_with_ghosts, 'foo')
793
vf = self.reopen_file()
794
# test key graph related apis: getncestry, _graph, get_parents
796
# - these are ghost unaware and must not be reflect ghosts
797
self.assertEqual(['notbxbfse'], vf.get_ancestry('notbxbfse'))
798
self.assertFalse(vf.has_version(parent_id_utf8))
799
# we have _with_ghost apis to give us ghost information.
800
self.assertEqual([parent_id_utf8, 'notbxbfse'], vf.get_ancestry_with_ghosts(['notbxbfse']))
801
self.assertEqual([parent_id_utf8], vf.get_parents_with_ghosts('notbxbfse'))
802
# if we add something that is a ghost of another, it should correct the
803
# results of the prior apis
804
vf.add_lines(parent_id_utf8, [], [])
805
self.assertEqual([parent_id_utf8, 'notbxbfse'], vf.get_ancestry(['notbxbfse']))
806
self.assertEqual({'notbxbfse':(parent_id_utf8,)},
807
vf.get_parent_map(['notbxbfse']))
808
self.assertTrue(vf.has_version(parent_id_utf8))
809
# we have _with_ghost apis to give us ghost information.
810
self.assertEqual([parent_id_utf8, 'notbxbfse'],
811
vf.get_ancestry_with_ghosts(['notbxbfse']))
812
self.assertEqual([parent_id_utf8], vf.get_parents_with_ghosts('notbxbfse'))
814
def test_add_lines_with_ghosts_after_normal_revs(self):
815
# some versioned file formats allow lines to be added with parent
816
# information that is > than that in the format. Formats that do
817
# not support this need to raise NotImplementedError on the
818
# add_lines_with_ghosts api.
820
# probe for ghost support
822
vf.add_lines_with_ghosts('base', [], ['line\n', 'line_b\n'])
823
except NotImplementedError:
825
vf.add_lines_with_ghosts('references_ghost',
827
['line\n', 'line_b\n', 'line_c\n'])
828
origins = vf.annotate('references_ghost')
829
self.assertEquals(('base', 'line\n'), origins[0])
830
self.assertEquals(('base', 'line_b\n'), origins[1])
831
self.assertEquals(('references_ghost', 'line_c\n'), origins[2])
833
def test_readonly_mode(self):
834
transport = get_transport(self.get_url('.'))
835
factory = self.get_factory()
836
vf = factory('id', transport, 0777, create=True, access_mode='w')
837
vf = factory('id', transport, access_mode='r')
838
self.assertRaises(errors.ReadOnlyError, vf.add_lines, 'base', [], [])
839
self.assertRaises(errors.ReadOnlyError,
840
vf.add_lines_with_ghosts,
845
def test_get_sha1s(self):
846
# check the sha1 data is available
849
vf.add_lines('a', [], ['a\n'])
850
# the same file, different metadata
851
vf.add_lines('b', ['a'], ['a\n'])
852
# a file differing only in last newline.
853
vf.add_lines('c', [], ['a'])
855
'a': '3f786850e387550fdab836ed7e6dc881de23001b',
856
'c': '86f7e437faa5a7fce15d1ddcb9eaeaea377667b8',
857
'b': '3f786850e387550fdab836ed7e6dc881de23001b',
859
vf.get_sha1s(['a', 'c', 'b']))
862
class TestWeave(TestCaseWithMemoryTransport, VersionedFileTestMixIn):
864
def get_file(self, name='foo'):
865
return WeaveFile(name, get_transport(self.get_url('.')), create=True,
866
get_scope=self.get_transaction)
868
def get_file_corrupted_text(self):
869
w = WeaveFile('foo', get_transport(self.get_url('.')), create=True,
870
get_scope=self.get_transaction)
871
w.add_lines('v1', [], ['hello\n'])
872
w.add_lines('v2', ['v1'], ['hello\n', 'there\n'])
874
# We are going to invasively corrupt the text
875
# Make sure the internals of weave are the same
876
self.assertEqual([('{', 0)
884
self.assertEqual(['f572d396fae9206628714fb2ce00f72e94f2258f'
885
, '90f265c6e75f1c8f9ab76dcf85528352c5f215ef'
890
w._weave[4] = 'There\n'
893
def get_file_corrupted_checksum(self):
894
w = self.get_file_corrupted_text()
896
w._weave[4] = 'there\n'
897
self.assertEqual('hello\nthere\n', w.get_text('v2'))
899
#Invalid checksum, first digit changed
900
w._sha1s[1] = 'f0f265c6e75f1c8f9ab76dcf85528352c5f215ef'
903
def reopen_file(self, name='foo', create=False):
904
return WeaveFile(name, get_transport(self.get_url('.')), create=create,
905
get_scope=self.get_transaction)
907
def test_no_implicit_create(self):
908
self.assertRaises(errors.NoSuchFile,
911
get_transport(self.get_url('.')),
912
get_scope=self.get_transaction)
914
def get_factory(self):
918
class TestPlanMergeVersionedFile(TestCaseWithMemoryTransport):
921
TestCaseWithMemoryTransport.setUp(self)
922
mapper = PrefixMapper()
923
factory = make_file_factory(True, mapper)
924
self.vf1 = factory(self.get_transport('root-1'))
925
self.vf2 = factory(self.get_transport('root-2'))
926
self.plan_merge_vf = versionedfile._PlanMergeVersionedFile('root')
927
self.plan_merge_vf.fallback_versionedfiles.extend([self.vf1, self.vf2])
929
def test_add_lines(self):
930
self.plan_merge_vf.add_lines(('root', 'a:'), [], [])
931
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
932
('root', 'a'), [], [])
933
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
934
('root', 'a:'), None, [])
935
self.assertRaises(ValueError, self.plan_merge_vf.add_lines,
936
('root', 'a:'), [], None)
938
def setup_abcde(self):
939
self.vf1.add_lines(('root', 'A'), [], ['a'])
940
self.vf1.add_lines(('root', 'B'), [('root', 'A')], ['b'])
941
self.vf2.add_lines(('root', 'C'), [], ['c'])
942
self.vf2.add_lines(('root', 'D'), [('root', 'C')], ['d'])
943
self.plan_merge_vf.add_lines(('root', 'E:'),
944
[('root', 'B'), ('root', 'D')], ['e'])
946
def test_get_parents(self):
948
self.assertEqual({('root', 'B'):(('root', 'A'),)},
949
self.plan_merge_vf.get_parent_map([('root', 'B')]))
950
self.assertEqual({('root', 'D'):(('root', 'C'),)},
951
self.plan_merge_vf.get_parent_map([('root', 'D')]))
952
self.assertEqual({('root', 'E:'):(('root', 'B'),('root', 'D'))},
953
self.plan_merge_vf.get_parent_map([('root', 'E:')]))
955
self.plan_merge_vf.get_parent_map([('root', 'F')]))
957
('root', 'B'):(('root', 'A'),),
958
('root', 'D'):(('root', 'C'),),
959
('root', 'E:'):(('root', 'B'),('root', 'D')),
961
self.plan_merge_vf.get_parent_map(
962
[('root', 'B'), ('root', 'D'), ('root', 'E:'), ('root', 'F')]))
964
def test_get_record_stream(self):
966
def get_record(suffix):
967
return self.plan_merge_vf.get_record_stream(
968
[('root', suffix)], 'unordered', True).next()
969
self.assertEqual('a', get_record('A').get_bytes_as('fulltext'))
970
self.assertEqual('c', get_record('C').get_bytes_as('fulltext'))
971
self.assertEqual('e', get_record('E:').get_bytes_as('fulltext'))
972
self.assertEqual('absent', get_record('F').storage_kind)
975
class TestReadonlyHttpMixin(object):
977
def get_transaction(self):
980
def test_readonly_http_works(self):
981
# we should be able to read from http with a versioned file.
983
# try an empty file access
984
readonly_vf = self.get_factory()('foo', get_transport(self.get_readonly_url('.')))
985
self.assertEqual([], readonly_vf.versions())
987
vf.add_lines('1', [], ['a\n'])
988
vf.add_lines('2', ['1'], ['b\n', 'a\n'])
989
readonly_vf = self.get_factory()('foo', get_transport(self.get_readonly_url('.')))
990
self.assertEqual(['1', '2'], vf.versions())
991
for version in readonly_vf.versions():
992
readonly_vf.get_lines(version)
995
class TestWeaveHTTP(TestCaseWithWebserver, TestReadonlyHttpMixin):
998
return WeaveFile('foo', get_transport(self.get_url('.')), create=True,
999
get_scope=self.get_transaction)
1001
def get_factory(self):
1005
class MergeCasesMixin(object):
1007
def doMerge(self, base, a, b, mp):
1008
from cStringIO import StringIO
1009
from textwrap import dedent
1015
w.add_lines('text0', [], map(addcrlf, base))
1016
w.add_lines('text1', ['text0'], map(addcrlf, a))
1017
w.add_lines('text2', ['text0'], map(addcrlf, b))
1019
self.log_contents(w)
1021
self.log('merge plan:')
1022
p = list(w.plan_merge('text1', 'text2'))
1023
for state, line in p:
1025
self.log('%12s | %s' % (state, line[:-1]))
1029
mt.writelines(w.weave_merge(p))
1031
self.log(mt.getvalue())
1033
mp = map(addcrlf, mp)
1034
self.assertEqual(mt.readlines(), mp)
1037
def testOneInsert(self):
1043
def testSeparateInserts(self):
1044
self.doMerge(['aaa', 'bbb', 'ccc'],
1045
['aaa', 'xxx', 'bbb', 'ccc'],
1046
['aaa', 'bbb', 'yyy', 'ccc'],
1047
['aaa', 'xxx', 'bbb', 'yyy', 'ccc'])
1049
def testSameInsert(self):
1050
self.doMerge(['aaa', 'bbb', 'ccc'],
1051
['aaa', 'xxx', 'bbb', 'ccc'],
1052
['aaa', 'xxx', 'bbb', 'yyy', 'ccc'],
1053
['aaa', 'xxx', 'bbb', 'yyy', 'ccc'])
1054
overlappedInsertExpected = ['aaa', 'xxx', 'yyy', 'bbb']
1055
def testOverlappedInsert(self):
1056
self.doMerge(['aaa', 'bbb'],
1057
['aaa', 'xxx', 'yyy', 'bbb'],
1058
['aaa', 'xxx', 'bbb'], self.overlappedInsertExpected)
1060
# really it ought to reduce this to
1061
# ['aaa', 'xxx', 'yyy', 'bbb']
1064
def testClashReplace(self):
1065
self.doMerge(['aaa'],
1068
['<<<<<<< ', 'xxx', '=======', 'yyy', 'zzz',
1071
def testNonClashInsert1(self):
1072
self.doMerge(['aaa'],
1075
['<<<<<<< ', 'xxx', 'aaa', '=======', 'yyy', 'zzz',
1078
def testNonClashInsert2(self):
1079
self.doMerge(['aaa'],
1085
def testDeleteAndModify(self):
1086
"""Clashing delete and modification.
1088
If one side modifies a region and the other deletes it then
1089
there should be a conflict with one side blank.
1092
#######################################
1093
# skippd, not working yet
1096
self.doMerge(['aaa', 'bbb', 'ccc'],
1097
['aaa', 'ddd', 'ccc'],
1099
['<<<<<<<< ', 'aaa', '=======', '>>>>>>> ', 'ccc'])
1101
def _test_merge_from_strings(self, base, a, b, expected):
1103
w.add_lines('text0', [], base.splitlines(True))
1104
w.add_lines('text1', ['text0'], a.splitlines(True))
1105
w.add_lines('text2', ['text0'], b.splitlines(True))
1106
self.log('merge plan:')
1107
p = list(w.plan_merge('text1', 'text2'))
1108
for state, line in p:
1110
self.log('%12s | %s' % (state, line[:-1]))
1111
self.log('merge result:')
1112
result_text = ''.join(w.weave_merge(p))
1113
self.log(result_text)
1114
self.assertEqualDiff(result_text, expected)
1116
def test_weave_merge_conflicts(self):
1117
# does weave merge properly handle plans that end with unchanged?
1118
result = ''.join(self.get_file().weave_merge([('new-a', 'hello\n')]))
1119
self.assertEqual(result, 'hello\n')
1121
def test_deletion_extended(self):
1122
"""One side deletes, the other deletes more.
1139
self._test_merge_from_strings(base, a, b, result)
1141
def test_deletion_overlap(self):
1142
"""Delete overlapping regions with no other conflict.
1144
Arguably it'd be better to treat these as agreement, rather than
1145
conflict, but for now conflict is safer.
1173
self._test_merge_from_strings(base, a, b, result)
1175
def test_agreement_deletion(self):
1176
"""Agree to delete some lines, without conflicts."""
1198
self._test_merge_from_strings(base, a, b, result)
1200
def test_sync_on_deletion(self):
1201
"""Specific case of merge where we can synchronize incorrectly.
1203
A previous version of the weave merge concluded that the two versions
1204
agreed on deleting line 2, and this could be a synchronization point.
1205
Line 1 was then considered in isolation, and thought to be deleted on
1208
It's better to consider the whole thing as a disagreement region.
1219
a's replacement line 2
1232
a's replacement line 2
1239
self._test_merge_from_strings(base, a, b, result)
1242
class TestWeaveMerge(TestCaseWithMemoryTransport, MergeCasesMixin):
1244
def get_file(self, name='foo'):
1245
return WeaveFile(name, get_transport(self.get_url('.')), create=True)
1247
def log_contents(self, w):
1248
self.log('weave is:')
1250
write_weave(w, tmpf)
1251
self.log(tmpf.getvalue())
1253
overlappedInsertExpected = ['aaa', '<<<<<<< ', 'xxx', 'yyy', '=======',
1254
'xxx', '>>>>>>> ', 'bbb']
1257
class TestContentFactoryAdaption(TestCaseWithMemoryTransport):
1259
def test_select_adaptor(self):
1260
"""Test expected adapters exist."""
1261
# One scenario for each lookup combination we expect to use.
1262
# Each is source_kind, requested_kind, adapter class
1264
('knit-delta-gz', 'fulltext', _mod_knit.DeltaPlainToFullText),
1265
('knit-ft-gz', 'fulltext', _mod_knit.FTPlainToFullText),
1266
('knit-annotated-delta-gz', 'knit-delta-gz',
1267
_mod_knit.DeltaAnnotatedToUnannotated),
1268
('knit-annotated-delta-gz', 'fulltext',
1269
_mod_knit.DeltaAnnotatedToFullText),
1270
('knit-annotated-ft-gz', 'knit-ft-gz',
1271
_mod_knit.FTAnnotatedToUnannotated),
1272
('knit-annotated-ft-gz', 'fulltext',
1273
_mod_knit.FTAnnotatedToFullText),
1275
for source, requested, klass in scenarios:
1276
adapter_factory = versionedfile.adapter_registry.get(
1277
(source, requested))
1278
adapter = adapter_factory(None)
1279
self.assertIsInstance(adapter, klass)
1281
def get_knit(self, annotated=True):
1282
mapper = ConstantMapper('knit')
1283
transport = self.get_transport()
1284
return make_file_factory(annotated, mapper)(transport)
1286
def helpGetBytes(self, f, ft_adapter, delta_adapter):
1287
"""Grab the interested adapted texts for tests."""
1288
# origin is a fulltext
1289
entries = f.get_record_stream([('origin',)], 'unordered', False)
1290
base = entries.next()
1291
ft_data = ft_adapter.get_bytes(base)
1292
# merged is both a delta and multiple parents.
1293
entries = f.get_record_stream([('merged',)], 'unordered', False)
1294
merged = entries.next()
1295
delta_data = delta_adapter.get_bytes(merged)
1296
return ft_data, delta_data
1298
def test_deannotation_noeol(self):
1299
"""Test converting annotated knits to unannotated knits."""
1300
# we need a full text, and a delta
1302
get_diamond_files(f, 1, trailing_eol=False)
1303
ft_data, delta_data = self.helpGetBytes(f,
1304
_mod_knit.FTAnnotatedToUnannotated(None),
1305
_mod_knit.DeltaAnnotatedToUnannotated(None))
1307
'version origin 1 b284f94827db1fa2970d9e2014f080413b547a7e\n'
1310
GzipFile(mode='rb', fileobj=StringIO(ft_data)).read())
1312
'version merged 4 32c2e79763b3f90e8ccde37f9710b6629c25a796\n'
1313
'1,2,3\nleft\nright\nmerged\nend merged\n',
1314
GzipFile(mode='rb', fileobj=StringIO(delta_data)).read())
1316
def test_deannotation(self):
1317
"""Test converting annotated knits to unannotated knits."""
1318
# we need a full text, and a delta
1320
get_diamond_files(f, 1)
1321
ft_data, delta_data = self.helpGetBytes(f,
1322
_mod_knit.FTAnnotatedToUnannotated(None),
1323
_mod_knit.DeltaAnnotatedToUnannotated(None))
1325
'version origin 1 00e364d235126be43292ab09cb4686cf703ddc17\n'
1328
GzipFile(mode='rb', fileobj=StringIO(ft_data)).read())
1330
'version merged 3 ed8bce375198ea62444dc71952b22cfc2b09226d\n'
1331
'2,2,2\nright\nmerged\nend merged\n',
1332
GzipFile(mode='rb', fileobj=StringIO(delta_data)).read())
1334
def test_annotated_to_fulltext_no_eol(self):
1335
"""Test adapting annotated knits to full texts (for -> weaves)."""
1336
# we need a full text, and a delta
1338
get_diamond_files(f, 1, trailing_eol=False)
1339
# Reconstructing a full text requires a backing versioned file, and it
1340
# must have the base lines requested from it.
1341
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1342
ft_data, delta_data = self.helpGetBytes(f,
1343
_mod_knit.FTAnnotatedToFullText(None),
1344
_mod_knit.DeltaAnnotatedToFullText(logged_vf))
1345
self.assertEqual('origin', ft_data)
1346
self.assertEqual('base\nleft\nright\nmerged', delta_data)
1347
self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1348
True)], logged_vf.calls)
1350
def test_annotated_to_fulltext(self):
1351
"""Test adapting annotated knits to full texts (for -> weaves)."""
1352
# we need a full text, and a delta
1354
get_diamond_files(f, 1)
1355
# Reconstructing a full text requires a backing versioned file, and it
1356
# must have the base lines requested from it.
1357
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1358
ft_data, delta_data = self.helpGetBytes(f,
1359
_mod_knit.FTAnnotatedToFullText(None),
1360
_mod_knit.DeltaAnnotatedToFullText(logged_vf))
1361
self.assertEqual('origin\n', ft_data)
1362
self.assertEqual('base\nleft\nright\nmerged\n', delta_data)
1363
self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1364
True)], logged_vf.calls)
1366
def test_unannotated_to_fulltext(self):
1367
"""Test adapting unannotated knits to full texts.
1369
This is used for -> weaves, and for -> annotated knits.
1371
# we need a full text, and a delta
1372
f = self.get_knit(annotated=False)
1373
get_diamond_files(f, 1)
1374
# Reconstructing a full text requires a backing versioned file, and it
1375
# must have the base lines requested from it.
1376
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1377
ft_data, delta_data = self.helpGetBytes(f,
1378
_mod_knit.FTPlainToFullText(None),
1379
_mod_knit.DeltaPlainToFullText(logged_vf))
1380
self.assertEqual('origin\n', ft_data)
1381
self.assertEqual('base\nleft\nright\nmerged\n', delta_data)
1382
self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1383
True)], logged_vf.calls)
1385
def test_unannotated_to_fulltext_no_eol(self):
1386
"""Test adapting unannotated knits to full texts.
1388
This is used for -> weaves, and for -> annotated knits.
1390
# we need a full text, and a delta
1391
f = self.get_knit(annotated=False)
1392
get_diamond_files(f, 1, trailing_eol=False)
1393
# Reconstructing a full text requires a backing versioned file, and it
1394
# must have the base lines requested from it.
1395
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1396
ft_data, delta_data = self.helpGetBytes(f,
1397
_mod_knit.FTPlainToFullText(None),
1398
_mod_knit.DeltaPlainToFullText(logged_vf))
1399
self.assertEqual('origin', ft_data)
1400
self.assertEqual('base\nleft\nright\nmerged', delta_data)
1401
self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1402
True)], logged_vf.calls)
1405
class TestKeyMapper(TestCaseWithMemoryTransport):
1406
"""Tests for various key mapping logic."""
1408
def test_identity_mapper(self):
1409
mapper = versionedfile.ConstantMapper("inventory")
1410
self.assertEqual("inventory", mapper.map(('foo@ar',)))
1411
self.assertEqual("inventory", mapper.map(('quux',)))
1413
def test_prefix_mapper(self):
1415
mapper = versionedfile.PrefixMapper()
1416
self.assertEqual("file-id", mapper.map(("file-id", "revision-id")))
1417
self.assertEqual("new-id", mapper.map(("new-id", "revision-id")))
1418
self.assertEqual(('file-id',), mapper.unmap("file-id"))
1419
self.assertEqual(('new-id',), mapper.unmap("new-id"))
1421
def test_hash_prefix_mapper(self):
1422
#format6: hash + plain
1423
mapper = versionedfile.HashPrefixMapper()
1424
self.assertEqual("9b/file-id", mapper.map(("file-id", "revision-id")))
1425
self.assertEqual("45/new-id", mapper.map(("new-id", "revision-id")))
1426
self.assertEqual(('file-id',), mapper.unmap("9b/file-id"))
1427
self.assertEqual(('new-id',), mapper.unmap("45/new-id"))
1429
def test_hash_escaped_mapper(self):
1430
#knit1: hash + escaped
1431
mapper = versionedfile.HashEscapedPrefixMapper()
1432
self.assertEqual("88/%2520", mapper.map((" ", "revision-id")))
1433
self.assertEqual("ed/fil%2545-%2549d", mapper.map(("filE-Id",
1435
self.assertEqual("88/ne%2557-%2549d", mapper.map(("neW-Id",
1437
self.assertEqual(('filE-Id',), mapper.unmap("ed/fil%2545-%2549d"))
1438
self.assertEqual(('neW-Id',), mapper.unmap("88/ne%2557-%2549d"))
1441
class TestVersionedFiles(TestCaseWithMemoryTransport):
1442
"""Tests for the multiple-file variant of VersionedFile."""
1444
def get_versionedfiles(self, relpath='files'):
1445
transport = self.get_transport(relpath)
1447
transport.mkdir('.')
1448
files = self.factory(transport)
1449
if self.cleanup is not None:
1450
self.addCleanup(lambda:self.cleanup(files))
1453
def test_annotate(self):
1454
files = self.get_versionedfiles()
1455
self.get_diamond_files(files)
1456
if self.key_length == 1:
1460
# introduced full text
1461
origins = files.annotate(prefix + ('origin',))
1463
(prefix + ('origin',), 'origin\n')],
1466
origins = files.annotate(prefix + ('base',))
1468
(prefix + ('base',), 'base\n')],
1471
origins = files.annotate(prefix + ('merged',))
1474
(prefix + ('base',), 'base\n'),
1475
(prefix + ('left',), 'left\n'),
1476
(prefix + ('right',), 'right\n'),
1477
(prefix + ('merged',), 'merged\n')
1481
# Without a graph everything is new.
1483
(prefix + ('merged',), 'base\n'),
1484
(prefix + ('merged',), 'left\n'),
1485
(prefix + ('merged',), 'right\n'),
1486
(prefix + ('merged',), 'merged\n')
1489
self.assertRaises(RevisionNotPresent,
1490
files.annotate, prefix + ('missing-key',))
1492
def test_construct(self):
1493
"""Each parameterised test can be constructed on a transport."""
1494
files = self.get_versionedfiles()
1496
def get_diamond_files(self, files, trailing_eol=True, left_only=False):
1497
return get_diamond_files(files, self.key_length,
1498
trailing_eol=trailing_eol, nograph=not self.graph,
1499
left_only=left_only)
1501
def test_add_lines_return(self):
1502
files = self.get_versionedfiles()
1503
# save code by using the stock data insertion helper.
1504
adds = self.get_diamond_files(files)
1506
# We can only validate the first 2 elements returned from add_lines.
1508
self.assertEqual(3, len(add))
1509
results.append(add[:2])
1510
if self.key_length == 1:
1512
('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1513
('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1514
('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1515
('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1516
('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1518
elif self.key_length == 2:
1520
('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1521
('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1522
('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1523
('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1524
('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1525
('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1526
('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1527
('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1528
('ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1529
('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1532
def test_empty_lines(self):
1533
"""Empty files can be stored."""
1534
f = self.get_versionedfiles()
1535
key_a = self.get_simple_key('a')
1536
f.add_lines(key_a, [], [])
1537
self.assertEqual('',
1538
f.get_record_stream([key_a], 'unordered', True
1539
).next().get_bytes_as('fulltext'))
1540
key_b = self.get_simple_key('b')
1541
f.add_lines(key_b, self.get_parents([key_a]), [])
1542
self.assertEqual('',
1543
f.get_record_stream([key_b], 'unordered', True
1544
).next().get_bytes_as('fulltext'))
1546
def test_newline_only(self):
1547
f = self.get_versionedfiles()
1548
key_a = self.get_simple_key('a')
1549
f.add_lines(key_a, [], ['\n'])
1550
self.assertEqual('\n',
1551
f.get_record_stream([key_a], 'unordered', True
1552
).next().get_bytes_as('fulltext'))
1553
key_b = self.get_simple_key('b')
1554
f.add_lines(key_b, self.get_parents([key_a]), ['\n'])
1555
self.assertEqual('\n',
1556
f.get_record_stream([key_b], 'unordered', True
1557
).next().get_bytes_as('fulltext'))
1559
def test_get_record_stream_empty(self):
1560
"""An empty stream can be requested without error."""
1561
f = self.get_versionedfiles()
1562
entries = f.get_record_stream([], 'unordered', False)
1563
self.assertEqual([], list(entries))
1565
def assertValidStorageKind(self, storage_kind):
1566
"""Assert that storage_kind is a valid storage_kind."""
1567
self.assertSubset([storage_kind],
1568
['mpdiff', 'knit-annotated-ft', 'knit-annotated-delta',
1569
'knit-ft', 'knit-delta', 'chunked', 'fulltext',
1570
'knit-annotated-ft-gz', 'knit-annotated-delta-gz', 'knit-ft-gz',
1572
'knit-delta-closure', 'knit-delta-closure-ref'])
1574
def capture_stream(self, f, entries, on_seen, parents):
1575
"""Capture a stream for testing."""
1576
for factory in entries:
1577
on_seen(factory.key)
1578
self.assertValidStorageKind(factory.storage_kind)
1579
self.assertEqual(f.get_sha1s([factory.key])[factory.key],
1581
self.assertEqual(parents[factory.key], factory.parents)
1582
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1585
def test_get_record_stream_interface(self):
1586
"""each item in a stream has to provide a regular interface."""
1587
files = self.get_versionedfiles()
1588
self.get_diamond_files(files)
1589
keys, _ = self.get_keys_and_sort_order()
1590
parent_map = files.get_parent_map(keys)
1591
entries = files.get_record_stream(keys, 'unordered', False)
1593
self.capture_stream(files, entries, seen.add, parent_map)
1594
self.assertEqual(set(keys), seen)
1596
def get_simple_key(self, suffix):
1597
"""Return a key for the object under test."""
1598
if self.key_length == 1:
1601
return ('FileA',) + (suffix,)
1603
def get_keys_and_sort_order(self):
1604
"""Get diamond test keys list, and their sort ordering."""
1605
if self.key_length == 1:
1606
keys = [('merged',), ('left',), ('right',), ('base',)]
1607
sort_order = {('merged',):2, ('left',):1, ('right',):1, ('base',):0}
1610
('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1612
('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1616
('FileA', 'merged'):2, ('FileA', 'left'):1, ('FileA', 'right'):1,
1617
('FileA', 'base'):0,
1618
('FileB', 'merged'):2, ('FileB', 'left'):1, ('FileB', 'right'):1,
1619
('FileB', 'base'):0,
1621
return keys, sort_order
1623
def test_get_record_stream_interface_ordered(self):
1624
"""each item in a stream has to provide a regular interface."""
1625
files = self.get_versionedfiles()
1626
self.get_diamond_files(files)
1627
keys, sort_order = self.get_keys_and_sort_order()
1628
parent_map = files.get_parent_map(keys)
1629
entries = files.get_record_stream(keys, 'topological', False)
1631
self.capture_stream(files, entries, seen.append, parent_map)
1632
self.assertStreamOrder(sort_order, seen, keys)
1634
def test_get_record_stream_interface_ordered_with_delta_closure(self):
1635
"""each item must be accessible as a fulltext."""
1636
files = self.get_versionedfiles()
1637
self.get_diamond_files(files)
1638
keys, sort_order = self.get_keys_and_sort_order()
1639
parent_map = files.get_parent_map(keys)
1640
entries = files.get_record_stream(keys, 'topological', True)
1642
for factory in entries:
1643
seen.append(factory.key)
1644
self.assertValidStorageKind(factory.storage_kind)
1645
self.assertSubset([factory.sha1],
1646
[None, files.get_sha1s([factory.key])[factory.key]])
1647
self.assertEqual(parent_map[factory.key], factory.parents)
1648
# self.assertEqual(files.get_text(factory.key),
1649
ft_bytes = factory.get_bytes_as('fulltext')
1650
self.assertIsInstance(ft_bytes, str)
1651
chunked_bytes = factory.get_bytes_as('chunked')
1652
self.assertEqualDiff(ft_bytes, ''.join(chunked_bytes))
1654
self.assertStreamOrder(sort_order, seen, keys)
1656
def assertStreamOrder(self, sort_order, seen, keys):
1657
self.assertEqual(len(set(seen)), len(keys))
1658
if self.key_length == 1:
1661
lows = {('FileA',):0, ('FileB',):0}
1663
self.assertEqual(set(keys), set(seen))
1666
sort_pos = sort_order[key]
1667
self.assertTrue(sort_pos >= lows[key[:-1]],
1668
"Out of order in sorted stream: %r, %r" % (key, seen))
1669
lows[key[:-1]] = sort_pos
1671
def test_get_record_stream_unknown_storage_kind_raises(self):
1672
"""Asking for a storage kind that the stream cannot supply raises."""
1673
files = self.get_versionedfiles()
1674
self.get_diamond_files(files)
1675
if self.key_length == 1:
1676
keys = [('merged',), ('left',), ('right',), ('base',)]
1679
('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1681
('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1684
parent_map = files.get_parent_map(keys)
1685
entries = files.get_record_stream(keys, 'unordered', False)
1686
# We track the contents because we should be able to try, fail a
1687
# particular kind and then ask for one that works and continue.
1689
for factory in entries:
1690
seen.add(factory.key)
1691
self.assertValidStorageKind(factory.storage_kind)
1692
self.assertEqual(files.get_sha1s([factory.key])[factory.key],
1694
self.assertEqual(parent_map[factory.key], factory.parents)
1695
# currently no stream emits mpdiff
1696
self.assertRaises(errors.UnavailableRepresentation,
1697
factory.get_bytes_as, 'mpdiff')
1698
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1700
self.assertEqual(set(keys), seen)
1702
def test_get_record_stream_missing_records_are_absent(self):
1703
files = self.get_versionedfiles()
1704
self.get_diamond_files(files)
1705
if self.key_length == 1:
1706
keys = [('merged',), ('left',), ('right',), ('absent',), ('base',)]
1709
('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1710
('FileA', 'absent'), ('FileA', 'base'),
1711
('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1712
('FileB', 'absent'), ('FileB', 'base'),
1713
('absent', 'absent'),
1715
parent_map = files.get_parent_map(keys)
1716
entries = files.get_record_stream(keys, 'unordered', False)
1717
self.assertAbsentRecord(files, keys, parent_map, entries)
1718
entries = files.get_record_stream(keys, 'topological', False)
1719
self.assertAbsentRecord(files, keys, parent_map, entries)
1721
def assertRecordHasContent(self, record, bytes):
1722
"""Assert that record has the bytes bytes."""
1723
self.assertEqual(bytes, record.get_bytes_as('fulltext'))
1724
self.assertEqual(bytes, ''.join(record.get_bytes_as('chunked')))
1726
def test_get_record_stream_native_formats_are_wire_ready_one_ft(self):
1727
files = self.get_versionedfiles()
1728
key = self.get_simple_key('foo')
1729
files.add_lines(key, (), ['my text\n', 'content'])
1730
stream = files.get_record_stream([key], 'unordered', False)
1731
record = stream.next()
1732
if record.storage_kind in ('chunked', 'fulltext'):
1733
# chunked and fulltext representations are for direct use not wire
1734
# serialisation: check they are able to be used directly. To send
1735
# such records over the wire translation will be needed.
1736
self.assertRecordHasContent(record, "my text\ncontent")
1738
bytes = [record.get_bytes_as(record.storage_kind)]
1739
network_stream = versionedfile.NetworkRecordStream(bytes).read()
1740
source_record = record
1742
for record in network_stream:
1743
records.append(record)
1744
self.assertEqual(source_record.storage_kind,
1745
record.storage_kind)
1746
self.assertEqual(source_record.parents, record.parents)
1748
source_record.get_bytes_as(source_record.storage_kind),
1749
record.get_bytes_as(record.storage_kind))
1750
self.assertEqual(1, len(records))
1752
def assertStreamMetaEqual(self, records, expected, stream):
1753
"""Assert that streams expected and stream have the same records.
1755
:param records: A list to collect the seen records.
1756
:return: A generator of the records in stream.
1758
# We make assertions during copying to catch things early for
1760
for record, ref_record in izip(stream, expected):
1761
records.append(record)
1762
self.assertEqual(ref_record.key, record.key)
1763
self.assertEqual(ref_record.storage_kind, record.storage_kind)
1764
self.assertEqual(ref_record.parents, record.parents)
1767
def stream_to_bytes_or_skip_counter(self, skipped_records, full_texts,
1769
"""Convert a stream to a bytes iterator.
1771
:param skipped_records: A list with one element to increment when a
1773
:param full_texts: A dict from key->fulltext representation, for
1774
checking chunked or fulltext stored records.
1775
:param stream: A record_stream.
1776
:return: An iterator over the bytes of each record.
1778
for record in stream:
1779
if record.storage_kind in ('chunked', 'fulltext'):
1780
skipped_records[0] += 1
1781
# check the content is correct for direct use.
1782
self.assertRecordHasContent(record, full_texts[record.key])
1784
yield record.get_bytes_as(record.storage_kind)
1786
def test_get_record_stream_native_formats_are_wire_ready_ft_delta(self):
1787
files = self.get_versionedfiles()
1788
target_files = self.get_versionedfiles('target')
1789
key = self.get_simple_key('ft')
1790
key_delta = self.get_simple_key('delta')
1791
files.add_lines(key, (), ['my text\n', 'content'])
1793
delta_parents = (key,)
1796
files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
1797
local = files.get_record_stream([key, key_delta], 'unordered', False)
1798
ref = files.get_record_stream([key, key_delta], 'unordered', False)
1799
skipped_records = [0]
1801
key: "my text\ncontent",
1802
key_delta: "different\ncontent\n",
1804
byte_stream = self.stream_to_bytes_or_skip_counter(
1805
skipped_records, full_texts, local)
1806
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
1808
# insert the stream from the network into a versioned files object so we can
1809
# check the content was carried across correctly without doing delta
1811
target_files.insert_record_stream(
1812
self.assertStreamMetaEqual(records, ref, network_stream))
1813
# No duplicates on the wire thank you!
1814
self.assertEqual(2, len(records) + skipped_records[0])
1816
# if any content was copied it all must have all been.
1817
self.assertIdenticalVersionedFile(files, target_files)
1819
def test_get_record_stream_native_formats_are_wire_ready_delta(self):
1820
# copy a delta over the wire
1821
files = self.get_versionedfiles()
1822
target_files = self.get_versionedfiles('target')
1823
key = self.get_simple_key('ft')
1824
key_delta = self.get_simple_key('delta')
1825
files.add_lines(key, (), ['my text\n', 'content'])
1827
delta_parents = (key,)
1830
files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
1831
# Copy the basis text across so we can reconstruct the delta during
1832
# insertion into target.
1833
target_files.insert_record_stream(files.get_record_stream([key],
1834
'unordered', False))
1835
local = files.get_record_stream([key_delta], 'unordered', False)
1836
ref = files.get_record_stream([key_delta], 'unordered', False)
1837
skipped_records = [0]
1839
key_delta: "different\ncontent\n",
1841
byte_stream = self.stream_to_bytes_or_skip_counter(
1842
skipped_records, full_texts, local)
1843
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
1845
# insert the stream from the network into a versioned files object so we can
1846
# check the content was carried across correctly without doing delta
1847
# inspection during check_stream.
1848
target_files.insert_record_stream(
1849
self.assertStreamMetaEqual(records, ref, network_stream))
1850
# No duplicates on the wire thank you!
1851
self.assertEqual(1, len(records) + skipped_records[0])
1853
# if any content was copied it all must have all been
1854
self.assertIdenticalVersionedFile(files, target_files)
1856
def test_get_record_stream_wire_ready_delta_closure_included(self):
1857
# copy a delta over the wire with the ability to get its full text.
1858
files = self.get_versionedfiles()
1859
key = self.get_simple_key('ft')
1860
key_delta = self.get_simple_key('delta')
1861
files.add_lines(key, (), ['my text\n', 'content'])
1863
delta_parents = (key,)
1866
files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
1867
local = files.get_record_stream([key_delta], 'unordered', True)
1868
ref = files.get_record_stream([key_delta], 'unordered', True)
1869
skipped_records = [0]
1871
key_delta: "different\ncontent\n",
1873
byte_stream = self.stream_to_bytes_or_skip_counter(
1874
skipped_records, full_texts, local)
1875
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
1877
# insert the stream from the network into a versioned files object so we can
1878
# check the content was carried across correctly without doing delta
1879
# inspection during check_stream.
1880
for record in self.assertStreamMetaEqual(records, ref, network_stream):
1881
# we have to be able to get the full text out:
1882
self.assertRecordHasContent(record, full_texts[record.key])
1883
# No duplicates on the wire thank you!
1884
self.assertEqual(1, len(records) + skipped_records[0])
1886
def assertAbsentRecord(self, files, keys, parents, entries):
1887
"""Helper for test_get_record_stream_missing_records_are_absent."""
1889
for factory in entries:
1890
seen.add(factory.key)
1891
if factory.key[-1] == 'absent':
1892
self.assertEqual('absent', factory.storage_kind)
1893
self.assertEqual(None, factory.sha1)
1894
self.assertEqual(None, factory.parents)
1896
self.assertValidStorageKind(factory.storage_kind)
1897
self.assertEqual(files.get_sha1s([factory.key])[factory.key],
1899
self.assertEqual(parents[factory.key], factory.parents)
1900
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1902
self.assertEqual(set(keys), seen)
1904
def test_filter_absent_records(self):
1905
"""Requested missing records can be filter trivially."""
1906
files = self.get_versionedfiles()
1907
self.get_diamond_files(files)
1908
keys, _ = self.get_keys_and_sort_order()
1909
parent_map = files.get_parent_map(keys)
1910
# Add an absent record in the middle of the present keys. (We don't ask
1911
# for just absent keys to ensure that content before and after the
1912
# absent keys is still delivered).
1913
present_keys = list(keys)
1914
if self.key_length == 1:
1915
keys.insert(2, ('extra',))
1917
keys.insert(2, ('extra', 'extra'))
1918
entries = files.get_record_stream(keys, 'unordered', False)
1920
self.capture_stream(files, versionedfile.filter_absent(entries), seen.add,
1922
self.assertEqual(set(present_keys), seen)
1924
def get_mapper(self):
1925
"""Get a mapper suitable for the key length of the test interface."""
1926
if self.key_length == 1:
1927
return ConstantMapper('source')
1929
return HashEscapedPrefixMapper()
1931
def get_parents(self, parents):
1932
"""Get parents, taking self.graph into consideration."""
1938
def test_get_parent_map(self):
1939
files = self.get_versionedfiles()
1940
if self.key_length == 1:
1942
(('r0',), self.get_parents(())),
1943
(('r1',), self.get_parents((('r0',),))),
1944
(('r2',), self.get_parents(())),
1945
(('r3',), self.get_parents(())),
1946
(('m',), self.get_parents((('r0',),('r1',),('r2',),('r3',)))),
1950
(('FileA', 'r0'), self.get_parents(())),
1951
(('FileA', 'r1'), self.get_parents((('FileA', 'r0'),))),
1952
(('FileA', 'r2'), self.get_parents(())),
1953
(('FileA', 'r3'), self.get_parents(())),
1954
(('FileA', 'm'), self.get_parents((('FileA', 'r0'),
1955
('FileA', 'r1'), ('FileA', 'r2'), ('FileA', 'r3')))),
1957
for key, parents in parent_details:
1958
files.add_lines(key, parents, [])
1959
# immediately after adding it should be queryable.
1960
self.assertEqual({key:parents}, files.get_parent_map([key]))
1961
# We can ask for an empty set
1962
self.assertEqual({}, files.get_parent_map([]))
1963
# We can ask for many keys
1964
all_parents = dict(parent_details)
1965
self.assertEqual(all_parents, files.get_parent_map(all_parents.keys()))
1966
# Absent keys are just not included in the result.
1967
keys = all_parents.keys()
1968
if self.key_length == 1:
1969
keys.insert(1, ('missing',))
1971
keys.insert(1, ('missing', 'missing'))
1972
# Absent keys are just ignored
1973
self.assertEqual(all_parents, files.get_parent_map(keys))
1975
def test_get_sha1s(self):
1976
files = self.get_versionedfiles()
1977
self.get_diamond_files(files)
1978
if self.key_length == 1:
1979
keys = [('base',), ('origin',), ('left',), ('merged',), ('right',)]
1981
# ask for shas from different prefixes.
1983
('FileA', 'base'), ('FileB', 'origin'), ('FileA', 'left'),
1984
('FileA', 'merged'), ('FileB', 'right'),
1987
keys[0]: '51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',
1988
keys[1]: '00e364d235126be43292ab09cb4686cf703ddc17',
1989
keys[2]: 'a8478686da38e370e32e42e8a0c220e33ee9132f',
1990
keys[3]: 'ed8bce375198ea62444dc71952b22cfc2b09226d',
1991
keys[4]: '9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',
1993
files.get_sha1s(keys))
1995
def test_insert_record_stream_empty(self):
1996
"""Inserting an empty record stream should work."""
1997
files = self.get_versionedfiles()
1998
files.insert_record_stream([])
2000
def assertIdenticalVersionedFile(self, expected, actual):
2001
"""Assert that left and right have the same contents."""
2002
self.assertEqual(set(actual.keys()), set(expected.keys()))
2003
actual_parents = actual.get_parent_map(actual.keys())
2005
self.assertEqual(actual_parents, expected.get_parent_map(expected.keys()))
2007
for key, parents in actual_parents.items():
2008
self.assertEqual(None, parents)
2009
for key in actual.keys():
2010
actual_text = actual.get_record_stream(
2011
[key], 'unordered', True).next().get_bytes_as('fulltext')
2012
expected_text = expected.get_record_stream(
2013
[key], 'unordered', True).next().get_bytes_as('fulltext')
2014
self.assertEqual(actual_text, expected_text)
2016
def test_insert_record_stream_fulltexts(self):
2017
"""Any file should accept a stream of fulltexts."""
2018
files = self.get_versionedfiles()
2019
mapper = self.get_mapper()
2020
source_transport = self.get_transport('source')
2021
source_transport.mkdir('.')
2022
# weaves always output fulltexts.
2023
source = make_versioned_files_factory(WeaveFile, mapper)(
2025
self.get_diamond_files(source, trailing_eol=False)
2026
stream = source.get_record_stream(source.keys(), 'topological',
2028
files.insert_record_stream(stream)
2029
self.assertIdenticalVersionedFile(source, files)
2031
def test_insert_record_stream_fulltexts_noeol(self):
2032
"""Any file should accept a stream of fulltexts."""
2033
files = self.get_versionedfiles()
2034
mapper = self.get_mapper()
2035
source_transport = self.get_transport('source')
2036
source_transport.mkdir('.')
2037
# weaves always output fulltexts.
2038
source = make_versioned_files_factory(WeaveFile, mapper)(
2040
self.get_diamond_files(source, trailing_eol=False)
2041
stream = source.get_record_stream(source.keys(), 'topological',
2043
files.insert_record_stream(stream)
2044
self.assertIdenticalVersionedFile(source, files)
2046
def test_insert_record_stream_annotated_knits(self):
2047
"""Any file should accept a stream from plain knits."""
2048
files = self.get_versionedfiles()
2049
mapper = self.get_mapper()
2050
source_transport = self.get_transport('source')
2051
source_transport.mkdir('.')
2052
source = make_file_factory(True, mapper)(source_transport)
2053
self.get_diamond_files(source)
2054
stream = source.get_record_stream(source.keys(), 'topological',
2056
files.insert_record_stream(stream)
2057
self.assertIdenticalVersionedFile(source, files)
2059
def test_insert_record_stream_annotated_knits_noeol(self):
2060
"""Any file should accept a stream from plain knits."""
2061
files = self.get_versionedfiles()
2062
mapper = self.get_mapper()
2063
source_transport = self.get_transport('source')
2064
source_transport.mkdir('.')
2065
source = make_file_factory(True, mapper)(source_transport)
2066
self.get_diamond_files(source, trailing_eol=False)
2067
stream = source.get_record_stream(source.keys(), 'topological',
2069
files.insert_record_stream(stream)
2070
self.assertIdenticalVersionedFile(source, files)
2072
def test_insert_record_stream_plain_knits(self):
2073
"""Any file should accept a stream from plain knits."""
2074
files = self.get_versionedfiles()
2075
mapper = self.get_mapper()
2076
source_transport = self.get_transport('source')
2077
source_transport.mkdir('.')
2078
source = make_file_factory(False, mapper)(source_transport)
2079
self.get_diamond_files(source)
2080
stream = source.get_record_stream(source.keys(), 'topological',
2082
files.insert_record_stream(stream)
2083
self.assertIdenticalVersionedFile(source, files)
2085
def test_insert_record_stream_plain_knits_noeol(self):
2086
"""Any file should accept a stream from plain knits."""
2087
files = self.get_versionedfiles()
2088
mapper = self.get_mapper()
2089
source_transport = self.get_transport('source')
2090
source_transport.mkdir('.')
2091
source = make_file_factory(False, mapper)(source_transport)
2092
self.get_diamond_files(source, trailing_eol=False)
2093
stream = source.get_record_stream(source.keys(), 'topological',
2095
files.insert_record_stream(stream)
2096
self.assertIdenticalVersionedFile(source, files)
2098
def test_insert_record_stream_existing_keys(self):
2099
"""Inserting keys already in a file should not error."""
2100
files = self.get_versionedfiles()
2101
source = self.get_versionedfiles('source')
2102
self.get_diamond_files(source)
2103
# insert some keys into f.
2104
self.get_diamond_files(files, left_only=True)
2105
stream = source.get_record_stream(source.keys(), 'topological',
2107
files.insert_record_stream(stream)
2108
self.assertIdenticalVersionedFile(source, files)
2110
def test_insert_record_stream_missing_keys(self):
2111
"""Inserting a stream with absent keys should raise an error."""
2112
files = self.get_versionedfiles()
2113
source = self.get_versionedfiles('source')
2114
stream = source.get_record_stream([('missing',) * self.key_length],
2115
'topological', False)
2116
self.assertRaises(errors.RevisionNotPresent, files.insert_record_stream,
2119
def test_insert_record_stream_out_of_order(self):
2120
"""An out of order stream can either error or work."""
2121
files = self.get_versionedfiles()
2122
source = self.get_versionedfiles('source')
2123
self.get_diamond_files(source)
2124
if self.key_length == 1:
2125
origin_keys = [('origin',)]
2126
end_keys = [('merged',), ('left',)]
2127
start_keys = [('right',), ('base',)]
2129
origin_keys = [('FileA', 'origin'), ('FileB', 'origin')]
2130
end_keys = [('FileA', 'merged',), ('FileA', 'left',),
2131
('FileB', 'merged',), ('FileB', 'left',)]
2132
start_keys = [('FileA', 'right',), ('FileA', 'base',),
2133
('FileB', 'right',), ('FileB', 'base',)]
2134
origin_entries = source.get_record_stream(origin_keys, 'unordered', False)
2135
end_entries = source.get_record_stream(end_keys, 'topological', False)
2136
start_entries = source.get_record_stream(start_keys, 'topological', False)
2137
entries = chain(origin_entries, end_entries, start_entries)
2139
files.insert_record_stream(entries)
2140
except RevisionNotPresent:
2141
# Must not have corrupted the file.
2144
self.assertIdenticalVersionedFile(source, files)
2146
def get_knit_delta_source(self):
2147
"""Get a source that can produce a stream with knit delta records,
2148
regardless of this test's scenario.
2150
mapper = self.get_mapper()
2151
source_transport = self.get_transport('source')
2152
source_transport.mkdir('.')
2153
source = make_file_factory(False, mapper)(source_transport)
2154
get_diamond_files(source, self.key_length, trailing_eol=True,
2155
nograph=False, left_only=False)
2158
def test_insert_record_stream_delta_missing_basis_no_corruption(self):
2159
"""Insertion where a needed basis is not included notifies the caller
2160
of the missing basis. In the meantime a record missing its basis is
2163
source = self.get_knit_delta_source()
2164
keys = [self.get_simple_key('origin'), self.get_simple_key('merged')]
2165
entries = source.get_record_stream(keys, 'unordered', False)
2166
files = self.get_versionedfiles()
2167
if self.support_partial_insertion:
2168
self.assertEqual([],
2169
list(files.get_missing_compression_parent_keys()))
2170
files.insert_record_stream(entries)
2171
missing_bases = files.get_missing_compression_parent_keys()
2172
self.assertEqual(set([self.get_simple_key('left')]),
2174
self.assertEqual(set(keys), set(files.get_parent_map(keys)))
2177
errors.RevisionNotPresent, files.insert_record_stream, entries)
2180
def test_insert_record_stream_delta_missing_basis_can_be_added_later(self):
2181
"""Insertion where a needed basis is not included notifies the caller
2182
of the missing basis. That basis can be added in a second
2183
insert_record_stream call that does not need to repeat records present
2184
in the previous stream. The record(s) that required that basis are
2185
fully inserted once their basis is no longer missing.
2187
if not self.support_partial_insertion:
2188
raise TestNotApplicable(
2189
'versioned file scenario does not support partial insertion')
2190
source = self.get_knit_delta_source()
2191
entries = source.get_record_stream([self.get_simple_key('origin'),
2192
self.get_simple_key('merged')], 'unordered', False)
2193
files = self.get_versionedfiles()
2194
files.insert_record_stream(entries)
2195
missing_bases = files.get_missing_compression_parent_keys()
2196
self.assertEqual(set([self.get_simple_key('left')]),
2198
# 'merged' is inserted (although a commit of a write group involving
2199
# this versionedfiles would fail).
2200
merged_key = self.get_simple_key('merged')
2202
[merged_key], files.get_parent_map([merged_key]).keys())
2203
# Add the full delta closure of the missing records
2204
missing_entries = source.get_record_stream(
2205
missing_bases, 'unordered', True)
2206
files.insert_record_stream(missing_entries)
2207
# Now 'merged' is fully inserted (and a commit would succeed).
2208
self.assertEqual([], list(files.get_missing_compression_parent_keys()))
2210
[merged_key], files.get_parent_map([merged_key]).keys())
2213
def test_iter_lines_added_or_present_in_keys(self):
2214
# test that we get at least an equalset of the lines added by
2215
# versions in the store.
2216
# the ordering here is to make a tree so that dumb searches have
2217
# more changes to muck up.
2219
class InstrumentedProgress(progress.DummyProgress):
2223
progress.DummyProgress.__init__(self)
2226
def update(self, msg=None, current=None, total=None):
2227
self.updates.append((msg, current, total))
2229
files = self.get_versionedfiles()
2230
# add a base to get included
2231
files.add_lines(self.get_simple_key('base'), (), ['base\n'])
2232
# add a ancestor to be included on one side
2233
files.add_lines(self.get_simple_key('lancestor'), (), ['lancestor\n'])
2234
# add a ancestor to be included on the other side
2235
files.add_lines(self.get_simple_key('rancestor'),
2236
self.get_parents([self.get_simple_key('base')]), ['rancestor\n'])
2237
# add a child of rancestor with no eofile-nl
2238
files.add_lines(self.get_simple_key('child'),
2239
self.get_parents([self.get_simple_key('rancestor')]),
2240
['base\n', 'child\n'])
2241
# add a child of lancestor and base to join the two roots
2242
files.add_lines(self.get_simple_key('otherchild'),
2243
self.get_parents([self.get_simple_key('lancestor'),
2244
self.get_simple_key('base')]),
2245
['base\n', 'lancestor\n', 'otherchild\n'])
2246
def iter_with_keys(keys, expected):
2247
# now we need to see what lines are returned, and how often.
2249
progress = InstrumentedProgress()
2250
# iterate over the lines
2251
for line in files.iter_lines_added_or_present_in_keys(keys,
2253
lines.setdefault(line, 0)
2255
if []!= progress.updates:
2256
self.assertEqual(expected, progress.updates)
2258
lines = iter_with_keys(
2259
[self.get_simple_key('child'), self.get_simple_key('otherchild')],
2260
[('Walking content.', 0, 2),
2261
('Walking content.', 1, 2),
2262
('Walking content.', 2, 2)])
2263
# we must see child and otherchild
2264
self.assertTrue(lines[('child\n', self.get_simple_key('child'))] > 0)
2266
lines[('otherchild\n', self.get_simple_key('otherchild'))] > 0)
2267
# we dont care if we got more than that.
2270
lines = iter_with_keys(files.keys(),
2271
[('Walking content.', 0, 5),
2272
('Walking content.', 1, 5),
2273
('Walking content.', 2, 5),
2274
('Walking content.', 3, 5),
2275
('Walking content.', 4, 5),
2276
('Walking content.', 5, 5)])
2277
# all lines must be seen at least once
2278
self.assertTrue(lines[('base\n', self.get_simple_key('base'))] > 0)
2280
lines[('lancestor\n', self.get_simple_key('lancestor'))] > 0)
2282
lines[('rancestor\n', self.get_simple_key('rancestor'))] > 0)
2283
self.assertTrue(lines[('child\n', self.get_simple_key('child'))] > 0)
2285
lines[('otherchild\n', self.get_simple_key('otherchild'))] > 0)
2287
def test_make_mpdiffs(self):
2288
from bzrlib import multiparent
2289
files = self.get_versionedfiles('source')
2290
# add texts that should trip the knit maximum delta chain threshold
2291
# as well as doing parallel chains of data in knits.
2292
# this is done by two chains of 25 insertions
2293
files.add_lines(self.get_simple_key('base'), [], ['line\n'])
2294
files.add_lines(self.get_simple_key('noeol'),
2295
self.get_parents([self.get_simple_key('base')]), ['line'])
2296
# detailed eol tests:
2297
# shared last line with parent no-eol
2298
files.add_lines(self.get_simple_key('noeolsecond'),
2299
self.get_parents([self.get_simple_key('noeol')]),
2301
# differing last line with parent, both no-eol
2302
files.add_lines(self.get_simple_key('noeolnotshared'),
2303
self.get_parents([self.get_simple_key('noeolsecond')]),
2304
['line\n', 'phone'])
2305
# add eol following a noneol parent, change content
2306
files.add_lines(self.get_simple_key('eol'),
2307
self.get_parents([self.get_simple_key('noeol')]), ['phone\n'])
2308
# add eol following a noneol parent, no change content
2309
files.add_lines(self.get_simple_key('eolline'),
2310
self.get_parents([self.get_simple_key('noeol')]), ['line\n'])
2311
# noeol with no parents:
2312
files.add_lines(self.get_simple_key('noeolbase'), [], ['line'])
2313
# noeol preceeding its leftmost parent in the output:
2314
# this is done by making it a merge of two parents with no common
2315
# anestry: noeolbase and noeol with the
2316
# later-inserted parent the leftmost.
2317
files.add_lines(self.get_simple_key('eolbeforefirstparent'),
2318
self.get_parents([self.get_simple_key('noeolbase'),
2319
self.get_simple_key('noeol')]),
2321
# two identical eol texts
2322
files.add_lines(self.get_simple_key('noeoldup'),
2323
self.get_parents([self.get_simple_key('noeol')]), ['line'])
2324
next_parent = self.get_simple_key('base')
2325
text_name = 'chain1-'
2327
sha1s = {0 :'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
2328
1 :'45e21ea146a81ea44a821737acdb4f9791c8abe7',
2329
2 :'e1f11570edf3e2a070052366c582837a4fe4e9fa',
2330
3 :'26b4b8626da827088c514b8f9bbe4ebf181edda1',
2331
4 :'e28a5510be25ba84d31121cff00956f9970ae6f6',
2332
5 :'d63ec0ce22e11dcf65a931b69255d3ac747a318d',
2333
6 :'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
2334
7 :'95c14da9cafbf828e3e74a6f016d87926ba234ab',
2335
8 :'779e9a0b28f9f832528d4b21e17e168c67697272',
2336
9 :'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
2337
10:'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
2338
11:'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
2339
12:'31a2286267f24d8bedaa43355f8ad7129509ea85',
2340
13:'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
2341
14:'2c4b1736566b8ca6051e668de68650686a3922f2',
2342
15:'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
2343
16:'b0d2e18d3559a00580f6b49804c23fea500feab3',
2344
17:'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
2345
18:'5cf64a3459ae28efa60239e44b20312d25b253f3',
2346
19:'1ebed371807ba5935958ad0884595126e8c4e823',
2347
20:'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
2348
21:'01edc447978004f6e4e962b417a4ae1955b6fe5d',
2349
22:'d8d8dc49c4bf0bab401e0298bb5ad827768618bb',
2350
23:'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
2351
24:'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
2352
25:'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
2354
for depth in range(26):
2355
new_version = self.get_simple_key(text_name + '%s' % depth)
2356
text = text + ['line\n']
2357
files.add_lines(new_version, self.get_parents([next_parent]), text)
2358
next_parent = new_version
2359
next_parent = self.get_simple_key('base')
2360
text_name = 'chain2-'
2362
for depth in range(26):
2363
new_version = self.get_simple_key(text_name + '%s' % depth)
2364
text = text + ['line\n']
2365
files.add_lines(new_version, self.get_parents([next_parent]), text)
2366
next_parent = new_version
2367
target = self.get_versionedfiles('target')
2368
for key in multiparent.topo_iter_keys(files, files.keys()):
2369
mpdiff = files.make_mpdiffs([key])[0]
2370
parents = files.get_parent_map([key])[key] or []
2372
[(key, parents, files.get_sha1s([key])[key], mpdiff)])
2373
self.assertEqualDiff(
2374
files.get_record_stream([key], 'unordered',
2375
True).next().get_bytes_as('fulltext'),
2376
target.get_record_stream([key], 'unordered',
2377
True).next().get_bytes_as('fulltext')
2380
def test_keys(self):
2381
# While use is discouraged, versions() is still needed by aspects of
2383
files = self.get_versionedfiles()
2384
self.assertEqual(set(), set(files.keys()))
2385
if self.key_length == 1:
2388
key = ('foo', 'bar',)
2389
files.add_lines(key, (), [])
2390
self.assertEqual(set([key]), set(files.keys()))
2393
class VirtualVersionedFilesTests(TestCase):
2394
"""Basic tests for the VirtualVersionedFiles implementations."""
2396
def _get_parent_map(self, keys):
2399
if k in self._parent_map:
2400
ret[k] = self._parent_map[k]
2404
TestCase.setUp(self)
2406
self._parent_map = {}
2407
self.texts = VirtualVersionedFiles(self._get_parent_map,
2410
def test_add_lines(self):
2411
self.assertRaises(NotImplementedError,
2412
self.texts.add_lines, "foo", [], [])
2414
def test_add_mpdiffs(self):
2415
self.assertRaises(NotImplementedError,
2416
self.texts.add_mpdiffs, [])
2418
def test_check(self):
2419
self.assertTrue(self.texts.check())
2421
def test_insert_record_stream(self):
2422
self.assertRaises(NotImplementedError, self.texts.insert_record_stream,
2425
def test_get_sha1s_nonexistent(self):
2426
self.assertEquals({}, self.texts.get_sha1s([("NONEXISTENT",)]))
2428
def test_get_sha1s(self):
2429
self._lines["key"] = ["dataline1", "dataline2"]
2430
self.assertEquals({("key",): osutils.sha_strings(self._lines["key"])},
2431
self.texts.get_sha1s([("key",)]))
2433
def test_get_parent_map(self):
2434
self._parent_map = {"G": ("A", "B")}
2435
self.assertEquals({("G",): (("A",),("B",))},
2436
self.texts.get_parent_map([("G",), ("L",)]))
2438
def test_get_record_stream(self):
2439
self._lines["A"] = ["FOO", "BAR"]
2440
it = self.texts.get_record_stream([("A",)], "unordered", True)
2442
self.assertEquals("chunked", record.storage_kind)
2443
self.assertEquals("FOOBAR", record.get_bytes_as("fulltext"))
2444
self.assertEquals(["FOO", "BAR"], record.get_bytes_as("chunked"))
2446
def test_get_record_stream_absent(self):
2447
it = self.texts.get_record_stream([("A",)], "unordered", True)
2449
self.assertEquals("absent", record.storage_kind)
2451
def test_iter_lines_added_or_present_in_keys(self):
2452
self._lines["A"] = ["FOO", "BAR"]
2453
self._lines["B"] = ["HEY"]
2454
self._lines["C"] = ["Alberta"]
2455
it = self.texts.iter_lines_added_or_present_in_keys([("A",), ("B",)])
2456
self.assertEquals(sorted([("FOO", "A"), ("BAR", "A"), ("HEY", "B")]),
2460
class TestOrderingVersionedFilesDecorator(TestCaseWithMemoryTransport):
2462
def get_ordering_vf(self, key_priority):
2463
builder = self.make_branch_builder('test')
2464
builder.start_series()
2465
builder.build_snapshot('A', None, [
2466
('add', ('', 'TREE_ROOT', 'directory', None))])
2467
builder.build_snapshot('B', ['A'], [])
2468
builder.build_snapshot('C', ['B'], [])
2469
builder.build_snapshot('D', ['C'], [])
2470
builder.finish_series()
2471
b = builder.get_branch()
2473
self.addCleanup(b.unlock)
2474
vf = b.repository.inventories
2475
return versionedfile.OrderingVersionedFilesDecorator(vf, key_priority)
2477
def test_get_empty(self):
2478
vf = self.get_ordering_vf({})
2479
self.assertEqual([], vf.calls)
2481
def test_get_record_stream_topological(self):
2482
vf = self.get_ordering_vf({('A',): 3, ('B',): 2, ('C',): 4, ('D',): 1})
2483
request_keys = [('B',), ('C',), ('D',), ('A',)]
2484
keys = [r.key for r in vf.get_record_stream(request_keys,
2485
'topological', False)]
2486
# We should have gotten the keys in topological order
2487
self.assertEqual([('A',), ('B',), ('C',), ('D',)], keys)
2488
# And recorded that the request was made
2489
self.assertEqual([('get_record_stream', request_keys, 'topological',
2492
def test_get_record_stream_ordered(self):
2493
vf = self.get_ordering_vf({('A',): 3, ('B',): 2, ('C',): 4, ('D',): 1})
2494
request_keys = [('B',), ('C',), ('D',), ('A',)]
2495
keys = [r.key for r in vf.get_record_stream(request_keys,
2496
'unordered', False)]
2497
# They should be returned based on their priority
2498
self.assertEqual([('D',), ('B',), ('A',), ('C',)], keys)
2499
# And the request recorded
2500
self.assertEqual([('get_record_stream', request_keys, 'unordered',
2503
def test_get_record_stream_implicit_order(self):
2504
vf = self.get_ordering_vf({('B',): 2, ('D',): 1})
2505
request_keys = [('B',), ('C',), ('D',), ('A',)]
2506
keys = [r.key for r in vf.get_record_stream(request_keys,
2507
'unordered', False)]
2508
# A and C are not in the map, so they get sorted to the front. A comes
2509
# before C alphabetically, so it comes back first
2510
self.assertEqual([('A',), ('C',), ('D',), ('B',)], keys)
2511
# And the request recorded
2512
self.assertEqual([('get_record_stream', request_keys, 'unordered',
added
removed
bzrlib/tests/test_versionedfile.py
