146
374
self.assertRaises(errors.ReservedId,
147
375
vf.add_lines, 'a:', [], ['a\n', 'b\n', 'c\n'])
149
self.assertRaises(errors.ReservedId,
150
vf.add_delta, 'a:', [], None, 'sha1', False, ((0, 0, 0, []),))
377
def test_add_lines_nostoresha(self):
378
"""When nostore_sha is supplied using old content raises."""
380
empty_text = ('a', [])
381
sample_text_nl = ('b', ["foo\n", "bar\n"])
382
sample_text_no_nl = ('c', ["foo\n", "bar"])
384
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
385
sha, _, _ = vf.add_lines(version, [], lines)
387
# we now have a copy of all the lines in the vf.
388
for sha, (version, lines) in zip(
389
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
390
self.assertRaises(errors.ExistingContent,
391
vf.add_lines, version + "2", [], lines,
393
# and no new version should have been added.
394
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
397
def test_add_lines_with_ghosts_nostoresha(self):
398
"""When nostore_sha is supplied using old content raises."""
400
empty_text = ('a', [])
401
sample_text_nl = ('b', ["foo\n", "bar\n"])
402
sample_text_no_nl = ('c', ["foo\n", "bar"])
404
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
405
sha, _, _ = vf.add_lines(version, [], lines)
407
# we now have a copy of all the lines in the vf.
408
# is the test applicable to this vf implementation?
410
vf.add_lines_with_ghosts('d', [], [])
411
except NotImplementedError:
412
raise TestSkipped("add_lines_with_ghosts is optional")
413
for sha, (version, lines) in zip(
414
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
415
self.assertRaises(errors.ExistingContent,
416
vf.add_lines_with_ghosts, version + "2", [], lines,
418
# and no new version should have been added.
419
self.assertRaises(errors.RevisionNotPresent, vf.get_lines,
422
def test_add_lines_return_value(self):
423
# add_lines should return the sha1 and the text size.
425
empty_text = ('a', [])
426
sample_text_nl = ('b', ["foo\n", "bar\n"])
427
sample_text_no_nl = ('c', ["foo\n", "bar"])
428
# check results for the three cases:
429
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
430
# the first two elements are the same for all versioned files:
431
# - the digest and the size of the text. For some versioned files
432
# additional data is returned in additional tuple elements.
433
result = vf.add_lines(version, [], lines)
434
self.assertEqual(3, len(result))
435
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
437
# parents should not affect the result:
438
lines = sample_text_nl[1]
439
self.assertEqual((osutils.sha_strings(lines), sum(map(len, lines))),
440
vf.add_lines('d', ['b', 'c'], lines)[0:2])
152
442
def test_get_reserved(self):
153
443
vf = self.get_file()
154
self.assertRaises(errors.ReservedId, vf.get_delta, 'b:')
155
444
self.assertRaises(errors.ReservedId, vf.get_texts, ['b:'])
156
445
self.assertRaises(errors.ReservedId, vf.get_lines, 'b:')
157
446
self.assertRaises(errors.ReservedId, vf.get_text, 'b:')
159
def test_get_delta(self):
161
sha1s = self._setup_for_deltas(f)
162
expected_delta = (None, '6bfa09d82ce3e898ad4641ae13dd4fdb9cf0d76b', False,
163
[(0, 0, 1, [('base', 'line\n')])])
164
self.assertEqual(expected_delta, f.get_delta('base'))
166
text_name = 'chain1-'
167
for depth in range(26):
168
new_version = text_name + '%s' % depth
169
expected_delta = (next_parent, sha1s[depth],
171
[(depth + 1, depth + 1, 1, [(new_version, 'line\n')])])
172
self.assertEqual(expected_delta, f.get_delta(new_version))
173
next_parent = new_version
175
text_name = 'chain2-'
176
for depth in range(26):
177
new_version = text_name + '%s' % depth
178
expected_delta = (next_parent, sha1s[depth], False,
179
[(depth + 1, depth + 1, 1, [(new_version, 'line\n')])])
180
self.assertEqual(expected_delta, f.get_delta(new_version))
181
next_parent = new_version
182
# smoke test for eol support
183
expected_delta = ('base', '264f39cab871e4cfd65b3a002f7255888bb5ed97', True, [])
184
self.assertEqual(['line'], f.get_lines('noeol'))
185
self.assertEqual(expected_delta, f.get_delta('noeol'))
187
def test_get_deltas(self):
189
sha1s = self._setup_for_deltas(f)
190
deltas = f.get_deltas(f.versions())
191
expected_delta = (None, '6bfa09d82ce3e898ad4641ae13dd4fdb9cf0d76b', False,
192
[(0, 0, 1, [('base', 'line\n')])])
193
self.assertEqual(expected_delta, deltas['base'])
195
text_name = 'chain1-'
196
for depth in range(26):
197
new_version = text_name + '%s' % depth
198
expected_delta = (next_parent, sha1s[depth],
200
[(depth + 1, depth + 1, 1, [(new_version, 'line\n')])])
201
self.assertEqual(expected_delta, deltas[new_version])
202
next_parent = new_version
204
text_name = 'chain2-'
205
for depth in range(26):
206
new_version = text_name + '%s' % depth
207
expected_delta = (next_parent, sha1s[depth], False,
208
[(depth + 1, depth + 1, 1, [(new_version, 'line\n')])])
209
self.assertEqual(expected_delta, deltas[new_version])
210
next_parent = new_version
211
# smoke tests for eol support
212
expected_delta = ('base', '264f39cab871e4cfd65b3a002f7255888bb5ed97', True, [])
213
self.assertEqual(['line'], f.get_lines('noeol'))
214
self.assertEqual(expected_delta, deltas['noeol'])
215
# smoke tests for eol support - two noeol in a row same content
216
expected_deltas = (('noeol', '3ad7ee82dbd8f29ecba073f96e43e414b3f70a4d', True,
217
[(0, 1, 2, [('noeolsecond', 'line\n'), ('noeolsecond', 'line\n')])]),
218
('noeol', '3ad7ee82dbd8f29ecba073f96e43e414b3f70a4d', True,
219
[(0, 0, 1, [('noeolsecond', 'line\n')]), (1, 1, 0, [])]))
220
self.assertEqual(['line\n', 'line'], f.get_lines('noeolsecond'))
221
self.assertTrue(deltas['noeolsecond'] in expected_deltas)
222
# two no-eol in a row, different content
223
expected_delta = ('noeolsecond', '8bb553a84e019ef1149db082d65f3133b195223b', True,
224
[(1, 2, 1, [('noeolnotshared', 'phone\n')])])
225
self.assertEqual(['line\n', 'phone'], f.get_lines('noeolnotshared'))
226
self.assertEqual(expected_delta, deltas['noeolnotshared'])
227
# eol folling a no-eol with content change
228
expected_delta = ('noeol', 'a61f6fb6cfc4596e8d88c34a308d1e724caf8977', False,
229
[(0, 1, 1, [('eol', 'phone\n')])])
230
self.assertEqual(['phone\n'], f.get_lines('eol'))
231
self.assertEqual(expected_delta, deltas['eol'])
232
# eol folling a no-eol with content change
233
expected_delta = ('noeol', '6bfa09d82ce3e898ad4641ae13dd4fdb9cf0d76b', False,
234
[(0, 1, 1, [('eolline', 'line\n')])])
235
self.assertEqual(['line\n'], f.get_lines('eolline'))
236
self.assertEqual(expected_delta, deltas['eolline'])
237
# eol with no parents
238
expected_delta = (None, '264f39cab871e4cfd65b3a002f7255888bb5ed97', True,
239
[(0, 0, 1, [('noeolbase', 'line\n')])])
240
self.assertEqual(['line'], f.get_lines('noeolbase'))
241
self.assertEqual(expected_delta, deltas['noeolbase'])
242
# eol with two parents, in inverse insertion order
243
expected_deltas = (('noeolbase', '264f39cab871e4cfd65b3a002f7255888bb5ed97', True,
244
[(0, 1, 1, [('eolbeforefirstparent', 'line\n')])]),
245
('noeolbase', '264f39cab871e4cfd65b3a002f7255888bb5ed97', True,
246
[(0, 1, 1, [('eolbeforefirstparent', 'line\n')])]))
247
self.assertEqual(['line'], f.get_lines('eolbeforefirstparent'))
248
#self.assertTrue(deltas['eolbeforefirstparent'] in expected_deltas)
448
def test_add_unchanged_last_line_noeol_snapshot(self):
449
"""Add a text with an unchanged last line with no eol should work."""
450
# Test adding this in a number of chain lengths; because the interface
451
# for VersionedFile does not allow forcing a specific chain length, we
452
# just use a small base to get the first snapshot, then a much longer
453
# first line for the next add (which will make the third add snapshot)
454
# and so on. 20 has been chosen as an aribtrary figure - knits use 200
455
# as a capped delta length, but ideally we would have some way of
456
# tuning the test to the store (e.g. keep going until a snapshot
458
for length in range(20):
460
vf = self.get_file('case-%d' % length)
463
for step in range(length):
464
version = prefix % step
465
lines = (['prelude \n'] * step) + ['line']
466
vf.add_lines(version, parents, lines)
467
version_lines[version] = lines
469
vf.add_lines('no-eol', parents, ['line'])
470
vf.get_texts(version_lines.keys())
471
self.assertEqualDiff('line', vf.get_text('no-eol'))
473
def test_get_texts_eol_variation(self):
474
# similar to the failure in <http://bugs.launchpad.net/234748>
476
sample_text_nl = ["line\n"]
477
sample_text_no_nl = ["line"]
484
lines = sample_text_nl
486
lines = sample_text_no_nl
487
# left_matching blocks is an internal api; it operates on the
488
# *internal* representation for a knit, which is with *all* lines
489
# being normalised to end with \n - even the final line in a no_nl
490
# file. Using it here ensures that a broken internal implementation
491
# (which is what this test tests) will generate a correct line
492
# delta (which is to say, an empty delta).
493
vf.add_lines(version, parents, lines,
494
left_matching_blocks=[(0, 0, 1)])
496
versions.append(version)
497
version_lines[version] = lines
499
vf.get_texts(versions)
500
vf.get_texts(reversed(versions))
502
def test_add_lines_with_matching_blocks_noeol_last_line(self):
503
"""Add a text with an unchanged last line with no eol should work."""
504
from bzrlib import multiparent
505
# Hand verified sha1 of the text we're adding.
506
sha1 = '6a1d115ec7b60afb664dc14890b5af5ce3c827a4'
507
# Create a mpdiff which adds a new line before the trailing line, and
508
# reuse the last line unaltered (which can cause annotation reuse).
509
# Test adding this in two situations:
510
# On top of a new insertion
511
vf = self.get_file('fulltext')
512
vf.add_lines('noeol', [], ['line'])
513
vf.add_lines('noeol2', ['noeol'], ['newline\n', 'line'],
514
left_matching_blocks=[(0, 1, 1)])
515
self.assertEqualDiff('newline\nline', vf.get_text('noeol2'))
517
vf = self.get_file('delta')
518
vf.add_lines('base', [], ['line'])
519
vf.add_lines('noeol', ['base'], ['prelude\n', 'line'])
520
vf.add_lines('noeol2', ['noeol'], ['newline\n', 'line'],
521
left_matching_blocks=[(1, 1, 1)])
522
self.assertEqualDiff('newline\nline', vf.get_text('noeol2'))
524
def test_make_mpdiffs(self):
525
from bzrlib import multiparent
526
vf = self.get_file('foo')
527
sha1s = self._setup_for_deltas(vf)
528
new_vf = self.get_file('bar')
529
for version in multiparent.topo_iter(vf):
530
mpdiff = vf.make_mpdiffs([version])[0]
531
new_vf.add_mpdiffs([(version, vf.get_parent_map([version])[version],
532
vf.get_sha1s([version])[version], mpdiff)])
533
self.assertEqualDiff(vf.get_text(version),
534
new_vf.get_text(version))
536
def test_make_mpdiffs_with_ghosts(self):
537
vf = self.get_file('foo')
539
vf.add_lines_with_ghosts('text', ['ghost'], ['line\n'])
540
except NotImplementedError:
541
# old Weave formats do not allow ghosts
543
self.assertRaises(errors.RevisionNotPresent, vf.make_mpdiffs, ['ghost'])
250
545
def _setup_for_deltas(self, f):
251
self.assertRaises(errors.RevisionNotPresent, f.get_delta, 'base')
546
self.assertFalse(f.has_version('base'))
252
547
# add texts that should trip the knit maximum delta chain threshold
253
548
# as well as doing parallel chains of data in knits.
254
549
# this is done by two chains of 25 insertions
1217
1271
write_weave(w, tmpf)
1218
1272
self.log(tmpf.getvalue())
1220
overlappedInsertExpected = ['aaa', '<<<<<<< ', 'xxx', 'yyy', '=======',
1274
overlappedInsertExpected = ['aaa', '<<<<<<< ', 'xxx', 'yyy', '=======',
1221
1275
'xxx', '>>>>>>> ', 'bbb']
1278
class TestContentFactoryAdaption(TestCaseWithMemoryTransport):
1280
def test_select_adaptor(self):
1281
"""Test expected adapters exist."""
1282
# One scenario for each lookup combination we expect to use.
1283
# Each is source_kind, requested_kind, adapter class
1285
('knit-delta-gz', 'fulltext', _mod_knit.DeltaPlainToFullText),
1286
('knit-ft-gz', 'fulltext', _mod_knit.FTPlainToFullText),
1287
('knit-annotated-delta-gz', 'knit-delta-gz',
1288
_mod_knit.DeltaAnnotatedToUnannotated),
1289
('knit-annotated-delta-gz', 'fulltext',
1290
_mod_knit.DeltaAnnotatedToFullText),
1291
('knit-annotated-ft-gz', 'knit-ft-gz',
1292
_mod_knit.FTAnnotatedToUnannotated),
1293
('knit-annotated-ft-gz', 'fulltext',
1294
_mod_knit.FTAnnotatedToFullText),
1296
for source, requested, klass in scenarios:
1297
adapter_factory = versionedfile.adapter_registry.get(
1298
(source, requested))
1299
adapter = adapter_factory(None)
1300
self.assertIsInstance(adapter, klass)
1302
def get_knit(self, annotated=True):
1303
mapper = ConstantMapper('knit')
1304
transport = self.get_transport()
1305
return make_file_factory(annotated, mapper)(transport)
1307
def helpGetBytes(self, f, ft_adapter, delta_adapter):
1308
"""Grab the interested adapted texts for tests."""
1309
# origin is a fulltext
1310
entries = f.get_record_stream([('origin',)], 'unordered', False)
1311
base = entries.next()
1312
ft_data = ft_adapter.get_bytes(base)
1313
# merged is both a delta and multiple parents.
1314
entries = f.get_record_stream([('merged',)], 'unordered', False)
1315
merged = entries.next()
1316
delta_data = delta_adapter.get_bytes(merged)
1317
return ft_data, delta_data
1319
def test_deannotation_noeol(self):
1320
"""Test converting annotated knits to unannotated knits."""
1321
# we need a full text, and a delta
1323
get_diamond_files(f, 1, trailing_eol=False)
1324
ft_data, delta_data = self.helpGetBytes(f,
1325
_mod_knit.FTAnnotatedToUnannotated(None),
1326
_mod_knit.DeltaAnnotatedToUnannotated(None))
1328
'version origin 1 b284f94827db1fa2970d9e2014f080413b547a7e\n'
1331
GzipFile(mode='rb', fileobj=StringIO(ft_data)).read())
1333
'version merged 4 32c2e79763b3f90e8ccde37f9710b6629c25a796\n'
1334
'1,2,3\nleft\nright\nmerged\nend merged\n',
1335
GzipFile(mode='rb', fileobj=StringIO(delta_data)).read())
1337
def test_deannotation(self):
1338
"""Test converting annotated knits to unannotated knits."""
1339
# we need a full text, and a delta
1341
get_diamond_files(f, 1)
1342
ft_data, delta_data = self.helpGetBytes(f,
1343
_mod_knit.FTAnnotatedToUnannotated(None),
1344
_mod_knit.DeltaAnnotatedToUnannotated(None))
1346
'version origin 1 00e364d235126be43292ab09cb4686cf703ddc17\n'
1349
GzipFile(mode='rb', fileobj=StringIO(ft_data)).read())
1351
'version merged 3 ed8bce375198ea62444dc71952b22cfc2b09226d\n'
1352
'2,2,2\nright\nmerged\nend merged\n',
1353
GzipFile(mode='rb', fileobj=StringIO(delta_data)).read())
1355
def test_annotated_to_fulltext_no_eol(self):
1356
"""Test adapting annotated knits to full texts (for -> weaves)."""
1357
# we need a full text, and a delta
1359
get_diamond_files(f, 1, trailing_eol=False)
1360
# Reconstructing a full text requires a backing versioned file, and it
1361
# must have the base lines requested from it.
1362
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1363
ft_data, delta_data = self.helpGetBytes(f,
1364
_mod_knit.FTAnnotatedToFullText(None),
1365
_mod_knit.DeltaAnnotatedToFullText(logged_vf))
1366
self.assertEqual('origin', ft_data)
1367
self.assertEqual('base\nleft\nright\nmerged', delta_data)
1368
self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1369
True)], logged_vf.calls)
1371
def test_annotated_to_fulltext(self):
1372
"""Test adapting annotated knits to full texts (for -> weaves)."""
1373
# we need a full text, and a delta
1375
get_diamond_files(f, 1)
1376
# Reconstructing a full text requires a backing versioned file, and it
1377
# must have the base lines requested from it.
1378
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1379
ft_data, delta_data = self.helpGetBytes(f,
1380
_mod_knit.FTAnnotatedToFullText(None),
1381
_mod_knit.DeltaAnnotatedToFullText(logged_vf))
1382
self.assertEqual('origin\n', ft_data)
1383
self.assertEqual('base\nleft\nright\nmerged\n', delta_data)
1384
self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1385
True)], logged_vf.calls)
1387
def test_unannotated_to_fulltext(self):
1388
"""Test adapting unannotated knits to full texts.
1390
This is used for -> weaves, and for -> annotated knits.
1392
# we need a full text, and a delta
1393
f = self.get_knit(annotated=False)
1394
get_diamond_files(f, 1)
1395
# Reconstructing a full text requires a backing versioned file, and it
1396
# must have the base lines requested from it.
1397
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1398
ft_data, delta_data = self.helpGetBytes(f,
1399
_mod_knit.FTPlainToFullText(None),
1400
_mod_knit.DeltaPlainToFullText(logged_vf))
1401
self.assertEqual('origin\n', ft_data)
1402
self.assertEqual('base\nleft\nright\nmerged\n', delta_data)
1403
self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1404
True)], logged_vf.calls)
1406
def test_unannotated_to_fulltext_no_eol(self):
1407
"""Test adapting unannotated knits to full texts.
1409
This is used for -> weaves, and for -> annotated knits.
1411
# we need a full text, and a delta
1412
f = self.get_knit(annotated=False)
1413
get_diamond_files(f, 1, trailing_eol=False)
1414
# Reconstructing a full text requires a backing versioned file, and it
1415
# must have the base lines requested from it.
1416
logged_vf = versionedfile.RecordingVersionedFilesDecorator(f)
1417
ft_data, delta_data = self.helpGetBytes(f,
1418
_mod_knit.FTPlainToFullText(None),
1419
_mod_knit.DeltaPlainToFullText(logged_vf))
1420
self.assertEqual('origin', ft_data)
1421
self.assertEqual('base\nleft\nright\nmerged', delta_data)
1422
self.assertEqual([('get_record_stream', [('left',)], 'unordered',
1423
True)], logged_vf.calls)
1426
class TestKeyMapper(TestCaseWithMemoryTransport):
1427
"""Tests for various key mapping logic."""
1429
def test_identity_mapper(self):
1430
mapper = versionedfile.ConstantMapper("inventory")
1431
self.assertEqual("inventory", mapper.map(('foo@ar',)))
1432
self.assertEqual("inventory", mapper.map(('quux',)))
1434
def test_prefix_mapper(self):
1436
mapper = versionedfile.PrefixMapper()
1437
self.assertEqual("file-id", mapper.map(("file-id", "revision-id")))
1438
self.assertEqual("new-id", mapper.map(("new-id", "revision-id")))
1439
self.assertEqual(('file-id',), mapper.unmap("file-id"))
1440
self.assertEqual(('new-id',), mapper.unmap("new-id"))
1442
def test_hash_prefix_mapper(self):
1443
#format6: hash + plain
1444
mapper = versionedfile.HashPrefixMapper()
1445
self.assertEqual("9b/file-id", mapper.map(("file-id", "revision-id")))
1446
self.assertEqual("45/new-id", mapper.map(("new-id", "revision-id")))
1447
self.assertEqual(('file-id',), mapper.unmap("9b/file-id"))
1448
self.assertEqual(('new-id',), mapper.unmap("45/new-id"))
1450
def test_hash_escaped_mapper(self):
1451
#knit1: hash + escaped
1452
mapper = versionedfile.HashEscapedPrefixMapper()
1453
self.assertEqual("88/%2520", mapper.map((" ", "revision-id")))
1454
self.assertEqual("ed/fil%2545-%2549d", mapper.map(("filE-Id",
1456
self.assertEqual("88/ne%2557-%2549d", mapper.map(("neW-Id",
1458
self.assertEqual(('filE-Id',), mapper.unmap("ed/fil%2545-%2549d"))
1459
self.assertEqual(('neW-Id',), mapper.unmap("88/ne%2557-%2549d"))
1462
class TestVersionedFiles(TestCaseWithMemoryTransport):
1463
"""Tests for the multiple-file variant of VersionedFile."""
1465
def get_versionedfiles(self, relpath='files'):
1466
transport = self.get_transport(relpath)
1468
transport.mkdir('.')
1469
files = self.factory(transport)
1470
if self.cleanup is not None:
1471
self.addCleanup(lambda:self.cleanup(files))
1474
def test_annotate(self):
1475
files = self.get_versionedfiles()
1476
self.get_diamond_files(files)
1477
if self.key_length == 1:
1481
# introduced full text
1482
origins = files.annotate(prefix + ('origin',))
1484
(prefix + ('origin',), 'origin\n')],
1487
origins = files.annotate(prefix + ('base',))
1489
(prefix + ('base',), 'base\n')],
1492
origins = files.annotate(prefix + ('merged',))
1495
(prefix + ('base',), 'base\n'),
1496
(prefix + ('left',), 'left\n'),
1497
(prefix + ('right',), 'right\n'),
1498
(prefix + ('merged',), 'merged\n')
1502
# Without a graph everything is new.
1504
(prefix + ('merged',), 'base\n'),
1505
(prefix + ('merged',), 'left\n'),
1506
(prefix + ('merged',), 'right\n'),
1507
(prefix + ('merged',), 'merged\n')
1510
self.assertRaises(RevisionNotPresent,
1511
files.annotate, prefix + ('missing-key',))
1513
def test_construct(self):
1514
"""Each parameterised test can be constructed on a transport."""
1515
files = self.get_versionedfiles()
1517
def get_diamond_files(self, files, trailing_eol=True, left_only=False,
1519
return get_diamond_files(files, self.key_length,
1520
trailing_eol=trailing_eol, nograph=not self.graph,
1521
left_only=left_only, nokeys=nokeys)
1523
def test_add_lines_nostoresha(self):
1524
"""When nostore_sha is supplied using old content raises."""
1525
vf = self.get_versionedfiles()
1526
empty_text = ('a', [])
1527
sample_text_nl = ('b', ["foo\n", "bar\n"])
1528
sample_text_no_nl = ('c', ["foo\n", "bar"])
1530
for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
1531
sha, _, _ = vf.add_lines(self.get_simple_key(version), [], lines)
1533
# we now have a copy of all the lines in the vf.
1534
for sha, (version, lines) in zip(
1535
shas, (empty_text, sample_text_nl, sample_text_no_nl)):
1536
new_key = self.get_simple_key(version + "2")
1537
self.assertRaises(errors.ExistingContent,
1538
vf.add_lines, new_key, [], lines,
1540
# and no new version should have been added.
1541
record = vf.get_record_stream([new_key], 'unordered', True).next()
1542
self.assertEqual('absent', record.storage_kind)
1544
def test_add_lines_return(self):
1545
files = self.get_versionedfiles()
1546
# save code by using the stock data insertion helper.
1547
adds = self.get_diamond_files(files)
1549
# We can only validate the first 2 elements returned from add_lines.
1551
self.assertEqual(3, len(add))
1552
results.append(add[:2])
1553
if self.key_length == 1:
1555
('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1556
('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1557
('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1558
('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1559
('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1561
elif self.key_length == 2:
1563
('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1564
('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1565
('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1566
('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1567
('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1568
('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1569
('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1570
('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1571
('ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1572
('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1575
def test_add_lines_no_key_generates_chk_key(self):
1576
files = self.get_versionedfiles()
1577
# save code by using the stock data insertion helper.
1578
adds = self.get_diamond_files(files, nokeys=True)
1580
# We can only validate the first 2 elements returned from add_lines.
1582
self.assertEqual(3, len(add))
1583
results.append(add[:2])
1584
if self.key_length == 1:
1586
('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1587
('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1588
('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1589
('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1590
('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1592
# Check the added items got CHK keys.
1593
self.assertEqual(set([
1594
('sha1:00e364d235126be43292ab09cb4686cf703ddc17',),
1595
('sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',),
1596
('sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',),
1597
('sha1:a8478686da38e370e32e42e8a0c220e33ee9132f',),
1598
('sha1:ed8bce375198ea62444dc71952b22cfc2b09226d',),
1601
elif self.key_length == 2:
1603
('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1604
('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1605
('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1606
('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1607
('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1608
('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1609
('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1610
('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1611
('ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1612
('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1614
# Check the added items got CHK keys.
1615
self.assertEqual(set([
1616
('FileA', 'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1617
('FileA', 'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1618
('FileA', 'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1619
('FileA', 'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1620
('FileA', 'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1621
('FileB', 'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1622
('FileB', 'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1623
('FileB', 'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1624
('FileB', 'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1625
('FileB', 'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1629
def test_empty_lines(self):
1630
"""Empty files can be stored."""
1631
f = self.get_versionedfiles()
1632
key_a = self.get_simple_key('a')
1633
f.add_lines(key_a, [], [])
1634
self.assertEqual('',
1635
f.get_record_stream([key_a], 'unordered', True
1636
).next().get_bytes_as('fulltext'))
1637
key_b = self.get_simple_key('b')
1638
f.add_lines(key_b, self.get_parents([key_a]), [])
1639
self.assertEqual('',
1640
f.get_record_stream([key_b], 'unordered', True
1641
).next().get_bytes_as('fulltext'))
1643
def test_newline_only(self):
1644
f = self.get_versionedfiles()
1645
key_a = self.get_simple_key('a')
1646
f.add_lines(key_a, [], ['\n'])
1647
self.assertEqual('\n',
1648
f.get_record_stream([key_a], 'unordered', True
1649
).next().get_bytes_as('fulltext'))
1650
key_b = self.get_simple_key('b')
1651
f.add_lines(key_b, self.get_parents([key_a]), ['\n'])
1652
self.assertEqual('\n',
1653
f.get_record_stream([key_b], 'unordered', True
1654
).next().get_bytes_as('fulltext'))
1656
def test_get_record_stream_empty(self):
1657
"""An empty stream can be requested without error."""
1658
f = self.get_versionedfiles()
1659
entries = f.get_record_stream([], 'unordered', False)
1660
self.assertEqual([], list(entries))
1662
def assertValidStorageKind(self, storage_kind):
1663
"""Assert that storage_kind is a valid storage_kind."""
1664
self.assertSubset([storage_kind],
1665
['mpdiff', 'knit-annotated-ft', 'knit-annotated-delta',
1666
'knit-ft', 'knit-delta', 'chunked', 'fulltext',
1667
'knit-annotated-ft-gz', 'knit-annotated-delta-gz', 'knit-ft-gz',
1669
'knit-delta-closure', 'knit-delta-closure-ref',
1670
'groupcompress-block', 'groupcompress-block-ref'])
1672
def capture_stream(self, f, entries, on_seen, parents):
1673
"""Capture a stream for testing."""
1674
for factory in entries:
1675
on_seen(factory.key)
1676
self.assertValidStorageKind(factory.storage_kind)
1677
if factory.sha1 is not None:
1678
self.assertEqual(f.get_sha1s([factory.key])[factory.key],
1680
self.assertEqual(parents[factory.key], factory.parents)
1681
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1684
def test_get_record_stream_interface(self):
1685
"""each item in a stream has to provide a regular interface."""
1686
files = self.get_versionedfiles()
1687
self.get_diamond_files(files)
1688
keys, _ = self.get_keys_and_sort_order()
1689
parent_map = files.get_parent_map(keys)
1690
entries = files.get_record_stream(keys, 'unordered', False)
1692
self.capture_stream(files, entries, seen.add, parent_map)
1693
self.assertEqual(set(keys), seen)
1695
def get_simple_key(self, suffix):
1696
"""Return a key for the object under test."""
1697
if self.key_length == 1:
1700
return ('FileA',) + (suffix,)
1702
def get_keys_and_sort_order(self):
1703
"""Get diamond test keys list, and their sort ordering."""
1704
if self.key_length == 1:
1705
keys = [('merged',), ('left',), ('right',), ('base',)]
1706
sort_order = {('merged',):2, ('left',):1, ('right',):1, ('base',):0}
1709
('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1711
('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1715
('FileA', 'merged'):2, ('FileA', 'left'):1, ('FileA', 'right'):1,
1716
('FileA', 'base'):0,
1717
('FileB', 'merged'):2, ('FileB', 'left'):1, ('FileB', 'right'):1,
1718
('FileB', 'base'):0,
1720
return keys, sort_order
1722
def get_keys_and_groupcompress_sort_order(self):
1723
"""Get diamond test keys list, and their groupcompress sort ordering."""
1724
if self.key_length == 1:
1725
keys = [('merged',), ('left',), ('right',), ('base',)]
1726
sort_order = {('merged',):0, ('left',):1, ('right',):1, ('base',):2}
1729
('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1731
('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1735
('FileA', 'merged'):0, ('FileA', 'left'):1, ('FileA', 'right'):1,
1736
('FileA', 'base'):2,
1737
('FileB', 'merged'):3, ('FileB', 'left'):4, ('FileB', 'right'):4,
1738
('FileB', 'base'):5,
1740
return keys, sort_order
1742
def test_get_record_stream_interface_ordered(self):
1743
"""each item in a stream has to provide a regular interface."""
1744
files = self.get_versionedfiles()
1745
self.get_diamond_files(files)
1746
keys, sort_order = self.get_keys_and_sort_order()
1747
parent_map = files.get_parent_map(keys)
1748
entries = files.get_record_stream(keys, 'topological', False)
1750
self.capture_stream(files, entries, seen.append, parent_map)
1751
self.assertStreamOrder(sort_order, seen, keys)
1753
def test_get_record_stream_interface_ordered_with_delta_closure(self):
1754
"""each item must be accessible as a fulltext."""
1755
files = self.get_versionedfiles()
1756
self.get_diamond_files(files)
1757
keys, sort_order = self.get_keys_and_sort_order()
1758
parent_map = files.get_parent_map(keys)
1759
entries = files.get_record_stream(keys, 'topological', True)
1761
for factory in entries:
1762
seen.append(factory.key)
1763
self.assertValidStorageKind(factory.storage_kind)
1764
self.assertSubset([factory.sha1],
1765
[None, files.get_sha1s([factory.key])[factory.key]])
1766
self.assertEqual(parent_map[factory.key], factory.parents)
1767
# self.assertEqual(files.get_text(factory.key),
1768
ft_bytes = factory.get_bytes_as('fulltext')
1769
self.assertIsInstance(ft_bytes, str)
1770
chunked_bytes = factory.get_bytes_as('chunked')
1771
self.assertEqualDiff(ft_bytes, ''.join(chunked_bytes))
1773
self.assertStreamOrder(sort_order, seen, keys)
1775
def test_get_record_stream_interface_groupcompress(self):
1776
"""each item in a stream has to provide a regular interface."""
1777
files = self.get_versionedfiles()
1778
self.get_diamond_files(files)
1779
keys, sort_order = self.get_keys_and_groupcompress_sort_order()
1780
parent_map = files.get_parent_map(keys)
1781
entries = files.get_record_stream(keys, 'groupcompress', False)
1783
self.capture_stream(files, entries, seen.append, parent_map)
1784
self.assertStreamOrder(sort_order, seen, keys)
1786
def assertStreamOrder(self, sort_order, seen, keys):
1787
self.assertEqual(len(set(seen)), len(keys))
1788
if self.key_length == 1:
1791
lows = {('FileA',):0, ('FileB',):0}
1793
self.assertEqual(set(keys), set(seen))
1796
sort_pos = sort_order[key]
1797
self.assertTrue(sort_pos >= lows[key[:-1]],
1798
"Out of order in sorted stream: %r, %r" % (key, seen))
1799
lows[key[:-1]] = sort_pos
1801
def test_get_record_stream_unknown_storage_kind_raises(self):
1802
"""Asking for a storage kind that the stream cannot supply raises."""
1803
files = self.get_versionedfiles()
1804
self.get_diamond_files(files)
1805
if self.key_length == 1:
1806
keys = [('merged',), ('left',), ('right',), ('base',)]
1809
('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1811
('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1814
parent_map = files.get_parent_map(keys)
1815
entries = files.get_record_stream(keys, 'unordered', False)
1816
# We track the contents because we should be able to try, fail a
1817
# particular kind and then ask for one that works and continue.
1819
for factory in entries:
1820
seen.add(factory.key)
1821
self.assertValidStorageKind(factory.storage_kind)
1822
if factory.sha1 is not None:
1823
self.assertEqual(files.get_sha1s([factory.key])[factory.key],
1825
self.assertEqual(parent_map[factory.key], factory.parents)
1826
# currently no stream emits mpdiff
1827
self.assertRaises(errors.UnavailableRepresentation,
1828
factory.get_bytes_as, 'mpdiff')
1829
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1831
self.assertEqual(set(keys), seen)
1833
def test_get_record_stream_missing_records_are_absent(self):
1834
files = self.get_versionedfiles()
1835
self.get_diamond_files(files)
1836
if self.key_length == 1:
1837
keys = [('merged',), ('left',), ('right',), ('absent',), ('base',)]
1840
('FileA', 'merged'), ('FileA', 'left'), ('FileA', 'right'),
1841
('FileA', 'absent'), ('FileA', 'base'),
1842
('FileB', 'merged'), ('FileB', 'left'), ('FileB', 'right'),
1843
('FileB', 'absent'), ('FileB', 'base'),
1844
('absent', 'absent'),
1846
parent_map = files.get_parent_map(keys)
1847
entries = files.get_record_stream(keys, 'unordered', False)
1848
self.assertAbsentRecord(files, keys, parent_map, entries)
1849
entries = files.get_record_stream(keys, 'topological', False)
1850
self.assertAbsentRecord(files, keys, parent_map, entries)
1852
def assertRecordHasContent(self, record, bytes):
1853
"""Assert that record has the bytes bytes."""
1854
self.assertEqual(bytes, record.get_bytes_as('fulltext'))
1855
self.assertEqual(bytes, ''.join(record.get_bytes_as('chunked')))
1857
def test_get_record_stream_native_formats_are_wire_ready_one_ft(self):
1858
files = self.get_versionedfiles()
1859
key = self.get_simple_key('foo')
1860
files.add_lines(key, (), ['my text\n', 'content'])
1861
stream = files.get_record_stream([key], 'unordered', False)
1862
record = stream.next()
1863
if record.storage_kind in ('chunked', 'fulltext'):
1864
# chunked and fulltext representations are for direct use not wire
1865
# serialisation: check they are able to be used directly. To send
1866
# such records over the wire translation will be needed.
1867
self.assertRecordHasContent(record, "my text\ncontent")
1869
bytes = [record.get_bytes_as(record.storage_kind)]
1870
network_stream = versionedfile.NetworkRecordStream(bytes).read()
1871
source_record = record
1873
for record in network_stream:
1874
records.append(record)
1875
self.assertEqual(source_record.storage_kind,
1876
record.storage_kind)
1877
self.assertEqual(source_record.parents, record.parents)
1879
source_record.get_bytes_as(source_record.storage_kind),
1880
record.get_bytes_as(record.storage_kind))
1881
self.assertEqual(1, len(records))
1883
def assertStreamMetaEqual(self, records, expected, stream):
1884
"""Assert that streams expected and stream have the same records.
1886
:param records: A list to collect the seen records.
1887
:return: A generator of the records in stream.
1889
# We make assertions during copying to catch things early for
1891
for record, ref_record in izip(stream, expected):
1892
records.append(record)
1893
self.assertEqual(ref_record.key, record.key)
1894
self.assertEqual(ref_record.storage_kind, record.storage_kind)
1895
self.assertEqual(ref_record.parents, record.parents)
1898
def stream_to_bytes_or_skip_counter(self, skipped_records, full_texts,
1900
"""Convert a stream to a bytes iterator.
1902
:param skipped_records: A list with one element to increment when a
1904
:param full_texts: A dict from key->fulltext representation, for
1905
checking chunked or fulltext stored records.
1906
:param stream: A record_stream.
1907
:return: An iterator over the bytes of each record.
1909
for record in stream:
1910
if record.storage_kind in ('chunked', 'fulltext'):
1911
skipped_records[0] += 1
1912
# check the content is correct for direct use.
1913
self.assertRecordHasContent(record, full_texts[record.key])
1915
yield record.get_bytes_as(record.storage_kind)
1917
def test_get_record_stream_native_formats_are_wire_ready_ft_delta(self):
1918
files = self.get_versionedfiles()
1919
target_files = self.get_versionedfiles('target')
1920
key = self.get_simple_key('ft')
1921
key_delta = self.get_simple_key('delta')
1922
files.add_lines(key, (), ['my text\n', 'content'])
1924
delta_parents = (key,)
1927
files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
1928
local = files.get_record_stream([key, key_delta], 'unordered', False)
1929
ref = files.get_record_stream([key, key_delta], 'unordered', False)
1930
skipped_records = [0]
1932
key: "my text\ncontent",
1933
key_delta: "different\ncontent\n",
1935
byte_stream = self.stream_to_bytes_or_skip_counter(
1936
skipped_records, full_texts, local)
1937
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
1939
# insert the stream from the network into a versioned files object so we can
1940
# check the content was carried across correctly without doing delta
1942
target_files.insert_record_stream(
1943
self.assertStreamMetaEqual(records, ref, network_stream))
1944
# No duplicates on the wire thank you!
1945
self.assertEqual(2, len(records) + skipped_records[0])
1947
# if any content was copied it all must have all been.
1948
self.assertIdenticalVersionedFile(files, target_files)
1950
def test_get_record_stream_native_formats_are_wire_ready_delta(self):
1951
# copy a delta over the wire
1952
files = self.get_versionedfiles()
1953
target_files = self.get_versionedfiles('target')
1954
key = self.get_simple_key('ft')
1955
key_delta = self.get_simple_key('delta')
1956
files.add_lines(key, (), ['my text\n', 'content'])
1958
delta_parents = (key,)
1961
files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
1962
# Copy the basis text across so we can reconstruct the delta during
1963
# insertion into target.
1964
target_files.insert_record_stream(files.get_record_stream([key],
1965
'unordered', False))
1966
local = files.get_record_stream([key_delta], 'unordered', False)
1967
ref = files.get_record_stream([key_delta], 'unordered', False)
1968
skipped_records = [0]
1970
key_delta: "different\ncontent\n",
1972
byte_stream = self.stream_to_bytes_or_skip_counter(
1973
skipped_records, full_texts, local)
1974
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
1976
# insert the stream from the network into a versioned files object so we can
1977
# check the content was carried across correctly without doing delta
1978
# inspection during check_stream.
1979
target_files.insert_record_stream(
1980
self.assertStreamMetaEqual(records, ref, network_stream))
1981
# No duplicates on the wire thank you!
1982
self.assertEqual(1, len(records) + skipped_records[0])
1984
# if any content was copied it all must have all been
1985
self.assertIdenticalVersionedFile(files, target_files)
1987
def test_get_record_stream_wire_ready_delta_closure_included(self):
1988
# copy a delta over the wire with the ability to get its full text.
1989
files = self.get_versionedfiles()
1990
key = self.get_simple_key('ft')
1991
key_delta = self.get_simple_key('delta')
1992
files.add_lines(key, (), ['my text\n', 'content'])
1994
delta_parents = (key,)
1997
files.add_lines(key_delta, delta_parents, ['different\n', 'content\n'])
1998
local = files.get_record_stream([key_delta], 'unordered', True)
1999
ref = files.get_record_stream([key_delta], 'unordered', True)
2000
skipped_records = [0]
2002
key_delta: "different\ncontent\n",
2004
byte_stream = self.stream_to_bytes_or_skip_counter(
2005
skipped_records, full_texts, local)
2006
network_stream = versionedfile.NetworkRecordStream(byte_stream).read()
2008
# insert the stream from the network into a versioned files object so we can
2009
# check the content was carried across correctly without doing delta
2010
# inspection during check_stream.
2011
for record in self.assertStreamMetaEqual(records, ref, network_stream):
2012
# we have to be able to get the full text out:
2013
self.assertRecordHasContent(record, full_texts[record.key])
2014
# No duplicates on the wire thank you!
2015
self.assertEqual(1, len(records) + skipped_records[0])
2017
def assertAbsentRecord(self, files, keys, parents, entries):
2018
"""Helper for test_get_record_stream_missing_records_are_absent."""
2020
for factory in entries:
2021
seen.add(factory.key)
2022
if factory.key[-1] == 'absent':
2023
self.assertEqual('absent', factory.storage_kind)
2024
self.assertEqual(None, factory.sha1)
2025
self.assertEqual(None, factory.parents)
2027
self.assertValidStorageKind(factory.storage_kind)
2028
if factory.sha1 is not None:
2029
sha1 = files.get_sha1s([factory.key])[factory.key]
2030
self.assertEqual(sha1, factory.sha1)
2031
self.assertEqual(parents[factory.key], factory.parents)
2032
self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
2034
self.assertEqual(set(keys), seen)
2036
def test_filter_absent_records(self):
2037
"""Requested missing records can be filter trivially."""
2038
files = self.get_versionedfiles()
2039
self.get_diamond_files(files)
2040
keys, _ = self.get_keys_and_sort_order()
2041
parent_map = files.get_parent_map(keys)
2042
# Add an absent record in the middle of the present keys. (We don't ask
2043
# for just absent keys to ensure that content before and after the
2044
# absent keys is still delivered).
2045
present_keys = list(keys)
2046
if self.key_length == 1:
2047
keys.insert(2, ('extra',))
2049
keys.insert(2, ('extra', 'extra'))
2050
entries = files.get_record_stream(keys, 'unordered', False)
2052
self.capture_stream(files, versionedfile.filter_absent(entries), seen.add,
2054
self.assertEqual(set(present_keys), seen)
2056
def get_mapper(self):
2057
"""Get a mapper suitable for the key length of the test interface."""
2058
if self.key_length == 1:
2059
return ConstantMapper('source')
2061
return HashEscapedPrefixMapper()
2063
def get_parents(self, parents):
2064
"""Get parents, taking self.graph into consideration."""
2070
def test_get_parent_map(self):
2071
files = self.get_versionedfiles()
2072
if self.key_length == 1:
2074
(('r0',), self.get_parents(())),
2075
(('r1',), self.get_parents((('r0',),))),
2076
(('r2',), self.get_parents(())),
2077
(('r3',), self.get_parents(())),
2078
(('m',), self.get_parents((('r0',),('r1',),('r2',),('r3',)))),
2082
(('FileA', 'r0'), self.get_parents(())),
2083
(('FileA', 'r1'), self.get_parents((('FileA', 'r0'),))),
2084
(('FileA', 'r2'), self.get_parents(())),
2085
(('FileA', 'r3'), self.get_parents(())),
2086
(('FileA', 'm'), self.get_parents((('FileA', 'r0'),
2087
('FileA', 'r1'), ('FileA', 'r2'), ('FileA', 'r3')))),
2089
for key, parents in parent_details:
2090
files.add_lines(key, parents, [])
2091
# immediately after adding it should be queryable.
2092
self.assertEqual({key:parents}, files.get_parent_map([key]))
2093
# We can ask for an empty set
2094
self.assertEqual({}, files.get_parent_map([]))
2095
# We can ask for many keys
2096
all_parents = dict(parent_details)
2097
self.assertEqual(all_parents, files.get_parent_map(all_parents.keys()))
2098
# Absent keys are just not included in the result.
2099
keys = all_parents.keys()
2100
if self.key_length == 1:
2101
keys.insert(1, ('missing',))
2103
keys.insert(1, ('missing', 'missing'))
2104
# Absent keys are just ignored
2105
self.assertEqual(all_parents, files.get_parent_map(keys))
2107
def test_get_sha1s(self):
2108
files = self.get_versionedfiles()
2109
self.get_diamond_files(files)
2110
if self.key_length == 1:
2111
keys = [('base',), ('origin',), ('left',), ('merged',), ('right',)]
2113
# ask for shas from different prefixes.
2115
('FileA', 'base'), ('FileB', 'origin'), ('FileA', 'left'),
2116
('FileA', 'merged'), ('FileB', 'right'),
2119
keys[0]: '51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',
2120
keys[1]: '00e364d235126be43292ab09cb4686cf703ddc17',
2121
keys[2]: 'a8478686da38e370e32e42e8a0c220e33ee9132f',
2122
keys[3]: 'ed8bce375198ea62444dc71952b22cfc2b09226d',
2123
keys[4]: '9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',
2125
files.get_sha1s(keys))
2127
def test_insert_record_stream_empty(self):
2128
"""Inserting an empty record stream should work."""
2129
files = self.get_versionedfiles()
2130
files.insert_record_stream([])
2132
def assertIdenticalVersionedFile(self, expected, actual):
2133
"""Assert that left and right have the same contents."""
2134
self.assertEqual(set(actual.keys()), set(expected.keys()))
2135
actual_parents = actual.get_parent_map(actual.keys())
2137
self.assertEqual(actual_parents, expected.get_parent_map(expected.keys()))
2139
for key, parents in actual_parents.items():
2140
self.assertEqual(None, parents)
2141
for key in actual.keys():
2142
actual_text = actual.get_record_stream(
2143
[key], 'unordered', True).next().get_bytes_as('fulltext')
2144
expected_text = expected.get_record_stream(
2145
[key], 'unordered', True).next().get_bytes_as('fulltext')
2146
self.assertEqual(actual_text, expected_text)
2148
def test_insert_record_stream_fulltexts(self):
2149
"""Any file should accept a stream of fulltexts."""
2150
files = self.get_versionedfiles()
2151
mapper = self.get_mapper()
2152
source_transport = self.get_transport('source')
2153
source_transport.mkdir('.')
2154
# weaves always output fulltexts.
2155
source = make_versioned_files_factory(WeaveFile, mapper)(
2157
self.get_diamond_files(source, trailing_eol=False)
2158
stream = source.get_record_stream(source.keys(), 'topological',
2160
files.insert_record_stream(stream)
2161
self.assertIdenticalVersionedFile(source, files)
2163
def test_insert_record_stream_fulltexts_noeol(self):
2164
"""Any file should accept a stream of fulltexts."""
2165
files = self.get_versionedfiles()
2166
mapper = self.get_mapper()
2167
source_transport = self.get_transport('source')
2168
source_transport.mkdir('.')
2169
# weaves always output fulltexts.
2170
source = make_versioned_files_factory(WeaveFile, mapper)(
2172
self.get_diamond_files(source, trailing_eol=False)
2173
stream = source.get_record_stream(source.keys(), 'topological',
2175
files.insert_record_stream(stream)
2176
self.assertIdenticalVersionedFile(source, files)
2178
def test_insert_record_stream_annotated_knits(self):
2179
"""Any file should accept a stream from plain knits."""
2180
files = self.get_versionedfiles()
2181
mapper = self.get_mapper()
2182
source_transport = self.get_transport('source')
2183
source_transport.mkdir('.')
2184
source = make_file_factory(True, mapper)(source_transport)
2185
self.get_diamond_files(source)
2186
stream = source.get_record_stream(source.keys(), 'topological',
2188
files.insert_record_stream(stream)
2189
self.assertIdenticalVersionedFile(source, files)
2191
def test_insert_record_stream_annotated_knits_noeol(self):
2192
"""Any file should accept a stream from plain knits."""
2193
files = self.get_versionedfiles()
2194
mapper = self.get_mapper()
2195
source_transport = self.get_transport('source')
2196
source_transport.mkdir('.')
2197
source = make_file_factory(True, mapper)(source_transport)
2198
self.get_diamond_files(source, trailing_eol=False)
2199
stream = source.get_record_stream(source.keys(), 'topological',
2201
files.insert_record_stream(stream)
2202
self.assertIdenticalVersionedFile(source, files)
2204
def test_insert_record_stream_plain_knits(self):
2205
"""Any file should accept a stream from plain knits."""
2206
files = self.get_versionedfiles()
2207
mapper = self.get_mapper()
2208
source_transport = self.get_transport('source')
2209
source_transport.mkdir('.')
2210
source = make_file_factory(False, mapper)(source_transport)
2211
self.get_diamond_files(source)
2212
stream = source.get_record_stream(source.keys(), 'topological',
2214
files.insert_record_stream(stream)
2215
self.assertIdenticalVersionedFile(source, files)
2217
def test_insert_record_stream_plain_knits_noeol(self):
2218
"""Any file should accept a stream from plain knits."""
2219
files = self.get_versionedfiles()
2220
mapper = self.get_mapper()
2221
source_transport = self.get_transport('source')
2222
source_transport.mkdir('.')
2223
source = make_file_factory(False, mapper)(source_transport)
2224
self.get_diamond_files(source, trailing_eol=False)
2225
stream = source.get_record_stream(source.keys(), 'topological',
2227
files.insert_record_stream(stream)
2228
self.assertIdenticalVersionedFile(source, files)
2230
def test_insert_record_stream_existing_keys(self):
2231
"""Inserting keys already in a file should not error."""
2232
files = self.get_versionedfiles()
2233
source = self.get_versionedfiles('source')
2234
self.get_diamond_files(source)
2235
# insert some keys into f.
2236
self.get_diamond_files(files, left_only=True)
2237
stream = source.get_record_stream(source.keys(), 'topological',
2239
files.insert_record_stream(stream)
2240
self.assertIdenticalVersionedFile(source, files)
2242
def test_insert_record_stream_missing_keys(self):
2243
"""Inserting a stream with absent keys should raise an error."""
2244
files = self.get_versionedfiles()
2245
source = self.get_versionedfiles('source')
2246
stream = source.get_record_stream([('missing',) * self.key_length],
2247
'topological', False)
2248
self.assertRaises(errors.RevisionNotPresent, files.insert_record_stream,
2251
def test_insert_record_stream_out_of_order(self):
2252
"""An out of order stream can either error or work."""
2253
files = self.get_versionedfiles()
2254
source = self.get_versionedfiles('source')
2255
self.get_diamond_files(source)
2256
if self.key_length == 1:
2257
origin_keys = [('origin',)]
2258
end_keys = [('merged',), ('left',)]
2259
start_keys = [('right',), ('base',)]
2261
origin_keys = [('FileA', 'origin'), ('FileB', 'origin')]
2262
end_keys = [('FileA', 'merged',), ('FileA', 'left',),
2263
('FileB', 'merged',), ('FileB', 'left',)]
2264
start_keys = [('FileA', 'right',), ('FileA', 'base',),
2265
('FileB', 'right',), ('FileB', 'base',)]
2266
origin_entries = source.get_record_stream(origin_keys, 'unordered', False)
2267
end_entries = source.get_record_stream(end_keys, 'topological', False)
2268
start_entries = source.get_record_stream(start_keys, 'topological', False)
2269
entries = chain(origin_entries, end_entries, start_entries)
2271
files.insert_record_stream(entries)
2272
except RevisionNotPresent:
2273
# Must not have corrupted the file.
2276
self.assertIdenticalVersionedFile(source, files)
2278
def get_knit_delta_source(self):
2279
"""Get a source that can produce a stream with knit delta records,
2280
regardless of this test's scenario.
2282
mapper = self.get_mapper()
2283
source_transport = self.get_transport('source')
2284
source_transport.mkdir('.')
2285
source = make_file_factory(False, mapper)(source_transport)
2286
get_diamond_files(source, self.key_length, trailing_eol=True,
2287
nograph=False, left_only=False)
2290
def test_insert_record_stream_delta_missing_basis_no_corruption(self):
2291
"""Insertion where a needed basis is not included notifies the caller
2292
of the missing basis. In the meantime a record missing its basis is
2295
source = self.get_knit_delta_source()
2296
keys = [self.get_simple_key('origin'), self.get_simple_key('merged')]
2297
entries = source.get_record_stream(keys, 'unordered', False)
2298
files = self.get_versionedfiles()
2299
if self.support_partial_insertion:
2300
self.assertEqual([],
2301
list(files.get_missing_compression_parent_keys()))
2302
files.insert_record_stream(entries)
2303
missing_bases = files.get_missing_compression_parent_keys()
2304
self.assertEqual(set([self.get_simple_key('left')]),
2306
self.assertEqual(set(keys), set(files.get_parent_map(keys)))
2309
errors.RevisionNotPresent, files.insert_record_stream, entries)
2312
def test_insert_record_stream_delta_missing_basis_can_be_added_later(self):
2313
"""Insertion where a needed basis is not included notifies the caller
2314
of the missing basis. That basis can be added in a second
2315
insert_record_stream call that does not need to repeat records present
2316
in the previous stream. The record(s) that required that basis are
2317
fully inserted once their basis is no longer missing.
2319
if not self.support_partial_insertion:
2320
raise TestNotApplicable(
2321
'versioned file scenario does not support partial insertion')
2322
source = self.get_knit_delta_source()
2323
entries = source.get_record_stream([self.get_simple_key('origin'),
2324
self.get_simple_key('merged')], 'unordered', False)
2325
files = self.get_versionedfiles()
2326
files.insert_record_stream(entries)
2327
missing_bases = files.get_missing_compression_parent_keys()
2328
self.assertEqual(set([self.get_simple_key('left')]),
2330
# 'merged' is inserted (although a commit of a write group involving
2331
# this versionedfiles would fail).
2332
merged_key = self.get_simple_key('merged')
2334
[merged_key], files.get_parent_map([merged_key]).keys())
2335
# Add the full delta closure of the missing records
2336
missing_entries = source.get_record_stream(
2337
missing_bases, 'unordered', True)
2338
files.insert_record_stream(missing_entries)
2339
# Now 'merged' is fully inserted (and a commit would succeed).
2340
self.assertEqual([], list(files.get_missing_compression_parent_keys()))
2342
[merged_key], files.get_parent_map([merged_key]).keys())
2345
def test_iter_lines_added_or_present_in_keys(self):
2346
# test that we get at least an equalset of the lines added by
2347
# versions in the store.
2348
# the ordering here is to make a tree so that dumb searches have
2349
# more changes to muck up.
2351
class InstrumentedProgress(progress.DummyProgress):
2355
progress.DummyProgress.__init__(self)
2358
def update(self, msg=None, current=None, total=None):
2359
self.updates.append((msg, current, total))
2361
files = self.get_versionedfiles()
2362
# add a base to get included
2363
files.add_lines(self.get_simple_key('base'), (), ['base\n'])
2364
# add a ancestor to be included on one side
2365
files.add_lines(self.get_simple_key('lancestor'), (), ['lancestor\n'])
2366
# add a ancestor to be included on the other side
2367
files.add_lines(self.get_simple_key('rancestor'),
2368
self.get_parents([self.get_simple_key('base')]), ['rancestor\n'])
2369
# add a child of rancestor with no eofile-nl
2370
files.add_lines(self.get_simple_key('child'),
2371
self.get_parents([self.get_simple_key('rancestor')]),
2372
['base\n', 'child\n'])
2373
# add a child of lancestor and base to join the two roots
2374
files.add_lines(self.get_simple_key('otherchild'),
2375
self.get_parents([self.get_simple_key('lancestor'),
2376
self.get_simple_key('base')]),
2377
['base\n', 'lancestor\n', 'otherchild\n'])
2378
def iter_with_keys(keys, expected):
2379
# now we need to see what lines are returned, and how often.
2381
progress = InstrumentedProgress()
2382
# iterate over the lines
2383
for line in files.iter_lines_added_or_present_in_keys(keys,
2385
lines.setdefault(line, 0)
2387
if []!= progress.updates:
2388
self.assertEqual(expected, progress.updates)
2390
lines = iter_with_keys(
2391
[self.get_simple_key('child'), self.get_simple_key('otherchild')],
2392
[('Walking content', 0, 2),
2393
('Walking content', 1, 2),
2394
('Walking content', 2, 2)])
2395
# we must see child and otherchild
2396
self.assertTrue(lines[('child\n', self.get_simple_key('child'))] > 0)
2398
lines[('otherchild\n', self.get_simple_key('otherchild'))] > 0)
2399
# we dont care if we got more than that.
2402
lines = iter_with_keys(files.keys(),
2403
[('Walking content', 0, 5),
2404
('Walking content', 1, 5),
2405
('Walking content', 2, 5),
2406
('Walking content', 3, 5),
2407
('Walking content', 4, 5),
2408
('Walking content', 5, 5)])
2409
# all lines must be seen at least once
2410
self.assertTrue(lines[('base\n', self.get_simple_key('base'))] > 0)
2412
lines[('lancestor\n', self.get_simple_key('lancestor'))] > 0)
2414
lines[('rancestor\n', self.get_simple_key('rancestor'))] > 0)
2415
self.assertTrue(lines[('child\n', self.get_simple_key('child'))] > 0)
2417
lines[('otherchild\n', self.get_simple_key('otherchild'))] > 0)
2419
def test_make_mpdiffs(self):
2420
from bzrlib import multiparent
2421
files = self.get_versionedfiles('source')
2422
# add texts that should trip the knit maximum delta chain threshold
2423
# as well as doing parallel chains of data in knits.
2424
# this is done by two chains of 25 insertions
2425
files.add_lines(self.get_simple_key('base'), [], ['line\n'])
2426
files.add_lines(self.get_simple_key('noeol'),
2427
self.get_parents([self.get_simple_key('base')]), ['line'])
2428
# detailed eol tests:
2429
# shared last line with parent no-eol
2430
files.add_lines(self.get_simple_key('noeolsecond'),
2431
self.get_parents([self.get_simple_key('noeol')]),
2433
# differing last line with parent, both no-eol
2434
files.add_lines(self.get_simple_key('noeolnotshared'),
2435
self.get_parents([self.get_simple_key('noeolsecond')]),
2436
['line\n', 'phone'])
2437
# add eol following a noneol parent, change content
2438
files.add_lines(self.get_simple_key('eol'),
2439
self.get_parents([self.get_simple_key('noeol')]), ['phone\n'])
2440
# add eol following a noneol parent, no change content
2441
files.add_lines(self.get_simple_key('eolline'),
2442
self.get_parents([self.get_simple_key('noeol')]), ['line\n'])
2443
# noeol with no parents:
2444
files.add_lines(self.get_simple_key('noeolbase'), [], ['line'])
2445
# noeol preceeding its leftmost parent in the output:
2446
# this is done by making it a merge of two parents with no common
2447
# anestry: noeolbase and noeol with the
2448
# later-inserted parent the leftmost.
2449
files.add_lines(self.get_simple_key('eolbeforefirstparent'),
2450
self.get_parents([self.get_simple_key('noeolbase'),
2451
self.get_simple_key('noeol')]),
2453
# two identical eol texts
2454
files.add_lines(self.get_simple_key('noeoldup'),
2455
self.get_parents([self.get_simple_key('noeol')]), ['line'])
2456
next_parent = self.get_simple_key('base')
2457
text_name = 'chain1-'
2459
sha1s = {0 :'da6d3141cb4a5e6f464bf6e0518042ddc7bfd079',
2460
1 :'45e21ea146a81ea44a821737acdb4f9791c8abe7',
2461
2 :'e1f11570edf3e2a070052366c582837a4fe4e9fa',
2462
3 :'26b4b8626da827088c514b8f9bbe4ebf181edda1',
2463
4 :'e28a5510be25ba84d31121cff00956f9970ae6f6',
2464
5 :'d63ec0ce22e11dcf65a931b69255d3ac747a318d',
2465
6 :'2c2888d288cb5e1d98009d822fedfe6019c6a4ea',
2466
7 :'95c14da9cafbf828e3e74a6f016d87926ba234ab',
2467
8 :'779e9a0b28f9f832528d4b21e17e168c67697272',
2468
9 :'1f8ff4e5c6ff78ac106fcfe6b1e8cb8740ff9a8f',
2469
10:'131a2ae712cf51ed62f143e3fbac3d4206c25a05',
2470
11:'c5a9d6f520d2515e1ec401a8f8a67e6c3c89f199',
2471
12:'31a2286267f24d8bedaa43355f8ad7129509ea85',
2472
13:'dc2a7fe80e8ec5cae920973973a8ee28b2da5e0a',
2473
14:'2c4b1736566b8ca6051e668de68650686a3922f2',
2474
15:'5912e4ecd9b0c07be4d013e7e2bdcf9323276cde',
2475
16:'b0d2e18d3559a00580f6b49804c23fea500feab3',
2476
17:'8e1d43ad72f7562d7cb8f57ee584e20eb1a69fc7',
2477
18:'5cf64a3459ae28efa60239e44b20312d25b253f3',
2478
19:'1ebed371807ba5935958ad0884595126e8c4e823',
2479
20:'2aa62a8b06fb3b3b892a3292a068ade69d5ee0d3',
2480
21:'01edc447978004f6e4e962b417a4ae1955b6fe5d',
2481
22:'d8d8dc49c4bf0bab401e0298bb5ad827768618bb',
2482
23:'c21f62b1c482862983a8ffb2b0c64b3451876e3f',
2483
24:'c0593fe795e00dff6b3c0fe857a074364d5f04fc',
2484
25:'dd1a1cf2ba9cc225c3aff729953e6364bf1d1855',
2486
for depth in range(26):
2487
new_version = self.get_simple_key(text_name + '%s' % depth)
2488
text = text + ['line\n']
2489
files.add_lines(new_version, self.get_parents([next_parent]), text)
2490
next_parent = new_version
2491
next_parent = self.get_simple_key('base')
2492
text_name = 'chain2-'
2494
for depth in range(26):
2495
new_version = self.get_simple_key(text_name + '%s' % depth)
2496
text = text + ['line\n']
2497
files.add_lines(new_version, self.get_parents([next_parent]), text)
2498
next_parent = new_version
2499
target = self.get_versionedfiles('target')
2500
for key in multiparent.topo_iter_keys(files, files.keys()):
2501
mpdiff = files.make_mpdiffs([key])[0]
2502
parents = files.get_parent_map([key])[key] or []
2504
[(key, parents, files.get_sha1s([key])[key], mpdiff)])
2505
self.assertEqualDiff(
2506
files.get_record_stream([key], 'unordered',
2507
True).next().get_bytes_as('fulltext'),
2508
target.get_record_stream([key], 'unordered',
2509
True).next().get_bytes_as('fulltext')
2512
def test_keys(self):
2513
# While use is discouraged, versions() is still needed by aspects of
2515
files = self.get_versionedfiles()
2516
self.assertEqual(set(), set(files.keys()))
2517
if self.key_length == 1:
2520
key = ('foo', 'bar',)
2521
files.add_lines(key, (), [])
2522
self.assertEqual(set([key]), set(files.keys()))
2525
class VirtualVersionedFilesTests(TestCase):
2526
"""Basic tests for the VirtualVersionedFiles implementations."""
2528
def _get_parent_map(self, keys):
2531
if k in self._parent_map:
2532
ret[k] = self._parent_map[k]
2536
TestCase.setUp(self)
2538
self._parent_map = {}
2539
self.texts = VirtualVersionedFiles(self._get_parent_map,
2542
def test_add_lines(self):
2543
self.assertRaises(NotImplementedError,
2544
self.texts.add_lines, "foo", [], [])
2546
def test_add_mpdiffs(self):
2547
self.assertRaises(NotImplementedError,
2548
self.texts.add_mpdiffs, [])
2550
def test_check(self):
2551
self.assertTrue(self.texts.check())
2553
def test_insert_record_stream(self):
2554
self.assertRaises(NotImplementedError, self.texts.insert_record_stream,
2557
def test_get_sha1s_nonexistent(self):
2558
self.assertEquals({}, self.texts.get_sha1s([("NONEXISTENT",)]))
2560
def test_get_sha1s(self):
2561
self._lines["key"] = ["dataline1", "dataline2"]
2562
self.assertEquals({("key",): osutils.sha_strings(self._lines["key"])},
2563
self.texts.get_sha1s([("key",)]))
2565
def test_get_parent_map(self):
2566
self._parent_map = {"G": ("A", "B")}
2567
self.assertEquals({("G",): (("A",),("B",))},
2568
self.texts.get_parent_map([("G",), ("L",)]))
2570
def test_get_record_stream(self):
2571
self._lines["A"] = ["FOO", "BAR"]
2572
it = self.texts.get_record_stream([("A",)], "unordered", True)
2574
self.assertEquals("chunked", record.storage_kind)
2575
self.assertEquals("FOOBAR", record.get_bytes_as("fulltext"))
2576
self.assertEquals(["FOO", "BAR"], record.get_bytes_as("chunked"))
2578
def test_get_record_stream_absent(self):
2579
it = self.texts.get_record_stream([("A",)], "unordered", True)
2581
self.assertEquals("absent", record.storage_kind)
2583
def test_iter_lines_added_or_present_in_keys(self):
2584
self._lines["A"] = ["FOO", "BAR"]
2585
self._lines["B"] = ["HEY"]
2586
self._lines["C"] = ["Alberta"]
2587
it = self.texts.iter_lines_added_or_present_in_keys([("A",), ("B",)])
2588
self.assertEquals(sorted([("FOO", "A"), ("BAR", "A"), ("HEY", "B")]),
2592
class TestOrderingVersionedFilesDecorator(TestCaseWithMemoryTransport):
2594
def get_ordering_vf(self, key_priority):
2595
builder = self.make_branch_builder('test')
2596
builder.start_series()
2597
builder.build_snapshot('A', None, [
2598
('add', ('', 'TREE_ROOT', 'directory', None))])
2599
builder.build_snapshot('B', ['A'], [])
2600
builder.build_snapshot('C', ['B'], [])
2601
builder.build_snapshot('D', ['C'], [])
2602
builder.finish_series()
2603
b = builder.get_branch()
2605
self.addCleanup(b.unlock)
2606
vf = b.repository.inventories
2607
return versionedfile.OrderingVersionedFilesDecorator(vf, key_priority)
2609
def test_get_empty(self):
2610
vf = self.get_ordering_vf({})
2611
self.assertEqual([], vf.calls)
2613
def test_get_record_stream_topological(self):
2614
vf = self.get_ordering_vf({('A',): 3, ('B',): 2, ('C',): 4, ('D',): 1})
2615
request_keys = [('B',), ('C',), ('D',), ('A',)]
2616
keys = [r.key for r in vf.get_record_stream(request_keys,
2617
'topological', False)]
2618
# We should have gotten the keys in topological order
2619
self.assertEqual([('A',), ('B',), ('C',), ('D',)], keys)
2620
# And recorded that the request was made
2621
self.assertEqual([('get_record_stream', request_keys, 'topological',
2624
def test_get_record_stream_ordered(self):
2625
vf = self.get_ordering_vf({('A',): 3, ('B',): 2, ('C',): 4, ('D',): 1})
2626
request_keys = [('B',), ('C',), ('D',), ('A',)]
2627
keys = [r.key for r in vf.get_record_stream(request_keys,
2628
'unordered', False)]
2629
# They should be returned based on their priority
2630
self.assertEqual([('D',), ('B',), ('A',), ('C',)], keys)
2631
# And the request recorded
2632
self.assertEqual([('get_record_stream', request_keys, 'unordered',
2635
def test_get_record_stream_implicit_order(self):
2636
vf = self.get_ordering_vf({('B',): 2, ('D',): 1})
2637
request_keys = [('B',), ('C',), ('D',), ('A',)]
2638
keys = [r.key for r in vf.get_record_stream(request_keys,
2639
'unordered', False)]
2640
# A and C are not in the map, so they get sorted to the front. A comes
2641
# before C alphabetically, so it comes back first
2642
self.assertEqual([('A',), ('C',), ('D',), ('B',)], keys)
2643
# And the request recorded
2644
self.assertEqual([('get_record_stream', request_keys, 'unordered',
added
removed
bzrlib/tests/test_versionedfile.py
