32
31
from bzrlib.osutils import sha_string
33
from bzrlib.tests.test__groupcompress import compiled_groupcompress_feature
34
from bzrlib.tests.scenarios import load_tests_apply_scenarios
37
def group_compress_implementation_scenarios():
32
from bzrlib.tests.test__groupcompress import CompiledGroupCompressFeature
35
def load_tests(standard_tests, module, loader):
36
"""Parameterize tests for all versions of groupcompress."""
37
to_adapt, result = tests.split_suite_by_condition(
38
standard_tests, tests.condition_isinstance(TestAllGroupCompressors))
39
40
('python', {'compressor': groupcompress.PythonGroupCompressor}),
41
if compiled_groupcompress_feature.available():
42
if CompiledGroupCompressFeature.available():
42
43
scenarios.append(('C',
43
44
{'compressor': groupcompress.PyrexGroupCompressor}))
47
load_tests = load_tests_apply_scenarios
45
return tests.multiply_tests(to_adapt, scenarios, result)
50
48
class TestGroupCompressor(tests.TestCase):
350
347
self.assertEqual(z_content, block._z_content)
351
348
self.assertEqual(content, block._content)
353
def test_to_chunks(self):
354
content_chunks = ['this is some content\n',
355
'this content will be compressed\n']
356
content_len = sum(map(len, content_chunks))
357
content = ''.join(content_chunks)
358
gcb = groupcompress.GroupCompressBlock()
359
gcb.set_chunked_content(content_chunks, content_len)
360
total_len, block_chunks = gcb.to_chunks()
361
block_bytes = ''.join(block_chunks)
362
self.assertEqual(gcb._z_content_length, len(gcb._z_content))
363
self.assertEqual(total_len, len(block_bytes))
364
self.assertEqual(gcb._content_length, content_len)
365
expected_header =('gcb1z\n' # group compress block v1 zlib
366
'%d\n' # Length of compressed content
367
'%d\n' # Length of uncompressed content
368
) % (gcb._z_content_length, gcb._content_length)
369
# The first chunk should be the header chunk. It is small, fixed size,
370
# and there is no compelling reason to split it up
371
self.assertEqual(expected_header, block_chunks[0])
372
self.assertStartsWith(block_bytes, expected_header)
373
remaining_bytes = block_bytes[len(expected_header):]
374
raw_bytes = zlib.decompress(remaining_bytes)
375
self.assertEqual(content, raw_bytes)
377
350
def test_to_bytes(self):
378
351
content = ('this is some content\n'
379
352
'this content will be compressed\n')
445
418
# And the decompressor is finalized
446
419
self.assertIs(None, block._z_content_decompressor)
448
def test__ensure_all_content(self):
421
def test_partial_decomp_no_known_length(self):
449
422
content_chunks = []
450
# We need a sufficient amount of data so that zlib.decompress has
451
# partial decompression to work with. Most auto-generated data
452
# compresses a bit too well, we want a combination, so we combine a sha
453
# hash with compressible data.
454
423
for i in xrange(2048):
455
424
next_content = '%d\nThis is a bit of duplicate text\n' % (i,)
456
425
content_chunks.append(next_content)
461
430
z_content = zlib.compress(content)
462
431
self.assertEqual(57182, len(z_content))
463
432
block = groupcompress.GroupCompressBlock()
464
block._z_content_chunks = (z_content,)
433
block._z_content = z_content
465
434
block._z_content_length = len(z_content)
466
435
block._compressor_name = 'zlib'
467
block._content_length = 158634
436
block._content_length = None # Don't tell the decompressed length
468
437
self.assertIs(None, block._content)
469
# The first _ensure_content got all of the required data
470
block._ensure_content(158634)
438
block._ensure_content(100)
439
self.assertIsNot(None, block._content)
440
# We have decompressed at least 100 bytes
441
self.assertTrue(len(block._content) >= 100)
442
# We have not decompressed the whole content
443
self.assertTrue(len(block._content) < 158634)
444
self.assertEqualDiff(content[:len(block._content)], block._content)
445
# ensuring content that we already have shouldn't cause any more data
447
cur_len = len(block._content)
448
block._ensure_content(cur_len - 10)
449
self.assertEqual(cur_len, len(block._content))
450
# Now we want a bit more content
452
block._ensure_content(cur_len)
453
self.assertTrue(len(block._content) >= cur_len)
454
self.assertTrue(len(block._content) < 158634)
455
self.assertEqualDiff(content[:len(block._content)], block._content)
456
# And now lets finish
457
block._ensure_content()
471
458
self.assertEqualDiff(content, block._content)
472
# And we should have released the _z_content_decompressor since it was
459
# And the decompressor is finalized
474
460
self.assertIs(None, block._z_content_decompressor)
476
462
def test__dump(self):
553
538
'as-requested', False)]
554
539
self.assertEqual([('b',), ('a',), ('d',), ('c',)], keys)
556
def test_get_record_stream_max_bytes_to_index_default(self):
557
vf = self.make_test_vf(True, dir='source')
558
vf.add_lines(('a',), (), ['lines\n'])
560
record = vf.get_record_stream([('a',)], 'unordered', True).next()
561
self.assertEqual(vf._DEFAULT_COMPRESSOR_SETTINGS,
562
record._manager._get_compressor_settings())
564
def test_get_record_stream_accesses_compressor_settings(self):
565
vf = self.make_test_vf(True, dir='source')
566
vf.add_lines(('a',), (), ['lines\n'])
568
vf._max_bytes_to_index = 1234
569
record = vf.get_record_stream([('a',)], 'unordered', True).next()
570
self.assertEqual(dict(max_bytes_to_index=1234),
571
record._manager._get_compressor_settings())
573
541
def test_insert_record_stream_reuses_blocks(self):
574
542
vf = self.make_test_vf(True, dir='source')
575
543
def grouped_stream(revision_ids, first_parents=()):
777
745
" \('b',\) \('42 32 0 8', \(\(\),\)\) \('74 32"
778
746
" 0 8', \(\(\('a',\),\),\)\)")
780
def test_clear_cache(self):
781
vf = self.make_source_with_b(True, 'source')
783
for record in vf.get_record_stream([('a',), ('b',)], 'unordered',
786
self.assertTrue(len(vf._group_cache) > 0)
788
self.assertEqual(0, len(vf._group_cache))
791
class TestGroupCompressConfig(tests.TestCaseWithTransport):
793
def make_test_vf(self):
794
t = self.get_transport('.')
796
factory = groupcompress.make_pack_factory(graph=True,
797
delta=False, keylength=1, inconsistency_fatal=True)
799
self.addCleanup(groupcompress.cleanup_pack_group, vf)
802
def test_max_bytes_to_index_default(self):
803
vf = self.make_test_vf()
804
gc = vf._make_group_compressor()
805
self.assertEqual(vf._DEFAULT_MAX_BYTES_TO_INDEX,
806
vf._max_bytes_to_index)
807
if isinstance(gc, groupcompress.PyrexGroupCompressor):
808
self.assertEqual(vf._DEFAULT_MAX_BYTES_TO_INDEX,
809
gc._delta_index._max_bytes_to_index)
811
def test_max_bytes_to_index_in_config(self):
812
c = config.GlobalConfig()
813
c.set_user_option('bzr.groupcompress.max_bytes_to_index', '10000')
814
vf = self.make_test_vf()
815
gc = vf._make_group_compressor()
816
self.assertEqual(10000, vf._max_bytes_to_index)
817
if isinstance(gc, groupcompress.PyrexGroupCompressor):
818
self.assertEqual(10000, gc._delta_index._max_bytes_to_index)
820
def test_max_bytes_to_index_bad_config(self):
821
c = config.GlobalConfig()
822
c.set_user_option('bzr.groupcompress.max_bytes_to_index', 'boogah')
823
vf = self.make_test_vf()
824
# TODO: This is triggering a warning, we might want to trap and make
825
# sure it is readable.
826
gc = vf._make_group_compressor()
827
self.assertEqual(vf._DEFAULT_MAX_BYTES_TO_INDEX,
828
vf._max_bytes_to_index)
829
if isinstance(gc, groupcompress.PyrexGroupCompressor):
830
self.assertEqual(vf._DEFAULT_MAX_BYTES_TO_INDEX,
831
gc._delta_index._max_bytes_to_index)
834
749
class StubGCVF(object):
835
750
def __init__(self, canned_get_blocks=None):
1106
1021
self.assertEqual(self._texts[record.key],
1107
1022
record.get_bytes_as('fulltext'))
1109
def test_manager_default_compressor_settings(self):
1110
locations, old_block = self.make_block(self._texts)
1111
manager = groupcompress._LazyGroupContentManager(old_block)
1112
gcvf = groupcompress.GroupCompressVersionedFiles
1113
# It doesn't greedily evaluate _max_bytes_to_index
1114
self.assertIs(None, manager._compressor_settings)
1115
self.assertEqual(gcvf._DEFAULT_COMPRESSOR_SETTINGS,
1116
manager._get_compressor_settings())
1118
def test_manager_custom_compressor_settings(self):
1119
locations, old_block = self.make_block(self._texts)
1121
def compressor_settings():
1122
called.append('called')
1124
manager = groupcompress._LazyGroupContentManager(old_block,
1125
get_compressor_settings=compressor_settings)
1126
gcvf = groupcompress.GroupCompressVersionedFiles
1127
# It doesn't greedily evaluate compressor_settings
1128
self.assertIs(None, manager._compressor_settings)
1129
self.assertEqual((10,), manager._get_compressor_settings())
1130
self.assertEqual((10,), manager._get_compressor_settings())
1131
self.assertEqual((10,), manager._compressor_settings)
1132
# Only called 1 time
1133
self.assertEqual(['called'], called)
1135
def test__rebuild_handles_compressor_settings(self):
1136
if not isinstance(groupcompress.GroupCompressor,
1137
groupcompress.PyrexGroupCompressor):
1138
raise tests.TestNotApplicable('pure-python compressor'
1139
' does not handle compressor_settings')
1140
locations, old_block = self.make_block(self._texts)
1141
manager = groupcompress._LazyGroupContentManager(old_block,
1142
get_compressor_settings=lambda: dict(max_bytes_to_index=32))
1143
gc = manager._make_group_compressor()
1144
self.assertEqual(32, gc._delta_index._max_bytes_to_index)
1145
self.add_key_to_manager(('key3',), locations, old_block, manager)
1146
self.add_key_to_manager(('key4',), locations, old_block, manager)
1147
action, last_byte, total_bytes = manager._check_rebuild_action()
1148
self.assertEqual('rebuild', action)
1149
manager._rebuild_block()
1150
new_block = manager._block
1151
self.assertIsNot(old_block, new_block)
1152
# Because of the new max_bytes_to_index, we do a poor job of
1153
# rebuilding. This is a side-effect of the change, but at least it does
1154
# show the setting had an effect.
1155
self.assertTrue(old_block._content_length < new_block._content_length)
1157
1024
def test_check_is_well_utilized_all_keys(self):
1158
1025
block, manager = self.make_block_and_full_manager(self._texts)
1159
1026
self.assertFalse(manager.check_is_well_utilized())
1201
1068
self.add_key_to_manager(('key4',), locations, block, manager)
1202
1069
self.assertTrue(manager.check_is_well_utilized())
1205
class Test_GCBuildDetails(tests.TestCase):
1207
def test_acts_like_tuple(self):
1208
# _GCBuildDetails inlines some of the data that used to be spread out
1209
# across a bunch of tuples
1210
bd = groupcompress._GCBuildDetails((('parent1',), ('parent2',)),
1211
('INDEX', 10, 20, 0, 5))
1212
self.assertEqual(4, len(bd))
1213
self.assertEqual(('INDEX', 10, 20, 0, 5), bd[0])
1214
self.assertEqual(None, bd[1]) # Compression Parent is always None
1215
self.assertEqual((('parent1',), ('parent2',)), bd[2])
1216
self.assertEqual(('group', None), bd[3]) # Record details
1218
def test__repr__(self):
1219
bd = groupcompress._GCBuildDetails((('parent1',), ('parent2',)),
1220
('INDEX', 10, 20, 0, 5))
1221
self.assertEqual("_GCBuildDetails(('INDEX', 10, 20, 0, 5),"
1222
" (('parent1',), ('parent2',)))",
added
removed
bzrlib/tests/test_groupcompress.py
