← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

(jelmer) Use the absolute_import feature everywhere in bzrlib,
 and add a source test to make sure it's used everywhere. (Jelmer Vernooij)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
 
1
# Copyright (C) 2006-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
20
 
import difflib
21
20
import gzip
22
21
import sys
23
22
 
24
23
from bzrlib import (
25
24
    errors,
26
 
    generate_ids,
27
25
    knit,
28
26
    multiparent,
29
27
    osutils,
30
28
    pack,
 
29
    tests,
 
30
    transport,
31
31
    )
32
32
from bzrlib.errors import (
33
 
    RevisionAlreadyPresent,
34
33
    KnitHeaderError,
35
 
    RevisionNotPresent,
36
34
    NoSuchFile,
37
35
    )
38
36
from bzrlib.index import *
39
37
from bzrlib.knit import (
40
38
    AnnotatedKnitContent,
41
39
    KnitContent,
42
 
    KnitSequenceMatcher,
43
40
    KnitVersionedFiles,
44
41
    PlainKnitContent,
45
42
    _VFContentMapGenerator,
46
 
    _DirectPackAccess,
47
43
    _KndxIndex,
48
44
    _KnitGraphIndex,
49
45
    _KnitKeyAccess,
50
46
    make_file_factory,
51
47
    )
52
 
from bzrlib.repofmt import pack_repo
 
48
from bzrlib.patiencediff import PatienceSequenceMatcher
 
49
from bzrlib.repofmt import (
 
50
    knitpack_repo,
 
51
    pack_repo,
 
52
    )
53
53
from bzrlib.tests import (
54
 
    Feature,
55
 
    KnownFailure,
56
54
    TestCase,
57
55
    TestCaseWithMemoryTransport,
58
56
    TestCaseWithTransport,
59
57
    TestNotApplicable,
60
58
    )
61
 
from bzrlib.transport import get_transport
62
 
from bzrlib.transport.memory import MemoryTransport
63
 
from bzrlib.tuned_gzip import GzipFile
64
59
from bzrlib.versionedfile import (
65
60
    AbsentContentFactory,
66
61
    ConstantMapper,
67
62
    network_bytes_to_kind_and_offset,
68
63
    RecordingVersionedFilesDecorator,
69
64
    )
70
 
 
71
 
 
72
 
class _CompiledKnitFeature(Feature):
73
 
 
74
 
    def _probe(self):
75
 
        try:
76
 
            import bzrlib._knit_load_data_pyx
77
 
        except ImportError:
78
 
            return False
79
 
        return True
80
 
 
81
 
    def feature_name(self):
82
 
        return 'bzrlib._knit_load_data_pyx'
83
 
 
84
 
CompiledKnitFeature = _CompiledKnitFeature()
 
65
from bzrlib.tests import (
 
66
    features,
 
67
    )
 
68
 
 
69
 
 
70
compiled_knit_feature = features.ModuleAvailableFeature(
 
71
    'bzrlib._knit_load_data_pyx')
85
72
 
86
73
 
87
74
class KnitContentTestsMixin(object):
116
103
        line_delta = source_content.line_delta(target_content)
117
104
        delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
118
105
            source_lines, target_lines))
119
 
        matcher = KnitSequenceMatcher(None, source_lines, target_lines)
120
 
        matcher_blocks = list(list(matcher.get_matching_blocks()))
 
106
        matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
 
107
        matcher_blocks = list(matcher.get_matching_blocks())
121
108
        self.assertEqual(matcher_blocks, delta_blocks)
122
109
 
123
110
    def test_get_line_delta_blocks(self):
343
330
            transport.append_bytes(packname, bytes)
344
331
        writer = pack.ContainerWriter(write_data)
345
332
        writer.begin()
346
 
        access = _DirectPackAccess({})
 
333
        access = pack_repo._DirectPackAccess({})
347
334
        access.set_writer(writer, index, (transport, packname))
348
335
        return access, writer
349
336
 
356
343
        writer.end()
357
344
        return memos
358
345
 
 
346
    def test_pack_collection_pack_retries(self):
 
347
        """An explicit pack of a pack collection succeeds even when a
 
348
        concurrent pack happens.
 
349
        """
 
350
        builder = self.make_branch_builder('.')
 
351
        builder.start_series()
 
352
        builder.build_snapshot('rev-1', None, [
 
353
            ('add', ('', 'root-id', 'directory', None)),
 
354
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
 
355
            ])
 
356
        builder.build_snapshot('rev-2', ['rev-1'], [
 
357
            ('modify', ('file-id', 'content\nrev 2\n')),
 
358
            ])
 
359
        builder.build_snapshot('rev-3', ['rev-2'], [
 
360
            ('modify', ('file-id', 'content\nrev 3\n')),
 
361
            ])
 
362
        self.addCleanup(builder.finish_series)
 
363
        b = builder.get_branch()
 
364
        self.addCleanup(b.lock_write().unlock)
 
365
        repo = b.repository
 
366
        collection = repo._pack_collection
 
367
        # Concurrently repack the repo.
 
368
        reopened_repo = repo.bzrdir.open_repository()
 
369
        reopened_repo.pack()
 
370
        # Pack the new pack.
 
371
        collection.pack()
 
372
 
359
373
    def make_vf_for_retrying(self):
360
374
        """Create 3 packs and a reload function.
361
375
 
388
402
        collection = repo._pack_collection
389
403
        collection.ensure_loaded()
390
404
        orig_packs = collection.packs
391
 
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
 
405
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
392
406
        new_pack = packer.pack()
393
407
        # forget about the new pack
394
408
        collection.reset()
433
447
        except _TestException, e:
434
448
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
435
449
                                                 exc_info=sys.exc_info())
 
450
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
436
451
        return retry_exc
437
452
 
438
453
    def test_read_from_several_packs(self):
447
462
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
448
463
        writer.end()
449
464
        transport = self.get_transport()
450
 
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
 
465
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
451
466
            "FOOBAR":(transport, 'pack2'),
452
467
            "BAZ":(transport, 'pack3')})
453
468
        self.assertEqual(['1234567890', '12345', 'alpha'],
463
478
 
464
479
    def test_set_writer(self):
465
480
        """The writer should be settable post construction."""
466
 
        access = _DirectPackAccess({})
 
481
        access = pack_repo._DirectPackAccess({})
467
482
        transport = self.get_transport()
468
483
        packname = 'packfile'
469
484
        index = 'foo'
481
496
        transport = self.get_transport()
482
497
        reload_called, reload_func = self.make_reload_func()
483
498
        # Note that the index key has changed from 'foo' to 'bar'
484
 
        access = _DirectPackAccess({'bar':(transport, 'packname')},
 
499
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
485
500
                                   reload_func=reload_func)
486
501
        e = self.assertListRaises(errors.RetryWithNewPacks,
487
502
                                  access.get_raw_records, memos)
496
511
        memos = self.make_pack_file()
497
512
        transport = self.get_transport()
498
513
        # Note that the index key has changed from 'foo' to 'bar'
499
 
        access = _DirectPackAccess({'bar':(transport, 'packname')})
 
514
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
500
515
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
501
516
 
502
517
    def test_missing_file_raises_retry(self):
504
519
        transport = self.get_transport()
505
520
        reload_called, reload_func = self.make_reload_func()
506
521
        # Note that the 'filename' has been changed to 'different-packname'
507
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
508
 
                                   reload_func=reload_func)
 
522
        access = pack_repo._DirectPackAccess(
 
523
            {'foo':(transport, 'different-packname')},
 
524
            reload_func=reload_func)
509
525
        e = self.assertListRaises(errors.RetryWithNewPacks,
510
526
                                  access.get_raw_records, memos)
511
527
        # The file has gone missing, so we assume we need to reload
519
535
        memos = self.make_pack_file()
520
536
        transport = self.get_transport()
521
537
        # Note that the 'filename' has been changed to 'different-packname'
522
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
 
538
        access = pack_repo._DirectPackAccess(
 
539
            {'foo': (transport, 'different-packname')})
523
540
        e = self.assertListRaises(errors.NoSuchFile,
524
541
                                  access.get_raw_records, memos)
525
542
 
529
546
        failing_transport = MockReadvFailingTransport(
530
547
                                [transport.get_bytes('packname')])
531
548
        reload_called, reload_func = self.make_reload_func()
532
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
533
 
                                   reload_func=reload_func)
 
549
        access = pack_repo._DirectPackAccess(
 
550
            {'foo': (failing_transport, 'packname')},
 
551
            reload_func=reload_func)
534
552
        # Asking for a single record will not trigger the Mock failure
535
553
        self.assertEqual(['1234567890'],
536
554
            list(access.get_raw_records(memos[:1])))
552
570
        failing_transport = MockReadvFailingTransport(
553
571
                                [transport.get_bytes('packname')])
554
572
        reload_called, reload_func = self.make_reload_func()
555
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
 
573
        access = pack_repo._DirectPackAccess(
 
574
            {'foo':(failing_transport, 'packname')})
556
575
        # Asking for a single record will not trigger the Mock failure
557
576
        self.assertEqual(['1234567890'],
558
577
            list(access.get_raw_records(memos[:1])))
563
582
                                  access.get_raw_records, memos)
564
583
 
565
584
    def test_reload_or_raise_no_reload(self):
566
 
        access = _DirectPackAccess({}, reload_func=None)
 
585
        access = pack_repo._DirectPackAccess({}, reload_func=None)
567
586
        retry_exc = self.make_retry_exception()
568
587
        # Without a reload_func, we will just re-raise the original exception
569
588
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
570
589
 
571
590
    def test_reload_or_raise_reload_changed(self):
572
591
        reload_called, reload_func = self.make_reload_func(return_val=True)
573
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
592
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
574
593
        retry_exc = self.make_retry_exception()
575
594
        access.reload_or_raise(retry_exc)
576
595
        self.assertEqual([1], reload_called)
580
599
 
581
600
    def test_reload_or_raise_reload_no_change(self):
582
601
        reload_called, reload_func = self.make_reload_func(return_val=False)
583
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
602
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
584
603
        retry_exc = self.make_retry_exception()
585
604
        # If reload_occurred is False, then we consider it an error to have
586
605
        # reload_func() return False (no changes).
717
736
 
718
737
    def make_multiple_records(self):
719
738
        """Create the content for multiple records."""
720
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
739
        sha1sum = osutils.sha_string('foo\nbar\n')
721
740
        total_txt = []
722
741
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
723
742
                                        'foo\n'
726
745
                                        % (sha1sum,))
727
746
        record_1 = (0, len(gz_txt), sha1sum)
728
747
        total_txt.append(gz_txt)
729
 
        sha1sum = osutils.sha('baz\n').hexdigest()
 
748
        sha1sum = osutils.sha_string('baz\n')
730
749
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
731
750
                                        'baz\n'
732
751
                                        'end rev-id-2\n'
736
755
        return total_txt, record_1, record_2
737
756
 
738
757
    def test_valid_knit_data(self):
739
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
758
        sha1sum = osutils.sha_string('foo\nbar\n')
740
759
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
741
760
                                        'foo\n'
742
761
                                        'bar\n'
773
792
                         raw_contents)
774
793
 
775
794
    def test_not_enough_lines(self):
776
 
        sha1sum = osutils.sha('foo\n').hexdigest()
 
795
        sha1sum = osutils.sha_string('foo\n')
777
796
        # record says 2 lines data says 1
778
797
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
779
798
                                        'foo\n'
791
810
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
792
811
 
793
812
    def test_too_many_lines(self):
794
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
813
        sha1sum = osutils.sha_string('foo\nbar\n')
795
814
        # record says 1 lines data says 2
796
815
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
797
816
                                        'foo\n'
810
829
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
811
830
 
812
831
    def test_mismatched_version_id(self):
813
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
832
        sha1sum = osutils.sha_string('foo\nbar\n')
814
833
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
815
834
                                        'foo\n'
816
835
                                        'bar\n'
829
848
            knit._read_records_iter_raw(records))
830
849
 
831
850
    def test_uncompressed_data(self):
832
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
851
        sha1sum = osutils.sha_string('foo\nbar\n')
833
852
        txt = ('version rev-id-1 2 %s\n'
834
853
               'foo\n'
835
854
               'bar\n'
849
868
            knit._read_records_iter_raw(records))
850
869
 
851
870
    def test_corrupted_data(self):
852
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
871
        sha1sum = osutils.sha_string('foo\nbar\n')
853
872
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
854
873
                                        'foo\n'
855
874
                                        'bar\n'
872
891
 
873
892
    def get_knit_index(self, transport, name, mode):
874
893
        mapper = ConstantMapper(name)
875
 
        orig = knit._load_data
876
 
        def reset():
877
 
            knit._load_data = orig
878
 
        self.addCleanup(reset)
879
894
        from bzrlib._knit_load_data_py import _load_data_py
880
 
        knit._load_data = _load_data_py
 
895
        self.overrideAttr(knit, '_load_data', _load_data_py)
881
896
        allow_writes = lambda: 'w' in mode
882
897
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
883
898
 
1181
1196
            self.assertRaises(errors.KnitCorrupt, index.keys)
1182
1197
        except TypeError, e:
1183
1198
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1184
 
                           ' not exceptions.IndexError')
1185
 
                and sys.version_info[0:2] >= (2,5)):
 
1199
                           ' not exceptions.IndexError')):
1186
1200
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1187
1201
                                  ' raising new style exceptions with python'
1188
1202
                                  ' >=2.5')
1201
1215
            self.assertRaises(errors.KnitCorrupt, index.keys)
1202
1216
        except TypeError, e:
1203
1217
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1204
 
                           ' not exceptions.ValueError')
1205
 
                and sys.version_info[0:2] >= (2,5)):
 
1218
                           ' not exceptions.ValueError')):
1206
1219
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1207
1220
                                  ' raising new style exceptions with python'
1208
1221
                                  ' >=2.5')
1221
1234
            self.assertRaises(errors.KnitCorrupt, index.keys)
1222
1235
        except TypeError, e:
1223
1236
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1224
 
                           ' not exceptions.ValueError')
1225
 
                and sys.version_info[0:2] >= (2,5)):
 
1237
                           ' not exceptions.ValueError')):
1226
1238
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1227
1239
                                  ' raising new style exceptions with python'
1228
1240
                                  ' >=2.5')
1239
1251
            self.assertRaises(errors.KnitCorrupt, index.keys)
1240
1252
        except TypeError, e:
1241
1253
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1242
 
                           ' not exceptions.ValueError')
1243
 
                and sys.version_info[0:2] >= (2,5)):
 
1254
                           ' not exceptions.ValueError')):
1244
1255
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1245
1256
                                  ' raising new style exceptions with python'
1246
1257
                                  ' >=2.5')
1257
1268
            self.assertRaises(errors.KnitCorrupt, index.keys)
1258
1269
        except TypeError, e:
1259
1270
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1260
 
                           ' not exceptions.ValueError')
1261
 
                and sys.version_info[0:2] >= (2,5)):
 
1271
                           ' not exceptions.ValueError')):
1262
1272
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1263
1273
                                  ' raising new style exceptions with python'
1264
1274
                                  ' >=2.5')
1308
1318
 
1309
1319
class LowLevelKnitIndexTests_c(LowLevelKnitIndexTests):
1310
1320
 
1311
 
    _test_needs_features = [CompiledKnitFeature]
 
1321
    _test_needs_features = [compiled_knit_feature]
1312
1322
 
1313
1323
    def get_knit_index(self, transport, name, mode):
1314
1324
        mapper = ConstantMapper(name)
1315
 
        orig = knit._load_data
1316
 
        def reset():
1317
 
            knit._load_data = orig
1318
 
        self.addCleanup(reset)
1319
1325
        from bzrlib._knit_load_data_pyx import _load_data_c
1320
 
        knit._load_data = _load_data_c
 
1326
        self.overrideAttr(knit, '_load_data', _load_data_c)
1321
1327
        allow_writes = lambda: mode == 'w'
1322
 
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
 
1328
        return _KndxIndex(transport, mapper, lambda:None,
 
1329
                          allow_writes, lambda:True)
1323
1330
 
1324
1331
 
1325
1332
class Test_KnitAnnotator(TestCaseWithMemoryTransport):
1596
1603
        # could leave an empty .kndx file, which bzr would later claim was a
1597
1604
        # corrupted file since the header was not present. In reality, the file
1598
1605
        # just wasn't created, so it should be ignored.
1599
 
        t = get_transport('.')
 
1606
        t = transport.get_transport_from_path('.')
1600
1607
        t.put_bytes('test.kndx', '')
1601
1608
 
1602
1609
        knit = self.make_test_knit()
1603
1610
 
1604
1611
    def test_knit_index_checks_header(self):
1605
 
        t = get_transport('.')
 
1612
        t = transport.get_transport_from_path('.')
1606
1613
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1607
1614
        k = self.make_test_knit()
1608
1615
        self.assertRaises(KnitHeaderError, k.keys)
2436
2443
        key_basis = ('bar',)
2437
2444
        key_missing = ('missing',)
2438
2445
        test.add_lines(key, (), ['foo\n'])
2439
 
        key_sha1sum = osutils.sha('foo\n').hexdigest()
 
2446
        key_sha1sum = osutils.sha_string('foo\n')
2440
2447
        sha1s = test.get_sha1s([key])
2441
2448
        self.assertEqual({key: key_sha1sum}, sha1s)
2442
2449
        self.assertEqual([], basis.calls)
2444
2451
        # directly (rather than via text reconstruction) so that remote servers
2445
2452
        # etc don't have to answer with full content.
2446
2453
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2447
 
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
2454
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
2448
2455
        basis.calls = []
2449
2456
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2450
2457
        self.assertEqual({key: key_sha1sum,