← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Vincent Ladeuil
  • Date: 2016-01-21 11:42:23 UTC
  • mto: This revision was merged to the branch mainline in revision 6610.
  • Revision ID: v.ladeuil+lp@free.fr-20160121114223-ngcvndi02ydiqs5z
Allow hyphens in option names to unbreak compatibility.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
 
1
# Copyright (C) 2006-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
20
 
import difflib
21
20
import gzip
22
21
import sys
23
22
 
24
23
from bzrlib import (
25
24
    errors,
26
 
    generate_ids,
27
25
    knit,
28
26
    multiparent,
29
27
    osutils,
30
28
    pack,
31
29
    tests,
 
30
    transport,
32
31
    )
33
32
from bzrlib.errors import (
34
 
    RevisionAlreadyPresent,
35
33
    KnitHeaderError,
36
 
    RevisionNotPresent,
37
34
    NoSuchFile,
38
35
    )
39
36
from bzrlib.index import *
40
37
from bzrlib.knit import (
41
38
    AnnotatedKnitContent,
42
39
    KnitContent,
43
 
    KnitSequenceMatcher,
44
40
    KnitVersionedFiles,
45
41
    PlainKnitContent,
46
42
    _VFContentMapGenerator,
47
 
    _DirectPackAccess,
48
43
    _KndxIndex,
49
44
    _KnitGraphIndex,
50
45
    _KnitKeyAccess,
51
46
    make_file_factory,
52
47
    )
53
 
from bzrlib.repofmt import pack_repo
 
48
from bzrlib.patiencediff import PatienceSequenceMatcher
 
49
from bzrlib.repofmt import (
 
50
    knitpack_repo,
 
51
    pack_repo,
 
52
    )
54
53
from bzrlib.tests import (
55
 
    Feature,
56
 
    KnownFailure,
57
54
    TestCase,
58
55
    TestCaseWithMemoryTransport,
59
56
    TestCaseWithTransport,
60
57
    TestNotApplicable,
61
58
    )
62
 
from bzrlib.transport import get_transport
63
 
from bzrlib.transport.memory import MemoryTransport
64
 
from bzrlib.tuned_gzip import GzipFile
65
59
from bzrlib.versionedfile import (
66
60
    AbsentContentFactory,
67
61
    ConstantMapper,
68
62
    network_bytes_to_kind_and_offset,
69
63
    RecordingVersionedFilesDecorator,
70
64
    )
71
 
 
72
 
 
73
 
compiled_knit_feature = tests.ModuleAvailableFeature(
74
 
                            'bzrlib._knit_load_data_pyx')
 
65
from bzrlib.tests import (
 
66
    features,
 
67
    )
 
68
 
 
69
 
 
70
compiled_knit_feature = features.ModuleAvailableFeature(
 
71
    'bzrlib._knit_load_data_pyx')
75
72
 
76
73
 
77
74
class KnitContentTestsMixin(object):
106
103
        line_delta = source_content.line_delta(target_content)
107
104
        delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
108
105
            source_lines, target_lines))
109
 
        matcher = KnitSequenceMatcher(None, source_lines, target_lines)
110
 
        matcher_blocks = list(list(matcher.get_matching_blocks()))
 
106
        matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
 
107
        matcher_blocks = list(matcher.get_matching_blocks())
111
108
        self.assertEqual(matcher_blocks, delta_blocks)
112
109
 
113
110
    def test_get_line_delta_blocks(self):
333
330
            transport.append_bytes(packname, bytes)
334
331
        writer = pack.ContainerWriter(write_data)
335
332
        writer.begin()
336
 
        access = _DirectPackAccess({})
 
333
        access = pack_repo._DirectPackAccess({})
337
334
        access.set_writer(writer, index, (transport, packname))
338
335
        return access, writer
339
336
 
346
343
        writer.end()
347
344
        return memos
348
345
 
 
346
    def test_pack_collection_pack_retries(self):
 
347
        """An explicit pack of a pack collection succeeds even when a
 
348
        concurrent pack happens.
 
349
        """
 
350
        builder = self.make_branch_builder('.')
 
351
        builder.start_series()
 
352
        builder.build_snapshot('rev-1', None, [
 
353
            ('add', ('', 'root-id', 'directory', None)),
 
354
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
 
355
            ])
 
356
        builder.build_snapshot('rev-2', ['rev-1'], [
 
357
            ('modify', ('file-id', 'content\nrev 2\n')),
 
358
            ])
 
359
        builder.build_snapshot('rev-3', ['rev-2'], [
 
360
            ('modify', ('file-id', 'content\nrev 3\n')),
 
361
            ])
 
362
        self.addCleanup(builder.finish_series)
 
363
        b = builder.get_branch()
 
364
        self.addCleanup(b.lock_write().unlock)
 
365
        repo = b.repository
 
366
        collection = repo._pack_collection
 
367
        # Concurrently repack the repo.
 
368
        reopened_repo = repo.bzrdir.open_repository()
 
369
        reopened_repo.pack()
 
370
        # Pack the new pack.
 
371
        collection.pack()
 
372
 
349
373
    def make_vf_for_retrying(self):
350
374
        """Create 3 packs and a reload function.
351
375
 
378
402
        collection = repo._pack_collection
379
403
        collection.ensure_loaded()
380
404
        orig_packs = collection.packs
381
 
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
 
405
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
382
406
        new_pack = packer.pack()
383
407
        # forget about the new pack
384
408
        collection.reset()
423
447
        except _TestException, e:
424
448
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
425
449
                                                 exc_info=sys.exc_info())
 
450
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
426
451
        return retry_exc
427
452
 
428
453
    def test_read_from_several_packs(self):
437
462
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
438
463
        writer.end()
439
464
        transport = self.get_transport()
440
 
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
 
465
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
441
466
            "FOOBAR":(transport, 'pack2'),
442
467
            "BAZ":(transport, 'pack3')})
443
468
        self.assertEqual(['1234567890', '12345', 'alpha'],
453
478
 
454
479
    def test_set_writer(self):
455
480
        """The writer should be settable post construction."""
456
 
        access = _DirectPackAccess({})
 
481
        access = pack_repo._DirectPackAccess({})
457
482
        transport = self.get_transport()
458
483
        packname = 'packfile'
459
484
        index = 'foo'
471
496
        transport = self.get_transport()
472
497
        reload_called, reload_func = self.make_reload_func()
473
498
        # Note that the index key has changed from 'foo' to 'bar'
474
 
        access = _DirectPackAccess({'bar':(transport, 'packname')},
 
499
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
475
500
                                   reload_func=reload_func)
476
501
        e = self.assertListRaises(errors.RetryWithNewPacks,
477
502
                                  access.get_raw_records, memos)
486
511
        memos = self.make_pack_file()
487
512
        transport = self.get_transport()
488
513
        # Note that the index key has changed from 'foo' to 'bar'
489
 
        access = _DirectPackAccess({'bar':(transport, 'packname')})
 
514
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
490
515
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
491
516
 
492
517
    def test_missing_file_raises_retry(self):
494
519
        transport = self.get_transport()
495
520
        reload_called, reload_func = self.make_reload_func()
496
521
        # Note that the 'filename' has been changed to 'different-packname'
497
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
498
 
                                   reload_func=reload_func)
 
522
        access = pack_repo._DirectPackAccess(
 
523
            {'foo':(transport, 'different-packname')},
 
524
            reload_func=reload_func)
499
525
        e = self.assertListRaises(errors.RetryWithNewPacks,
500
526
                                  access.get_raw_records, memos)
501
527
        # The file has gone missing, so we assume we need to reload
509
535
        memos = self.make_pack_file()
510
536
        transport = self.get_transport()
511
537
        # Note that the 'filename' has been changed to 'different-packname'
512
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
 
538
        access = pack_repo._DirectPackAccess(
 
539
            {'foo': (transport, 'different-packname')})
513
540
        e = self.assertListRaises(errors.NoSuchFile,
514
541
                                  access.get_raw_records, memos)
515
542
 
519
546
        failing_transport = MockReadvFailingTransport(
520
547
                                [transport.get_bytes('packname')])
521
548
        reload_called, reload_func = self.make_reload_func()
522
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
523
 
                                   reload_func=reload_func)
 
549
        access = pack_repo._DirectPackAccess(
 
550
            {'foo': (failing_transport, 'packname')},
 
551
            reload_func=reload_func)
524
552
        # Asking for a single record will not trigger the Mock failure
525
553
        self.assertEqual(['1234567890'],
526
554
            list(access.get_raw_records(memos[:1])))
542
570
        failing_transport = MockReadvFailingTransport(
543
571
                                [transport.get_bytes('packname')])
544
572
        reload_called, reload_func = self.make_reload_func()
545
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
 
573
        access = pack_repo._DirectPackAccess(
 
574
            {'foo':(failing_transport, 'packname')})
546
575
        # Asking for a single record will not trigger the Mock failure
547
576
        self.assertEqual(['1234567890'],
548
577
            list(access.get_raw_records(memos[:1])))
553
582
                                  access.get_raw_records, memos)
554
583
 
555
584
    def test_reload_or_raise_no_reload(self):
556
 
        access = _DirectPackAccess({}, reload_func=None)
 
585
        access = pack_repo._DirectPackAccess({}, reload_func=None)
557
586
        retry_exc = self.make_retry_exception()
558
587
        # Without a reload_func, we will just re-raise the original exception
559
588
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
560
589
 
561
590
    def test_reload_or_raise_reload_changed(self):
562
591
        reload_called, reload_func = self.make_reload_func(return_val=True)
563
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
592
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
564
593
        retry_exc = self.make_retry_exception()
565
594
        access.reload_or_raise(retry_exc)
566
595
        self.assertEqual([1], reload_called)
570
599
 
571
600
    def test_reload_or_raise_reload_no_change(self):
572
601
        reload_called, reload_func = self.make_reload_func(return_val=False)
573
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
602
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
574
603
        retry_exc = self.make_retry_exception()
575
604
        # If reload_occurred is False, then we consider it an error to have
576
605
        # reload_func() return False (no changes).
707
736
 
708
737
    def make_multiple_records(self):
709
738
        """Create the content for multiple records."""
710
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
739
        sha1sum = osutils.sha_string('foo\nbar\n')
711
740
        total_txt = []
712
741
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
713
742
                                        'foo\n'
716
745
                                        % (sha1sum,))
717
746
        record_1 = (0, len(gz_txt), sha1sum)
718
747
        total_txt.append(gz_txt)
719
 
        sha1sum = osutils.sha('baz\n').hexdigest()
 
748
        sha1sum = osutils.sha_string('baz\n')
720
749
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
721
750
                                        'baz\n'
722
751
                                        'end rev-id-2\n'
726
755
        return total_txt, record_1, record_2
727
756
 
728
757
    def test_valid_knit_data(self):
729
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
758
        sha1sum = osutils.sha_string('foo\nbar\n')
730
759
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
731
760
                                        'foo\n'
732
761
                                        'bar\n'
763
792
                         raw_contents)
764
793
 
765
794
    def test_not_enough_lines(self):
766
 
        sha1sum = osutils.sha('foo\n').hexdigest()
 
795
        sha1sum = osutils.sha_string('foo\n')
767
796
        # record says 2 lines data says 1
768
797
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
769
798
                                        'foo\n'
781
810
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
782
811
 
783
812
    def test_too_many_lines(self):
784
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
813
        sha1sum = osutils.sha_string('foo\nbar\n')
785
814
        # record says 1 lines data says 2
786
815
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
787
816
                                        'foo\n'
800
829
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
801
830
 
802
831
    def test_mismatched_version_id(self):
803
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
832
        sha1sum = osutils.sha_string('foo\nbar\n')
804
833
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
805
834
                                        'foo\n'
806
835
                                        'bar\n'
819
848
            knit._read_records_iter_raw(records))
820
849
 
821
850
    def test_uncompressed_data(self):
822
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
851
        sha1sum = osutils.sha_string('foo\nbar\n')
823
852
        txt = ('version rev-id-1 2 %s\n'
824
853
               'foo\n'
825
854
               'bar\n'
839
868
            knit._read_records_iter_raw(records))
840
869
 
841
870
    def test_corrupted_data(self):
842
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
871
        sha1sum = osutils.sha_string('foo\nbar\n')
843
872
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
844
873
                                        'foo\n'
845
874
                                        'bar\n'
862
891
 
863
892
    def get_knit_index(self, transport, name, mode):
864
893
        mapper = ConstantMapper(name)
865
 
        orig = knit._load_data
866
 
        def reset():
867
 
            knit._load_data = orig
868
 
        self.addCleanup(reset)
869
894
        from bzrlib._knit_load_data_py import _load_data_py
870
 
        knit._load_data = _load_data_py
 
895
        self.overrideAttr(knit, '_load_data', _load_data_py)
871
896
        allow_writes = lambda: 'w' in mode
872
897
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
873
898
 
1171
1196
            self.assertRaises(errors.KnitCorrupt, index.keys)
1172
1197
        except TypeError, e:
1173
1198
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1174
 
                           ' not exceptions.IndexError')
1175
 
                and sys.version_info[0:2] >= (2,5)):
 
1199
                           ' not exceptions.IndexError')):
1176
1200
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1177
1201
                                  ' raising new style exceptions with python'
1178
1202
                                  ' >=2.5')
1191
1215
            self.assertRaises(errors.KnitCorrupt, index.keys)
1192
1216
        except TypeError, e:
1193
1217
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1194
 
                           ' not exceptions.ValueError')
1195
 
                and sys.version_info[0:2] >= (2,5)):
 
1218
                           ' not exceptions.ValueError')):
1196
1219
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1197
1220
                                  ' raising new style exceptions with python'
1198
1221
                                  ' >=2.5')
1211
1234
            self.assertRaises(errors.KnitCorrupt, index.keys)
1212
1235
        except TypeError, e:
1213
1236
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1214
 
                           ' not exceptions.ValueError')
1215
 
                and sys.version_info[0:2] >= (2,5)):
 
1237
                           ' not exceptions.ValueError')):
1216
1238
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1217
1239
                                  ' raising new style exceptions with python'
1218
1240
                                  ' >=2.5')
1229
1251
            self.assertRaises(errors.KnitCorrupt, index.keys)
1230
1252
        except TypeError, e:
1231
1253
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1232
 
                           ' not exceptions.ValueError')
1233
 
                and sys.version_info[0:2] >= (2,5)):
 
1254
                           ' not exceptions.ValueError')):
1234
1255
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1235
1256
                                  ' raising new style exceptions with python'
1236
1257
                                  ' >=2.5')
1247
1268
            self.assertRaises(errors.KnitCorrupt, index.keys)
1248
1269
        except TypeError, e:
1249
1270
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1250
 
                           ' not exceptions.ValueError')
1251
 
                and sys.version_info[0:2] >= (2,5)):
 
1271
                           ' not exceptions.ValueError')):
1252
1272
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1253
1273
                                  ' raising new style exceptions with python'
1254
1274
                                  ' >=2.5')
1302
1322
 
1303
1323
    def get_knit_index(self, transport, name, mode):
1304
1324
        mapper = ConstantMapper(name)
1305
 
        orig = knit._load_data
1306
 
        def reset():
1307
 
            knit._load_data = orig
1308
 
        self.addCleanup(reset)
1309
1325
        from bzrlib._knit_load_data_pyx import _load_data_c
1310
 
        knit._load_data = _load_data_c
 
1326
        self.overrideAttr(knit, '_load_data', _load_data_c)
1311
1327
        allow_writes = lambda: mode == 'w'
1312
 
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
 
1328
        return _KndxIndex(transport, mapper, lambda:None,
 
1329
                          allow_writes, lambda:True)
1313
1330
 
1314
1331
 
1315
1332
class Test_KnitAnnotator(TestCaseWithMemoryTransport):
1586
1603
        # could leave an empty .kndx file, which bzr would later claim was a
1587
1604
        # corrupted file since the header was not present. In reality, the file
1588
1605
        # just wasn't created, so it should be ignored.
1589
 
        t = get_transport('.')
 
1606
        t = transport.get_transport_from_path('.')
1590
1607
        t.put_bytes('test.kndx', '')
1591
1608
 
1592
1609
        knit = self.make_test_knit()
1593
1610
 
1594
1611
    def test_knit_index_checks_header(self):
1595
 
        t = get_transport('.')
 
1612
        t = transport.get_transport_from_path('.')
1596
1613
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1597
1614
        k = self.make_test_knit()
1598
1615
        self.assertRaises(KnitHeaderError, k.keys)
2426
2443
        key_basis = ('bar',)
2427
2444
        key_missing = ('missing',)
2428
2445
        test.add_lines(key, (), ['foo\n'])
2429
 
        key_sha1sum = osutils.sha('foo\n').hexdigest()
 
2446
        key_sha1sum = osutils.sha_string('foo\n')
2430
2447
        sha1s = test.get_sha1s([key])
2431
2448
        self.assertEqual({key: key_sha1sum}, sha1s)
2432
2449
        self.assertEqual([], basis.calls)
2434
2451
        # directly (rather than via text reconstruction) so that remote servers
2435
2452
        # etc don't have to answer with full content.
2436
2453
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2437
 
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
2454
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
2438
2455
        basis.calls = []
2439
2456
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2440
2457
        self.assertEqual({key: key_sha1sum,