← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Patch Queue Manager
  • Date: 2016-02-01 19:56:05 UTC
  • mfrom: (6615.1.1 trunk)
  • Revision ID: pqm@pqm.ubuntu.com-20160201195605-o7rl92wf6uyum3fk
(vila) Open trunk again as 2.8b1 (Vincent Ladeuil)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
 
1
# Copyright (C) 2006-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
20
 
import difflib
21
20
import gzip
22
21
import sys
23
22
 
24
23
from bzrlib import (
25
24
    errors,
26
 
    generate_ids,
27
25
    knit,
28
26
    multiparent,
29
27
    osutils,
30
28
    pack,
31
29
    tests,
 
30
    transport,
32
31
    )
33
32
from bzrlib.errors import (
34
 
    RevisionAlreadyPresent,
35
33
    KnitHeaderError,
36
 
    RevisionNotPresent,
37
34
    NoSuchFile,
38
35
    )
39
36
from bzrlib.index import *
40
37
from bzrlib.knit import (
41
38
    AnnotatedKnitContent,
42
39
    KnitContent,
43
 
    KnitSequenceMatcher,
44
40
    KnitVersionedFiles,
45
41
    PlainKnitContent,
46
42
    _VFContentMapGenerator,
47
 
    _DirectPackAccess,
48
43
    _KndxIndex,
49
44
    _KnitGraphIndex,
50
45
    _KnitKeyAccess,
51
46
    make_file_factory,
52
47
    )
53
 
from bzrlib.repofmt import pack_repo
 
48
from bzrlib.patiencediff import PatienceSequenceMatcher
 
49
from bzrlib.repofmt import (
 
50
    knitpack_repo,
 
51
    pack_repo,
 
52
    )
54
53
from bzrlib.tests import (
55
 
    Feature,
56
 
    KnownFailure,
57
54
    TestCase,
58
55
    TestCaseWithMemoryTransport,
59
56
    TestCaseWithTransport,
60
57
    TestNotApplicable,
61
58
    )
62
 
from bzrlib.transport import get_transport
63
 
from bzrlib.transport.memory import MemoryTransport
64
 
from bzrlib.tuned_gzip import GzipFile
65
59
from bzrlib.versionedfile import (
66
60
    AbsentContentFactory,
67
61
    ConstantMapper,
68
62
    network_bytes_to_kind_and_offset,
69
63
    RecordingVersionedFilesDecorator,
70
64
    )
71
 
 
72
 
 
73
 
compiled_knit_feature = tests.ModuleAvailableFeature(
74
 
                            'bzrlib._knit_load_data_pyx')
 
65
from bzrlib.tests import (
 
66
    features,
 
67
    )
 
68
 
 
69
 
 
70
compiled_knit_feature = features.ModuleAvailableFeature(
 
71
    'bzrlib._knit_load_data_pyx')
75
72
 
76
73
 
77
74
class KnitContentTestsMixin(object):
106
103
        line_delta = source_content.line_delta(target_content)
107
104
        delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
108
105
            source_lines, target_lines))
109
 
        matcher = KnitSequenceMatcher(None, source_lines, target_lines)
110
 
        matcher_blocks = list(list(matcher.get_matching_blocks()))
 
106
        matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
 
107
        matcher_blocks = list(matcher.get_matching_blocks())
111
108
        self.assertEqual(matcher_blocks, delta_blocks)
112
109
 
113
110
    def test_get_line_delta_blocks(self):
333
330
            transport.append_bytes(packname, bytes)
334
331
        writer = pack.ContainerWriter(write_data)
335
332
        writer.begin()
336
 
        access = _DirectPackAccess({})
 
333
        access = pack_repo._DirectPackAccess({})
337
334
        access.set_writer(writer, index, (transport, packname))
338
335
        return access, writer
339
336
 
346
343
        writer.end()
347
344
        return memos
348
345
 
 
346
    def test_pack_collection_pack_retries(self):
 
347
        """An explicit pack of a pack collection succeeds even when a
 
348
        concurrent pack happens.
 
349
        """
 
350
        builder = self.make_branch_builder('.')
 
351
        builder.start_series()
 
352
        builder.build_snapshot('rev-1', None, [
 
353
            ('add', ('', 'root-id', 'directory', None)),
 
354
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
 
355
            ])
 
356
        builder.build_snapshot('rev-2', ['rev-1'], [
 
357
            ('modify', ('file-id', 'content\nrev 2\n')),
 
358
            ])
 
359
        builder.build_snapshot('rev-3', ['rev-2'], [
 
360
            ('modify', ('file-id', 'content\nrev 3\n')),
 
361
            ])
 
362
        self.addCleanup(builder.finish_series)
 
363
        b = builder.get_branch()
 
364
        self.addCleanup(b.lock_write().unlock)
 
365
        repo = b.repository
 
366
        collection = repo._pack_collection
 
367
        # Concurrently repack the repo.
 
368
        reopened_repo = repo.bzrdir.open_repository()
 
369
        reopened_repo.pack()
 
370
        # Pack the new pack.
 
371
        collection.pack()
 
372
 
349
373
    def make_vf_for_retrying(self):
350
374
        """Create 3 packs and a reload function.
351
375
 
378
402
        collection = repo._pack_collection
379
403
        collection.ensure_loaded()
380
404
        orig_packs = collection.packs
381
 
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
 
405
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
382
406
        new_pack = packer.pack()
383
407
        # forget about the new pack
384
408
        collection.reset()
423
447
        except _TestException, e:
424
448
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
425
449
                                                 exc_info=sys.exc_info())
 
450
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
426
451
        return retry_exc
427
452
 
428
453
    def test_read_from_several_packs(self):
437
462
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
438
463
        writer.end()
439
464
        transport = self.get_transport()
440
 
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
 
465
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
441
466
            "FOOBAR":(transport, 'pack2'),
442
467
            "BAZ":(transport, 'pack3')})
443
468
        self.assertEqual(['1234567890', '12345', 'alpha'],
453
478
 
454
479
    def test_set_writer(self):
455
480
        """The writer should be settable post construction."""
456
 
        access = _DirectPackAccess({})
 
481
        access = pack_repo._DirectPackAccess({})
457
482
        transport = self.get_transport()
458
483
        packname = 'packfile'
459
484
        index = 'foo'
471
496
        transport = self.get_transport()
472
497
        reload_called, reload_func = self.make_reload_func()
473
498
        # Note that the index key has changed from 'foo' to 'bar'
474
 
        access = _DirectPackAccess({'bar':(transport, 'packname')},
 
499
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
475
500
                                   reload_func=reload_func)
476
501
        e = self.assertListRaises(errors.RetryWithNewPacks,
477
502
                                  access.get_raw_records, memos)
486
511
        memos = self.make_pack_file()
487
512
        transport = self.get_transport()
488
513
        # Note that the index key has changed from 'foo' to 'bar'
489
 
        access = _DirectPackAccess({'bar':(transport, 'packname')})
 
514
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
490
515
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
491
516
 
492
517
    def test_missing_file_raises_retry(self):
494
519
        transport = self.get_transport()
495
520
        reload_called, reload_func = self.make_reload_func()
496
521
        # Note that the 'filename' has been changed to 'different-packname'
497
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
498
 
                                   reload_func=reload_func)
 
522
        access = pack_repo._DirectPackAccess(
 
523
            {'foo':(transport, 'different-packname')},
 
524
            reload_func=reload_func)
499
525
        e = self.assertListRaises(errors.RetryWithNewPacks,
500
526
                                  access.get_raw_records, memos)
501
527
        # The file has gone missing, so we assume we need to reload
509
535
        memos = self.make_pack_file()
510
536
        transport = self.get_transport()
511
537
        # Note that the 'filename' has been changed to 'different-packname'
512
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
 
538
        access = pack_repo._DirectPackAccess(
 
539
            {'foo': (transport, 'different-packname')})
513
540
        e = self.assertListRaises(errors.NoSuchFile,
514
541
                                  access.get_raw_records, memos)
515
542
 
519
546
        failing_transport = MockReadvFailingTransport(
520
547
                                [transport.get_bytes('packname')])
521
548
        reload_called, reload_func = self.make_reload_func()
522
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
523
 
                                   reload_func=reload_func)
 
549
        access = pack_repo._DirectPackAccess(
 
550
            {'foo': (failing_transport, 'packname')},
 
551
            reload_func=reload_func)
524
552
        # Asking for a single record will not trigger the Mock failure
525
553
        self.assertEqual(['1234567890'],
526
554
            list(access.get_raw_records(memos[:1])))
542
570
        failing_transport = MockReadvFailingTransport(
543
571
                                [transport.get_bytes('packname')])
544
572
        reload_called, reload_func = self.make_reload_func()
545
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
 
573
        access = pack_repo._DirectPackAccess(
 
574
            {'foo':(failing_transport, 'packname')})
546
575
        # Asking for a single record will not trigger the Mock failure
547
576
        self.assertEqual(['1234567890'],
548
577
            list(access.get_raw_records(memos[:1])))
553
582
                                  access.get_raw_records, memos)
554
583
 
555
584
    def test_reload_or_raise_no_reload(self):
556
 
        access = _DirectPackAccess({}, reload_func=None)
 
585
        access = pack_repo._DirectPackAccess({}, reload_func=None)
557
586
        retry_exc = self.make_retry_exception()
558
587
        # Without a reload_func, we will just re-raise the original exception
559
588
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
560
589
 
561
590
    def test_reload_or_raise_reload_changed(self):
562
591
        reload_called, reload_func = self.make_reload_func(return_val=True)
563
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
592
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
564
593
        retry_exc = self.make_retry_exception()
565
594
        access.reload_or_raise(retry_exc)
566
595
        self.assertEqual([1], reload_called)
570
599
 
571
600
    def test_reload_or_raise_reload_no_change(self):
572
601
        reload_called, reload_func = self.make_reload_func(return_val=False)
573
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
602
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
574
603
        retry_exc = self.make_retry_exception()
575
604
        # If reload_occurred is False, then we consider it an error to have
576
605
        # reload_func() return False (no changes).
707
736
 
708
737
    def make_multiple_records(self):
709
738
        """Create the content for multiple records."""
710
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
739
        sha1sum = osutils.sha_string('foo\nbar\n')
711
740
        total_txt = []
712
741
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
713
742
                                        'foo\n'
716
745
                                        % (sha1sum,))
717
746
        record_1 = (0, len(gz_txt), sha1sum)
718
747
        total_txt.append(gz_txt)
719
 
        sha1sum = osutils.sha('baz\n').hexdigest()
 
748
        sha1sum = osutils.sha_string('baz\n')
720
749
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
721
750
                                        'baz\n'
722
751
                                        'end rev-id-2\n'
726
755
        return total_txt, record_1, record_2
727
756
 
728
757
    def test_valid_knit_data(self):
729
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
758
        sha1sum = osutils.sha_string('foo\nbar\n')
730
759
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
731
760
                                        'foo\n'
732
761
                                        'bar\n'
763
792
                         raw_contents)
764
793
 
765
794
    def test_not_enough_lines(self):
766
 
        sha1sum = osutils.sha('foo\n').hexdigest()
 
795
        sha1sum = osutils.sha_string('foo\n')
767
796
        # record says 2 lines data says 1
768
797
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
769
798
                                        'foo\n'
781
810
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
782
811
 
783
812
    def test_too_many_lines(self):
784
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
813
        sha1sum = osutils.sha_string('foo\nbar\n')
785
814
        # record says 1 lines data says 2
786
815
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
787
816
                                        'foo\n'
800
829
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
801
830
 
802
831
    def test_mismatched_version_id(self):
803
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
832
        sha1sum = osutils.sha_string('foo\nbar\n')
804
833
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
805
834
                                        'foo\n'
806
835
                                        'bar\n'
819
848
            knit._read_records_iter_raw(records))
820
849
 
821
850
    def test_uncompressed_data(self):
822
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
851
        sha1sum = osutils.sha_string('foo\nbar\n')
823
852
        txt = ('version rev-id-1 2 %s\n'
824
853
               'foo\n'
825
854
               'bar\n'
839
868
            knit._read_records_iter_raw(records))
840
869
 
841
870
    def test_corrupted_data(self):
842
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
871
        sha1sum = osutils.sha_string('foo\nbar\n')
843
872
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
844
873
                                        'foo\n'
845
874
                                        'bar\n'
1167
1196
            self.assertRaises(errors.KnitCorrupt, index.keys)
1168
1197
        except TypeError, e:
1169
1198
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1170
 
                           ' not exceptions.IndexError')
1171
 
                and sys.version_info[0:2] >= (2,5)):
 
1199
                           ' not exceptions.IndexError')):
1172
1200
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1173
1201
                                  ' raising new style exceptions with python'
1174
1202
                                  ' >=2.5')
1187
1215
            self.assertRaises(errors.KnitCorrupt, index.keys)
1188
1216
        except TypeError, e:
1189
1217
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1190
 
                           ' not exceptions.ValueError')
1191
 
                and sys.version_info[0:2] >= (2,5)):
 
1218
                           ' not exceptions.ValueError')):
1192
1219
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1193
1220
                                  ' raising new style exceptions with python'
1194
1221
                                  ' >=2.5')
1207
1234
            self.assertRaises(errors.KnitCorrupt, index.keys)
1208
1235
        except TypeError, e:
1209
1236
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1210
 
                           ' not exceptions.ValueError')
1211
 
                and sys.version_info[0:2] >= (2,5)):
 
1237
                           ' not exceptions.ValueError')):
1212
1238
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1213
1239
                                  ' raising new style exceptions with python'
1214
1240
                                  ' >=2.5')
1225
1251
            self.assertRaises(errors.KnitCorrupt, index.keys)
1226
1252
        except TypeError, e:
1227
1253
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1228
 
                           ' not exceptions.ValueError')
1229
 
                and sys.version_info[0:2] >= (2,5)):
 
1254
                           ' not exceptions.ValueError')):
1230
1255
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1231
1256
                                  ' raising new style exceptions with python'
1232
1257
                                  ' >=2.5')
1243
1268
            self.assertRaises(errors.KnitCorrupt, index.keys)
1244
1269
        except TypeError, e:
1245
1270
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1246
 
                           ' not exceptions.ValueError')
1247
 
                and sys.version_info[0:2] >= (2,5)):
 
1271
                           ' not exceptions.ValueError')):
1248
1272
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1249
1273
                                  ' raising new style exceptions with python'
1250
1274
                                  ' >=2.5')
1579
1603
        # could leave an empty .kndx file, which bzr would later claim was a
1580
1604
        # corrupted file since the header was not present. In reality, the file
1581
1605
        # just wasn't created, so it should be ignored.
1582
 
        t = get_transport('.')
 
1606
        t = transport.get_transport_from_path('.')
1583
1607
        t.put_bytes('test.kndx', '')
1584
1608
 
1585
1609
        knit = self.make_test_knit()
1586
1610
 
1587
1611
    def test_knit_index_checks_header(self):
1588
 
        t = get_transport('.')
 
1612
        t = transport.get_transport_from_path('.')
1589
1613
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1590
1614
        k = self.make_test_knit()
1591
1615
        self.assertRaises(KnitHeaderError, k.keys)
2419
2443
        key_basis = ('bar',)
2420
2444
        key_missing = ('missing',)
2421
2445
        test.add_lines(key, (), ['foo\n'])
2422
 
        key_sha1sum = osutils.sha('foo\n').hexdigest()
 
2446
        key_sha1sum = osutils.sha_string('foo\n')
2423
2447
        sha1s = test.get_sha1s([key])
2424
2448
        self.assertEqual({key: key_sha1sum}, sha1s)
2425
2449
        self.assertEqual([], basis.calls)
2427
2451
        # directly (rather than via text reconstruction) so that remote servers
2428
2452
        # etc don't have to answer with full content.
2429
2453
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2430
 
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
2454
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
2431
2455
        basis.calls = []
2432
2456
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2433
2457
        self.assertEqual({key: key_sha1sum,