← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Martin Pool
  • Date: 2010-02-25 06:17:27 UTC
  • mfrom: (5055 +trunk)
  • mto: This revision was merged to the branch mainline in revision 5057.
  • Revision ID: mbp@sourcefrog.net-20100225061727-4sd9lt0qmdc6087t
merge news

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2006-2011 Canonical Ltd
 
1
# Copyright (C) 2006-2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
 
20
import difflib
20
21
import gzip
21
22
import sys
22
23
 
23
24
from bzrlib import (
24
25
    errors,
 
26
    generate_ids,
25
27
    knit,
26
28
    multiparent,
27
29
    osutils,
28
30
    pack,
29
31
    tests,
30
 
    transport,
31
32
    )
32
33
from bzrlib.errors import (
 
34
    RevisionAlreadyPresent,
33
35
    KnitHeaderError,
 
36
    RevisionNotPresent,
34
37
    NoSuchFile,
35
38
    )
36
39
from bzrlib.index import *
37
40
from bzrlib.knit import (
38
41
    AnnotatedKnitContent,
39
42
    KnitContent,
 
43
    KnitSequenceMatcher,
40
44
    KnitVersionedFiles,
41
45
    PlainKnitContent,
42
46
    _VFContentMapGenerator,
 
47
    _DirectPackAccess,
43
48
    _KndxIndex,
44
49
    _KnitGraphIndex,
45
50
    _KnitKeyAccess,
46
51
    make_file_factory,
47
52
    )
48
 
from bzrlib.patiencediff import PatienceSequenceMatcher
49
 
from bzrlib.repofmt import (
50
 
    knitpack_repo,
51
 
    pack_repo,
52
 
    )
 
53
from bzrlib.repofmt import pack_repo
53
54
from bzrlib.tests import (
 
55
    Feature,
 
56
    KnownFailure,
54
57
    TestCase,
55
58
    TestCaseWithMemoryTransport,
56
59
    TestCaseWithTransport,
57
60
    TestNotApplicable,
58
61
    )
 
62
from bzrlib.transport import get_transport
 
63
from bzrlib.transport.memory import MemoryTransport
 
64
from bzrlib.tuned_gzip import GzipFile
59
65
from bzrlib.versionedfile import (
60
66
    AbsentContentFactory,
61
67
    ConstantMapper,
62
68
    network_bytes_to_kind_and_offset,
63
69
    RecordingVersionedFilesDecorator,
64
70
    )
65
 
from bzrlib.tests import (
66
 
    features,
67
 
    )
68
 
 
69
 
 
70
 
compiled_knit_feature = features.ModuleAvailableFeature(
71
 
    'bzrlib._knit_load_data_pyx')
 
71
 
 
72
 
 
73
compiled_knit_feature = tests.ModuleAvailableFeature(
 
74
                            'bzrlib._knit_load_data_pyx')
72
75
 
73
76
 
74
77
class KnitContentTestsMixin(object):
103
106
        line_delta = source_content.line_delta(target_content)
104
107
        delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
105
108
            source_lines, target_lines))
106
 
        matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
107
 
        matcher_blocks = list(matcher.get_matching_blocks())
 
109
        matcher = KnitSequenceMatcher(None, source_lines, target_lines)
 
110
        matcher_blocks = list(list(matcher.get_matching_blocks()))
108
111
        self.assertEqual(matcher_blocks, delta_blocks)
109
112
 
110
113
    def test_get_line_delta_blocks(self):
330
333
            transport.append_bytes(packname, bytes)
331
334
        writer = pack.ContainerWriter(write_data)
332
335
        writer.begin()
333
 
        access = pack_repo._DirectPackAccess({})
 
336
        access = _DirectPackAccess({})
334
337
        access.set_writer(writer, index, (transport, packname))
335
338
        return access, writer
336
339
 
343
346
        writer.end()
344
347
        return memos
345
348
 
346
 
    def test_pack_collection_pack_retries(self):
347
 
        """An explicit pack of a pack collection succeeds even when a
348
 
        concurrent pack happens.
349
 
        """
350
 
        builder = self.make_branch_builder('.')
351
 
        builder.start_series()
352
 
        builder.build_snapshot('rev-1', None, [
353
 
            ('add', ('', 'root-id', 'directory', None)),
354
 
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
355
 
            ])
356
 
        builder.build_snapshot('rev-2', ['rev-1'], [
357
 
            ('modify', ('file-id', 'content\nrev 2\n')),
358
 
            ])
359
 
        builder.build_snapshot('rev-3', ['rev-2'], [
360
 
            ('modify', ('file-id', 'content\nrev 3\n')),
361
 
            ])
362
 
        self.addCleanup(builder.finish_series)
363
 
        b = builder.get_branch()
364
 
        self.addCleanup(b.lock_write().unlock)
365
 
        repo = b.repository
366
 
        collection = repo._pack_collection
367
 
        # Concurrently repack the repo.
368
 
        reopened_repo = repo.bzrdir.open_repository()
369
 
        reopened_repo.pack()
370
 
        # Pack the new pack.
371
 
        collection.pack()
372
 
 
373
349
    def make_vf_for_retrying(self):
374
350
        """Create 3 packs and a reload function.
375
351
 
402
378
        collection = repo._pack_collection
403
379
        collection.ensure_loaded()
404
380
        orig_packs = collection.packs
405
 
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
 
381
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
406
382
        new_pack = packer.pack()
407
383
        # forget about the new pack
408
384
        collection.reset()
447
423
        except _TestException, e:
448
424
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
449
425
                                                 exc_info=sys.exc_info())
450
 
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
451
426
        return retry_exc
452
427
 
453
428
    def test_read_from_several_packs(self):
462
437
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
463
438
        writer.end()
464
439
        transport = self.get_transport()
465
 
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
 
440
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
466
441
            "FOOBAR":(transport, 'pack2'),
467
442
            "BAZ":(transport, 'pack3')})
468
443
        self.assertEqual(['1234567890', '12345', 'alpha'],
478
453
 
479
454
    def test_set_writer(self):
480
455
        """The writer should be settable post construction."""
481
 
        access = pack_repo._DirectPackAccess({})
 
456
        access = _DirectPackAccess({})
482
457
        transport = self.get_transport()
483
458
        packname = 'packfile'
484
459
        index = 'foo'
496
471
        transport = self.get_transport()
497
472
        reload_called, reload_func = self.make_reload_func()
498
473
        # Note that the index key has changed from 'foo' to 'bar'
499
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
 
474
        access = _DirectPackAccess({'bar':(transport, 'packname')},
500
475
                                   reload_func=reload_func)
501
476
        e = self.assertListRaises(errors.RetryWithNewPacks,
502
477
                                  access.get_raw_records, memos)
511
486
        memos = self.make_pack_file()
512
487
        transport = self.get_transport()
513
488
        # Note that the index key has changed from 'foo' to 'bar'
514
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
 
489
        access = _DirectPackAccess({'bar':(transport, 'packname')})
515
490
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
516
491
 
517
492
    def test_missing_file_raises_retry(self):
519
494
        transport = self.get_transport()
520
495
        reload_called, reload_func = self.make_reload_func()
521
496
        # Note that the 'filename' has been changed to 'different-packname'
522
 
        access = pack_repo._DirectPackAccess(
523
 
            {'foo':(transport, 'different-packname')},
524
 
            reload_func=reload_func)
 
497
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
 
498
                                   reload_func=reload_func)
525
499
        e = self.assertListRaises(errors.RetryWithNewPacks,
526
500
                                  access.get_raw_records, memos)
527
501
        # The file has gone missing, so we assume we need to reload
535
509
        memos = self.make_pack_file()
536
510
        transport = self.get_transport()
537
511
        # Note that the 'filename' has been changed to 'different-packname'
538
 
        access = pack_repo._DirectPackAccess(
539
 
            {'foo': (transport, 'different-packname')})
 
512
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
540
513
        e = self.assertListRaises(errors.NoSuchFile,
541
514
                                  access.get_raw_records, memos)
542
515
 
546
519
        failing_transport = MockReadvFailingTransport(
547
520
                                [transport.get_bytes('packname')])
548
521
        reload_called, reload_func = self.make_reload_func()
549
 
        access = pack_repo._DirectPackAccess(
550
 
            {'foo': (failing_transport, 'packname')},
551
 
            reload_func=reload_func)
 
522
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
 
523
                                   reload_func=reload_func)
552
524
        # Asking for a single record will not trigger the Mock failure
553
525
        self.assertEqual(['1234567890'],
554
526
            list(access.get_raw_records(memos[:1])))
570
542
        failing_transport = MockReadvFailingTransport(
571
543
                                [transport.get_bytes('packname')])
572
544
        reload_called, reload_func = self.make_reload_func()
573
 
        access = pack_repo._DirectPackAccess(
574
 
            {'foo':(failing_transport, 'packname')})
 
545
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
575
546
        # Asking for a single record will not trigger the Mock failure
576
547
        self.assertEqual(['1234567890'],
577
548
            list(access.get_raw_records(memos[:1])))
582
553
                                  access.get_raw_records, memos)
583
554
 
584
555
    def test_reload_or_raise_no_reload(self):
585
 
        access = pack_repo._DirectPackAccess({}, reload_func=None)
 
556
        access = _DirectPackAccess({}, reload_func=None)
586
557
        retry_exc = self.make_retry_exception()
587
558
        # Without a reload_func, we will just re-raise the original exception
588
559
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
589
560
 
590
561
    def test_reload_or_raise_reload_changed(self):
591
562
        reload_called, reload_func = self.make_reload_func(return_val=True)
592
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
563
        access = _DirectPackAccess({}, reload_func=reload_func)
593
564
        retry_exc = self.make_retry_exception()
594
565
        access.reload_or_raise(retry_exc)
595
566
        self.assertEqual([1], reload_called)
599
570
 
600
571
    def test_reload_or_raise_reload_no_change(self):
601
572
        reload_called, reload_func = self.make_reload_func(return_val=False)
602
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
573
        access = _DirectPackAccess({}, reload_func=reload_func)
603
574
        retry_exc = self.make_retry_exception()
604
575
        # If reload_occurred is False, then we consider it an error to have
605
576
        # reload_func() return False (no changes).
736
707
 
737
708
    def make_multiple_records(self):
738
709
        """Create the content for multiple records."""
739
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
710
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
740
711
        total_txt = []
741
712
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
742
713
                                        'foo\n'
745
716
                                        % (sha1sum,))
746
717
        record_1 = (0, len(gz_txt), sha1sum)
747
718
        total_txt.append(gz_txt)
748
 
        sha1sum = osutils.sha_string('baz\n')
 
719
        sha1sum = osutils.sha('baz\n').hexdigest()
749
720
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
750
721
                                        'baz\n'
751
722
                                        'end rev-id-2\n'
755
726
        return total_txt, record_1, record_2
756
727
 
757
728
    def test_valid_knit_data(self):
758
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
729
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
759
730
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
760
731
                                        'foo\n'
761
732
                                        'bar\n'
792
763
                         raw_contents)
793
764
 
794
765
    def test_not_enough_lines(self):
795
 
        sha1sum = osutils.sha_string('foo\n')
 
766
        sha1sum = osutils.sha('foo\n').hexdigest()
796
767
        # record says 2 lines data says 1
797
768
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
798
769
                                        'foo\n'
810
781
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
811
782
 
812
783
    def test_too_many_lines(self):
813
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
784
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
814
785
        # record says 1 lines data says 2
815
786
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
816
787
                                        'foo\n'
829
800
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
830
801
 
831
802
    def test_mismatched_version_id(self):
832
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
803
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
833
804
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
834
805
                                        'foo\n'
835
806
                                        'bar\n'
848
819
            knit._read_records_iter_raw(records))
849
820
 
850
821
    def test_uncompressed_data(self):
851
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
822
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
852
823
        txt = ('version rev-id-1 2 %s\n'
853
824
               'foo\n'
854
825
               'bar\n'
868
839
            knit._read_records_iter_raw(records))
869
840
 
870
841
    def test_corrupted_data(self):
871
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
842
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
872
843
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
873
844
                                        'foo\n'
874
845
                                        'bar\n'
1196
1167
            self.assertRaises(errors.KnitCorrupt, index.keys)
1197
1168
        except TypeError, e:
1198
1169
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1199
 
                           ' not exceptions.IndexError')):
 
1170
                           ' not exceptions.IndexError')
 
1171
                and sys.version_info[0:2] >= (2,5)):
1200
1172
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1201
1173
                                  ' raising new style exceptions with python'
1202
1174
                                  ' >=2.5')
1215
1187
            self.assertRaises(errors.KnitCorrupt, index.keys)
1216
1188
        except TypeError, e:
1217
1189
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1218
 
                           ' not exceptions.ValueError')):
 
1190
                           ' not exceptions.ValueError')
 
1191
                and sys.version_info[0:2] >= (2,5)):
1219
1192
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1220
1193
                                  ' raising new style exceptions with python'
1221
1194
                                  ' >=2.5')
1234
1207
            self.assertRaises(errors.KnitCorrupt, index.keys)
1235
1208
        except TypeError, e:
1236
1209
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1237
 
                           ' not exceptions.ValueError')):
 
1210
                           ' not exceptions.ValueError')
 
1211
                and sys.version_info[0:2] >= (2,5)):
1238
1212
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1239
1213
                                  ' raising new style exceptions with python'
1240
1214
                                  ' >=2.5')
1251
1225
            self.assertRaises(errors.KnitCorrupt, index.keys)
1252
1226
        except TypeError, e:
1253
1227
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1254
 
                           ' not exceptions.ValueError')):
 
1228
                           ' not exceptions.ValueError')
 
1229
                and sys.version_info[0:2] >= (2,5)):
1255
1230
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1256
1231
                                  ' raising new style exceptions with python'
1257
1232
                                  ' >=2.5')
1268
1243
            self.assertRaises(errors.KnitCorrupt, index.keys)
1269
1244
        except TypeError, e:
1270
1245
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1271
 
                           ' not exceptions.ValueError')):
 
1246
                           ' not exceptions.ValueError')
 
1247
                and sys.version_info[0:2] >= (2,5)):
1272
1248
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1273
1249
                                  ' raising new style exceptions with python'
1274
1250
                                  ' >=2.5')
1603
1579
        # could leave an empty .kndx file, which bzr would later claim was a
1604
1580
        # corrupted file since the header was not present. In reality, the file
1605
1581
        # just wasn't created, so it should be ignored.
1606
 
        t = transport.get_transport_from_path('.')
 
1582
        t = get_transport('.')
1607
1583
        t.put_bytes('test.kndx', '')
1608
1584
 
1609
1585
        knit = self.make_test_knit()
1610
1586
 
1611
1587
    def test_knit_index_checks_header(self):
1612
 
        t = transport.get_transport_from_path('.')
 
1588
        t = get_transport('.')
1613
1589
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1614
1590
        k = self.make_test_knit()
1615
1591
        self.assertRaises(KnitHeaderError, k.keys)
2443
2419
        key_basis = ('bar',)
2444
2420
        key_missing = ('missing',)
2445
2421
        test.add_lines(key, (), ['foo\n'])
2446
 
        key_sha1sum = osutils.sha_string('foo\n')
 
2422
        key_sha1sum = osutils.sha('foo\n').hexdigest()
2447
2423
        sha1s = test.get_sha1s([key])
2448
2424
        self.assertEqual({key: key_sha1sum}, sha1s)
2449
2425
        self.assertEqual([], basis.calls)
2451
2427
        # directly (rather than via text reconstruction) so that remote servers
2452
2428
        # etc don't have to answer with full content.
2453
2429
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2454
 
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
 
2430
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
2455
2431
        basis.calls = []
2456
2432
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2457
2433
        self.assertEqual({key: key_sha1sum,