← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2010-01-22 08:05:06 UTC
  • mfrom: (4980.1.2 news-details)
  • Revision ID: pqm@pqm.ubuntu.com-20100122080506-7p37edg2vai36qb4
(nmb) Clarify NEWS entries handling

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2006-2011 Canonical Ltd
 
1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
 
20
import difflib
20
21
import gzip
21
22
import sys
22
23
 
23
24
from bzrlib import (
24
25
    errors,
 
26
    generate_ids,
25
27
    knit,
26
28
    multiparent,
27
29
    osutils,
28
30
    pack,
29
31
    tests,
30
 
    transport,
31
32
    )
32
33
from bzrlib.errors import (
 
34
    RevisionAlreadyPresent,
33
35
    KnitHeaderError,
 
36
    RevisionNotPresent,
34
37
    NoSuchFile,
35
38
    )
36
39
from bzrlib.index import *
37
40
from bzrlib.knit import (
38
41
    AnnotatedKnitContent,
39
42
    KnitContent,
 
43
    KnitSequenceMatcher,
40
44
    KnitVersionedFiles,
41
45
    PlainKnitContent,
42
46
    _VFContentMapGenerator,
 
47
    _DirectPackAccess,
43
48
    _KndxIndex,
44
49
    _KnitGraphIndex,
45
50
    _KnitKeyAccess,
46
51
    make_file_factory,
47
52
    )
48
 
from bzrlib.patiencediff import PatienceSequenceMatcher
49
 
from bzrlib.repofmt import (
50
 
    knitpack_repo,
51
 
    pack_repo,
52
 
    )
 
53
from bzrlib.repofmt import pack_repo
53
54
from bzrlib.tests import (
 
55
    Feature,
 
56
    KnownFailure,
54
57
    TestCase,
55
58
    TestCaseWithMemoryTransport,
56
59
    TestCaseWithTransport,
57
60
    TestNotApplicable,
58
61
    )
 
62
from bzrlib.transport import get_transport
 
63
from bzrlib.transport.memory import MemoryTransport
 
64
from bzrlib.tuned_gzip import GzipFile
59
65
from bzrlib.versionedfile import (
60
66
    AbsentContentFactory,
61
67
    ConstantMapper,
62
68
    network_bytes_to_kind_and_offset,
63
69
    RecordingVersionedFilesDecorator,
64
70
    )
65
 
from bzrlib.tests import (
66
 
    features,
67
 
    )
68
 
 
69
 
 
70
 
compiled_knit_feature = features.ModuleAvailableFeature(
71
 
    'bzrlib._knit_load_data_pyx')
 
71
 
 
72
 
 
73
compiled_knit_feature = tests.ModuleAvailableFeature(
 
74
                            'bzrlib._knit_load_data_pyx')
72
75
 
73
76
 
74
77
class KnitContentTestsMixin(object):
103
106
        line_delta = source_content.line_delta(target_content)
104
107
        delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
105
108
            source_lines, target_lines))
106
 
        matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
107
 
        matcher_blocks = list(matcher.get_matching_blocks())
 
109
        matcher = KnitSequenceMatcher(None, source_lines, target_lines)
 
110
        matcher_blocks = list(list(matcher.get_matching_blocks()))
108
111
        self.assertEqual(matcher_blocks, delta_blocks)
109
112
 
110
113
    def test_get_line_delta_blocks(self):
330
333
            transport.append_bytes(packname, bytes)
331
334
        writer = pack.ContainerWriter(write_data)
332
335
        writer.begin()
333
 
        access = pack_repo._DirectPackAccess({})
 
336
        access = _DirectPackAccess({})
334
337
        access.set_writer(writer, index, (transport, packname))
335
338
        return access, writer
336
339
 
343
346
        writer.end()
344
347
        return memos
345
348
 
346
 
    def test_pack_collection_pack_retries(self):
347
 
        """An explicit pack of a pack collection succeeds even when a
348
 
        concurrent pack happens.
349
 
        """
350
 
        builder = self.make_branch_builder('.')
351
 
        builder.start_series()
352
 
        builder.build_snapshot('rev-1', None, [
353
 
            ('add', ('', 'root-id', 'directory', None)),
354
 
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
355
 
            ])
356
 
        builder.build_snapshot('rev-2', ['rev-1'], [
357
 
            ('modify', ('file-id', 'content\nrev 2\n')),
358
 
            ])
359
 
        builder.build_snapshot('rev-3', ['rev-2'], [
360
 
            ('modify', ('file-id', 'content\nrev 3\n')),
361
 
            ])
362
 
        self.addCleanup(builder.finish_series)
363
 
        b = builder.get_branch()
364
 
        self.addCleanup(b.lock_write().unlock)
365
 
        repo = b.repository
366
 
        collection = repo._pack_collection
367
 
        # Concurrently repack the repo.
368
 
        reopened_repo = repo.bzrdir.open_repository()
369
 
        reopened_repo.pack()
370
 
        # Pack the new pack.
371
 
        collection.pack()
372
 
 
373
349
    def make_vf_for_retrying(self):
374
350
        """Create 3 packs and a reload function.
375
351
 
402
378
        collection = repo._pack_collection
403
379
        collection.ensure_loaded()
404
380
        orig_packs = collection.packs
405
 
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
 
381
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
406
382
        new_pack = packer.pack()
407
383
        # forget about the new pack
408
384
        collection.reset()
447
423
        except _TestException, e:
448
424
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
449
425
                                                 exc_info=sys.exc_info())
450
 
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
451
426
        return retry_exc
452
427
 
453
428
    def test_read_from_several_packs(self):
462
437
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
463
438
        writer.end()
464
439
        transport = self.get_transport()
465
 
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
 
440
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
466
441
            "FOOBAR":(transport, 'pack2'),
467
442
            "BAZ":(transport, 'pack3')})
468
443
        self.assertEqual(['1234567890', '12345', 'alpha'],
478
453
 
479
454
    def test_set_writer(self):
480
455
        """The writer should be settable post construction."""
481
 
        access = pack_repo._DirectPackAccess({})
 
456
        access = _DirectPackAccess({})
482
457
        transport = self.get_transport()
483
458
        packname = 'packfile'
484
459
        index = 'foo'
496
471
        transport = self.get_transport()
497
472
        reload_called, reload_func = self.make_reload_func()
498
473
        # Note that the index key has changed from 'foo' to 'bar'
499
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
 
474
        access = _DirectPackAccess({'bar':(transport, 'packname')},
500
475
                                   reload_func=reload_func)
501
476
        e = self.assertListRaises(errors.RetryWithNewPacks,
502
477
                                  access.get_raw_records, memos)
511
486
        memos = self.make_pack_file()
512
487
        transport = self.get_transport()
513
488
        # Note that the index key has changed from 'foo' to 'bar'
514
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
 
489
        access = _DirectPackAccess({'bar':(transport, 'packname')})
515
490
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
516
491
 
517
492
    def test_missing_file_raises_retry(self):
519
494
        transport = self.get_transport()
520
495
        reload_called, reload_func = self.make_reload_func()
521
496
        # Note that the 'filename' has been changed to 'different-packname'
522
 
        access = pack_repo._DirectPackAccess(
523
 
            {'foo':(transport, 'different-packname')},
524
 
            reload_func=reload_func)
 
497
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
 
498
                                   reload_func=reload_func)
525
499
        e = self.assertListRaises(errors.RetryWithNewPacks,
526
500
                                  access.get_raw_records, memos)
527
501
        # The file has gone missing, so we assume we need to reload
535
509
        memos = self.make_pack_file()
536
510
        transport = self.get_transport()
537
511
        # Note that the 'filename' has been changed to 'different-packname'
538
 
        access = pack_repo._DirectPackAccess(
539
 
            {'foo': (transport, 'different-packname')})
 
512
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
540
513
        e = self.assertListRaises(errors.NoSuchFile,
541
514
                                  access.get_raw_records, memos)
542
515
 
546
519
        failing_transport = MockReadvFailingTransport(
547
520
                                [transport.get_bytes('packname')])
548
521
        reload_called, reload_func = self.make_reload_func()
549
 
        access = pack_repo._DirectPackAccess(
550
 
            {'foo': (failing_transport, 'packname')},
551
 
            reload_func=reload_func)
 
522
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
 
523
                                   reload_func=reload_func)
552
524
        # Asking for a single record will not trigger the Mock failure
553
525
        self.assertEqual(['1234567890'],
554
526
            list(access.get_raw_records(memos[:1])))
570
542
        failing_transport = MockReadvFailingTransport(
571
543
                                [transport.get_bytes('packname')])
572
544
        reload_called, reload_func = self.make_reload_func()
573
 
        access = pack_repo._DirectPackAccess(
574
 
            {'foo':(failing_transport, 'packname')})
 
545
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
575
546
        # Asking for a single record will not trigger the Mock failure
576
547
        self.assertEqual(['1234567890'],
577
548
            list(access.get_raw_records(memos[:1])))
582
553
                                  access.get_raw_records, memos)
583
554
 
584
555
    def test_reload_or_raise_no_reload(self):
585
 
        access = pack_repo._DirectPackAccess({}, reload_func=None)
 
556
        access = _DirectPackAccess({}, reload_func=None)
586
557
        retry_exc = self.make_retry_exception()
587
558
        # Without a reload_func, we will just re-raise the original exception
588
559
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
589
560
 
590
561
    def test_reload_or_raise_reload_changed(self):
591
562
        reload_called, reload_func = self.make_reload_func(return_val=True)
592
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
563
        access = _DirectPackAccess({}, reload_func=reload_func)
593
564
        retry_exc = self.make_retry_exception()
594
565
        access.reload_or_raise(retry_exc)
595
566
        self.assertEqual([1], reload_called)
599
570
 
600
571
    def test_reload_or_raise_reload_no_change(self):
601
572
        reload_called, reload_func = self.make_reload_func(return_val=False)
602
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
573
        access = _DirectPackAccess({}, reload_func=reload_func)
603
574
        retry_exc = self.make_retry_exception()
604
575
        # If reload_occurred is False, then we consider it an error to have
605
576
        # reload_func() return False (no changes).
736
707
 
737
708
    def make_multiple_records(self):
738
709
        """Create the content for multiple records."""
739
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
710
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
740
711
        total_txt = []
741
712
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
742
713
                                        'foo\n'
745
716
                                        % (sha1sum,))
746
717
        record_1 = (0, len(gz_txt), sha1sum)
747
718
        total_txt.append(gz_txt)
748
 
        sha1sum = osutils.sha_string('baz\n')
 
719
        sha1sum = osutils.sha('baz\n').hexdigest()
749
720
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
750
721
                                        'baz\n'
751
722
                                        'end rev-id-2\n'
755
726
        return total_txt, record_1, record_2
756
727
 
757
728
    def test_valid_knit_data(self):
758
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
729
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
759
730
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
760
731
                                        'foo\n'
761
732
                                        'bar\n'
792
763
                         raw_contents)
793
764
 
794
765
    def test_not_enough_lines(self):
795
 
        sha1sum = osutils.sha_string('foo\n')
 
766
        sha1sum = osutils.sha('foo\n').hexdigest()
796
767
        # record says 2 lines data says 1
797
768
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
798
769
                                        'foo\n'
810
781
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
811
782
 
812
783
    def test_too_many_lines(self):
813
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
784
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
814
785
        # record says 1 lines data says 2
815
786
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
816
787
                                        'foo\n'
829
800
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
830
801
 
831
802
    def test_mismatched_version_id(self):
832
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
803
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
833
804
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
834
805
                                        'foo\n'
835
806
                                        'bar\n'
848
819
            knit._read_records_iter_raw(records))
849
820
 
850
821
    def test_uncompressed_data(self):
851
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
822
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
852
823
        txt = ('version rev-id-1 2 %s\n'
853
824
               'foo\n'
854
825
               'bar\n'
868
839
            knit._read_records_iter_raw(records))
869
840
 
870
841
    def test_corrupted_data(self):
871
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
842
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
872
843
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
873
844
                                        'foo\n'
874
845
                                        'bar\n'
891
862
 
892
863
    def get_knit_index(self, transport, name, mode):
893
864
        mapper = ConstantMapper(name)
 
865
        orig = knit._load_data
 
866
        def reset():
 
867
            knit._load_data = orig
 
868
        self.addCleanup(reset)
894
869
        from bzrlib._knit_load_data_py import _load_data_py
895
 
        self.overrideAttr(knit, '_load_data', _load_data_py)
 
870
        knit._load_data = _load_data_py
896
871
        allow_writes = lambda: 'w' in mode
897
872
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
898
873
 
1196
1171
            self.assertRaises(errors.KnitCorrupt, index.keys)
1197
1172
        except TypeError, e:
1198
1173
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1199
 
                           ' not exceptions.IndexError')):
 
1174
                           ' not exceptions.IndexError')
 
1175
                and sys.version_info[0:2] >= (2,5)):
1200
1176
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1201
1177
                                  ' raising new style exceptions with python'
1202
1178
                                  ' >=2.5')
1215
1191
            self.assertRaises(errors.KnitCorrupt, index.keys)
1216
1192
        except TypeError, e:
1217
1193
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1218
 
                           ' not exceptions.ValueError')):
 
1194
                           ' not exceptions.ValueError')
 
1195
                and sys.version_info[0:2] >= (2,5)):
1219
1196
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1220
1197
                                  ' raising new style exceptions with python'
1221
1198
                                  ' >=2.5')
1234
1211
            self.assertRaises(errors.KnitCorrupt, index.keys)
1235
1212
        except TypeError, e:
1236
1213
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1237
 
                           ' not exceptions.ValueError')):
 
1214
                           ' not exceptions.ValueError')
 
1215
                and sys.version_info[0:2] >= (2,5)):
1238
1216
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1239
1217
                                  ' raising new style exceptions with python'
1240
1218
                                  ' >=2.5')
1251
1229
            self.assertRaises(errors.KnitCorrupt, index.keys)
1252
1230
        except TypeError, e:
1253
1231
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1254
 
                           ' not exceptions.ValueError')):
 
1232
                           ' not exceptions.ValueError')
 
1233
                and sys.version_info[0:2] >= (2,5)):
1255
1234
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1256
1235
                                  ' raising new style exceptions with python'
1257
1236
                                  ' >=2.5')
1268
1247
            self.assertRaises(errors.KnitCorrupt, index.keys)
1269
1248
        except TypeError, e:
1270
1249
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1271
 
                           ' not exceptions.ValueError')):
 
1250
                           ' not exceptions.ValueError')
 
1251
                and sys.version_info[0:2] >= (2,5)):
1272
1252
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1273
1253
                                  ' raising new style exceptions with python'
1274
1254
                                  ' >=2.5')
1322
1302
 
1323
1303
    def get_knit_index(self, transport, name, mode):
1324
1304
        mapper = ConstantMapper(name)
 
1305
        orig = knit._load_data
 
1306
        def reset():
 
1307
            knit._load_data = orig
 
1308
        self.addCleanup(reset)
1325
1309
        from bzrlib._knit_load_data_pyx import _load_data_c
1326
 
        self.overrideAttr(knit, '_load_data', _load_data_c)
 
1310
        knit._load_data = _load_data_c
1327
1311
        allow_writes = lambda: mode == 'w'
1328
 
        return _KndxIndex(transport, mapper, lambda:None,
1329
 
                          allow_writes, lambda:True)
 
1312
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
1330
1313
 
1331
1314
 
1332
1315
class Test_KnitAnnotator(TestCaseWithMemoryTransport):
1603
1586
        # could leave an empty .kndx file, which bzr would later claim was a
1604
1587
        # corrupted file since the header was not present. In reality, the file
1605
1588
        # just wasn't created, so it should be ignored.
1606
 
        t = transport.get_transport('.')
 
1589
        t = get_transport('.')
1607
1590
        t.put_bytes('test.kndx', '')
1608
1591
 
1609
1592
        knit = self.make_test_knit()
1610
1593
 
1611
1594
    def test_knit_index_checks_header(self):
1612
 
        t = transport.get_transport('.')
 
1595
        t = get_transport('.')
1613
1596
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1614
1597
        k = self.make_test_knit()
1615
1598
        self.assertRaises(KnitHeaderError, k.keys)
2443
2426
        key_basis = ('bar',)
2444
2427
        key_missing = ('missing',)
2445
2428
        test.add_lines(key, (), ['foo\n'])
2446
 
        key_sha1sum = osutils.sha_string('foo\n')
 
2429
        key_sha1sum = osutils.sha('foo\n').hexdigest()
2447
2430
        sha1s = test.get_sha1s([key])
2448
2431
        self.assertEqual({key: key_sha1sum}, sha1s)
2449
2432
        self.assertEqual([], basis.calls)
2451
2434
        # directly (rather than via text reconstruction) so that remote servers
2452
2435
        # etc don't have to answer with full content.
2453
2436
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2454
 
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
 
2437
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
2455
2438
        basis.calls = []
2456
2439
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2457
2440
        self.assertEqual({key: key_sha1sum,