← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Martin Pool
  • Date: 2011-01-20 23:07:25 UTC
  • mfrom: (5626 +trunk)
  • mto: This revision was merged to the branch mainline in revision 5630.
  • Revision ID: mbp@canonical.com-20110120230725-12l7ltnko5x3fgnz
merge news

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
40
40
    KnitVersionedFiles,
41
41
    PlainKnitContent,
42
42
    _VFContentMapGenerator,
 
43
    _DirectPackAccess,
43
44
    _KndxIndex,
44
45
    _KnitGraphIndex,
45
46
    _KnitKeyAccess,
46
47
    make_file_factory,
47
48
    )
48
49
from bzrlib.patiencediff import PatienceSequenceMatcher
49
 
from bzrlib.repofmt import (
50
 
    knitpack_repo,
51
 
    pack_repo,
52
 
    )
 
50
from bzrlib.repofmt import pack_repo
53
51
from bzrlib.tests import (
54
52
    TestCase,
55
53
    TestCaseWithMemoryTransport,
62
60
    network_bytes_to_kind_and_offset,
63
61
    RecordingVersionedFilesDecorator,
64
62
    )
65
 
from bzrlib.tests import (
66
 
    features,
67
 
    )
68
 
 
69
 
 
70
 
compiled_knit_feature = features.ModuleAvailableFeature(
71
 
    'bzrlib._knit_load_data_pyx')
 
63
 
 
64
 
 
65
compiled_knit_feature = tests.ModuleAvailableFeature(
 
66
                            'bzrlib._knit_load_data_pyx')
72
67
 
73
68
 
74
69
class KnitContentTestsMixin(object):
330
325
            transport.append_bytes(packname, bytes)
331
326
        writer = pack.ContainerWriter(write_data)
332
327
        writer.begin()
333
 
        access = pack_repo._DirectPackAccess({})
 
328
        access = _DirectPackAccess({})
334
329
        access.set_writer(writer, index, (transport, packname))
335
330
        return access, writer
336
331
 
343
338
        writer.end()
344
339
        return memos
345
340
 
346
 
    def test_pack_collection_pack_retries(self):
347
 
        """An explicit pack of a pack collection succeeds even when a
348
 
        concurrent pack happens.
349
 
        """
350
 
        builder = self.make_branch_builder('.')
351
 
        builder.start_series()
352
 
        builder.build_snapshot('rev-1', None, [
353
 
            ('add', ('', 'root-id', 'directory', None)),
354
 
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
355
 
            ])
356
 
        builder.build_snapshot('rev-2', ['rev-1'], [
357
 
            ('modify', ('file-id', 'content\nrev 2\n')),
358
 
            ])
359
 
        builder.build_snapshot('rev-3', ['rev-2'], [
360
 
            ('modify', ('file-id', 'content\nrev 3\n')),
361
 
            ])
362
 
        self.addCleanup(builder.finish_series)
363
 
        b = builder.get_branch()
364
 
        self.addCleanup(b.lock_write().unlock)
365
 
        repo = b.repository
366
 
        collection = repo._pack_collection
367
 
        # Concurrently repack the repo.
368
 
        reopened_repo = repo.bzrdir.open_repository()
369
 
        reopened_repo.pack()
370
 
        # Pack the new pack.
371
 
        collection.pack()
372
 
 
373
341
    def make_vf_for_retrying(self):
374
342
        """Create 3 packs and a reload function.
375
343
 
402
370
        collection = repo._pack_collection
403
371
        collection.ensure_loaded()
404
372
        orig_packs = collection.packs
405
 
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
 
373
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
406
374
        new_pack = packer.pack()
407
375
        # forget about the new pack
408
376
        collection.reset()
447
415
        except _TestException, e:
448
416
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
449
417
                                                 exc_info=sys.exc_info())
450
 
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
451
418
        return retry_exc
452
419
 
453
420
    def test_read_from_several_packs(self):
462
429
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
463
430
        writer.end()
464
431
        transport = self.get_transport()
465
 
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
 
432
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
466
433
            "FOOBAR":(transport, 'pack2'),
467
434
            "BAZ":(transport, 'pack3')})
468
435
        self.assertEqual(['1234567890', '12345', 'alpha'],
478
445
 
479
446
    def test_set_writer(self):
480
447
        """The writer should be settable post construction."""
481
 
        access = pack_repo._DirectPackAccess({})
 
448
        access = _DirectPackAccess({})
482
449
        transport = self.get_transport()
483
450
        packname = 'packfile'
484
451
        index = 'foo'
496
463
        transport = self.get_transport()
497
464
        reload_called, reload_func = self.make_reload_func()
498
465
        # Note that the index key has changed from 'foo' to 'bar'
499
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
 
466
        access = _DirectPackAccess({'bar':(transport, 'packname')},
500
467
                                   reload_func=reload_func)
501
468
        e = self.assertListRaises(errors.RetryWithNewPacks,
502
469
                                  access.get_raw_records, memos)
511
478
        memos = self.make_pack_file()
512
479
        transport = self.get_transport()
513
480
        # Note that the index key has changed from 'foo' to 'bar'
514
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
 
481
        access = _DirectPackAccess({'bar':(transport, 'packname')})
515
482
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
516
483
 
517
484
    def test_missing_file_raises_retry(self):
519
486
        transport = self.get_transport()
520
487
        reload_called, reload_func = self.make_reload_func()
521
488
        # Note that the 'filename' has been changed to 'different-packname'
522
 
        access = pack_repo._DirectPackAccess(
523
 
            {'foo':(transport, 'different-packname')},
524
 
            reload_func=reload_func)
 
489
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
 
490
                                   reload_func=reload_func)
525
491
        e = self.assertListRaises(errors.RetryWithNewPacks,
526
492
                                  access.get_raw_records, memos)
527
493
        # The file has gone missing, so we assume we need to reload
535
501
        memos = self.make_pack_file()
536
502
        transport = self.get_transport()
537
503
        # Note that the 'filename' has been changed to 'different-packname'
538
 
        access = pack_repo._DirectPackAccess(
539
 
            {'foo': (transport, 'different-packname')})
 
504
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
540
505
        e = self.assertListRaises(errors.NoSuchFile,
541
506
                                  access.get_raw_records, memos)
542
507
 
546
511
        failing_transport = MockReadvFailingTransport(
547
512
                                [transport.get_bytes('packname')])
548
513
        reload_called, reload_func = self.make_reload_func()
549
 
        access = pack_repo._DirectPackAccess(
550
 
            {'foo': (failing_transport, 'packname')},
551
 
            reload_func=reload_func)
 
514
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
 
515
                                   reload_func=reload_func)
552
516
        # Asking for a single record will not trigger the Mock failure
553
517
        self.assertEqual(['1234567890'],
554
518
            list(access.get_raw_records(memos[:1])))
570
534
        failing_transport = MockReadvFailingTransport(
571
535
                                [transport.get_bytes('packname')])
572
536
        reload_called, reload_func = self.make_reload_func()
573
 
        access = pack_repo._DirectPackAccess(
574
 
            {'foo':(failing_transport, 'packname')})
 
537
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
575
538
        # Asking for a single record will not trigger the Mock failure
576
539
        self.assertEqual(['1234567890'],
577
540
            list(access.get_raw_records(memos[:1])))
582
545
                                  access.get_raw_records, memos)
583
546
 
584
547
    def test_reload_or_raise_no_reload(self):
585
 
        access = pack_repo._DirectPackAccess({}, reload_func=None)
 
548
        access = _DirectPackAccess({}, reload_func=None)
586
549
        retry_exc = self.make_retry_exception()
587
550
        # Without a reload_func, we will just re-raise the original exception
588
551
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
589
552
 
590
553
    def test_reload_or_raise_reload_changed(self):
591
554
        reload_called, reload_func = self.make_reload_func(return_val=True)
592
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
555
        access = _DirectPackAccess({}, reload_func=reload_func)
593
556
        retry_exc = self.make_retry_exception()
594
557
        access.reload_or_raise(retry_exc)
595
558
        self.assertEqual([1], reload_called)
599
562
 
600
563
    def test_reload_or_raise_reload_no_change(self):
601
564
        reload_called, reload_func = self.make_reload_func(return_val=False)
602
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
565
        access = _DirectPackAccess({}, reload_func=reload_func)
603
566
        retry_exc = self.make_retry_exception()
604
567
        # If reload_occurred is False, then we consider it an error to have
605
568
        # reload_func() return False (no changes).
736
699
 
737
700
    def make_multiple_records(self):
738
701
        """Create the content for multiple records."""
739
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
702
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
740
703
        total_txt = []
741
704
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
742
705
                                        'foo\n'
745
708
                                        % (sha1sum,))
746
709
        record_1 = (0, len(gz_txt), sha1sum)
747
710
        total_txt.append(gz_txt)
748
 
        sha1sum = osutils.sha_string('baz\n')
 
711
        sha1sum = osutils.sha('baz\n').hexdigest()
749
712
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
750
713
                                        'baz\n'
751
714
                                        'end rev-id-2\n'
755
718
        return total_txt, record_1, record_2
756
719
 
757
720
    def test_valid_knit_data(self):
758
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
721
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
759
722
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
760
723
                                        'foo\n'
761
724
                                        'bar\n'
792
755
                         raw_contents)
793
756
 
794
757
    def test_not_enough_lines(self):
795
 
        sha1sum = osutils.sha_string('foo\n')
 
758
        sha1sum = osutils.sha('foo\n').hexdigest()
796
759
        # record says 2 lines data says 1
797
760
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
798
761
                                        'foo\n'
810
773
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
811
774
 
812
775
    def test_too_many_lines(self):
813
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
776
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
814
777
        # record says 1 lines data says 2
815
778
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
816
779
                                        'foo\n'
829
792
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
830
793
 
831
794
    def test_mismatched_version_id(self):
832
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
795
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
833
796
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
834
797
                                        'foo\n'
835
798
                                        'bar\n'
848
811
            knit._read_records_iter_raw(records))
849
812
 
850
813
    def test_uncompressed_data(self):
851
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
814
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
852
815
        txt = ('version rev-id-1 2 %s\n'
853
816
               'foo\n'
854
817
               'bar\n'
868
831
            knit._read_records_iter_raw(records))
869
832
 
870
833
    def test_corrupted_data(self):
871
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
834
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
872
835
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
873
836
                                        'foo\n'
874
837
                                        'bar\n'
1196
1159
            self.assertRaises(errors.KnitCorrupt, index.keys)
1197
1160
        except TypeError, e:
1198
1161
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1199
 
                           ' not exceptions.IndexError')):
 
1162
                           ' not exceptions.IndexError')
 
1163
                and sys.version_info[0:2] >= (2,5)):
1200
1164
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1201
1165
                                  ' raising new style exceptions with python'
1202
1166
                                  ' >=2.5')
1215
1179
            self.assertRaises(errors.KnitCorrupt, index.keys)
1216
1180
        except TypeError, e:
1217
1181
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1218
 
                           ' not exceptions.ValueError')):
 
1182
                           ' not exceptions.ValueError')
 
1183
                and sys.version_info[0:2] >= (2,5)):
1219
1184
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1220
1185
                                  ' raising new style exceptions with python'
1221
1186
                                  ' >=2.5')
1234
1199
            self.assertRaises(errors.KnitCorrupt, index.keys)
1235
1200
        except TypeError, e:
1236
1201
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1237
 
                           ' not exceptions.ValueError')):
 
1202
                           ' not exceptions.ValueError')
 
1203
                and sys.version_info[0:2] >= (2,5)):
1238
1204
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1239
1205
                                  ' raising new style exceptions with python'
1240
1206
                                  ' >=2.5')
1251
1217
            self.assertRaises(errors.KnitCorrupt, index.keys)
1252
1218
        except TypeError, e:
1253
1219
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1254
 
                           ' not exceptions.ValueError')):
 
1220
                           ' not exceptions.ValueError')
 
1221
                and sys.version_info[0:2] >= (2,5)):
1255
1222
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1256
1223
                                  ' raising new style exceptions with python'
1257
1224
                                  ' >=2.5')
1268
1235
            self.assertRaises(errors.KnitCorrupt, index.keys)
1269
1236
        except TypeError, e:
1270
1237
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1271
 
                           ' not exceptions.ValueError')):
 
1238
                           ' not exceptions.ValueError')
 
1239
                and sys.version_info[0:2] >= (2,5)):
1272
1240
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1273
1241
                                  ' raising new style exceptions with python'
1274
1242
                                  ' >=2.5')
1603
1571
        # could leave an empty .kndx file, which bzr would later claim was a
1604
1572
        # corrupted file since the header was not present. In reality, the file
1605
1573
        # just wasn't created, so it should be ignored.
1606
 
        t = transport.get_transport_from_path('.')
 
1574
        t = transport.get_transport('.')
1607
1575
        t.put_bytes('test.kndx', '')
1608
1576
 
1609
1577
        knit = self.make_test_knit()
1610
1578
 
1611
1579
    def test_knit_index_checks_header(self):
1612
 
        t = transport.get_transport_from_path('.')
 
1580
        t = transport.get_transport('.')
1613
1581
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1614
1582
        k = self.make_test_knit()
1615
1583
        self.assertRaises(KnitHeaderError, k.keys)
2443
2411
        key_basis = ('bar',)
2444
2412
        key_missing = ('missing',)
2445
2413
        test.add_lines(key, (), ['foo\n'])
2446
 
        key_sha1sum = osutils.sha_string('foo\n')
 
2414
        key_sha1sum = osutils.sha('foo\n').hexdigest()
2447
2415
        sha1s = test.get_sha1s([key])
2448
2416
        self.assertEqual({key: key_sha1sum}, sha1s)
2449
2417
        self.assertEqual([], basis.calls)
2451
2419
        # directly (rather than via text reconstruction) so that remote servers
2452
2420
        # etc don't have to answer with full content.
2453
2421
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2454
 
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
 
2422
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
2455
2423
        basis.calls = []
2456
2424
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2457
2425
        self.assertEqual({key: key_sha1sum,