← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Andrew Bennetts
  • Date: 2010-11-22 03:35:24 UTC
  • mto: This revision was merged to the branch mainline in revision 5547.
  • Revision ID: andrew.bennetts@canonical.com-20101122033524-ouxj0onm3gtkimx3
Remove RepositoryFormatCHK1 and RepositoryFormatCHK2.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2006-2011 Canonical Ltd
 
1
# Copyright (C) 2006-2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
20
 
import gzip
21
20
import sys
22
21
 
23
22
from bzrlib import (
28
27
    pack,
29
28
    tests,
30
29
    transport,
 
30
    tuned_gzip,
31
31
    )
32
32
from bzrlib.errors import (
33
33
    KnitHeaderError,
40
40
    KnitVersionedFiles,
41
41
    PlainKnitContent,
42
42
    _VFContentMapGenerator,
 
43
    _DirectPackAccess,
43
44
    _KndxIndex,
44
45
    _KnitGraphIndex,
45
46
    _KnitKeyAccess,
46
47
    make_file_factory,
47
48
    )
48
49
from bzrlib.patiencediff import PatienceSequenceMatcher
49
 
from bzrlib.repofmt import (
50
 
    knitpack_repo,
51
 
    pack_repo,
52
 
    )
 
50
from bzrlib.repofmt import pack_repo
53
51
from bzrlib.tests import (
54
52
    TestCase,
55
53
    TestCaseWithMemoryTransport,
62
60
    network_bytes_to_kind_and_offset,
63
61
    RecordingVersionedFilesDecorator,
64
62
    )
65
 
from bzrlib.tests import (
66
 
    features,
67
 
    )
68
 
 
69
 
 
70
 
compiled_knit_feature = features.ModuleAvailableFeature(
71
 
    'bzrlib._knit_load_data_pyx')
 
63
 
 
64
 
 
65
compiled_knit_feature = tests.ModuleAvailableFeature(
 
66
                            'bzrlib._knit_load_data_pyx')
72
67
 
73
68
 
74
69
class KnitContentTestsMixin(object):
330
325
            transport.append_bytes(packname, bytes)
331
326
        writer = pack.ContainerWriter(write_data)
332
327
        writer.begin()
333
 
        access = pack_repo._DirectPackAccess({})
 
328
        access = _DirectPackAccess({})
334
329
        access.set_writer(writer, index, (transport, packname))
335
330
        return access, writer
336
331
 
343
338
        writer.end()
344
339
        return memos
345
340
 
346
 
    def test_pack_collection_pack_retries(self):
347
 
        """An explicit pack of a pack collection succeeds even when a
348
 
        concurrent pack happens.
349
 
        """
350
 
        builder = self.make_branch_builder('.')
351
 
        builder.start_series()
352
 
        builder.build_snapshot('rev-1', None, [
353
 
            ('add', ('', 'root-id', 'directory', None)),
354
 
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
355
 
            ])
356
 
        builder.build_snapshot('rev-2', ['rev-1'], [
357
 
            ('modify', ('file-id', 'content\nrev 2\n')),
358
 
            ])
359
 
        builder.build_snapshot('rev-3', ['rev-2'], [
360
 
            ('modify', ('file-id', 'content\nrev 3\n')),
361
 
            ])
362
 
        self.addCleanup(builder.finish_series)
363
 
        b = builder.get_branch()
364
 
        self.addCleanup(b.lock_write().unlock)
365
 
        repo = b.repository
366
 
        collection = repo._pack_collection
367
 
        # Concurrently repack the repo.
368
 
        reopened_repo = repo.bzrdir.open_repository()
369
 
        reopened_repo.pack()
370
 
        # Pack the new pack.
371
 
        collection.pack()
372
 
 
373
341
    def make_vf_for_retrying(self):
374
342
        """Create 3 packs and a reload function.
375
343
 
402
370
        collection = repo._pack_collection
403
371
        collection.ensure_loaded()
404
372
        orig_packs = collection.packs
405
 
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
 
373
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
406
374
        new_pack = packer.pack()
407
375
        # forget about the new pack
408
376
        collection.reset()
447
415
        except _TestException, e:
448
416
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
449
417
                                                 exc_info=sys.exc_info())
450
 
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
451
418
        return retry_exc
452
419
 
453
420
    def test_read_from_several_packs(self):
462
429
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
463
430
        writer.end()
464
431
        transport = self.get_transport()
465
 
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
 
432
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
466
433
            "FOOBAR":(transport, 'pack2'),
467
434
            "BAZ":(transport, 'pack3')})
468
435
        self.assertEqual(['1234567890', '12345', 'alpha'],
478
445
 
479
446
    def test_set_writer(self):
480
447
        """The writer should be settable post construction."""
481
 
        access = pack_repo._DirectPackAccess({})
 
448
        access = _DirectPackAccess({})
482
449
        transport = self.get_transport()
483
450
        packname = 'packfile'
484
451
        index = 'foo'
496
463
        transport = self.get_transport()
497
464
        reload_called, reload_func = self.make_reload_func()
498
465
        # Note that the index key has changed from 'foo' to 'bar'
499
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
 
466
        access = _DirectPackAccess({'bar':(transport, 'packname')},
500
467
                                   reload_func=reload_func)
501
468
        e = self.assertListRaises(errors.RetryWithNewPacks,
502
469
                                  access.get_raw_records, memos)
511
478
        memos = self.make_pack_file()
512
479
        transport = self.get_transport()
513
480
        # Note that the index key has changed from 'foo' to 'bar'
514
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
 
481
        access = _DirectPackAccess({'bar':(transport, 'packname')})
515
482
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
516
483
 
517
484
    def test_missing_file_raises_retry(self):
519
486
        transport = self.get_transport()
520
487
        reload_called, reload_func = self.make_reload_func()
521
488
        # Note that the 'filename' has been changed to 'different-packname'
522
 
        access = pack_repo._DirectPackAccess(
523
 
            {'foo':(transport, 'different-packname')},
524
 
            reload_func=reload_func)
 
489
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
 
490
                                   reload_func=reload_func)
525
491
        e = self.assertListRaises(errors.RetryWithNewPacks,
526
492
                                  access.get_raw_records, memos)
527
493
        # The file has gone missing, so we assume we need to reload
535
501
        memos = self.make_pack_file()
536
502
        transport = self.get_transport()
537
503
        # Note that the 'filename' has been changed to 'different-packname'
538
 
        access = pack_repo._DirectPackAccess(
539
 
            {'foo': (transport, 'different-packname')})
 
504
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
540
505
        e = self.assertListRaises(errors.NoSuchFile,
541
506
                                  access.get_raw_records, memos)
542
507
 
546
511
        failing_transport = MockReadvFailingTransport(
547
512
                                [transport.get_bytes('packname')])
548
513
        reload_called, reload_func = self.make_reload_func()
549
 
        access = pack_repo._DirectPackAccess(
550
 
            {'foo': (failing_transport, 'packname')},
551
 
            reload_func=reload_func)
 
514
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
 
515
                                   reload_func=reload_func)
552
516
        # Asking for a single record will not trigger the Mock failure
553
517
        self.assertEqual(['1234567890'],
554
518
            list(access.get_raw_records(memos[:1])))
570
534
        failing_transport = MockReadvFailingTransport(
571
535
                                [transport.get_bytes('packname')])
572
536
        reload_called, reload_func = self.make_reload_func()
573
 
        access = pack_repo._DirectPackAccess(
574
 
            {'foo':(failing_transport, 'packname')})
 
537
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
575
538
        # Asking for a single record will not trigger the Mock failure
576
539
        self.assertEqual(['1234567890'],
577
540
            list(access.get_raw_records(memos[:1])))
582
545
                                  access.get_raw_records, memos)
583
546
 
584
547
    def test_reload_or_raise_no_reload(self):
585
 
        access = pack_repo._DirectPackAccess({}, reload_func=None)
 
548
        access = _DirectPackAccess({}, reload_func=None)
586
549
        retry_exc = self.make_retry_exception()
587
550
        # Without a reload_func, we will just re-raise the original exception
588
551
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
589
552
 
590
553
    def test_reload_or_raise_reload_changed(self):
591
554
        reload_called, reload_func = self.make_reload_func(return_val=True)
592
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
555
        access = _DirectPackAccess({}, reload_func=reload_func)
593
556
        retry_exc = self.make_retry_exception()
594
557
        access.reload_or_raise(retry_exc)
595
558
        self.assertEqual([1], reload_called)
599
562
 
600
563
    def test_reload_or_raise_reload_no_change(self):
601
564
        reload_called, reload_func = self.make_reload_func(return_val=False)
602
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
565
        access = _DirectPackAccess({}, reload_func=reload_func)
603
566
        retry_exc = self.make_retry_exception()
604
567
        # If reload_occurred is False, then we consider it an error to have
605
568
        # reload_func() return False (no changes).
729
692
 
730
693
    def create_gz_content(self, text):
731
694
        sio = StringIO()
732
 
        gz_file = gzip.GzipFile(mode='wb', fileobj=sio)
 
695
        gz_file = tuned_gzip.GzipFile(mode='wb', fileobj=sio)
733
696
        gz_file.write(text)
734
697
        gz_file.close()
735
698
        return sio.getvalue()
736
699
 
737
700
    def make_multiple_records(self):
738
701
        """Create the content for multiple records."""
739
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
702
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
740
703
        total_txt = []
741
704
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
742
705
                                        'foo\n'
745
708
                                        % (sha1sum,))
746
709
        record_1 = (0, len(gz_txt), sha1sum)
747
710
        total_txt.append(gz_txt)
748
 
        sha1sum = osutils.sha_string('baz\n')
 
711
        sha1sum = osutils.sha('baz\n').hexdigest()
749
712
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
750
713
                                        'baz\n'
751
714
                                        'end rev-id-2\n'
755
718
        return total_txt, record_1, record_2
756
719
 
757
720
    def test_valid_knit_data(self):
758
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
721
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
759
722
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
760
723
                                        'foo\n'
761
724
                                        'bar\n'
792
755
                         raw_contents)
793
756
 
794
757
    def test_not_enough_lines(self):
795
 
        sha1sum = osutils.sha_string('foo\n')
 
758
        sha1sum = osutils.sha('foo\n').hexdigest()
796
759
        # record says 2 lines data says 1
797
760
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
798
761
                                        'foo\n'
810
773
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
811
774
 
812
775
    def test_too_many_lines(self):
813
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
776
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
814
777
        # record says 1 lines data says 2
815
778
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
816
779
                                        'foo\n'
829
792
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
830
793
 
831
794
    def test_mismatched_version_id(self):
832
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
795
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
833
796
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
834
797
                                        'foo\n'
835
798
                                        'bar\n'
848
811
            knit._read_records_iter_raw(records))
849
812
 
850
813
    def test_uncompressed_data(self):
851
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
814
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
852
815
        txt = ('version rev-id-1 2 %s\n'
853
816
               'foo\n'
854
817
               'bar\n'
868
831
            knit._read_records_iter_raw(records))
869
832
 
870
833
    def test_corrupted_data(self):
871
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
834
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
872
835
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
873
836
                                        'foo\n'
874
837
                                        'bar\n'
1196
1159
            self.assertRaises(errors.KnitCorrupt, index.keys)
1197
1160
        except TypeError, e:
1198
1161
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1199
 
                           ' not exceptions.IndexError')):
 
1162
                           ' not exceptions.IndexError')
 
1163
                and sys.version_info[0:2] >= (2,5)):
1200
1164
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1201
1165
                                  ' raising new style exceptions with python'
1202
1166
                                  ' >=2.5')
1215
1179
            self.assertRaises(errors.KnitCorrupt, index.keys)
1216
1180
        except TypeError, e:
1217
1181
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1218
 
                           ' not exceptions.ValueError')):
 
1182
                           ' not exceptions.ValueError')
 
1183
                and sys.version_info[0:2] >= (2,5)):
1219
1184
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1220
1185
                                  ' raising new style exceptions with python'
1221
1186
                                  ' >=2.5')
1234
1199
            self.assertRaises(errors.KnitCorrupt, index.keys)
1235
1200
        except TypeError, e:
1236
1201
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1237
 
                           ' not exceptions.ValueError')):
 
1202
                           ' not exceptions.ValueError')
 
1203
                and sys.version_info[0:2] >= (2,5)):
1238
1204
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1239
1205
                                  ' raising new style exceptions with python'
1240
1206
                                  ' >=2.5')
1251
1217
            self.assertRaises(errors.KnitCorrupt, index.keys)
1252
1218
        except TypeError, e:
1253
1219
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1254
 
                           ' not exceptions.ValueError')):
 
1220
                           ' not exceptions.ValueError')
 
1221
                and sys.version_info[0:2] >= (2,5)):
1255
1222
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1256
1223
                                  ' raising new style exceptions with python'
1257
1224
                                  ' >=2.5')
1268
1235
            self.assertRaises(errors.KnitCorrupt, index.keys)
1269
1236
        except TypeError, e:
1270
1237
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1271
 
                           ' not exceptions.ValueError')):
 
1238
                           ' not exceptions.ValueError')
 
1239
                and sys.version_info[0:2] >= (2,5)):
1272
1240
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1273
1241
                                  ' raising new style exceptions with python'
1274
1242
                                  ' >=2.5')
1603
1571
        # could leave an empty .kndx file, which bzr would later claim was a
1604
1572
        # corrupted file since the header was not present. In reality, the file
1605
1573
        # just wasn't created, so it should be ignored.
1606
 
        t = transport.get_transport_from_path('.')
 
1574
        t = transport.get_transport('.')
1607
1575
        t.put_bytes('test.kndx', '')
1608
1576
 
1609
1577
        knit = self.make_test_knit()
1610
1578
 
1611
1579
    def test_knit_index_checks_header(self):
1612
 
        t = transport.get_transport_from_path('.')
 
1580
        t = transport.get_transport('.')
1613
1581
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1614
1582
        k = self.make_test_knit()
1615
1583
        self.assertRaises(KnitHeaderError, k.keys)
2443
2411
        key_basis = ('bar',)
2444
2412
        key_missing = ('missing',)
2445
2413
        test.add_lines(key, (), ['foo\n'])
2446
 
        key_sha1sum = osutils.sha_string('foo\n')
 
2414
        key_sha1sum = osutils.sha('foo\n').hexdigest()
2447
2415
        sha1s = test.get_sha1s([key])
2448
2416
        self.assertEqual({key: key_sha1sum}, sha1s)
2449
2417
        self.assertEqual([], basis.calls)
2451
2419
        # directly (rather than via text reconstruction) so that remote servers
2452
2420
        # etc don't have to answer with full content.
2453
2421
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2454
 
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
 
2422
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
2455
2423
        basis.calls = []
2456
2424
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2457
2425
        self.assertEqual({key: key_sha1sum,