← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

Merge bzr.dev, update to use new hooks.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2006-2010 Canonical Ltd
 
1
# Copyright (C) 2006-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
 
20
import gzip
20
21
import sys
21
22
 
22
23
from bzrlib import (
27
28
    pack,
28
29
    tests,
29
30
    transport,
30
 
    tuned_gzip,
31
31
    )
32
32
from bzrlib.errors import (
33
33
    KnitHeaderError,
40
40
    KnitVersionedFiles,
41
41
    PlainKnitContent,
42
42
    _VFContentMapGenerator,
43
 
    _DirectPackAccess,
44
43
    _KndxIndex,
45
44
    _KnitGraphIndex,
46
45
    _KnitKeyAccess,
47
46
    make_file_factory,
48
47
    )
49
48
from bzrlib.patiencediff import PatienceSequenceMatcher
50
 
from bzrlib.repofmt import pack_repo
 
49
from bzrlib.repofmt import (
 
50
    knitpack_repo,
 
51
    pack_repo,
 
52
    )
51
53
from bzrlib.tests import (
52
54
    TestCase,
53
55
    TestCaseWithMemoryTransport,
60
62
    network_bytes_to_kind_and_offset,
61
63
    RecordingVersionedFilesDecorator,
62
64
    )
63
 
 
64
 
 
65
 
compiled_knit_feature = tests.ModuleAvailableFeature(
66
 
                            'bzrlib._knit_load_data_pyx')
 
65
from bzrlib.tests import (
 
66
    features,
 
67
    )
 
68
 
 
69
 
 
70
compiled_knit_feature = features.ModuleAvailableFeature(
 
71
    'bzrlib._knit_load_data_pyx')
67
72
 
68
73
 
69
74
class KnitContentTestsMixin(object):
325
330
            transport.append_bytes(packname, bytes)
326
331
        writer = pack.ContainerWriter(write_data)
327
332
        writer.begin()
328
 
        access = _DirectPackAccess({})
 
333
        access = pack_repo._DirectPackAccess({})
329
334
        access.set_writer(writer, index, (transport, packname))
330
335
        return access, writer
331
336
 
338
343
        writer.end()
339
344
        return memos
340
345
 
 
346
    def test_pack_collection_pack_retries(self):
 
347
        """An explicit pack of a pack collection succeeds even when a
 
348
        concurrent pack happens.
 
349
        """
 
350
        builder = self.make_branch_builder('.')
 
351
        builder.start_series()
 
352
        builder.build_snapshot('rev-1', None, [
 
353
            ('add', ('', 'root-id', 'directory', None)),
 
354
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
 
355
            ])
 
356
        builder.build_snapshot('rev-2', ['rev-1'], [
 
357
            ('modify', ('file-id', 'content\nrev 2\n')),
 
358
            ])
 
359
        builder.build_snapshot('rev-3', ['rev-2'], [
 
360
            ('modify', ('file-id', 'content\nrev 3\n')),
 
361
            ])
 
362
        self.addCleanup(builder.finish_series)
 
363
        b = builder.get_branch()
 
364
        self.addCleanup(b.lock_write().unlock)
 
365
        repo = b.repository
 
366
        collection = repo._pack_collection
 
367
        # Concurrently repack the repo.
 
368
        reopened_repo = repo.bzrdir.open_repository()
 
369
        reopened_repo.pack()
 
370
        # Pack the new pack.
 
371
        collection.pack()
 
372
 
341
373
    def make_vf_for_retrying(self):
342
374
        """Create 3 packs and a reload function.
343
375
 
370
402
        collection = repo._pack_collection
371
403
        collection.ensure_loaded()
372
404
        orig_packs = collection.packs
373
 
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
 
405
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
374
406
        new_pack = packer.pack()
375
407
        # forget about the new pack
376
408
        collection.reset()
415
447
        except _TestException, e:
416
448
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
417
449
                                                 exc_info=sys.exc_info())
 
450
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
418
451
        return retry_exc
419
452
 
420
453
    def test_read_from_several_packs(self):
429
462
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
430
463
        writer.end()
431
464
        transport = self.get_transport()
432
 
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
 
465
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
433
466
            "FOOBAR":(transport, 'pack2'),
434
467
            "BAZ":(transport, 'pack3')})
435
468
        self.assertEqual(['1234567890', '12345', 'alpha'],
445
478
 
446
479
    def test_set_writer(self):
447
480
        """The writer should be settable post construction."""
448
 
        access = _DirectPackAccess({})
 
481
        access = pack_repo._DirectPackAccess({})
449
482
        transport = self.get_transport()
450
483
        packname = 'packfile'
451
484
        index = 'foo'
463
496
        transport = self.get_transport()
464
497
        reload_called, reload_func = self.make_reload_func()
465
498
        # Note that the index key has changed from 'foo' to 'bar'
466
 
        access = _DirectPackAccess({'bar':(transport, 'packname')},
 
499
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
467
500
                                   reload_func=reload_func)
468
501
        e = self.assertListRaises(errors.RetryWithNewPacks,
469
502
                                  access.get_raw_records, memos)
478
511
        memos = self.make_pack_file()
479
512
        transport = self.get_transport()
480
513
        # Note that the index key has changed from 'foo' to 'bar'
481
 
        access = _DirectPackAccess({'bar':(transport, 'packname')})
 
514
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
482
515
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
483
516
 
484
517
    def test_missing_file_raises_retry(self):
486
519
        transport = self.get_transport()
487
520
        reload_called, reload_func = self.make_reload_func()
488
521
        # Note that the 'filename' has been changed to 'different-packname'
489
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
490
 
                                   reload_func=reload_func)
 
522
        access = pack_repo._DirectPackAccess(
 
523
            {'foo':(transport, 'different-packname')},
 
524
            reload_func=reload_func)
491
525
        e = self.assertListRaises(errors.RetryWithNewPacks,
492
526
                                  access.get_raw_records, memos)
493
527
        # The file has gone missing, so we assume we need to reload
501
535
        memos = self.make_pack_file()
502
536
        transport = self.get_transport()
503
537
        # Note that the 'filename' has been changed to 'different-packname'
504
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
 
538
        access = pack_repo._DirectPackAccess(
 
539
            {'foo': (transport, 'different-packname')})
505
540
        e = self.assertListRaises(errors.NoSuchFile,
506
541
                                  access.get_raw_records, memos)
507
542
 
511
546
        failing_transport = MockReadvFailingTransport(
512
547
                                [transport.get_bytes('packname')])
513
548
        reload_called, reload_func = self.make_reload_func()
514
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
515
 
                                   reload_func=reload_func)
 
549
        access = pack_repo._DirectPackAccess(
 
550
            {'foo': (failing_transport, 'packname')},
 
551
            reload_func=reload_func)
516
552
        # Asking for a single record will not trigger the Mock failure
517
553
        self.assertEqual(['1234567890'],
518
554
            list(access.get_raw_records(memos[:1])))
534
570
        failing_transport = MockReadvFailingTransport(
535
571
                                [transport.get_bytes('packname')])
536
572
        reload_called, reload_func = self.make_reload_func()
537
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
 
573
        access = pack_repo._DirectPackAccess(
 
574
            {'foo':(failing_transport, 'packname')})
538
575
        # Asking for a single record will not trigger the Mock failure
539
576
        self.assertEqual(['1234567890'],
540
577
            list(access.get_raw_records(memos[:1])))
545
582
                                  access.get_raw_records, memos)
546
583
 
547
584
    def test_reload_or_raise_no_reload(self):
548
 
        access = _DirectPackAccess({}, reload_func=None)
 
585
        access = pack_repo._DirectPackAccess({}, reload_func=None)
549
586
        retry_exc = self.make_retry_exception()
550
587
        # Without a reload_func, we will just re-raise the original exception
551
588
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
552
589
 
553
590
    def test_reload_or_raise_reload_changed(self):
554
591
        reload_called, reload_func = self.make_reload_func(return_val=True)
555
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
592
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
556
593
        retry_exc = self.make_retry_exception()
557
594
        access.reload_or_raise(retry_exc)
558
595
        self.assertEqual([1], reload_called)
562
599
 
563
600
    def test_reload_or_raise_reload_no_change(self):
564
601
        reload_called, reload_func = self.make_reload_func(return_val=False)
565
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
602
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
566
603
        retry_exc = self.make_retry_exception()
567
604
        # If reload_occurred is False, then we consider it an error to have
568
605
        # reload_func() return False (no changes).
692
729
 
693
730
    def create_gz_content(self, text):
694
731
        sio = StringIO()
695
 
        gz_file = tuned_gzip.GzipFile(mode='wb', fileobj=sio)
 
732
        gz_file = gzip.GzipFile(mode='wb', fileobj=sio)
696
733
        gz_file.write(text)
697
734
        gz_file.close()
698
735
        return sio.getvalue()
699
736
 
700
737
    def make_multiple_records(self):
701
738
        """Create the content for multiple records."""
702
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
739
        sha1sum = osutils.sha_string('foo\nbar\n')
703
740
        total_txt = []
704
741
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
705
742
                                        'foo\n'
708
745
                                        % (sha1sum,))
709
746
        record_1 = (0, len(gz_txt), sha1sum)
710
747
        total_txt.append(gz_txt)
711
 
        sha1sum = osutils.sha('baz\n').hexdigest()
 
748
        sha1sum = osutils.sha_string('baz\n')
712
749
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
713
750
                                        'baz\n'
714
751
                                        'end rev-id-2\n'
718
755
        return total_txt, record_1, record_2
719
756
 
720
757
    def test_valid_knit_data(self):
721
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
758
        sha1sum = osutils.sha_string('foo\nbar\n')
722
759
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
723
760
                                        'foo\n'
724
761
                                        'bar\n'
755
792
                         raw_contents)
756
793
 
757
794
    def test_not_enough_lines(self):
758
 
        sha1sum = osutils.sha('foo\n').hexdigest()
 
795
        sha1sum = osutils.sha_string('foo\n')
759
796
        # record says 2 lines data says 1
760
797
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
761
798
                                        'foo\n'
773
810
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
774
811
 
775
812
    def test_too_many_lines(self):
776
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
813
        sha1sum = osutils.sha_string('foo\nbar\n')
777
814
        # record says 1 lines data says 2
778
815
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
779
816
                                        'foo\n'
792
829
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
793
830
 
794
831
    def test_mismatched_version_id(self):
795
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
832
        sha1sum = osutils.sha_string('foo\nbar\n')
796
833
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
797
834
                                        'foo\n'
798
835
                                        'bar\n'
811
848
            knit._read_records_iter_raw(records))
812
849
 
813
850
    def test_uncompressed_data(self):
814
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
851
        sha1sum = osutils.sha_string('foo\nbar\n')
815
852
        txt = ('version rev-id-1 2 %s\n'
816
853
               'foo\n'
817
854
               'bar\n'
831
868
            knit._read_records_iter_raw(records))
832
869
 
833
870
    def test_corrupted_data(self):
834
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
871
        sha1sum = osutils.sha_string('foo\nbar\n')
835
872
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
836
873
                                        'foo\n'
837
874
                                        'bar\n'
1159
1196
            self.assertRaises(errors.KnitCorrupt, index.keys)
1160
1197
        except TypeError, e:
1161
1198
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1162
 
                           ' not exceptions.IndexError')
1163
 
                and sys.version_info[0:2] >= (2,5)):
 
1199
                           ' not exceptions.IndexError')):
1164
1200
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1165
1201
                                  ' raising new style exceptions with python'
1166
1202
                                  ' >=2.5')
1179
1215
            self.assertRaises(errors.KnitCorrupt, index.keys)
1180
1216
        except TypeError, e:
1181
1217
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1182
 
                           ' not exceptions.ValueError')
1183
 
                and sys.version_info[0:2] >= (2,5)):
 
1218
                           ' not exceptions.ValueError')):
1184
1219
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1185
1220
                                  ' raising new style exceptions with python'
1186
1221
                                  ' >=2.5')
1199
1234
            self.assertRaises(errors.KnitCorrupt, index.keys)
1200
1235
        except TypeError, e:
1201
1236
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1202
 
                           ' not exceptions.ValueError')
1203
 
                and sys.version_info[0:2] >= (2,5)):
 
1237
                           ' not exceptions.ValueError')):
1204
1238
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1205
1239
                                  ' raising new style exceptions with python'
1206
1240
                                  ' >=2.5')
1217
1251
            self.assertRaises(errors.KnitCorrupt, index.keys)
1218
1252
        except TypeError, e:
1219
1253
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1220
 
                           ' not exceptions.ValueError')
1221
 
                and sys.version_info[0:2] >= (2,5)):
 
1254
                           ' not exceptions.ValueError')):
1222
1255
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1223
1256
                                  ' raising new style exceptions with python'
1224
1257
                                  ' >=2.5')
1235
1268
            self.assertRaises(errors.KnitCorrupt, index.keys)
1236
1269
        except TypeError, e:
1237
1270
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1238
 
                           ' not exceptions.ValueError')
1239
 
                and sys.version_info[0:2] >= (2,5)):
 
1271
                           ' not exceptions.ValueError')):
1240
1272
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1241
1273
                                  ' raising new style exceptions with python'
1242
1274
                                  ' >=2.5')
1571
1603
        # could leave an empty .kndx file, which bzr would later claim was a
1572
1604
        # corrupted file since the header was not present. In reality, the file
1573
1605
        # just wasn't created, so it should be ignored.
1574
 
        t = transport.get_transport('.')
 
1606
        t = transport.get_transport_from_path('.')
1575
1607
        t.put_bytes('test.kndx', '')
1576
1608
 
1577
1609
        knit = self.make_test_knit()
1578
1610
 
1579
1611
    def test_knit_index_checks_header(self):
1580
 
        t = transport.get_transport('.')
 
1612
        t = transport.get_transport_from_path('.')
1581
1613
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1582
1614
        k = self.make_test_knit()
1583
1615
        self.assertRaises(KnitHeaderError, k.keys)
2411
2443
        key_basis = ('bar',)
2412
2444
        key_missing = ('missing',)
2413
2445
        test.add_lines(key, (), ['foo\n'])
2414
 
        key_sha1sum = osutils.sha('foo\n').hexdigest()
 
2446
        key_sha1sum = osutils.sha_string('foo\n')
2415
2447
        sha1s = test.get_sha1s([key])
2416
2448
        self.assertEqual({key: key_sha1sum}, sha1s)
2417
2449
        self.assertEqual([], basis.calls)
2419
2451
        # directly (rather than via text reconstruction) so that remote servers
2420
2452
        # etc don't have to answer with full content.
2421
2453
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2422
 
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
2454
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
2423
2455
        basis.calls = []
2424
2456
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2425
2457
        self.assertEqual({key: key_sha1sum,