← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Andrew Bennetts
  • Date: 2010-10-13 00:26:41 UTC
  • mto: This revision was merged to the branch mainline in revision 5498.
  • Revision ID: andrew.bennetts@canonical.com-20101013002641-9tlh9k89mlj1666m
Keep docs-plain working.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2006-2011 Canonical Ltd
 
1
# Copyright (C) 2006-2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
20
 
import gzip
21
20
import sys
22
21
 
23
22
from bzrlib import (
28
27
    pack,
29
28
    tests,
30
29
    transport,
 
30
    tuned_gzip,
31
31
    )
32
32
from bzrlib.errors import (
33
33
    KnitHeaderError,
40
40
    KnitVersionedFiles,
41
41
    PlainKnitContent,
42
42
    _VFContentMapGenerator,
 
43
    _DirectPackAccess,
43
44
    _KndxIndex,
44
45
    _KnitGraphIndex,
45
46
    _KnitKeyAccess,
46
47
    make_file_factory,
47
48
    )
48
49
from bzrlib.patiencediff import PatienceSequenceMatcher
49
 
from bzrlib.repofmt import (
50
 
    knitpack_repo,
51
 
    pack_repo,
52
 
    )
 
50
from bzrlib.repofmt import pack_repo
53
51
from bzrlib.tests import (
54
52
    TestCase,
55
53
    TestCaseWithMemoryTransport,
327
325
            transport.append_bytes(packname, bytes)
328
326
        writer = pack.ContainerWriter(write_data)
329
327
        writer.begin()
330
 
        access = pack_repo._DirectPackAccess({})
 
328
        access = _DirectPackAccess({})
331
329
        access.set_writer(writer, index, (transport, packname))
332
330
        return access, writer
333
331
 
340
338
        writer.end()
341
339
        return memos
342
340
 
343
 
    def test_pack_collection_pack_retries(self):
344
 
        """An explicit pack of a pack collection succeeds even when a
345
 
        concurrent pack happens.
346
 
        """
347
 
        builder = self.make_branch_builder('.')
348
 
        builder.start_series()
349
 
        builder.build_snapshot('rev-1', None, [
350
 
            ('add', ('', 'root-id', 'directory', None)),
351
 
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
352
 
            ])
353
 
        builder.build_snapshot('rev-2', ['rev-1'], [
354
 
            ('modify', ('file-id', 'content\nrev 2\n')),
355
 
            ])
356
 
        builder.build_snapshot('rev-3', ['rev-2'], [
357
 
            ('modify', ('file-id', 'content\nrev 3\n')),
358
 
            ])
359
 
        self.addCleanup(builder.finish_series)
360
 
        b = builder.get_branch()
361
 
        self.addCleanup(b.lock_write().unlock)
362
 
        repo = b.repository
363
 
        collection = repo._pack_collection
364
 
        # Concurrently repack the repo.
365
 
        reopened_repo = repo.bzrdir.open_repository()
366
 
        reopened_repo.pack()
367
 
        # Pack the new pack.
368
 
        collection.pack()
369
 
 
370
341
    def make_vf_for_retrying(self):
371
342
        """Create 3 packs and a reload function.
372
343
 
399
370
        collection = repo._pack_collection
400
371
        collection.ensure_loaded()
401
372
        orig_packs = collection.packs
402
 
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
 
373
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
403
374
        new_pack = packer.pack()
404
375
        # forget about the new pack
405
376
        collection.reset()
444
415
        except _TestException, e:
445
416
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
446
417
                                                 exc_info=sys.exc_info())
447
 
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
448
418
        return retry_exc
449
419
 
450
420
    def test_read_from_several_packs(self):
459
429
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
460
430
        writer.end()
461
431
        transport = self.get_transport()
462
 
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
 
432
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
463
433
            "FOOBAR":(transport, 'pack2'),
464
434
            "BAZ":(transport, 'pack3')})
465
435
        self.assertEqual(['1234567890', '12345', 'alpha'],
475
445
 
476
446
    def test_set_writer(self):
477
447
        """The writer should be settable post construction."""
478
 
        access = pack_repo._DirectPackAccess({})
 
448
        access = _DirectPackAccess({})
479
449
        transport = self.get_transport()
480
450
        packname = 'packfile'
481
451
        index = 'foo'
493
463
        transport = self.get_transport()
494
464
        reload_called, reload_func = self.make_reload_func()
495
465
        # Note that the index key has changed from 'foo' to 'bar'
496
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
 
466
        access = _DirectPackAccess({'bar':(transport, 'packname')},
497
467
                                   reload_func=reload_func)
498
468
        e = self.assertListRaises(errors.RetryWithNewPacks,
499
469
                                  access.get_raw_records, memos)
508
478
        memos = self.make_pack_file()
509
479
        transport = self.get_transport()
510
480
        # Note that the index key has changed from 'foo' to 'bar'
511
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
 
481
        access = _DirectPackAccess({'bar':(transport, 'packname')})
512
482
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
513
483
 
514
484
    def test_missing_file_raises_retry(self):
516
486
        transport = self.get_transport()
517
487
        reload_called, reload_func = self.make_reload_func()
518
488
        # Note that the 'filename' has been changed to 'different-packname'
519
 
        access = pack_repo._DirectPackAccess(
520
 
            {'foo':(transport, 'different-packname')},
521
 
            reload_func=reload_func)
 
489
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
 
490
                                   reload_func=reload_func)
522
491
        e = self.assertListRaises(errors.RetryWithNewPacks,
523
492
                                  access.get_raw_records, memos)
524
493
        # The file has gone missing, so we assume we need to reload
532
501
        memos = self.make_pack_file()
533
502
        transport = self.get_transport()
534
503
        # Note that the 'filename' has been changed to 'different-packname'
535
 
        access = pack_repo._DirectPackAccess(
536
 
            {'foo': (transport, 'different-packname')})
 
504
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
537
505
        e = self.assertListRaises(errors.NoSuchFile,
538
506
                                  access.get_raw_records, memos)
539
507
 
543
511
        failing_transport = MockReadvFailingTransport(
544
512
                                [transport.get_bytes('packname')])
545
513
        reload_called, reload_func = self.make_reload_func()
546
 
        access = pack_repo._DirectPackAccess(
547
 
            {'foo': (failing_transport, 'packname')},
548
 
            reload_func=reload_func)
 
514
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
 
515
                                   reload_func=reload_func)
549
516
        # Asking for a single record will not trigger the Mock failure
550
517
        self.assertEqual(['1234567890'],
551
518
            list(access.get_raw_records(memos[:1])))
567
534
        failing_transport = MockReadvFailingTransport(
568
535
                                [transport.get_bytes('packname')])
569
536
        reload_called, reload_func = self.make_reload_func()
570
 
        access = pack_repo._DirectPackAccess(
571
 
            {'foo':(failing_transport, 'packname')})
 
537
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
572
538
        # Asking for a single record will not trigger the Mock failure
573
539
        self.assertEqual(['1234567890'],
574
540
            list(access.get_raw_records(memos[:1])))
579
545
                                  access.get_raw_records, memos)
580
546
 
581
547
    def test_reload_or_raise_no_reload(self):
582
 
        access = pack_repo._DirectPackAccess({}, reload_func=None)
 
548
        access = _DirectPackAccess({}, reload_func=None)
583
549
        retry_exc = self.make_retry_exception()
584
550
        # Without a reload_func, we will just re-raise the original exception
585
551
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
586
552
 
587
553
    def test_reload_or_raise_reload_changed(self):
588
554
        reload_called, reload_func = self.make_reload_func(return_val=True)
589
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
555
        access = _DirectPackAccess({}, reload_func=reload_func)
590
556
        retry_exc = self.make_retry_exception()
591
557
        access.reload_or_raise(retry_exc)
592
558
        self.assertEqual([1], reload_called)
596
562
 
597
563
    def test_reload_or_raise_reload_no_change(self):
598
564
        reload_called, reload_func = self.make_reload_func(return_val=False)
599
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
565
        access = _DirectPackAccess({}, reload_func=reload_func)
600
566
        retry_exc = self.make_retry_exception()
601
567
        # If reload_occurred is False, then we consider it an error to have
602
568
        # reload_func() return False (no changes).
726
692
 
727
693
    def create_gz_content(self, text):
728
694
        sio = StringIO()
729
 
        gz_file = gzip.GzipFile(mode='wb', fileobj=sio)
 
695
        gz_file = tuned_gzip.GzipFile(mode='wb', fileobj=sio)
730
696
        gz_file.write(text)
731
697
        gz_file.close()
732
698
        return sio.getvalue()
733
699
 
734
700
    def make_multiple_records(self):
735
701
        """Create the content for multiple records."""
736
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
702
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
737
703
        total_txt = []
738
704
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
739
705
                                        'foo\n'
742
708
                                        % (sha1sum,))
743
709
        record_1 = (0, len(gz_txt), sha1sum)
744
710
        total_txt.append(gz_txt)
745
 
        sha1sum = osutils.sha_string('baz\n')
 
711
        sha1sum = osutils.sha('baz\n').hexdigest()
746
712
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
747
713
                                        'baz\n'
748
714
                                        'end rev-id-2\n'
752
718
        return total_txt, record_1, record_2
753
719
 
754
720
    def test_valid_knit_data(self):
755
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
721
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
756
722
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
757
723
                                        'foo\n'
758
724
                                        'bar\n'
789
755
                         raw_contents)
790
756
 
791
757
    def test_not_enough_lines(self):
792
 
        sha1sum = osutils.sha_string('foo\n')
 
758
        sha1sum = osutils.sha('foo\n').hexdigest()
793
759
        # record says 2 lines data says 1
794
760
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
795
761
                                        'foo\n'
807
773
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
808
774
 
809
775
    def test_too_many_lines(self):
810
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
776
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
811
777
        # record says 1 lines data says 2
812
778
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
813
779
                                        'foo\n'
826
792
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
827
793
 
828
794
    def test_mismatched_version_id(self):
829
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
795
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
830
796
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
831
797
                                        'foo\n'
832
798
                                        'bar\n'
845
811
            knit._read_records_iter_raw(records))
846
812
 
847
813
    def test_uncompressed_data(self):
848
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
814
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
849
815
        txt = ('version rev-id-1 2 %s\n'
850
816
               'foo\n'
851
817
               'bar\n'
865
831
            knit._read_records_iter_raw(records))
866
832
 
867
833
    def test_corrupted_data(self):
868
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
834
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
869
835
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
870
836
                                        'foo\n'
871
837
                                        'bar\n'
1193
1159
            self.assertRaises(errors.KnitCorrupt, index.keys)
1194
1160
        except TypeError, e:
1195
1161
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1196
 
                           ' not exceptions.IndexError')):
 
1162
                           ' not exceptions.IndexError')
 
1163
                and sys.version_info[0:2] >= (2,5)):
1197
1164
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1198
1165
                                  ' raising new style exceptions with python'
1199
1166
                                  ' >=2.5')
1212
1179
            self.assertRaises(errors.KnitCorrupt, index.keys)
1213
1180
        except TypeError, e:
1214
1181
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1215
 
                           ' not exceptions.ValueError')):
 
1182
                           ' not exceptions.ValueError')
 
1183
                and sys.version_info[0:2] >= (2,5)):
1216
1184
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1217
1185
                                  ' raising new style exceptions with python'
1218
1186
                                  ' >=2.5')
1231
1199
            self.assertRaises(errors.KnitCorrupt, index.keys)
1232
1200
        except TypeError, e:
1233
1201
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1234
 
                           ' not exceptions.ValueError')):
 
1202
                           ' not exceptions.ValueError')
 
1203
                and sys.version_info[0:2] >= (2,5)):
1235
1204
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1236
1205
                                  ' raising new style exceptions with python'
1237
1206
                                  ' >=2.5')
1248
1217
            self.assertRaises(errors.KnitCorrupt, index.keys)
1249
1218
        except TypeError, e:
1250
1219
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1251
 
                           ' not exceptions.ValueError')):
 
1220
                           ' not exceptions.ValueError')
 
1221
                and sys.version_info[0:2] >= (2,5)):
1252
1222
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1253
1223
                                  ' raising new style exceptions with python'
1254
1224
                                  ' >=2.5')
1265
1235
            self.assertRaises(errors.KnitCorrupt, index.keys)
1266
1236
        except TypeError, e:
1267
1237
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1268
 
                           ' not exceptions.ValueError')):
 
1238
                           ' not exceptions.ValueError')
 
1239
                and sys.version_info[0:2] >= (2,5)):
1269
1240
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1270
1241
                                  ' raising new style exceptions with python'
1271
1242
                                  ' >=2.5')
2440
2411
        key_basis = ('bar',)
2441
2412
        key_missing = ('missing',)
2442
2413
        test.add_lines(key, (), ['foo\n'])
2443
 
        key_sha1sum = osutils.sha_string('foo\n')
 
2414
        key_sha1sum = osutils.sha('foo\n').hexdigest()
2444
2415
        sha1s = test.get_sha1s([key])
2445
2416
        self.assertEqual({key: key_sha1sum}, sha1s)
2446
2417
        self.assertEqual([], basis.calls)
2448
2419
        # directly (rather than via text reconstruction) so that remote servers
2449
2420
        # etc don't have to answer with full content.
2450
2421
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2451
 
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
 
2422
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
2452
2423
        basis.calls = []
2453
2424
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2454
2425
        self.assertEqual({key: key_sha1sum,