← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

  • Committer: Canonical.com Patch Queue Manager
  • Date: 2011-05-16 17:33:27 UTC
  • mfrom: (5755.2.10 2.4-max-entries-gc-602614)
  • Revision ID: pqm@pqm.ubuntu.com-20110516173327-5ehst0ttceohsf5w
(jameinel) Add bzr.groupcompress.max_bytes_to_index to limit peak memory
 when delta-compressing large files (bug #602614) (John A Meinel)

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2006-2010 Canonical Ltd
 
1
# Copyright (C) 2006-2011 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
 
20
import gzip
20
21
import sys
21
22
 
22
23
from bzrlib import (
27
28
    pack,
28
29
    tests,
29
30
    transport,
30
 
    tuned_gzip,
31
31
    )
32
32
from bzrlib.errors import (
33
33
    KnitHeaderError,
40
40
    KnitVersionedFiles,
41
41
    PlainKnitContent,
42
42
    _VFContentMapGenerator,
43
 
    _DirectPackAccess,
44
43
    _KndxIndex,
45
44
    _KnitGraphIndex,
46
45
    _KnitKeyAccess,
47
46
    make_file_factory,
48
47
    )
49
48
from bzrlib.patiencediff import PatienceSequenceMatcher
50
 
from bzrlib.repofmt import pack_repo
 
49
from bzrlib.repofmt import (
 
50
    knitpack_repo,
 
51
    pack_repo,
 
52
    )
51
53
from bzrlib.tests import (
52
54
    TestCase,
53
55
    TestCaseWithMemoryTransport,
325
327
            transport.append_bytes(packname, bytes)
326
328
        writer = pack.ContainerWriter(write_data)
327
329
        writer.begin()
328
 
        access = _DirectPackAccess({})
 
330
        access = pack_repo._DirectPackAccess({})
329
331
        access.set_writer(writer, index, (transport, packname))
330
332
        return access, writer
331
333
 
338
340
        writer.end()
339
341
        return memos
340
342
 
 
343
    def test_pack_collection_pack_retries(self):
 
344
        """An explicit pack of a pack collection succeeds even when a
 
345
        concurrent pack happens.
 
346
        """
 
347
        builder = self.make_branch_builder('.')
 
348
        builder.start_series()
 
349
        builder.build_snapshot('rev-1', None, [
 
350
            ('add', ('', 'root-id', 'directory', None)),
 
351
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
 
352
            ])
 
353
        builder.build_snapshot('rev-2', ['rev-1'], [
 
354
            ('modify', ('file-id', 'content\nrev 2\n')),
 
355
            ])
 
356
        builder.build_snapshot('rev-3', ['rev-2'], [
 
357
            ('modify', ('file-id', 'content\nrev 3\n')),
 
358
            ])
 
359
        self.addCleanup(builder.finish_series)
 
360
        b = builder.get_branch()
 
361
        self.addCleanup(b.lock_write().unlock)
 
362
        repo = b.repository
 
363
        collection = repo._pack_collection
 
364
        # Concurrently repack the repo.
 
365
        reopened_repo = repo.bzrdir.open_repository()
 
366
        reopened_repo.pack()
 
367
        # Pack the new pack.
 
368
        collection.pack()
 
369
 
341
370
    def make_vf_for_retrying(self):
342
371
        """Create 3 packs and a reload function.
343
372
 
370
399
        collection = repo._pack_collection
371
400
        collection.ensure_loaded()
372
401
        orig_packs = collection.packs
373
 
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
 
402
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
374
403
        new_pack = packer.pack()
375
404
        # forget about the new pack
376
405
        collection.reset()
429
458
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
430
459
        writer.end()
431
460
        transport = self.get_transport()
432
 
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
 
461
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
433
462
            "FOOBAR":(transport, 'pack2'),
434
463
            "BAZ":(transport, 'pack3')})
435
464
        self.assertEqual(['1234567890', '12345', 'alpha'],
445
474
 
446
475
    def test_set_writer(self):
447
476
        """The writer should be settable post construction."""
448
 
        access = _DirectPackAccess({})
 
477
        access = pack_repo._DirectPackAccess({})
449
478
        transport = self.get_transport()
450
479
        packname = 'packfile'
451
480
        index = 'foo'
463
492
        transport = self.get_transport()
464
493
        reload_called, reload_func = self.make_reload_func()
465
494
        # Note that the index key has changed from 'foo' to 'bar'
466
 
        access = _DirectPackAccess({'bar':(transport, 'packname')},
 
495
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
467
496
                                   reload_func=reload_func)
468
497
        e = self.assertListRaises(errors.RetryWithNewPacks,
469
498
                                  access.get_raw_records, memos)
478
507
        memos = self.make_pack_file()
479
508
        transport = self.get_transport()
480
509
        # Note that the index key has changed from 'foo' to 'bar'
481
 
        access = _DirectPackAccess({'bar':(transport, 'packname')})
 
510
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
482
511
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
483
512
 
484
513
    def test_missing_file_raises_retry(self):
486
515
        transport = self.get_transport()
487
516
        reload_called, reload_func = self.make_reload_func()
488
517
        # Note that the 'filename' has been changed to 'different-packname'
489
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
490
 
                                   reload_func=reload_func)
 
518
        access = pack_repo._DirectPackAccess(
 
519
            {'foo':(transport, 'different-packname')},
 
520
            reload_func=reload_func)
491
521
        e = self.assertListRaises(errors.RetryWithNewPacks,
492
522
                                  access.get_raw_records, memos)
493
523
        # The file has gone missing, so we assume we need to reload
501
531
        memos = self.make_pack_file()
502
532
        transport = self.get_transport()
503
533
        # Note that the 'filename' has been changed to 'different-packname'
504
 
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
 
534
        access = pack_repo._DirectPackAccess(
 
535
            {'foo': (transport, 'different-packname')})
505
536
        e = self.assertListRaises(errors.NoSuchFile,
506
537
                                  access.get_raw_records, memos)
507
538
 
511
542
        failing_transport = MockReadvFailingTransport(
512
543
                                [transport.get_bytes('packname')])
513
544
        reload_called, reload_func = self.make_reload_func()
514
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
515
 
                                   reload_func=reload_func)
 
545
        access = pack_repo._DirectPackAccess(
 
546
            {'foo': (failing_transport, 'packname')},
 
547
            reload_func=reload_func)
516
548
        # Asking for a single record will not trigger the Mock failure
517
549
        self.assertEqual(['1234567890'],
518
550
            list(access.get_raw_records(memos[:1])))
534
566
        failing_transport = MockReadvFailingTransport(
535
567
                                [transport.get_bytes('packname')])
536
568
        reload_called, reload_func = self.make_reload_func()
537
 
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
 
569
        access = pack_repo._DirectPackAccess(
 
570
            {'foo':(failing_transport, 'packname')})
538
571
        # Asking for a single record will not trigger the Mock failure
539
572
        self.assertEqual(['1234567890'],
540
573
            list(access.get_raw_records(memos[:1])))
545
578
                                  access.get_raw_records, memos)
546
579
 
547
580
    def test_reload_or_raise_no_reload(self):
548
 
        access = _DirectPackAccess({}, reload_func=None)
 
581
        access = pack_repo._DirectPackAccess({}, reload_func=None)
549
582
        retry_exc = self.make_retry_exception()
550
583
        # Without a reload_func, we will just re-raise the original exception
551
584
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
552
585
 
553
586
    def test_reload_or_raise_reload_changed(self):
554
587
        reload_called, reload_func = self.make_reload_func(return_val=True)
555
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
588
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
556
589
        retry_exc = self.make_retry_exception()
557
590
        access.reload_or_raise(retry_exc)
558
591
        self.assertEqual([1], reload_called)
562
595
 
563
596
    def test_reload_or_raise_reload_no_change(self):
564
597
        reload_called, reload_func = self.make_reload_func(return_val=False)
565
 
        access = _DirectPackAccess({}, reload_func=reload_func)
 
598
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
566
599
        retry_exc = self.make_retry_exception()
567
600
        # If reload_occurred is False, then we consider it an error to have
568
601
        # reload_func() return False (no changes).
692
725
 
693
726
    def create_gz_content(self, text):
694
727
        sio = StringIO()
695
 
        gz_file = tuned_gzip.GzipFile(mode='wb', fileobj=sio)
 
728
        gz_file = gzip.GzipFile(mode='wb', fileobj=sio)
696
729
        gz_file.write(text)
697
730
        gz_file.close()
698
731
        return sio.getvalue()
699
732
 
700
733
    def make_multiple_records(self):
701
734
        """Create the content for multiple records."""
702
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
735
        sha1sum = osutils.sha_string('foo\nbar\n')
703
736
        total_txt = []
704
737
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
705
738
                                        'foo\n'
708
741
                                        % (sha1sum,))
709
742
        record_1 = (0, len(gz_txt), sha1sum)
710
743
        total_txt.append(gz_txt)
711
 
        sha1sum = osutils.sha('baz\n').hexdigest()
 
744
        sha1sum = osutils.sha_string('baz\n')
712
745
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
713
746
                                        'baz\n'
714
747
                                        'end rev-id-2\n'
718
751
        return total_txt, record_1, record_2
719
752
 
720
753
    def test_valid_knit_data(self):
721
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
754
        sha1sum = osutils.sha_string('foo\nbar\n')
722
755
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
723
756
                                        'foo\n'
724
757
                                        'bar\n'
755
788
                         raw_contents)
756
789
 
757
790
    def test_not_enough_lines(self):
758
 
        sha1sum = osutils.sha('foo\n').hexdigest()
 
791
        sha1sum = osutils.sha_string('foo\n')
759
792
        # record says 2 lines data says 1
760
793
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
761
794
                                        'foo\n'
773
806
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
774
807
 
775
808
    def test_too_many_lines(self):
776
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
809
        sha1sum = osutils.sha_string('foo\nbar\n')
777
810
        # record says 1 lines data says 2
778
811
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
779
812
                                        'foo\n'
792
825
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
793
826
 
794
827
    def test_mismatched_version_id(self):
795
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
828
        sha1sum = osutils.sha_string('foo\nbar\n')
796
829
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
797
830
                                        'foo\n'
798
831
                                        'bar\n'
811
844
            knit._read_records_iter_raw(records))
812
845
 
813
846
    def test_uncompressed_data(self):
814
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
847
        sha1sum = osutils.sha_string('foo\nbar\n')
815
848
        txt = ('version rev-id-1 2 %s\n'
816
849
               'foo\n'
817
850
               'bar\n'
831
864
            knit._read_records_iter_raw(records))
832
865
 
833
866
    def test_corrupted_data(self):
834
 
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
867
        sha1sum = osutils.sha_string('foo\nbar\n')
835
868
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
836
869
                                        'foo\n'
837
870
                                        'bar\n'
1159
1192
            self.assertRaises(errors.KnitCorrupt, index.keys)
1160
1193
        except TypeError, e:
1161
1194
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1162
 
                           ' not exceptions.IndexError')
1163
 
                and sys.version_info[0:2] >= (2,5)):
 
1195
                           ' not exceptions.IndexError')):
1164
1196
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1165
1197
                                  ' raising new style exceptions with python'
1166
1198
                                  ' >=2.5')
1179
1211
            self.assertRaises(errors.KnitCorrupt, index.keys)
1180
1212
        except TypeError, e:
1181
1213
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1182
 
                           ' not exceptions.ValueError')
1183
 
                and sys.version_info[0:2] >= (2,5)):
 
1214
                           ' not exceptions.ValueError')):
1184
1215
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1185
1216
                                  ' raising new style exceptions with python'
1186
1217
                                  ' >=2.5')
1199
1230
            self.assertRaises(errors.KnitCorrupt, index.keys)
1200
1231
        except TypeError, e:
1201
1232
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1202
 
                           ' not exceptions.ValueError')
1203
 
                and sys.version_info[0:2] >= (2,5)):
 
1233
                           ' not exceptions.ValueError')):
1204
1234
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1205
1235
                                  ' raising new style exceptions with python'
1206
1236
                                  ' >=2.5')
1217
1247
            self.assertRaises(errors.KnitCorrupt, index.keys)
1218
1248
        except TypeError, e:
1219
1249
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1220
 
                           ' not exceptions.ValueError')
1221
 
                and sys.version_info[0:2] >= (2,5)):
 
1250
                           ' not exceptions.ValueError')):
1222
1251
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1223
1252
                                  ' raising new style exceptions with python'
1224
1253
                                  ' >=2.5')
1235
1264
            self.assertRaises(errors.KnitCorrupt, index.keys)
1236
1265
        except TypeError, e:
1237
1266
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1238
 
                           ' not exceptions.ValueError')
1239
 
                and sys.version_info[0:2] >= (2,5)):
 
1267
                           ' not exceptions.ValueError')):
1240
1268
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1241
1269
                                  ' raising new style exceptions with python'
1242
1270
                                  ' >=2.5')
2411
2439
        key_basis = ('bar',)
2412
2440
        key_missing = ('missing',)
2413
2441
        test.add_lines(key, (), ['foo\n'])
2414
 
        key_sha1sum = osutils.sha('foo\n').hexdigest()
 
2442
        key_sha1sum = osutils.sha_string('foo\n')
2415
2443
        sha1s = test.get_sha1s([key])
2416
2444
        self.assertEqual({key: key_sha1sum}, sha1s)
2417
2445
        self.assertEqual([], basis.calls)
2419
2447
        # directly (rather than via text reconstruction) so that remote servers
2420
2448
        # etc don't have to answer with full content.
2421
2449
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2422
 
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
 
2450
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
2423
2451
        basis.calls = []
2424
2452
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2425
2453
        self.assertEqual({key: key_sha1sum,