← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_knit.py

Modify test_tsort_partial to accept multiple valid orderings.

This test previously checked for an exact match on the result of
tsort.topo_sort, while only a partial ordering is garantueed.  The current
implementation of topo_sort indeed returns the graph in lexicographical order,
but this depends on the order in which dict.popitem() pops and that is a
Python implementation detail not to be relied on.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_knit.py
1
 
# Copyright (C) 2006-2011 Canonical Ltd
 
1
# Copyright (C) 2005, 2006, 2007 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
17
17
"""Tests for Knit data structure"""
18
18
 
19
19
from cStringIO import StringIO
 
20
import difflib
20
21
import gzip
21
22
import sys
22
23
 
23
24
from bzrlib import (
24
25
    errors,
 
26
    generate_ids,
25
27
    knit,
26
28
    multiparent,
27
29
    osutils,
28
30
    pack,
29
 
    tests,
30
 
    transport,
31
31
    )
32
32
from bzrlib.errors import (
 
33
    RevisionAlreadyPresent,
33
34
    KnitHeaderError,
 
35
    RevisionNotPresent,
34
36
    NoSuchFile,
35
37
    )
36
38
from bzrlib.index import *
37
39
from bzrlib.knit import (
38
40
    AnnotatedKnitContent,
39
41
    KnitContent,
 
42
    KnitSequenceMatcher,
40
43
    KnitVersionedFiles,
41
44
    PlainKnitContent,
42
45
    _VFContentMapGenerator,
 
46
    _DirectPackAccess,
43
47
    _KndxIndex,
44
48
    _KnitGraphIndex,
45
49
    _KnitKeyAccess,
46
50
    make_file_factory,
47
51
    )
48
 
from bzrlib.patiencediff import PatienceSequenceMatcher
49
 
from bzrlib.repofmt import (
50
 
    knitpack_repo,
51
 
    pack_repo,
52
 
    )
 
52
from bzrlib.repofmt import pack_repo
53
53
from bzrlib.tests import (
 
54
    Feature,
 
55
    KnownFailure,
54
56
    TestCase,
55
57
    TestCaseWithMemoryTransport,
56
58
    TestCaseWithTransport,
57
59
    TestNotApplicable,
58
60
    )
 
61
from bzrlib.transport import get_transport
 
62
from bzrlib.transport.memory import MemoryTransport
 
63
from bzrlib.tuned_gzip import GzipFile
59
64
from bzrlib.versionedfile import (
60
65
    AbsentContentFactory,
61
66
    ConstantMapper,
62
67
    network_bytes_to_kind_and_offset,
63
68
    RecordingVersionedFilesDecorator,
64
69
    )
65
 
from bzrlib.tests import (
66
 
    features,
67
 
    )
68
 
 
69
 
 
70
 
compiled_knit_feature = features.ModuleAvailableFeature(
71
 
    'bzrlib._knit_load_data_pyx')
 
70
 
 
71
 
 
72
class _CompiledKnitFeature(Feature):
 
73
 
 
74
    def _probe(self):
 
75
        try:
 
76
            import bzrlib._knit_load_data_pyx
 
77
        except ImportError:
 
78
            return False
 
79
        return True
 
80
 
 
81
    def feature_name(self):
 
82
        return 'bzrlib._knit_load_data_pyx'
 
83
 
 
84
CompiledKnitFeature = _CompiledKnitFeature()
72
85
 
73
86
 
74
87
class KnitContentTestsMixin(object):
103
116
        line_delta = source_content.line_delta(target_content)
104
117
        delta_blocks = list(KnitContent.get_line_delta_blocks(line_delta,
105
118
            source_lines, target_lines))
106
 
        matcher = PatienceSequenceMatcher(None, source_lines, target_lines)
107
 
        matcher_blocks = list(matcher.get_matching_blocks())
 
119
        matcher = KnitSequenceMatcher(None, source_lines, target_lines)
 
120
        matcher_blocks = list(list(matcher.get_matching_blocks()))
108
121
        self.assertEqual(matcher_blocks, delta_blocks)
109
122
 
110
123
    def test_get_line_delta_blocks(self):
330
343
            transport.append_bytes(packname, bytes)
331
344
        writer = pack.ContainerWriter(write_data)
332
345
        writer.begin()
333
 
        access = pack_repo._DirectPackAccess({})
 
346
        access = _DirectPackAccess({})
334
347
        access.set_writer(writer, index, (transport, packname))
335
348
        return access, writer
336
349
 
343
356
        writer.end()
344
357
        return memos
345
358
 
346
 
    def test_pack_collection_pack_retries(self):
347
 
        """An explicit pack of a pack collection succeeds even when a
348
 
        concurrent pack happens.
349
 
        """
350
 
        builder = self.make_branch_builder('.')
351
 
        builder.start_series()
352
 
        builder.build_snapshot('rev-1', None, [
353
 
            ('add', ('', 'root-id', 'directory', None)),
354
 
            ('add', ('file', 'file-id', 'file', 'content\nrev 1\n')),
355
 
            ])
356
 
        builder.build_snapshot('rev-2', ['rev-1'], [
357
 
            ('modify', ('file-id', 'content\nrev 2\n')),
358
 
            ])
359
 
        builder.build_snapshot('rev-3', ['rev-2'], [
360
 
            ('modify', ('file-id', 'content\nrev 3\n')),
361
 
            ])
362
 
        self.addCleanup(builder.finish_series)
363
 
        b = builder.get_branch()
364
 
        self.addCleanup(b.lock_write().unlock)
365
 
        repo = b.repository
366
 
        collection = repo._pack_collection
367
 
        # Concurrently repack the repo.
368
 
        reopened_repo = repo.bzrdir.open_repository()
369
 
        reopened_repo.pack()
370
 
        # Pack the new pack.
371
 
        collection.pack()
372
 
 
373
359
    def make_vf_for_retrying(self):
374
360
        """Create 3 packs and a reload function.
375
361
 
380
366
        :return: (versioned_file, reload_counter)
381
367
            versioned_file  a KnitVersionedFiles using the packs for access
382
368
        """
383
 
        builder = self.make_branch_builder('.', format="1.9")
 
369
        builder = self.make_branch_builder('.')
384
370
        builder.start_series()
385
371
        builder.build_snapshot('rev-1', None, [
386
372
            ('add', ('', 'root-id', 'directory', None)),
402
388
        collection = repo._pack_collection
403
389
        collection.ensure_loaded()
404
390
        orig_packs = collection.packs
405
 
        packer = knitpack_repo.KnitPacker(collection, orig_packs, '.testpack')
 
391
        packer = pack_repo.Packer(collection, orig_packs, '.testpack')
406
392
        new_pack = packer.pack()
407
393
        # forget about the new pack
408
394
        collection.reset()
447
433
        except _TestException, e:
448
434
            retry_exc = errors.RetryWithNewPacks(None, reload_occurred=False,
449
435
                                                 exc_info=sys.exc_info())
450
 
        # GZ 2010-08-10: Cycle with exc_info affects 3 tests
451
436
        return retry_exc
452
437
 
453
438
    def test_read_from_several_packs(self):
462
447
        memos.extend(access.add_raw_records([('key', 5)], 'alpha'))
463
448
        writer.end()
464
449
        transport = self.get_transport()
465
 
        access = pack_repo._DirectPackAccess({"FOO":(transport, 'packfile'),
 
450
        access = _DirectPackAccess({"FOO":(transport, 'packfile'),
466
451
            "FOOBAR":(transport, 'pack2'),
467
452
            "BAZ":(transport, 'pack3')})
468
453
        self.assertEqual(['1234567890', '12345', 'alpha'],
478
463
 
479
464
    def test_set_writer(self):
480
465
        """The writer should be settable post construction."""
481
 
        access = pack_repo._DirectPackAccess({})
 
466
        access = _DirectPackAccess({})
482
467
        transport = self.get_transport()
483
468
        packname = 'packfile'
484
469
        index = 'foo'
496
481
        transport = self.get_transport()
497
482
        reload_called, reload_func = self.make_reload_func()
498
483
        # Note that the index key has changed from 'foo' to 'bar'
499
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')},
 
484
        access = _DirectPackAccess({'bar':(transport, 'packname')},
500
485
                                   reload_func=reload_func)
501
486
        e = self.assertListRaises(errors.RetryWithNewPacks,
502
487
                                  access.get_raw_records, memos)
511
496
        memos = self.make_pack_file()
512
497
        transport = self.get_transport()
513
498
        # Note that the index key has changed from 'foo' to 'bar'
514
 
        access = pack_repo._DirectPackAccess({'bar':(transport, 'packname')})
 
499
        access = _DirectPackAccess({'bar':(transport, 'packname')})
515
500
        e = self.assertListRaises(KeyError, access.get_raw_records, memos)
516
501
 
517
502
    def test_missing_file_raises_retry(self):
519
504
        transport = self.get_transport()
520
505
        reload_called, reload_func = self.make_reload_func()
521
506
        # Note that the 'filename' has been changed to 'different-packname'
522
 
        access = pack_repo._DirectPackAccess(
523
 
            {'foo':(transport, 'different-packname')},
524
 
            reload_func=reload_func)
 
507
        access = _DirectPackAccess({'foo':(transport, 'different-packname')},
 
508
                                   reload_func=reload_func)
525
509
        e = self.assertListRaises(errors.RetryWithNewPacks,
526
510
                                  access.get_raw_records, memos)
527
511
        # The file has gone missing, so we assume we need to reload
535
519
        memos = self.make_pack_file()
536
520
        transport = self.get_transport()
537
521
        # Note that the 'filename' has been changed to 'different-packname'
538
 
        access = pack_repo._DirectPackAccess(
539
 
            {'foo': (transport, 'different-packname')})
 
522
        access = _DirectPackAccess({'foo':(transport, 'different-packname')})
540
523
        e = self.assertListRaises(errors.NoSuchFile,
541
524
                                  access.get_raw_records, memos)
542
525
 
546
529
        failing_transport = MockReadvFailingTransport(
547
530
                                [transport.get_bytes('packname')])
548
531
        reload_called, reload_func = self.make_reload_func()
549
 
        access = pack_repo._DirectPackAccess(
550
 
            {'foo': (failing_transport, 'packname')},
551
 
            reload_func=reload_func)
 
532
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')},
 
533
                                   reload_func=reload_func)
552
534
        # Asking for a single record will not trigger the Mock failure
553
535
        self.assertEqual(['1234567890'],
554
536
            list(access.get_raw_records(memos[:1])))
570
552
        failing_transport = MockReadvFailingTransport(
571
553
                                [transport.get_bytes('packname')])
572
554
        reload_called, reload_func = self.make_reload_func()
573
 
        access = pack_repo._DirectPackAccess(
574
 
            {'foo':(failing_transport, 'packname')})
 
555
        access = _DirectPackAccess({'foo':(failing_transport, 'packname')})
575
556
        # Asking for a single record will not trigger the Mock failure
576
557
        self.assertEqual(['1234567890'],
577
558
            list(access.get_raw_records(memos[:1])))
582
563
                                  access.get_raw_records, memos)
583
564
 
584
565
    def test_reload_or_raise_no_reload(self):
585
 
        access = pack_repo._DirectPackAccess({}, reload_func=None)
 
566
        access = _DirectPackAccess({}, reload_func=None)
586
567
        retry_exc = self.make_retry_exception()
587
568
        # Without a reload_func, we will just re-raise the original exception
588
569
        self.assertRaises(_TestException, access.reload_or_raise, retry_exc)
589
570
 
590
571
    def test_reload_or_raise_reload_changed(self):
591
572
        reload_called, reload_func = self.make_reload_func(return_val=True)
592
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
573
        access = _DirectPackAccess({}, reload_func=reload_func)
593
574
        retry_exc = self.make_retry_exception()
594
575
        access.reload_or_raise(retry_exc)
595
576
        self.assertEqual([1], reload_called)
599
580
 
600
581
    def test_reload_or_raise_reload_no_change(self):
601
582
        reload_called, reload_func = self.make_reload_func(return_val=False)
602
 
        access = pack_repo._DirectPackAccess({}, reload_func=reload_func)
 
583
        access = _DirectPackAccess({}, reload_func=reload_func)
603
584
        retry_exc = self.make_retry_exception()
604
585
        # If reload_occurred is False, then we consider it an error to have
605
586
        # reload_func() return False (no changes).
736
717
 
737
718
    def make_multiple_records(self):
738
719
        """Create the content for multiple records."""
739
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
720
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
740
721
        total_txt = []
741
722
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
742
723
                                        'foo\n'
745
726
                                        % (sha1sum,))
746
727
        record_1 = (0, len(gz_txt), sha1sum)
747
728
        total_txt.append(gz_txt)
748
 
        sha1sum = osutils.sha_string('baz\n')
 
729
        sha1sum = osutils.sha('baz\n').hexdigest()
749
730
        gz_txt = self.create_gz_content('version rev-id-2 1 %s\n'
750
731
                                        'baz\n'
751
732
                                        'end rev-id-2\n'
755
736
        return total_txt, record_1, record_2
756
737
 
757
738
    def test_valid_knit_data(self):
758
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
739
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
759
740
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
760
741
                                        'foo\n'
761
742
                                        'bar\n'
792
773
                         raw_contents)
793
774
 
794
775
    def test_not_enough_lines(self):
795
 
        sha1sum = osutils.sha_string('foo\n')
 
776
        sha1sum = osutils.sha('foo\n').hexdigest()
796
777
        # record says 2 lines data says 1
797
778
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
798
779
                                        'foo\n'
810
791
        self.assertEqual([(('rev-id-1',),  gz_txt, sha1sum)], raw_contents)
811
792
 
812
793
    def test_too_many_lines(self):
813
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
794
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
814
795
        # record says 1 lines data says 2
815
796
        gz_txt = self.create_gz_content('version rev-id-1 1 %s\n'
816
797
                                        'foo\n'
829
810
        self.assertEqual([(('rev-id-1',), gz_txt, sha1sum)], raw_contents)
830
811
 
831
812
    def test_mismatched_version_id(self):
832
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
813
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
833
814
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
834
815
                                        'foo\n'
835
816
                                        'bar\n'
848
829
            knit._read_records_iter_raw(records))
849
830
 
850
831
    def test_uncompressed_data(self):
851
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
832
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
852
833
        txt = ('version rev-id-1 2 %s\n'
853
834
               'foo\n'
854
835
               'bar\n'
868
849
            knit._read_records_iter_raw(records))
869
850
 
870
851
    def test_corrupted_data(self):
871
 
        sha1sum = osutils.sha_string('foo\nbar\n')
 
852
        sha1sum = osutils.sha('foo\nbar\n').hexdigest()
872
853
        gz_txt = self.create_gz_content('version rev-id-1 2 %s\n'
873
854
                                        'foo\n'
874
855
                                        'bar\n'
891
872
 
892
873
    def get_knit_index(self, transport, name, mode):
893
874
        mapper = ConstantMapper(name)
 
875
        orig = knit._load_data
 
876
        def reset():
 
877
            knit._load_data = orig
 
878
        self.addCleanup(reset)
894
879
        from bzrlib._knit_load_data_py import _load_data_py
895
 
        self.overrideAttr(knit, '_load_data', _load_data_py)
 
880
        knit._load_data = _load_data_py
896
881
        allow_writes = lambda: 'w' in mode
897
882
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
898
883
 
1196
1181
            self.assertRaises(errors.KnitCorrupt, index.keys)
1197
1182
        except TypeError, e:
1198
1183
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1199
 
                           ' not exceptions.IndexError')):
 
1184
                           ' not exceptions.IndexError')
 
1185
                and sys.version_info[0:2] >= (2,5)):
1200
1186
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1201
1187
                                  ' raising new style exceptions with python'
1202
1188
                                  ' >=2.5')
1215
1201
            self.assertRaises(errors.KnitCorrupt, index.keys)
1216
1202
        except TypeError, e:
1217
1203
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1218
 
                           ' not exceptions.ValueError')):
 
1204
                           ' not exceptions.ValueError')
 
1205
                and sys.version_info[0:2] >= (2,5)):
1219
1206
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1220
1207
                                  ' raising new style exceptions with python'
1221
1208
                                  ' >=2.5')
1234
1221
            self.assertRaises(errors.KnitCorrupt, index.keys)
1235
1222
        except TypeError, e:
1236
1223
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1237
 
                           ' not exceptions.ValueError')):
 
1224
                           ' not exceptions.ValueError')
 
1225
                and sys.version_info[0:2] >= (2,5)):
1238
1226
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1239
1227
                                  ' raising new style exceptions with python'
1240
1228
                                  ' >=2.5')
1251
1239
            self.assertRaises(errors.KnitCorrupt, index.keys)
1252
1240
        except TypeError, e:
1253
1241
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1254
 
                           ' not exceptions.ValueError')):
 
1242
                           ' not exceptions.ValueError')
 
1243
                and sys.version_info[0:2] >= (2,5)):
1255
1244
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1256
1245
                                  ' raising new style exceptions with python'
1257
1246
                                  ' >=2.5')
1268
1257
            self.assertRaises(errors.KnitCorrupt, index.keys)
1269
1258
        except TypeError, e:
1270
1259
            if (str(e) == ('exceptions must be strings, classes, or instances,'
1271
 
                           ' not exceptions.ValueError')):
 
1260
                           ' not exceptions.ValueError')
 
1261
                and sys.version_info[0:2] >= (2,5)):
1272
1262
                self.knownFailure('Pyrex <0.9.5 fails with TypeError when'
1273
1263
                                  ' raising new style exceptions with python'
1274
1264
                                  ' >=2.5')
1318
1308
 
1319
1309
class LowLevelKnitIndexTests_c(LowLevelKnitIndexTests):
1320
1310
 
1321
 
    _test_needs_features = [compiled_knit_feature]
 
1311
    _test_needs_features = [CompiledKnitFeature]
1322
1312
 
1323
1313
    def get_knit_index(self, transport, name, mode):
1324
1314
        mapper = ConstantMapper(name)
 
1315
        orig = knit._load_data
 
1316
        def reset():
 
1317
            knit._load_data = orig
 
1318
        self.addCleanup(reset)
1325
1319
        from bzrlib._knit_load_data_pyx import _load_data_c
1326
 
        self.overrideAttr(knit, '_load_data', _load_data_c)
 
1320
        knit._load_data = _load_data_c
1327
1321
        allow_writes = lambda: mode == 'w'
1328
 
        return _KndxIndex(transport, mapper, lambda:None,
1329
 
                          allow_writes, lambda:True)
 
1322
        return _KndxIndex(transport, mapper, lambda:None, allow_writes, lambda:True)
1330
1323
 
1331
1324
 
1332
1325
class Test_KnitAnnotator(TestCaseWithMemoryTransport):
1603
1596
        # could leave an empty .kndx file, which bzr would later claim was a
1604
1597
        # corrupted file since the header was not present. In reality, the file
1605
1598
        # just wasn't created, so it should be ignored.
1606
 
        t = transport.get_transport_from_path('.')
 
1599
        t = get_transport('.')
1607
1600
        t.put_bytes('test.kndx', '')
1608
1601
 
1609
1602
        knit = self.make_test_knit()
1610
1603
 
1611
1604
    def test_knit_index_checks_header(self):
1612
 
        t = transport.get_transport_from_path('.')
 
1605
        t = get_transport('.')
1613
1606
        t.put_bytes('test.kndx', '# not really a knit header\n\n')
1614
1607
        k = self.make_test_knit()
1615
1608
        self.assertRaises(KnitHeaderError, k.keys)
2443
2436
        key_basis = ('bar',)
2444
2437
        key_missing = ('missing',)
2445
2438
        test.add_lines(key, (), ['foo\n'])
2446
 
        key_sha1sum = osutils.sha_string('foo\n')
 
2439
        key_sha1sum = osutils.sha('foo\n').hexdigest()
2447
2440
        sha1s = test.get_sha1s([key])
2448
2441
        self.assertEqual({key: key_sha1sum}, sha1s)
2449
2442
        self.assertEqual([], basis.calls)
2451
2444
        # directly (rather than via text reconstruction) so that remote servers
2452
2445
        # etc don't have to answer with full content.
2453
2446
        basis.add_lines(key_basis, (), ['foo\n', 'bar\n'])
2454
 
        basis_sha1sum = osutils.sha_string('foo\nbar\n')
 
2447
        basis_sha1sum = osutils.sha('foo\nbar\n').hexdigest()
2455
2448
        basis.calls = []
2456
2449
        sha1s = test.get_sha1s([key, key_missing, key_basis])
2457
2450
        self.assertEqual({key: key_sha1sum,