← Back to branch summary

~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_versionedfile.py

  • Committer: Robert Collins
  • Date: 2009-03-27 04:10:25 UTC
  • mfrom: (4208 +trunk)
  • mto: This revision was merged to the branch mainline in revision 4216.
  • Revision ID: robertc@robertcollins.net-20090327041025-rgutx4q03xo4pq6l
Resolve NEWS conflicts.

Show diffs side-by-side

added added

removed removed

Lines of Context:
bzrlib/tests/test_versionedfile.py
1
 
# Copyright (C) 2006-2011 Canonical Ltd
 
1
# Copyright (C) 2005, 2009 Canonical Ltd
2
2
#
3
3
# Authors:
4
4
#   Johan Rydberg <jrydberg@gnu.org>
21
21
# TODO: might be nice to create a versionedfile with some type of corruption
22
22
# considered typical and check that it can be detected/corrected.
23
23
 
24
 
from gzip import GzipFile
25
24
from itertools import chain, izip
26
25
from StringIO import StringIO
27
26
 
 
27
import bzrlib
28
28
from bzrlib import (
29
29
    errors,
30
 
    graph as _mod_graph,
31
 
    groupcompress,
32
 
    knit as _mod_knit,
33
30
    osutils,
34
31
    progress,
35
 
    transport,
36
 
    ui,
37
32
    )
38
33
from bzrlib.errors import (
39
34
                           RevisionNotPresent,
40
35
                           RevisionAlreadyPresent,
 
36
                           WeaveParentMismatch
41
37
                           )
 
38
from bzrlib import knit as _mod_knit
42
39
from bzrlib.knit import (
43
40
    cleanup_pack_knit,
44
41
    make_file_factory,
45
42
    make_pack_factory,
 
43
    KnitAnnotateFactory,
 
44
    KnitPlainFactory,
46
45
    )
47
46
from bzrlib.tests import (
48
47
    TestCase,
49
48
    TestCaseWithMemoryTransport,
50
49
    TestNotApplicable,
51
50
    TestSkipped,
 
51
    condition_isinstance,
 
52
    split_suite_by_condition,
 
53
    multiply_tests,
52
54
    )
53
55
from bzrlib.tests.http_utils import TestCaseWithWebserver
 
56
from bzrlib.trace import mutter
 
57
from bzrlib.transport import get_transport
54
58
from bzrlib.transport.memory import MemoryTransport
 
59
from bzrlib.tsort import topo_sort
 
60
from bzrlib.tuned_gzip import GzipFile
55
61
import bzrlib.versionedfile as versionedfile
56
62
from bzrlib.versionedfile import (
57
63
    ConstantMapper,
61
67
    make_versioned_files_factory,
62
68
    )
63
69
from bzrlib.weave import WeaveFile
64
 
from bzrlib.weavefile import write_weave
65
 
from bzrlib.tests.scenarios import load_tests_apply_scenarios
66
 
 
67
 
 
68
 
load_tests = load_tests_apply_scenarios
 
70
from bzrlib.weavefile import read_weave, write_weave
 
71
 
 
72
 
 
73
def load_tests(standard_tests, module, loader):
 
74
    """Parameterize VersionedFiles tests for different implementations."""
 
75
    to_adapt, result = split_suite_by_condition(
 
76
        standard_tests, condition_isinstance(TestVersionedFiles))
 
77
    # We want to be sure of behaviour for:
 
78
    # weaves prefix layout (weave texts)
 
79
    # individually named weaves (weave inventories)
 
80
    # annotated knits - prefix|hash|hash-escape layout, we test the third only
 
81
    #                   as it is the most complex mapper.
 
82
    # individually named knits
 
83
    # individual no-graph knits in packs (signatures)
 
84
    # individual graph knits in packs (inventories)
 
85
    # individual graph nocompression knits in packs (revisions)
 
86
    # plain text knits in packs (texts)
 
87
    len_one_scenarios = [
 
88
        ('weave-named', {
 
89
            'cleanup':None,
 
90
            'factory':make_versioned_files_factory(WeaveFile,
 
91
                ConstantMapper('inventory')),
 
92
            'graph':True,
 
93
            'key_length':1,
 
94
            'support_partial_insertion': False,
 
95
            }),
 
96
        ('named-knit', {
 
97
            'cleanup':None,
 
98
            'factory':make_file_factory(False, ConstantMapper('revisions')),
 
99
            'graph':True,
 
100
            'key_length':1,
 
101
            'support_partial_insertion': False,
 
102
            }),
 
103
        ('named-nograph-nodelta-knit-pack', {
 
104
            'cleanup':cleanup_pack_knit,
 
105
            'factory':make_pack_factory(False, False, 1),
 
106
            'graph':False,
 
107
            'key_length':1,
 
108
            'support_partial_insertion': False,
 
109
            }),
 
110
        ('named-graph-knit-pack', {
 
111
            'cleanup':cleanup_pack_knit,
 
112
            'factory':make_pack_factory(True, True, 1),
 
113
            'graph':True,
 
114
            'key_length':1,
 
115
            'support_partial_insertion': True,
 
116
            }),
 
117
        ('named-graph-nodelta-knit-pack', {
 
118
            'cleanup':cleanup_pack_knit,
 
119
            'factory':make_pack_factory(True, False, 1),
 
120
            'graph':True,
 
121
            'key_length':1,
 
122
            'support_partial_insertion': False,
 
123
            }),
 
124
        ]
 
125
    len_two_scenarios = [
 
126
        ('weave-prefix', {
 
127
            'cleanup':None,
 
128
            'factory':make_versioned_files_factory(WeaveFile,
 
129
                PrefixMapper()),
 
130
            'graph':True,
 
131
            'key_length':2,
 
132
            'support_partial_insertion': False,
 
133
            }),
 
134
        ('annotated-knit-escape', {
 
135
            'cleanup':None,
 
136
            'factory':make_file_factory(True, HashEscapedPrefixMapper()),
 
137
            'graph':True,
 
138
            'key_length':2,
 
139
            'support_partial_insertion': False,
 
140
            }),
 
141
        ('plain-knit-pack', {
 
142
            'cleanup':cleanup_pack_knit,
 
143
            'factory':make_pack_factory(True, True, 2),
 
144
            'graph':True,
 
145
            'key_length':2,
 
146
            'support_partial_insertion': True,
 
147
            }),
 
148
        ]
 
149
    scenarios = len_one_scenarios + len_two_scenarios
 
150
    return multiply_tests(to_adapt, scenarios, result)
69
151
 
70
152
 
71
153
def get_diamond_vf(f, trailing_eol=True, left_only=False):
97
179
 
98
180
 
99
181
def get_diamond_files(files, key_length, trailing_eol=True, left_only=False,
100
 
    nograph=False, nokeys=False):
 
182
    nograph=False):
101
183
    """Get a diamond graph to exercise deltas and merges.
102
184
 
103
185
    This creates a 5-node graph in files. If files supports 2-length keys two
110
192
    :param nograph: If True, do not provide parents to the add_lines calls;
111
193
        this is useful for tests that need inserted data but have graphless
112
194
        stores.
113
 
    :param nokeys: If True, pass None is as the key for all insertions.
114
 
        Currently implies nograph.
115
195
    :return: The results of the add_lines calls.
116
196
    """
117
 
    if nokeys:
118
 
        nograph = True
119
197
    if key_length == 1:
120
198
        prefixes = [()]
121
199
    else:
132
210
        else:
133
211
            result = [prefix + suffix for suffix in suffix_list]
134
212
            return result
135
 
    def get_key(suffix):
136
 
        if nokeys:
137
 
            return (None, )
138
 
        else:
139
 
            return (suffix,)
140
213
    # we loop over each key because that spreads the inserts across prefixes,
141
214
    # which is how commit operates.
142
215
    for prefix in prefixes:
143
 
        result.append(files.add_lines(prefix + get_key('origin'), (),
 
216
        result.append(files.add_lines(prefix + ('origin',), (),
144
217
            ['origin' + last_char]))
145
218
    for prefix in prefixes:
146
 
        result.append(files.add_lines(prefix + get_key('base'),
 
219
        result.append(files.add_lines(prefix + ('base',),
147
220
            get_parents([('origin',)]), ['base' + last_char]))
148
221
    for prefix in prefixes:
149
 
        result.append(files.add_lines(prefix + get_key('left'),
 
222
        result.append(files.add_lines(prefix + ('left',),
150
223
            get_parents([('base',)]),
151
224
            ['base\n', 'left' + last_char]))
152
225
    if not left_only:
153
226
        for prefix in prefixes:
154
 
            result.append(files.add_lines(prefix + get_key('right'),
 
227
            result.append(files.add_lines(prefix + ('right',),
155
228
                get_parents([('base',)]),
156
229
                ['base\n', 'right' + last_char]))
157
230
        for prefix in prefixes:
158
 
            result.append(files.add_lines(prefix + get_key('merged'),
 
231
            result.append(files.add_lines(prefix + ('merged',),
159
232
                get_parents([('left',), ('right',)]),
160
233
                ['base\n', 'left\n', 'right\n', 'merged' + last_char]))
161
234
    return result
636
709
        # the ordering here is to make a tree so that dumb searches have
637
710
        # more changes to muck up.
638
711
 
639
 
        class InstrumentedProgress(progress.ProgressTask):
 
712
        class InstrumentedProgress(progress.DummyProgress):
640
713
 
641
714
            def __init__(self):
642
 
                progress.ProgressTask.__init__(self)
 
715
 
 
716
                progress.DummyProgress.__init__(self)
643
717
                self.updates = []
644
718
 
645
719
            def update(self, msg=None, current=None, total=None):
751
825
        self.assertEquals(('references_ghost', 'line_c\n'), origins[2])
752
826
 
753
827
    def test_readonly_mode(self):
754
 
        t = self.get_transport()
 
828
        transport = get_transport(self.get_url('.'))
755
829
        factory = self.get_factory()
756
 
        vf = factory('id', t, 0777, create=True, access_mode='w')
757
 
        vf = factory('id', t, access_mode='r')
 
830
        vf = factory('id', transport, 0777, create=True, access_mode='w')
 
831
        vf = factory('id', transport, access_mode='r')
758
832
        self.assertRaises(errors.ReadOnlyError, vf.add_lines, 'base', [], [])
759
833
        self.assertRaises(errors.ReadOnlyError,
760
834
                          vf.add_lines_with_ghosts,
782
856
class TestWeave(TestCaseWithMemoryTransport, VersionedFileTestMixIn):
783
857
 
784
858
    def get_file(self, name='foo'):
785
 
        return WeaveFile(name, self.get_transport(),
786
 
                         create=True,
787
 
                         get_scope=self.get_transaction)
 
859
        return WeaveFile(name, get_transport(self.get_url('.')), create=True,
 
860
            get_scope=self.get_transaction)
788
861
 
789
862
    def get_file_corrupted_text(self):
790
 
        w = WeaveFile('foo', self.get_transport(),
791
 
                      create=True,
792
 
                      get_scope=self.get_transaction)
 
863
        w = WeaveFile('foo', get_transport(self.get_url('.')), create=True,
 
864
            get_scope=self.get_transaction)
793
865
        w.add_lines('v1', [], ['hello\n'])
794
866
        w.add_lines('v2', ['v1'], ['hello\n', 'there\n'])
795
867
 
823
895
        return w
824
896
 
825
897
    def reopen_file(self, name='foo', create=False):
826
 
        return WeaveFile(name, self.get_transport(),
827
 
                         create=create,
828
 
                         get_scope=self.get_transaction)
 
898
        return WeaveFile(name, get_transport(self.get_url('.')), create=create,
 
899
            get_scope=self.get_transaction)
829
900
 
830
901
    def test_no_implicit_create(self):
831
902
        self.assertRaises(errors.NoSuchFile,
832
903
                          WeaveFile,
833
904
                          'foo',
834
 
                          self.get_transport(),
 
905
                          get_transport(self.get_url('.')),
835
906
                          get_scope=self.get_transaction)
836
907
 
837
908
    def get_factory(self):
904
975
        # we should be able to read from http with a versioned file.
905
976
        vf = self.get_file()
906
977
        # try an empty file access
907
 
        readonly_vf = self.get_factory()('foo', transport.get_transport(
908
 
                self.get_readonly_url('.')))
 
978
        readonly_vf = self.get_factory()('foo', get_transport(self.get_readonly_url('.')))
909
979
        self.assertEqual([], readonly_vf.versions())
910
 
 
911
 
    def test_readonly_http_works_with_feeling(self):
912
 
        # we should be able to read from http with a versioned file.
913
 
        vf = self.get_file()
914
980
        # now with feeling.
915
981
        vf.add_lines('1', [], ['a\n'])
916
982
        vf.add_lines('2', ['1'], ['b\n', 'a\n'])
917
 
        readonly_vf = self.get_factory()('foo', transport.get_transport(
918
 
                self.get_readonly_url('.')))
 
983
        readonly_vf = self.get_factory()('foo', get_transport(self.get_readonly_url('.')))
919
984
        self.assertEqual(['1', '2'], vf.versions())
920
 
        self.assertEqual(['1', '2'], readonly_vf.versions())
921
985
        for version in readonly_vf.versions():
922
986
            readonly_vf.get_lines(version)
923
987
 
925
989
class TestWeaveHTTP(TestCaseWithWebserver, TestReadonlyHttpMixin):
926
990
 
927
991
    def get_file(self):
928
 
        return WeaveFile('foo', self.get_transport(),
929
 
                         create=True,
930
 
                         get_scope=self.get_transaction)
 
992
        return WeaveFile('foo', get_transport(self.get_url('.')), create=True,
 
993
            get_scope=self.get_transaction)
931
994
 
932
995
    def get_factory(self):
933
996
        return WeaveFile
1066
1129
            """
1067
1130
        result = """\
1068
1131
            line 1
1069
 
<<<<<<<\x20
1070
 
            line 2
1071
 
=======
1072
 
>>>>>>>\x20
1073
1132
            """
1074
1133
        self._test_merge_from_strings(base, a, b, result)
1075
1134
 
1177
1236
class TestWeaveMerge(TestCaseWithMemoryTransport, MergeCasesMixin):
1178
1237
 
1179
1238
    def get_file(self, name='foo'):
1180
 
        return WeaveFile(name, self.get_transport(),
1181
 
                         create=True)
 
1239
        return WeaveFile(name, get_transport(self.get_url('.')), create=True)
1182
1240
 
1183
1241
    def log_contents(self, w):
1184
1242
        self.log('weave is:')
1377
1435
class TestVersionedFiles(TestCaseWithMemoryTransport):
1378
1436
    """Tests for the multiple-file variant of VersionedFile."""
1379
1437
 
1380
 
    # We want to be sure of behaviour for:
1381
 
    # weaves prefix layout (weave texts)
1382
 
    # individually named weaves (weave inventories)
1383
 
    # annotated knits - prefix|hash|hash-escape layout, we test the third only
1384
 
    #                   as it is the most complex mapper.
1385
 
    # individually named knits
1386
 
    # individual no-graph knits in packs (signatures)
1387
 
    # individual graph knits in packs (inventories)
1388
 
    # individual graph nocompression knits in packs (revisions)
1389
 
    # plain text knits in packs (texts)
1390
 
    len_one_scenarios = [
1391
 
        ('weave-named', {
1392
 
            'cleanup':None,
1393
 
            'factory':make_versioned_files_factory(WeaveFile,
1394
 
                ConstantMapper('inventory')),
1395
 
            'graph':True,
1396
 
            'key_length':1,
1397
 
            'support_partial_insertion': False,
1398
 
            }),
1399
 
        ('named-knit', {
1400
 
            'cleanup':None,
1401
 
            'factory':make_file_factory(False, ConstantMapper('revisions')),
1402
 
            'graph':True,
1403
 
            'key_length':1,
1404
 
            'support_partial_insertion': False,
1405
 
            }),
1406
 
        ('named-nograph-nodelta-knit-pack', {
1407
 
            'cleanup':cleanup_pack_knit,
1408
 
            'factory':make_pack_factory(False, False, 1),
1409
 
            'graph':False,
1410
 
            'key_length':1,
1411
 
            'support_partial_insertion': False,
1412
 
            }),
1413
 
        ('named-graph-knit-pack', {
1414
 
            'cleanup':cleanup_pack_knit,
1415
 
            'factory':make_pack_factory(True, True, 1),
1416
 
            'graph':True,
1417
 
            'key_length':1,
1418
 
            'support_partial_insertion': True,
1419
 
            }),
1420
 
        ('named-graph-nodelta-knit-pack', {
1421
 
            'cleanup':cleanup_pack_knit,
1422
 
            'factory':make_pack_factory(True, False, 1),
1423
 
            'graph':True,
1424
 
            'key_length':1,
1425
 
            'support_partial_insertion': False,
1426
 
            }),
1427
 
        ('groupcompress-nograph', {
1428
 
            'cleanup':groupcompress.cleanup_pack_group,
1429
 
            'factory':groupcompress.make_pack_factory(False, False, 1),
1430
 
            'graph': False,
1431
 
            'key_length':1,
1432
 
            'support_partial_insertion':False,
1433
 
            }),
1434
 
        ]
1435
 
    len_two_scenarios = [
1436
 
        ('weave-prefix', {
1437
 
            'cleanup':None,
1438
 
            'factory':make_versioned_files_factory(WeaveFile,
1439
 
                PrefixMapper()),
1440
 
            'graph':True,
1441
 
            'key_length':2,
1442
 
            'support_partial_insertion': False,
1443
 
            }),
1444
 
        ('annotated-knit-escape', {
1445
 
            'cleanup':None,
1446
 
            'factory':make_file_factory(True, HashEscapedPrefixMapper()),
1447
 
            'graph':True,
1448
 
            'key_length':2,
1449
 
            'support_partial_insertion': False,
1450
 
            }),
1451
 
        ('plain-knit-pack', {
1452
 
            'cleanup':cleanup_pack_knit,
1453
 
            'factory':make_pack_factory(True, True, 2),
1454
 
            'graph':True,
1455
 
            'key_length':2,
1456
 
            'support_partial_insertion': True,
1457
 
            }),
1458
 
        ('groupcompress', {
1459
 
            'cleanup':groupcompress.cleanup_pack_group,
1460
 
            'factory':groupcompress.make_pack_factory(True, False, 1),
1461
 
            'graph': True,
1462
 
            'key_length':1,
1463
 
            'support_partial_insertion':False,
1464
 
            }),
1465
 
        ]
1466
 
 
1467
 
    scenarios = len_one_scenarios + len_two_scenarios
1468
 
 
1469
1438
    def get_versionedfiles(self, relpath='files'):
1470
1439
        transport = self.get_transport(relpath)
1471
1440
        if relpath != '.':
1472
1441
            transport.mkdir('.')
1473
1442
        files = self.factory(transport)
1474
1443
        if self.cleanup is not None:
1475
 
            self.addCleanup(self.cleanup, files)
 
1444
            self.addCleanup(lambda:self.cleanup(files))
1476
1445
        return files
1477
1446
 
1478
 
    def get_simple_key(self, suffix):
1479
 
        """Return a key for the object under test."""
1480
 
        if self.key_length == 1:
1481
 
            return (suffix,)
1482
 
        else:
1483
 
            return ('FileA',) + (suffix,)
1484
 
 
1485
 
    def test_add_fallback_implies_without_fallbacks(self):
1486
 
        f = self.get_versionedfiles('files')
1487
 
        if getattr(f, 'add_fallback_versioned_files', None) is None:
1488
 
            raise TestNotApplicable("%s doesn't support fallbacks"
1489
 
                                    % (f.__class__.__name__,))
1490
 
        g = self.get_versionedfiles('fallback')
1491
 
        key_a = self.get_simple_key('a')
1492
 
        g.add_lines(key_a, [], ['\n'])
1493
 
        f.add_fallback_versioned_files(g)
1494
 
        self.assertTrue(key_a in f.get_parent_map([key_a]))
1495
 
        self.assertFalse(key_a in f.without_fallbacks().get_parent_map([key_a]))
1496
 
 
1497
 
    def test_add_lines(self):
1498
 
        f = self.get_versionedfiles()
1499
 
        key0 = self.get_simple_key('r0')
1500
 
        key1 = self.get_simple_key('r1')
1501
 
        key2 = self.get_simple_key('r2')
1502
 
        keyf = self.get_simple_key('foo')
1503
 
        f.add_lines(key0, [], ['a\n', 'b\n'])
1504
 
        if self.graph:
1505
 
            f.add_lines(key1, [key0], ['b\n', 'c\n'])
1506
 
        else:
1507
 
            f.add_lines(key1, [], ['b\n', 'c\n'])
1508
 
        keys = f.keys()
1509
 
        self.assertTrue(key0 in keys)
1510
 
        self.assertTrue(key1 in keys)
1511
 
        records = []
1512
 
        for record in f.get_record_stream([key0, key1], 'unordered', True):
1513
 
            records.append((record.key, record.get_bytes_as('fulltext')))
1514
 
        records.sort()
1515
 
        self.assertEqual([(key0, 'a\nb\n'), (key1, 'b\nc\n')], records)
1516
 
 
1517
 
    def test__add_text(self):
1518
 
        f = self.get_versionedfiles()
1519
 
        key0 = self.get_simple_key('r0')
1520
 
        key1 = self.get_simple_key('r1')
1521
 
        key2 = self.get_simple_key('r2')
1522
 
        keyf = self.get_simple_key('foo')
1523
 
        f._add_text(key0, [], 'a\nb\n')
1524
 
        if self.graph:
1525
 
            f._add_text(key1, [key0], 'b\nc\n')
1526
 
        else:
1527
 
            f._add_text(key1, [], 'b\nc\n')
1528
 
        keys = f.keys()
1529
 
        self.assertTrue(key0 in keys)
1530
 
        self.assertTrue(key1 in keys)
1531
 
        records = []
1532
 
        for record in f.get_record_stream([key0, key1], 'unordered', True):
1533
 
            records.append((record.key, record.get_bytes_as('fulltext')))
1534
 
        records.sort()
1535
 
        self.assertEqual([(key0, 'a\nb\n'), (key1, 'b\nc\n')], records)
1536
 
 
1537
1447
    def test_annotate(self):
1538
1448
        files = self.get_versionedfiles()
1539
1449
        self.get_diamond_files(files)
1573
1483
        self.assertRaises(RevisionNotPresent,
1574
1484
            files.annotate, prefix + ('missing-key',))
1575
1485
 
1576
 
    def test_check_no_parameters(self):
1577
 
        files = self.get_versionedfiles()
1578
 
 
1579
 
    def test_check_progressbar_parameter(self):
1580
 
        """A progress bar can be supplied because check can be a generator."""
1581
 
        pb = ui.ui_factory.nested_progress_bar()
1582
 
        self.addCleanup(pb.finished)
1583
 
        files = self.get_versionedfiles()
1584
 
        files.check(progress_bar=pb)
1585
 
 
1586
 
    def test_check_with_keys_becomes_generator(self):
1587
 
        files = self.get_versionedfiles()
1588
 
        self.get_diamond_files(files)
1589
 
        keys = files.keys()
1590
 
        entries = files.check(keys=keys)
1591
 
        seen = set()
1592
 
        # Texts output should be fulltexts.
1593
 
        self.capture_stream(files, entries, seen.add,
1594
 
            files.get_parent_map(keys), require_fulltext=True)
1595
 
        # All texts should be output.
1596
 
        self.assertEqual(set(keys), seen)
1597
 
 
1598
 
    def test_clear_cache(self):
1599
 
        files = self.get_versionedfiles()
1600
 
        files.clear_cache()
1601
 
 
1602
1486
    def test_construct(self):
1603
1487
        """Each parameterised test can be constructed on a transport."""
1604
1488
        files = self.get_versionedfiles()
1605
1489
 
1606
 
    def get_diamond_files(self, files, trailing_eol=True, left_only=False,
1607
 
        nokeys=False):
 
1490
    def get_diamond_files(self, files, trailing_eol=True, left_only=False):
1608
1491
        return get_diamond_files(files, self.key_length,
1609
1492
            trailing_eol=trailing_eol, nograph=not self.graph,
1610
 
            left_only=left_only, nokeys=nokeys)
 
1493
            left_only=left_only)
1611
1494
 
1612
 
    def _add_content_nostoresha(self, add_lines):
 
1495
    def test_add_lines_nostoresha(self):
1613
1496
        """When nostore_sha is supplied using old content raises."""
1614
1497
        vf = self.get_versionedfiles()
1615
1498
        empty_text = ('a', [])
1617
1500
        sample_text_no_nl = ('c', ["foo\n", "bar"])
1618
1501
        shas = []
1619
1502
        for version, lines in (empty_text, sample_text_nl, sample_text_no_nl):
1620
 
            if add_lines:
1621
 
                sha, _, _ = vf.add_lines(self.get_simple_key(version), [],
1622
 
                                         lines)
1623
 
            else:
1624
 
                sha, _, _ = vf._add_text(self.get_simple_key(version), [],
1625
 
                                         ''.join(lines))
 
1503
            sha, _, _ = vf.add_lines(self.get_simple_key(version), [], lines)
1626
1504
            shas.append(sha)
1627
1505
        # we now have a copy of all the lines in the vf.
1628
1506
        for sha, (version, lines) in zip(
1631
1509
            self.assertRaises(errors.ExistingContent,
1632
1510
                vf.add_lines, new_key, [], lines,
1633
1511
                nostore_sha=sha)
1634
 
            self.assertRaises(errors.ExistingContent,
1635
 
                vf._add_text, new_key, [], ''.join(lines),
1636
 
                nostore_sha=sha)
1637
1512
            # and no new version should have been added.
1638
1513
            record = vf.get_record_stream([new_key], 'unordered', True).next()
1639
1514
            self.assertEqual('absent', record.storage_kind)
1640
1515
 
1641
 
    def test_add_lines_nostoresha(self):
1642
 
        self._add_content_nostoresha(add_lines=True)
1643
 
 
1644
 
    def test__add_text_nostoresha(self):
1645
 
        self._add_content_nostoresha(add_lines=False)
1646
 
 
1647
1516
    def test_add_lines_return(self):
1648
1517
        files = self.get_versionedfiles()
1649
1518
        # save code by using the stock data insertion helper.
1675
1544
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1676
1545
                results)
1677
1546
 
1678
 
    def test_add_lines_no_key_generates_chk_key(self):
1679
 
        files = self.get_versionedfiles()
1680
 
        # save code by using the stock data insertion helper.
1681
 
        adds = self.get_diamond_files(files, nokeys=True)
1682
 
        results = []
1683
 
        # We can only validate the first 2 elements returned from add_lines.
1684
 
        for add in adds:
1685
 
            self.assertEqual(3, len(add))
1686
 
            results.append(add[:2])
1687
 
        if self.key_length == 1:
1688
 
            self.assertEqual([
1689
 
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1690
 
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1691
 
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1692
 
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1693
 
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1694
 
                results)
1695
 
            # Check the added items got CHK keys.
1696
 
            self.assertEqual(set([
1697
 
                ('sha1:00e364d235126be43292ab09cb4686cf703ddc17',),
1698
 
                ('sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44',),
1699
 
                ('sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1',),
1700
 
                ('sha1:a8478686da38e370e32e42e8a0c220e33ee9132f',),
1701
 
                ('sha1:ed8bce375198ea62444dc71952b22cfc2b09226d',),
1702
 
                ]),
1703
 
                files.keys())
1704
 
        elif self.key_length == 2:
1705
 
            self.assertEqual([
1706
 
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1707
 
                ('00e364d235126be43292ab09cb4686cf703ddc17', 7),
1708
 
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1709
 
                ('51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44', 5),
1710
 
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1711
 
                ('a8478686da38e370e32e42e8a0c220e33ee9132f', 10),
1712
 
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1713
 
                ('9ef09dfa9d86780bdec9219a22560c6ece8e0ef1', 11),
1714
 
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23),
1715
 
                ('ed8bce375198ea62444dc71952b22cfc2b09226d', 23)],
1716
 
                results)
1717
 
            # Check the added items got CHK keys.
1718
 
            self.assertEqual(set([
1719
 
                ('FileA', 'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1720
 
                ('FileA', 'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1721
 
                ('FileA', 'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1722
 
                ('FileA', 'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1723
 
                ('FileA', 'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1724
 
                ('FileB', 'sha1:00e364d235126be43292ab09cb4686cf703ddc17'),
1725
 
                ('FileB', 'sha1:51c64a6f4fc375daf0d24aafbabe4d91b6f4bb44'),
1726
 
                ('FileB', 'sha1:9ef09dfa9d86780bdec9219a22560c6ece8e0ef1'),
1727
 
                ('FileB', 'sha1:a8478686da38e370e32e42e8a0c220e33ee9132f'),
1728
 
                ('FileB', 'sha1:ed8bce375198ea62444dc71952b22cfc2b09226d'),
1729
 
                ]),
1730
 
                files.keys())
1731
 
 
1732
1547
    def test_empty_lines(self):
1733
1548
        """Empty files can be stored."""
1734
1549
        f = self.get_versionedfiles()
1756
1571
            f.get_record_stream([key_b], 'unordered', True
1757
1572
                ).next().get_bytes_as('fulltext'))
1758
1573
 
1759
 
    def test_get_known_graph_ancestry(self):
1760
 
        f = self.get_versionedfiles()
1761
 
        if not self.graph:
1762
 
            raise TestNotApplicable('ancestry info only relevant with graph.')
1763
 
        key_a = self.get_simple_key('a')
1764
 
        key_b = self.get_simple_key('b')
1765
 
        key_c = self.get_simple_key('c')
1766
 
        # A
1767
 
        # |\
1768
 
        # | B
1769
 
        # |/
1770
 
        # C
1771
 
        f.add_lines(key_a, [], ['\n'])
1772
 
        f.add_lines(key_b, [key_a], ['\n'])
1773
 
        f.add_lines(key_c, [key_a, key_b], ['\n'])
1774
 
        kg = f.get_known_graph_ancestry([key_c])
1775
 
        self.assertIsInstance(kg, _mod_graph.KnownGraph)
1776
 
        self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1777
 
 
1778
 
    def test_known_graph_with_fallbacks(self):
1779
 
        f = self.get_versionedfiles('files')
1780
 
        if not self.graph:
1781
 
            raise TestNotApplicable('ancestry info only relevant with graph.')
1782
 
        if getattr(f, 'add_fallback_versioned_files', None) is None:
1783
 
            raise TestNotApplicable("%s doesn't support fallbacks"
1784
 
                                    % (f.__class__.__name__,))
1785
 
        key_a = self.get_simple_key('a')
1786
 
        key_b = self.get_simple_key('b')
1787
 
        key_c = self.get_simple_key('c')
1788
 
        # A     only in fallback
1789
 
        # |\
1790
 
        # | B
1791
 
        # |/
1792
 
        # C
1793
 
        g = self.get_versionedfiles('fallback')
1794
 
        g.add_lines(key_a, [], ['\n'])
1795
 
        f.add_fallback_versioned_files(g)
1796
 
        f.add_lines(key_b, [key_a], ['\n'])
1797
 
        f.add_lines(key_c, [key_a, key_b], ['\n'])
1798
 
        kg = f.get_known_graph_ancestry([key_c])
1799
 
        self.assertEqual([key_a, key_b, key_c], list(kg.topo_sort()))
1800
 
 
1801
1574
    def test_get_record_stream_empty(self):
1802
1575
        """An empty stream can be requested without error."""
1803
1576
        f = self.get_versionedfiles()
1811
1584
             'knit-ft', 'knit-delta', 'chunked', 'fulltext',
1812
1585
             'knit-annotated-ft-gz', 'knit-annotated-delta-gz', 'knit-ft-gz',
1813
1586
             'knit-delta-gz',
1814
 
             'knit-delta-closure', 'knit-delta-closure-ref',
1815
 
             'groupcompress-block', 'groupcompress-block-ref'])
 
1587
             'knit-delta-closure', 'knit-delta-closure-ref'])
1816
1588
 
1817
 
    def capture_stream(self, f, entries, on_seen, parents,
1818
 
        require_fulltext=False):
 
1589
    def capture_stream(self, f, entries, on_seen, parents):
1819
1590
        """Capture a stream for testing."""
1820
1591
        for factory in entries:
1821
1592
            on_seen(factory.key)
1822
1593
            self.assertValidStorageKind(factory.storage_kind)
1823
 
            if factory.sha1 is not None:
1824
 
                self.assertEqual(f.get_sha1s([factory.key])[factory.key],
1825
 
                    factory.sha1)
 
1594
            self.assertEqual(f.get_sha1s([factory.key])[factory.key],
 
1595
                factory.sha1)
1826
1596
            self.assertEqual(parents[factory.key], factory.parents)
1827
1597
            self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
1828
1598
                str)
1829
 
            if require_fulltext:
1830
 
                factory.get_bytes_as('fulltext')
1831
1599
 
1832
1600
    def test_get_record_stream_interface(self):
1833
1601
        """each item in a stream has to provide a regular interface."""
1840
1608
        self.capture_stream(files, entries, seen.add, parent_map)
1841
1609
        self.assertEqual(set(keys), seen)
1842
1610
 
 
1611
    def get_simple_key(self, suffix):
 
1612
        """Return a key for the object under test."""
 
1613
        if self.key_length == 1:
 
1614
            return (suffix,)
 
1615
        else:
 
1616
            return ('FileA',) + (suffix,)
 
1617
 
1843
1618
    def get_keys_and_sort_order(self):
1844
1619
        """Get diamond test keys list, and their sort ordering."""
1845
1620
        if self.key_length == 1:
1960
1735
        for factory in entries:
1961
1736
            seen.add(factory.key)
1962
1737
            self.assertValidStorageKind(factory.storage_kind)
1963
 
            if factory.sha1 is not None:
1964
 
                self.assertEqual(files.get_sha1s([factory.key])[factory.key],
1965
 
                                 factory.sha1)
 
1738
            self.assertEqual(files.get_sha1s([factory.key])[factory.key],
 
1739
                factory.sha1)
1966
1740
            self.assertEqual(parent_map[factory.key], factory.parents)
1967
1741
            # currently no stream emits mpdiff
1968
1742
            self.assertRaises(errors.UnavailableRepresentation,
2166
1940
                self.assertEqual(None, factory.parents)
2167
1941
            else:
2168
1942
                self.assertValidStorageKind(factory.storage_kind)
2169
 
                if factory.sha1 is not None:
2170
 
                    sha1 = files.get_sha1s([factory.key])[factory.key]
2171
 
                    self.assertEqual(sha1, factory.sha1)
 
1943
                self.assertEqual(files.get_sha1s([factory.key])[factory.key],
 
1944
                    factory.sha1)
2172
1945
                self.assertEqual(parents[factory.key], factory.parents)
2173
1946
                self.assertIsInstance(factory.get_bytes_as(factory.storage_kind),
2174
1947
                    str)
2208
1981
        else:
2209
1982
            return None
2210
1983
 
2211
 
    def test_get_annotator(self):
2212
 
        files = self.get_versionedfiles()
2213
 
        self.get_diamond_files(files)
2214
 
        origin_key = self.get_simple_key('origin')
2215
 
        base_key = self.get_simple_key('base')
2216
 
        left_key = self.get_simple_key('left')
2217
 
        right_key = self.get_simple_key('right')
2218
 
        merged_key = self.get_simple_key('merged')
2219
 
        # annotator = files.get_annotator()
2220
 
        # introduced full text
2221
 
        origins, lines = files.get_annotator().annotate(origin_key)
2222
 
        self.assertEqual([(origin_key,)], origins)
2223
 
        self.assertEqual(['origin\n'], lines)
2224
 
        # a delta
2225
 
        origins, lines = files.get_annotator().annotate(base_key)
2226
 
        self.assertEqual([(base_key,)], origins)
2227
 
        # a merge
2228
 
        origins, lines = files.get_annotator().annotate(merged_key)
2229
 
        if self.graph:
2230
 
            self.assertEqual([
2231
 
                (base_key,),
2232
 
                (left_key,),
2233
 
                (right_key,),
2234
 
                (merged_key,),
2235
 
                ], origins)
2236
 
        else:
2237
 
            # Without a graph everything is new.
2238
 
            self.assertEqual([
2239
 
                (merged_key,),
2240
 
                (merged_key,),
2241
 
                (merged_key,),
2242
 
                (merged_key,),
2243
 
                ], origins)
2244
 
        self.assertRaises(RevisionNotPresent,
2245
 
            files.get_annotator().annotate, self.get_simple_key('missing-key'))
2246
 
 
2247
1984
    def test_get_parent_map(self):
2248
1985
        files = self.get_versionedfiles()
2249
1986
        if self.key_length == 1:
2452
2189
        else:
2453
2190
            self.assertIdenticalVersionedFile(source, files)
2454
2191
 
2455
 
    def test_insert_record_stream_long_parent_chain_out_of_order(self):
2456
 
        """An out of order stream can either error or work."""
2457
 
        if not self.graph:
2458
 
            raise TestNotApplicable('ancestry info only relevant with graph.')
2459
 
        # Create a reasonably long chain of records based on each other, where
2460
 
        # most will be deltas.
2461
 
        source = self.get_versionedfiles('source')
2462
 
        parents = ()
2463
 
        keys = []
2464
 
        content = [('same same %d\n' % n) for n in range(500)]
2465
 
        for letter in 'abcdefghijklmnopqrstuvwxyz':
2466
 
            key = ('key-' + letter,)
2467
 
            if self.key_length == 2:
2468
 
                key = ('prefix',) + key
2469
 
            content.append('content for ' + letter + '\n')
2470
 
            source.add_lines(key, parents, content)
2471
 
            keys.append(key)
2472
 
            parents = (key,)
2473
 
        # Create a stream of these records, excluding the first record that the
2474
 
        # rest ultimately depend upon, and insert it into a new vf.
2475
 
        streams = []
2476
 
        for key in reversed(keys):
2477
 
            streams.append(source.get_record_stream([key], 'unordered', False))
2478
 
        deltas = chain(*streams[:-1])
2479
 
        files = self.get_versionedfiles()
2480
 
        try:
2481
 
            files.insert_record_stream(deltas)
2482
 
        except RevisionNotPresent:
2483
 
            # Must not have corrupted the file.
2484
 
            files.check()
2485
 
        else:
2486
 
            # Must only report either just the first key as a missing parent,
2487
 
            # no key as missing (for nodelta scenarios).
2488
 
            missing = set(files.get_missing_compression_parent_keys())
2489
 
            missing.discard(keys[0])
2490
 
            self.assertEqual(set(), missing)
2491
 
 
2492
2192
    def get_knit_delta_source(self):
2493
2193
        """Get a source that can produce a stream with knit delta records,
2494
2194
        regardless of this test's scenario.
2562
2262
        # the ordering here is to make a tree so that dumb searches have
2563
2263
        # more changes to muck up.
2564
2264
 
2565
 
        class InstrumentedProgress(progress.ProgressTask):
 
2265
        class InstrumentedProgress(progress.DummyProgress):
2566
2266
 
2567
2267
            def __init__(self):
2568
 
                progress.ProgressTask.__init__(self)
 
2268
 
 
2269
                progress.DummyProgress.__init__(self)
2569
2270
                self.updates = []
2570
2271
 
2571
2272
            def update(self, msg=None, current=None, total=None):
2760
2461
        self.assertRaises(NotImplementedError,
2761
2462
                self.texts.add_mpdiffs, [])
2762
2463
 
2763
 
    def test_check_noerrors(self):
2764
 
        self.texts.check()
 
2464
    def test_check(self):
 
2465
        self.assertTrue(self.texts.check())
2765
2466
 
2766
2467
    def test_insert_record_stream(self):
2767
2468
        self.assertRaises(NotImplementedError, self.texts.insert_record_stream,