~bzr-pqm/bzr/bzr.dev

« back to all changes in this revision

Viewing changes to bzrlib/tests/test_repository.py

  • Committer: Robert Collins
  • Date: 2010-04-08 04:34:03 UTC
  • mfrom: (5138 +trunk)
  • mto: This revision was merged to the branch mainline in revision 5139.
  • Revision ID: robertc@robertcollins.net-20100408043403-56z0d07vdqrx7f3t
Update bugfix for 528114 to trunk.

Show diffs side-by-side

added added

removed removed

Lines of Context:
1
 
# Copyright (C) 2006, 2007 Canonical Ltd
 
1
# Copyright (C) 2006-2010 Canonical Ltd
2
2
#
3
3
# This program is free software; you can redistribute it and/or modify
4
4
# it under the terms of the GNU General Public License as published by
12
12
#
13
13
# You should have received a copy of the GNU General Public License
14
14
# along with this program; if not, write to the Free Software
15
 
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
15
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
 
17
17
"""Tests for the Repository facility that are not interface tests.
18
18
 
19
 
For interface tests see tests/repository_implementations/*.py.
 
19
For interface tests see tests/per_repository/*.py.
20
20
 
21
21
For concrete class tests see this file, and for storage formats tests
22
22
also see this file.
23
23
"""
24
24
 
25
 
import md5
26
25
from stat import S_ISDIR
27
26
from StringIO import StringIO
 
27
import sys
28
28
 
29
29
import bzrlib
30
30
from bzrlib.errors import (NotBranchError,
32
32
                           UnknownFormatError,
33
33
                           UnsupportedFormatError,
34
34
                           )
 
35
from bzrlib import (
 
36
    graph,
 
37
    tests,
 
38
    )
 
39
from bzrlib.branchbuilder import BranchBuilder
 
40
from bzrlib.btree_index import BTreeBuilder, BTreeGraphIndex
35
41
from bzrlib.index import GraphIndex, InMemoryGraphIndex
36
42
from bzrlib.repository import RepositoryFormat
37
43
from bzrlib.smart import server
38
44
from bzrlib.tests import (
39
45
    TestCase,
40
46
    TestCaseWithTransport,
 
47
    TestSkipped,
41
48
    test_knit,
42
49
    )
43
 
from bzrlib.transport import get_transport
44
 
from bzrlib.transport.memory import MemoryServer
45
 
from bzrlib.util import bencode
 
50
from bzrlib.transport import (
 
51
    fakenfs,
 
52
    get_transport,
 
53
    )
46
54
from bzrlib import (
 
55
    bencode,
47
56
    bzrdir,
48
57
    errors,
49
58
    inventory,
 
59
    osutils,
 
60
    progress,
50
61
    repository,
51
62
    revision as _mod_revision,
52
63
    symbol_versioning,
53
64
    upgrade,
 
65
    versionedfile,
54
66
    workingtree,
55
67
    )
56
 
from bzrlib.repofmt import knitrepo, weaverepo, pack_repo
 
68
from bzrlib.repofmt import (
 
69
    groupcompress_repo,
 
70
    knitrepo,
 
71
    pack_repo,
 
72
    weaverepo,
 
73
    )
57
74
 
58
75
 
59
76
class TestDefaultFormat(TestCase):
88
105
class SampleRepositoryFormat(repository.RepositoryFormat):
89
106
    """A sample format
90
107
 
91
 
    this format is initializable, unsupported to aid in testing the 
 
108
    this format is initializable, unsupported to aid in testing the
92
109
    open and open(unsupported=True) routines.
93
110
    """
94
111
 
115
132
    def test_find_format(self):
116
133
        # is the right format object found for a repository?
117
134
        # create a branch with a few known format objects.
118
 
        # this is not quite the same as 
 
135
        # this is not quite the same as
119
136
        self.build_tree(["foo/", "bar/"])
120
137
        def check_format(format, url):
121
138
            dir = format._matchingbzrdir.initialize(url)
124
141
            found_format = repository.RepositoryFormat.find_format(dir)
125
142
            self.failUnless(isinstance(found_format, format.__class__))
126
143
        check_format(weaverepo.RepositoryFormat7(), "bar")
127
 
        
 
144
 
128
145
    def test_find_format_no_repository(self):
129
146
        dir = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
130
147
        self.assertRaises(errors.NoRepositoryPresent,
156
173
 
157
174
class TestFormat6(TestCaseWithTransport):
158
175
 
 
176
    def test_attribute__fetch_order(self):
 
177
        """Weaves need topological data insertion."""
 
178
        control = bzrdir.BzrDirFormat6().initialize(self.get_url())
 
179
        repo = weaverepo.RepositoryFormat6().initialize(control)
 
180
        self.assertEqual('topological', repo._format._fetch_order)
 
181
 
 
182
    def test_attribute__fetch_uses_deltas(self):
 
183
        """Weaves do not reuse deltas."""
 
184
        control = bzrdir.BzrDirFormat6().initialize(self.get_url())
 
185
        repo = weaverepo.RepositoryFormat6().initialize(control)
 
186
        self.assertEqual(False, repo._format._fetch_uses_deltas)
 
187
 
 
188
    def test_attribute__fetch_reconcile(self):
 
189
        """Weave repositories need a reconcile after fetch."""
 
190
        control = bzrdir.BzrDirFormat6().initialize(self.get_url())
 
191
        repo = weaverepo.RepositoryFormat6().initialize(control)
 
192
        self.assertEqual(True, repo._format._fetch_reconcile)
 
193
 
159
194
    def test_no_ancestry_weave(self):
160
195
        control = bzrdir.BzrDirFormat6().initialize(self.get_url())
161
196
        repo = weaverepo.RepositoryFormat6().initialize(control)
165
200
                          control.transport.get,
166
201
                          'ancestry.weave')
167
202
 
168
 
    def test_exposed_versioned_files_are_marked_dirty(self):
 
203
    def test_supports_external_lookups(self):
169
204
        control = bzrdir.BzrDirFormat6().initialize(self.get_url())
170
205
        repo = weaverepo.RepositoryFormat6().initialize(control)
171
 
        repo.lock_write()
172
 
        inv = repo.get_inventory_weave()
173
 
        repo.unlock()
174
 
        self.assertRaises(errors.OutSideTransaction,
175
 
            inv.add_lines, 'foo', [], [])
 
206
        self.assertFalse(repo._format.supports_external_lookups)
176
207
 
177
208
 
178
209
class TestFormat7(TestCaseWithTransport):
179
 
    
 
210
 
 
211
    def test_attribute__fetch_order(self):
 
212
        """Weaves need topological data insertion."""
 
213
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
 
214
        repo = weaverepo.RepositoryFormat7().initialize(control)
 
215
        self.assertEqual('topological', repo._format._fetch_order)
 
216
 
 
217
    def test_attribute__fetch_uses_deltas(self):
 
218
        """Weaves do not reuse deltas."""
 
219
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
 
220
        repo = weaverepo.RepositoryFormat7().initialize(control)
 
221
        self.assertEqual(False, repo._format._fetch_uses_deltas)
 
222
 
 
223
    def test_attribute__fetch_reconcile(self):
 
224
        """Weave repositories need a reconcile after fetch."""
 
225
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
 
226
        repo = weaverepo.RepositoryFormat7().initialize(control)
 
227
        self.assertEqual(True, repo._format._fetch_reconcile)
 
228
 
180
229
    def test_disk_layout(self):
181
230
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
182
231
        repo = weaverepo.RepositoryFormat7().initialize(control)
198
247
                             'w\n'
199
248
                             'W\n',
200
249
                             t.get('inventory.weave').read())
 
250
        # Creating a file with id Foo:Bar results in a non-escaped file name on
 
251
        # disk.
 
252
        control.create_branch()
 
253
        tree = control.create_workingtree()
 
254
        tree.add(['foo'], ['Foo:Bar'], ['file'])
 
255
        tree.put_file_bytes_non_atomic('Foo:Bar', 'content\n')
 
256
        try:
 
257
            tree.commit('first post', rev_id='first')
 
258
        except errors.IllegalPath:
 
259
            if sys.platform != 'win32':
 
260
                raise
 
261
            self.knownFailure('Foo:Bar cannot be used as a file-id on windows'
 
262
                              ' in repo format 7')
 
263
            return
 
264
        self.assertEqualDiff(
 
265
            '# bzr weave file v5\n'
 
266
            'i\n'
 
267
            '1 7fe70820e08a1aac0ef224d9c66ab66831cc4ab1\n'
 
268
            'n first\n'
 
269
            '\n'
 
270
            'w\n'
 
271
            '{ 0\n'
 
272
            '. content\n'
 
273
            '}\n'
 
274
            'W\n',
 
275
            t.get('weaves/74/Foo%3ABar.weave').read())
201
276
 
202
277
    def test_shared_disk_layout(self):
203
278
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
226
301
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
227
302
        repo = weaverepo.RepositoryFormat7().initialize(control, shared=True)
228
303
        t = control.get_repository_transport(None)
229
 
        # TODO: Should check there is a 'lock' toplevel directory, 
 
304
        # TODO: Should check there is a 'lock' toplevel directory,
230
305
        # regardless of contents
231
306
        self.assertFalse(t.has('lock/held/info'))
232
307
        repo.lock_write()
278
353
                             'W\n',
279
354
                             t.get('inventory.weave').read())
280
355
 
281
 
    def test_exposed_versioned_files_are_marked_dirty(self):
 
356
    def test_supports_external_lookups(self):
282
357
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
283
358
        repo = weaverepo.RepositoryFormat7().initialize(control)
284
 
        repo.lock_write()
285
 
        inv = repo.get_inventory_weave()
286
 
        repo.unlock()
287
 
        self.assertRaises(errors.OutSideTransaction,
288
 
            inv.add_lines, 'foo', [], [])
 
359
        self.assertFalse(repo._format.supports_external_lookups)
289
360
 
290
361
 
291
362
class TestFormatKnit1(TestCaseWithTransport):
292
 
    
 
363
 
 
364
    def test_attribute__fetch_order(self):
 
365
        """Knits need topological data insertion."""
 
366
        repo = self.make_repository('.',
 
367
                format=bzrdir.format_registry.get('knit')())
 
368
        self.assertEqual('topological', repo._format._fetch_order)
 
369
 
 
370
    def test_attribute__fetch_uses_deltas(self):
 
371
        """Knits reuse deltas."""
 
372
        repo = self.make_repository('.',
 
373
                format=bzrdir.format_registry.get('knit')())
 
374
        self.assertEqual(True, repo._format._fetch_uses_deltas)
 
375
 
293
376
    def test_disk_layout(self):
294
377
        control = bzrdir.BzrDirMetaFormat1().initialize(self.get_url())
295
378
        repo = knitrepo.RepositoryFormatKnit1().initialize(control)
309
392
        # self.assertEqualDiff('', t.get('lock').read())
310
393
        self.assertTrue(S_ISDIR(t.stat('knits').st_mode))
311
394
        self.check_knits(t)
 
395
        # Check per-file knits.
 
396
        branch = control.create_branch()
 
397
        tree = control.create_workingtree()
 
398
        tree.add(['foo'], ['Nasty-IdC:'], ['file'])
 
399
        tree.put_file_bytes_non_atomic('Nasty-IdC:', '')
 
400
        tree.commit('1st post', rev_id='foo')
 
401
        self.assertHasKnit(t, 'knits/e8/%254easty-%2549d%2543%253a',
 
402
            '\nfoo fulltext 0 81  :')
312
403
 
313
 
    def assertHasKnit(self, t, knit_name):
 
404
    def assertHasKnit(self, t, knit_name, extra_content=''):
314
405
        """Assert that knit_name exists on t."""
315
 
        self.assertEqualDiff('# bzr knit index 8\n',
 
406
        self.assertEqualDiff('# bzr knit index 8\n' + extra_content,
316
407
                             t.get(knit_name + '.kndx').read())
317
 
        # no default content
318
 
        self.assertTrue(t.has(knit_name + '.knit'))
319
408
 
320
409
    def check_knits(self, t):
321
410
        """check knit content for a repository."""
365
454
        self.assertTrue(S_ISDIR(t.stat('knits').st_mode))
366
455
        self.check_knits(t)
367
456
 
368
 
    def test_exposed_versioned_files_are_marked_dirty(self):
369
 
        format = bzrdir.BzrDirMetaFormat1()
370
 
        format.repository_format = knitrepo.RepositoryFormatKnit1()
371
 
        repo = self.make_repository('.', format=format)
372
 
        repo.lock_write()
373
 
        inv = repo.get_inventory_weave()
374
 
        repo.unlock()
375
 
        self.assertRaises(errors.OutSideTransaction,
376
 
            inv.add_lines, 'foo', [], [])
377
 
 
378
457
    def test_deserialise_sets_root_revision(self):
379
458
        """We must have a inventory.root.revision
380
459
 
385
464
        repo = self.make_repository('.',
386
465
                format=bzrdir.format_registry.get('knit')())
387
466
        inv_xml = '<inventory format="5">\n</inventory>\n'
388
 
        inv = repo.deserialise_inventory('test-rev-id', inv_xml)
 
467
        inv = repo._deserialise_inventory('test-rev-id', inv_xml)
389
468
        self.assertEqual('test-rev-id', inv.root.revision)
390
469
 
391
470
    def test_deserialise_uses_global_revision_id(self):
397
476
        # Arguably, the deserialise_inventory should detect a mismatch, and
398
477
        # raise an error, rather than silently using one revision_id over the
399
478
        # other.
400
 
        inv = repo.deserialise_inventory('test-rev-id', inv_xml)
 
479
        self.assertRaises(AssertionError, repo._deserialise_inventory,
 
480
            'test-rev-id', inv_xml)
 
481
        inv = repo._deserialise_inventory('other-rev-id', inv_xml)
401
482
        self.assertEqual('other-rev-id', inv.root.revision)
402
483
 
403
 
 
404
 
class KnitRepositoryStreamTests(test_knit.KnitTests):
405
 
    """Tests for knitrepo._get_stream_as_bytes."""
406
 
 
407
 
    def test_get_stream_as_bytes(self):
408
 
        # Make a simple knit
409
 
        k1 = self.make_test_knit()
410
 
        k1.add_lines('text-a', [], test_knit.split_lines(test_knit.TEXT_1))
411
 
        
412
 
        # Serialise it, check the output.
413
 
        bytes = knitrepo._get_stream_as_bytes(k1, ['text-a'])
414
 
        data = bencode.bdecode(bytes)
415
 
        format, record = data
416
 
        self.assertEqual('knit-plain', format)
417
 
        self.assertEqual(['text-a', ['fulltext'], []], record[:3])
418
 
        self.assertRecordContentEqual(k1, 'text-a', record[3])
419
 
 
420
 
    def test_get_stream_as_bytes_all(self):
421
 
        """Get a serialised data stream for all the records in a knit.
422
 
 
423
 
        Much like test_get_stream_all, except for get_stream_as_bytes.
424
 
        """
425
 
        k1 = self.make_test_knit()
426
 
        # Insert the same data as BasicKnitTests.test_knit_join, as they seem
427
 
        # to cover a range of cases (no parents, one parent, multiple parents).
428
 
        test_data = [
429
 
            ('text-a', [], test_knit.TEXT_1),
430
 
            ('text-b', ['text-a'], test_knit.TEXT_1),
431
 
            ('text-c', [], test_knit.TEXT_1),
432
 
            ('text-d', ['text-c'], test_knit.TEXT_1),
433
 
            ('text-m', ['text-b', 'text-d'], test_knit.TEXT_1),
434
 
           ]
435
 
        # This test is actually a bit strict as the order in which they're
436
 
        # returned is not defined.  This matches the current (deterministic)
437
 
        # behaviour.
438
 
        expected_data_list = [
439
 
            # version, options, parents
440
 
            ('text-a', ['fulltext'], []),
441
 
            ('text-b', ['line-delta'], ['text-a']),
442
 
            ('text-m', ['line-delta'], ['text-b', 'text-d']),
443
 
            ('text-c', ['fulltext'], []),
444
 
            ('text-d', ['line-delta'], ['text-c']),
445
 
            ]
446
 
        for version_id, parents, lines in test_data:
447
 
            k1.add_lines(version_id, parents, test_knit.split_lines(lines))
448
 
 
449
 
        bytes = knitrepo._get_stream_as_bytes(
450
 
            k1, ['text-a', 'text-b', 'text-m', 'text-c', 'text-d', ])
451
 
 
452
 
        data = bencode.bdecode(bytes)
453
 
        format = data.pop(0)
454
 
        self.assertEqual('knit-plain', format)
455
 
 
456
 
        for expected, actual in zip(expected_data_list, data):
457
 
            expected_version = expected[0]
458
 
            expected_options = expected[1]
459
 
            expected_parents = expected[2]
460
 
            version, options, parents, bytes = actual
461
 
            self.assertEqual(expected_version, version)
462
 
            self.assertEqual(expected_options, options)
463
 
            self.assertEqual(expected_parents, parents)
464
 
            self.assertRecordContentEqual(k1, version, bytes)
 
484
    def test_supports_external_lookups(self):
 
485
        repo = self.make_repository('.',
 
486
                format=bzrdir.format_registry.get('knit')())
 
487
        self.assertFalse(repo._format.supports_external_lookups)
465
488
 
466
489
 
467
490
class DummyRepository(object):
468
491
    """A dummy repository for testing."""
469
492
 
 
493
    _format = None
470
494
    _serializer = None
471
495
 
472
496
    def supports_rich_root(self):
 
497
        if self._format is not None:
 
498
            return self._format.rich_root_data
473
499
        return False
474
500
 
 
501
    def get_graph(self):
 
502
        raise NotImplementedError
 
503
 
 
504
    def get_parent_map(self, revision_ids):
 
505
        raise NotImplementedError
 
506
 
475
507
 
476
508
class InterDummy(repository.InterRepository):
477
509
    """An inter-repository optimised code path for DummyRepository.
484
516
    @staticmethod
485
517
    def is_compatible(repo_source, repo_target):
486
518
        """InterDummy is compatible with DummyRepository."""
487
 
        return (isinstance(repo_source, DummyRepository) and 
 
519
        return (isinstance(repo_source, DummyRepository) and
488
520
            isinstance(repo_target, DummyRepository))
489
521
 
490
522
 
503
535
 
504
536
    def assertGetsDefaultInterRepository(self, repo_a, repo_b):
505
537
        """Asserts that InterRepository.get(repo_a, repo_b) -> the default.
506
 
        
 
538
 
507
539
        The effective default is now InterSameDataRepository because there is
508
540
        no actual sane default in the presence of incompatible data models.
509
541
        """
520
552
        # pair that it returns true on for the is_compatible static method
521
553
        # check
522
554
        dummy_a = DummyRepository()
 
555
        dummy_a._format = RepositoryFormat()
523
556
        dummy_b = DummyRepository()
 
557
        dummy_b._format = RepositoryFormat()
524
558
        repo = self.make_repository('.')
525
559
        # hack dummies to look like repo somewhat.
526
560
        dummy_a._serializer = repo._serializer
 
561
        dummy_a._format.supports_tree_reference = repo._format.supports_tree_reference
 
562
        dummy_a._format.rich_root_data = repo._format.rich_root_data
527
563
        dummy_b._serializer = repo._serializer
 
564
        dummy_b._format.supports_tree_reference = repo._format.supports_tree_reference
 
565
        dummy_b._format.rich_root_data = repo._format.rich_root_data
528
566
        repository.InterRepository.register_optimiser(InterDummy)
529
567
        try:
530
568
            # we should get the default for something InterDummy returns False
574
612
                                                        repo_b).__class__)
575
613
 
576
614
 
577
 
class TestInterRemoteToOther(TestCaseWithTransport):
578
 
 
579
 
    def make_remote_repository(self, path, backing_format=None):
580
 
        """Make a RemoteRepository object backed by a real repository that will
581
 
        be created at the given path."""
582
 
        self.make_repository(path, format=backing_format)
583
 
        smart_server = server.SmartTCPServer_for_testing()
584
 
        smart_server.setUp()
585
 
        remote_transport = get_transport(smart_server.get_url()).clone(path)
586
 
        self.addCleanup(smart_server.tearDown)
587
 
        remote_bzrdir = bzrdir.BzrDir.open_from_transport(remote_transport)
588
 
        remote_repo = remote_bzrdir.open_repository()
589
 
        return remote_repo
590
 
 
591
 
    def test_is_compatible_same_format(self):
592
 
        """InterRemoteToOther is compatible with a remote repository and a
593
 
        second repository that have the same format."""
594
 
        local_repo = self.make_repository('local')
595
 
        remote_repo = self.make_remote_repository('remote')
596
 
        is_compatible = repository.InterRemoteToOther.is_compatible
597
 
        self.assertTrue(
598
 
            is_compatible(remote_repo, local_repo),
599
 
            "InterRemoteToOther(%r, %r) is false" % (remote_repo, local_repo))
600
 
          
601
 
    def test_is_incompatible_different_format(self):
602
 
        local_repo = self.make_repository('local', 'dirstate')
603
 
        remote_repo = self.make_remote_repository('a', 'dirstate-with-subtree')
604
 
        is_compatible = repository.InterRemoteToOther.is_compatible
605
 
        self.assertFalse(
606
 
            is_compatible(remote_repo, local_repo),
607
 
            "InterRemoteToOther(%r, %r) is true" % (local_repo, remote_repo))
608
 
 
609
 
    def test_is_incompatible_different_format_both_remote(self):
610
 
        remote_repo_a = self.make_remote_repository(
611
 
            'a', 'dirstate-with-subtree')
612
 
        remote_repo_b = self.make_remote_repository('b', 'dirstate')
613
 
        is_compatible = repository.InterRemoteToOther.is_compatible
614
 
        self.assertFalse(
615
 
            is_compatible(remote_repo_a, remote_repo_b),
616
 
            "InterRemoteToOther(%r, %r) is true"
617
 
            % (remote_repo_a, remote_repo_b))
618
 
 
619
 
 
620
615
class TestRepositoryConverter(TestCaseWithTransport):
621
616
 
622
617
    def test_convert_empty(self):
636
631
 
637
632
 
638
633
class TestMisc(TestCase):
639
 
    
 
634
 
640
635
    def test_unescape_xml(self):
641
636
        """We get some kind of error when malformed entities are passed"""
642
 
        self.assertRaises(KeyError, repository._unescape_xml, 'foo&bar;') 
 
637
        self.assertRaises(KeyError, repository._unescape_xml, 'foo&bar;')
643
638
 
644
639
 
645
640
class TestRepositoryFormatKnit3(TestCaseWithTransport):
646
641
 
 
642
    def test_attribute__fetch_order(self):
 
643
        """Knits need topological data insertion."""
 
644
        format = bzrdir.BzrDirMetaFormat1()
 
645
        format.repository_format = knitrepo.RepositoryFormatKnit3()
 
646
        repo = self.make_repository('.', format=format)
 
647
        self.assertEqual('topological', repo._format._fetch_order)
 
648
 
 
649
    def test_attribute__fetch_uses_deltas(self):
 
650
        """Knits reuse deltas."""
 
651
        format = bzrdir.BzrDirMetaFormat1()
 
652
        format.repository_format = knitrepo.RepositoryFormatKnit3()
 
653
        repo = self.make_repository('.', format=format)
 
654
        self.assertEqual(True, repo._format._fetch_uses_deltas)
 
655
 
647
656
    def test_convert(self):
648
657
        """Ensure the upgrade adds weaves for roots"""
649
658
        format = bzrdir.BzrDirMetaFormat1()
651
660
        tree = self.make_branch_and_tree('.', format)
652
661
        tree.commit("Dull commit", rev_id="dull")
653
662
        revision_tree = tree.branch.repository.revision_tree('dull')
654
 
        self.assertRaises(errors.NoSuchFile, revision_tree.get_file_lines,
655
 
            revision_tree.inventory.root.file_id)
 
663
        revision_tree.lock_read()
 
664
        try:
 
665
            self.assertRaises(errors.NoSuchFile, revision_tree.get_file_lines,
 
666
                revision_tree.inventory.root.file_id)
 
667
        finally:
 
668
            revision_tree.unlock()
656
669
        format = bzrdir.BzrDirMetaFormat1()
657
670
        format.repository_format = knitrepo.RepositoryFormatKnit3()
658
671
        upgrade.Convert('.', format)
659
672
        tree = workingtree.WorkingTree.open('.')
660
673
        revision_tree = tree.branch.repository.revision_tree('dull')
661
 
        revision_tree.get_file_lines(revision_tree.inventory.root.file_id)
 
674
        revision_tree.lock_read()
 
675
        try:
 
676
            revision_tree.get_file_lines(revision_tree.inventory.root.file_id)
 
677
        finally:
 
678
            revision_tree.unlock()
662
679
        tree.commit("Another dull commit", rev_id='dull2')
663
680
        revision_tree = tree.branch.repository.revision_tree('dull2')
 
681
        revision_tree.lock_read()
 
682
        self.addCleanup(revision_tree.unlock)
664
683
        self.assertEqual('dull', revision_tree.inventory.root.revision)
665
684
 
666
 
    def test_exposed_versioned_files_are_marked_dirty(self):
 
685
    def test_supports_external_lookups(self):
667
686
        format = bzrdir.BzrDirMetaFormat1()
668
687
        format.repository_format = knitrepo.RepositoryFormatKnit3()
669
688
        repo = self.make_repository('.', format=format)
670
 
        repo.lock_write()
671
 
        inv = repo.get_inventory_weave()
672
 
        repo.unlock()
673
 
        self.assertRaises(errors.OutSideTransaction,
674
 
            inv.add_lines, 'foo', [], [])
 
689
        self.assertFalse(repo._format.supports_external_lookups)
 
690
 
 
691
 
 
692
class Test2a(tests.TestCaseWithMemoryTransport):
 
693
 
 
694
    def test_fetch_combines_groups(self):
 
695
        builder = self.make_branch_builder('source', format='2a')
 
696
        builder.start_series()
 
697
        builder.build_snapshot('1', None, [
 
698
            ('add', ('', 'root-id', 'directory', '')),
 
699
            ('add', ('file', 'file-id', 'file', 'content\n'))])
 
700
        builder.build_snapshot('2', ['1'], [
 
701
            ('modify', ('file-id', 'content-2\n'))])
 
702
        builder.finish_series()
 
703
        source = builder.get_branch()
 
704
        target = self.make_repository('target', format='2a')
 
705
        target.fetch(source.repository)
 
706
        target.lock_read()
 
707
        self.addCleanup(target.unlock)
 
708
        details = target.texts._index.get_build_details(
 
709
            [('file-id', '1',), ('file-id', '2',)])
 
710
        file_1_details = details[('file-id', '1')]
 
711
        file_2_details = details[('file-id', '2')]
 
712
        # The index, and what to read off disk, should be the same for both
 
713
        # versions of the file.
 
714
        self.assertEqual(file_1_details[0][:3], file_2_details[0][:3])
 
715
 
 
716
    def test_fetch_combines_groups(self):
 
717
        builder = self.make_branch_builder('source', format='2a')
 
718
        builder.start_series()
 
719
        builder.build_snapshot('1', None, [
 
720
            ('add', ('', 'root-id', 'directory', '')),
 
721
            ('add', ('file', 'file-id', 'file', 'content\n'))])
 
722
        builder.build_snapshot('2', ['1'], [
 
723
            ('modify', ('file-id', 'content-2\n'))])
 
724
        builder.finish_series()
 
725
        source = builder.get_branch()
 
726
        target = self.make_repository('target', format='2a')
 
727
        target.fetch(source.repository)
 
728
        target.lock_read()
 
729
        self.addCleanup(target.unlock)
 
730
        details = target.texts._index.get_build_details(
 
731
            [('file-id', '1',), ('file-id', '2',)])
 
732
        file_1_details = details[('file-id', '1')]
 
733
        file_2_details = details[('file-id', '2')]
 
734
        # The index, and what to read off disk, should be the same for both
 
735
        # versions of the file.
 
736
        self.assertEqual(file_1_details[0][:3], file_2_details[0][:3])
 
737
 
 
738
    def test_fetch_combines_groups(self):
 
739
        builder = self.make_branch_builder('source', format='2a')
 
740
        builder.start_series()
 
741
        builder.build_snapshot('1', None, [
 
742
            ('add', ('', 'root-id', 'directory', '')),
 
743
            ('add', ('file', 'file-id', 'file', 'content\n'))])
 
744
        builder.build_snapshot('2', ['1'], [
 
745
            ('modify', ('file-id', 'content-2\n'))])
 
746
        builder.finish_series()
 
747
        source = builder.get_branch()
 
748
        target = self.make_repository('target', format='2a')
 
749
        target.fetch(source.repository)
 
750
        target.lock_read()
 
751
        self.addCleanup(target.unlock)
 
752
        details = target.texts._index.get_build_details(
 
753
            [('file-id', '1',), ('file-id', '2',)])
 
754
        file_1_details = details[('file-id', '1')]
 
755
        file_2_details = details[('file-id', '2')]
 
756
        # The index, and what to read off disk, should be the same for both
 
757
        # versions of the file.
 
758
        self.assertEqual(file_1_details[0][:3], file_2_details[0][:3])
 
759
 
 
760
    def test_format_pack_compresses_True(self):
 
761
        repo = self.make_repository('repo', format='2a')
 
762
        self.assertTrue(repo._format.pack_compresses)
 
763
 
 
764
    def test_inventories_use_chk_map_with_parent_base_dict(self):
 
765
        tree = self.make_branch_and_memory_tree('repo', format="2a")
 
766
        tree.lock_write()
 
767
        tree.add([''], ['TREE_ROOT'])
 
768
        revid = tree.commit("foo")
 
769
        tree.unlock()
 
770
        tree.lock_read()
 
771
        self.addCleanup(tree.unlock)
 
772
        inv = tree.branch.repository.get_inventory(revid)
 
773
        self.assertNotEqual(None, inv.parent_id_basename_to_file_id)
 
774
        inv.parent_id_basename_to_file_id._ensure_root()
 
775
        inv.id_to_entry._ensure_root()
 
776
        self.assertEqual(65536, inv.id_to_entry._root_node.maximum_size)
 
777
        self.assertEqual(65536,
 
778
            inv.parent_id_basename_to_file_id._root_node.maximum_size)
 
779
 
 
780
    def test_autopack_unchanged_chk_nodes(self):
 
781
        # at 20 unchanged commits, chk pages are packed that are split into
 
782
        # two groups such that the new pack being made doesn't have all its
 
783
        # pages in the source packs (though they are in the repository).
 
784
        # Use a memory backed repository, we don't need to hit disk for this
 
785
        tree = self.make_branch_and_memory_tree('tree', format='2a')
 
786
        tree.lock_write()
 
787
        self.addCleanup(tree.unlock)
 
788
        tree.add([''], ['TREE_ROOT'])
 
789
        for pos in range(20):
 
790
            tree.commit(str(pos))
 
791
 
 
792
    def test_pack_with_hint(self):
 
793
        tree = self.make_branch_and_memory_tree('tree', format='2a')
 
794
        tree.lock_write()
 
795
        self.addCleanup(tree.unlock)
 
796
        tree.add([''], ['TREE_ROOT'])
 
797
        # 1 commit to leave untouched
 
798
        tree.commit('1')
 
799
        to_keep = tree.branch.repository._pack_collection.names()
 
800
        # 2 to combine
 
801
        tree.commit('2')
 
802
        tree.commit('3')
 
803
        all = tree.branch.repository._pack_collection.names()
 
804
        combine = list(set(all) - set(to_keep))
 
805
        self.assertLength(3, all)
 
806
        self.assertLength(2, combine)
 
807
        tree.branch.repository.pack(hint=combine)
 
808
        final = tree.branch.repository._pack_collection.names()
 
809
        self.assertLength(2, final)
 
810
        self.assertFalse(combine[0] in final)
 
811
        self.assertFalse(combine[1] in final)
 
812
        self.assertSubset(to_keep, final)
 
813
 
 
814
    def test_stream_source_to_gc(self):
 
815
        source = self.make_repository('source', format='2a')
 
816
        target = self.make_repository('target', format='2a')
 
817
        stream = source._get_source(target._format)
 
818
        self.assertIsInstance(stream, groupcompress_repo.GroupCHKStreamSource)
 
819
 
 
820
    def test_stream_source_to_non_gc(self):
 
821
        source = self.make_repository('source', format='2a')
 
822
        target = self.make_repository('target', format='rich-root-pack')
 
823
        stream = source._get_source(target._format)
 
824
        # We don't want the child GroupCHKStreamSource
 
825
        self.assertIs(type(stream), repository.StreamSource)
 
826
 
 
827
    def test_get_stream_for_missing_keys_includes_all_chk_refs(self):
 
828
        source_builder = self.make_branch_builder('source',
 
829
                            format='2a')
 
830
        # We have to build a fairly large tree, so that we are sure the chk
 
831
        # pages will have split into multiple pages.
 
832
        entries = [('add', ('', 'a-root-id', 'directory', None))]
 
833
        for i in 'abcdefghijklmnopqrstuvwxyz123456789':
 
834
            for j in 'abcdefghijklmnopqrstuvwxyz123456789':
 
835
                fname = i + j
 
836
                fid = fname + '-id'
 
837
                content = 'content for %s\n' % (fname,)
 
838
                entries.append(('add', (fname, fid, 'file', content)))
 
839
        source_builder.start_series()
 
840
        source_builder.build_snapshot('rev-1', None, entries)
 
841
        # Now change a few of them, so we get a few new pages for the second
 
842
        # revision
 
843
        source_builder.build_snapshot('rev-2', ['rev-1'], [
 
844
            ('modify', ('aa-id', 'new content for aa-id\n')),
 
845
            ('modify', ('cc-id', 'new content for cc-id\n')),
 
846
            ('modify', ('zz-id', 'new content for zz-id\n')),
 
847
            ])
 
848
        source_builder.finish_series()
 
849
        source_branch = source_builder.get_branch()
 
850
        source_branch.lock_read()
 
851
        self.addCleanup(source_branch.unlock)
 
852
        target = self.make_repository('target', format='2a')
 
853
        source = source_branch.repository._get_source(target._format)
 
854
        self.assertIsInstance(source, groupcompress_repo.GroupCHKStreamSource)
 
855
 
 
856
        # On a regular pass, getting the inventories and chk pages for rev-2
 
857
        # would only get the newly created chk pages
 
858
        search = graph.SearchResult(set(['rev-2']), set(['rev-1']), 1,
 
859
                                    set(['rev-2']))
 
860
        simple_chk_records = []
 
861
        for vf_name, substream in source.get_stream(search):
 
862
            if vf_name == 'chk_bytes':
 
863
                for record in substream:
 
864
                    simple_chk_records.append(record.key)
 
865
            else:
 
866
                for _ in substream:
 
867
                    continue
 
868
        # 3 pages, the root (InternalNode), + 2 pages which actually changed
 
869
        self.assertEqual([('sha1:91481f539e802c76542ea5e4c83ad416bf219f73',),
 
870
                          ('sha1:4ff91971043668583985aec83f4f0ab10a907d3f',),
 
871
                          ('sha1:81e7324507c5ca132eedaf2d8414ee4bb2226187',),
 
872
                          ('sha1:b101b7da280596c71a4540e9a1eeba8045985ee0',)],
 
873
                         simple_chk_records)
 
874
        # Now, when we do a similar call using 'get_stream_for_missing_keys'
 
875
        # we should get a much larger set of pages.
 
876
        missing = [('inventories', 'rev-2')]
 
877
        full_chk_records = []
 
878
        for vf_name, substream in source.get_stream_for_missing_keys(missing):
 
879
            if vf_name == 'inventories':
 
880
                for record in substream:
 
881
                    self.assertEqual(('rev-2',), record.key)
 
882
            elif vf_name == 'chk_bytes':
 
883
                for record in substream:
 
884
                    full_chk_records.append(record.key)
 
885
            else:
 
886
                self.fail('Should not be getting a stream of %s' % (vf_name,))
 
887
        # We have 257 records now. This is because we have 1 root page, and 256
 
888
        # leaf pages in a complete listing.
 
889
        self.assertEqual(257, len(full_chk_records))
 
890
        self.assertSubset(simple_chk_records, full_chk_records)
 
891
 
 
892
    def test_inconsistency_fatal(self):
 
893
        repo = self.make_repository('repo', format='2a')
 
894
        self.assertTrue(repo.revisions._index._inconsistency_fatal)
 
895
        self.assertFalse(repo.texts._index._inconsistency_fatal)
 
896
        self.assertFalse(repo.inventories._index._inconsistency_fatal)
 
897
        self.assertFalse(repo.signatures._index._inconsistency_fatal)
 
898
        self.assertFalse(repo.chk_bytes._index._inconsistency_fatal)
 
899
 
 
900
 
 
901
class TestKnitPackStreamSource(tests.TestCaseWithMemoryTransport):
 
902
 
 
903
    def test_source_to_exact_pack_092(self):
 
904
        source = self.make_repository('source', format='pack-0.92')
 
905
        target = self.make_repository('target', format='pack-0.92')
 
906
        stream_source = source._get_source(target._format)
 
907
        self.assertIsInstance(stream_source, pack_repo.KnitPackStreamSource)
 
908
 
 
909
    def test_source_to_exact_pack_rich_root_pack(self):
 
910
        source = self.make_repository('source', format='rich-root-pack')
 
911
        target = self.make_repository('target', format='rich-root-pack')
 
912
        stream_source = source._get_source(target._format)
 
913
        self.assertIsInstance(stream_source, pack_repo.KnitPackStreamSource)
 
914
 
 
915
    def test_source_to_exact_pack_19(self):
 
916
        source = self.make_repository('source', format='1.9')
 
917
        target = self.make_repository('target', format='1.9')
 
918
        stream_source = source._get_source(target._format)
 
919
        self.assertIsInstance(stream_source, pack_repo.KnitPackStreamSource)
 
920
 
 
921
    def test_source_to_exact_pack_19_rich_root(self):
 
922
        source = self.make_repository('source', format='1.9-rich-root')
 
923
        target = self.make_repository('target', format='1.9-rich-root')
 
924
        stream_source = source._get_source(target._format)
 
925
        self.assertIsInstance(stream_source, pack_repo.KnitPackStreamSource)
 
926
 
 
927
    def test_source_to_remote_exact_pack_19(self):
 
928
        trans = self.make_smart_server('target')
 
929
        trans.ensure_base()
 
930
        source = self.make_repository('source', format='1.9')
 
931
        target = self.make_repository('target', format='1.9')
 
932
        target = repository.Repository.open(trans.base)
 
933
        stream_source = source._get_source(target._format)
 
934
        self.assertIsInstance(stream_source, pack_repo.KnitPackStreamSource)
 
935
 
 
936
    def test_stream_source_to_non_exact(self):
 
937
        source = self.make_repository('source', format='pack-0.92')
 
938
        target = self.make_repository('target', format='1.9')
 
939
        stream = source._get_source(target._format)
 
940
        self.assertIs(type(stream), repository.StreamSource)
 
941
 
 
942
    def test_stream_source_to_non_exact_rich_root(self):
 
943
        source = self.make_repository('source', format='1.9')
 
944
        target = self.make_repository('target', format='1.9-rich-root')
 
945
        stream = source._get_source(target._format)
 
946
        self.assertIs(type(stream), repository.StreamSource)
 
947
 
 
948
    def test_source_to_remote_non_exact_pack_19(self):
 
949
        trans = self.make_smart_server('target')
 
950
        trans.ensure_base()
 
951
        source = self.make_repository('source', format='1.9')
 
952
        target = self.make_repository('target', format='1.6')
 
953
        target = repository.Repository.open(trans.base)
 
954
        stream_source = source._get_source(target._format)
 
955
        self.assertIs(type(stream_source), repository.StreamSource)
 
956
 
 
957
    def test_stream_source_to_knit(self):
 
958
        source = self.make_repository('source', format='pack-0.92')
 
959
        target = self.make_repository('target', format='dirstate')
 
960
        stream = source._get_source(target._format)
 
961
        self.assertIs(type(stream), repository.StreamSource)
 
962
 
 
963
 
 
964
class TestDevelopment6FindParentIdsOfRevisions(TestCaseWithTransport):
 
965
    """Tests for _find_parent_ids_of_revisions."""
 
966
 
 
967
    def setUp(self):
 
968
        super(TestDevelopment6FindParentIdsOfRevisions, self).setUp()
 
969
        self.builder = self.make_branch_builder('source',
 
970
            format='development6-rich-root')
 
971
        self.builder.start_series()
 
972
        self.builder.build_snapshot('initial', None,
 
973
            [('add', ('', 'tree-root', 'directory', None))])
 
974
        self.repo = self.builder.get_branch().repository
 
975
        self.addCleanup(self.builder.finish_series)
 
976
 
 
977
    def assertParentIds(self, expected_result, rev_set):
 
978
        self.assertEqual(sorted(expected_result),
 
979
            sorted(self.repo._find_parent_ids_of_revisions(rev_set)))
 
980
 
 
981
    def test_simple(self):
 
982
        self.builder.build_snapshot('revid1', None, [])
 
983
        self.builder.build_snapshot('revid2', ['revid1'], [])
 
984
        rev_set = ['revid2']
 
985
        self.assertParentIds(['revid1'], rev_set)
 
986
 
 
987
    def test_not_first_parent(self):
 
988
        self.builder.build_snapshot('revid1', None, [])
 
989
        self.builder.build_snapshot('revid2', ['revid1'], [])
 
990
        self.builder.build_snapshot('revid3', ['revid2'], [])
 
991
        rev_set = ['revid3', 'revid2']
 
992
        self.assertParentIds(['revid1'], rev_set)
 
993
 
 
994
    def test_not_null(self):
 
995
        rev_set = ['initial']
 
996
        self.assertParentIds([], rev_set)
 
997
 
 
998
    def test_not_null_set(self):
 
999
        self.builder.build_snapshot('revid1', None, [])
 
1000
        rev_set = [_mod_revision.NULL_REVISION]
 
1001
        self.assertParentIds([], rev_set)
 
1002
 
 
1003
    def test_ghost(self):
 
1004
        self.builder.build_snapshot('revid1', None, [])
 
1005
        rev_set = ['ghost', 'revid1']
 
1006
        self.assertParentIds(['initial'], rev_set)
 
1007
 
 
1008
    def test_ghost_parent(self):
 
1009
        self.builder.build_snapshot('revid1', None, [])
 
1010
        self.builder.build_snapshot('revid2', ['revid1', 'ghost'], [])
 
1011
        rev_set = ['revid2', 'revid1']
 
1012
        self.assertParentIds(['ghost', 'initial'], rev_set)
 
1013
 
 
1014
    def test_righthand_parent(self):
 
1015
        self.builder.build_snapshot('revid1', None, [])
 
1016
        self.builder.build_snapshot('revid2a', ['revid1'], [])
 
1017
        self.builder.build_snapshot('revid2b', ['revid1'], [])
 
1018
        self.builder.build_snapshot('revid3', ['revid2a', 'revid2b'], [])
 
1019
        rev_set = ['revid3', 'revid2a']
 
1020
        self.assertParentIds(['revid1', 'revid2b'], rev_set)
675
1021
 
676
1022
 
677
1023
class TestWithBrokenRepo(TestCaseWithTransport):
692
1038
            inv = inventory.Inventory(revision_id='rev1a')
693
1039
            inv.root.revision = 'rev1a'
694
1040
            self.add_file(repo, inv, 'file1', 'rev1a', [])
 
1041
            repo.texts.add_lines((inv.root.file_id, 'rev1a'), [], [])
695
1042
            repo.add_inventory('rev1a', inv, [])
696
1043
            revision = _mod_revision.Revision('rev1a',
697
1044
                committer='jrandom@example.com', timestamp=0,
732
1079
    def add_revision(self, repo, revision_id, inv, parent_ids):
733
1080
        inv.revision_id = revision_id
734
1081
        inv.root.revision = revision_id
 
1082
        repo.texts.add_lines((inv.root.file_id, revision_id), [], [])
735
1083
        repo.add_inventory(revision_id, inv, parent_ids)
736
1084
        revision = _mod_revision.Revision(revision_id,
737
1085
            committer='jrandom@example.com', timestamp=0, inventory_sha1='',
744
1092
        entry.revision = revision
745
1093
        entry.text_size = 0
746
1094
        inv.add(entry)
747
 
        vf = repo.weave_store.get_weave_or_empty(file_id,
748
 
                                                 repo.get_transaction())
749
 
        vf.add_lines(revision, parents, ['line\n'])
 
1095
        text_key = (file_id, revision)
 
1096
        parent_keys = [(file_id, parent) for parent in parents]
 
1097
        repo.texts.add_lines(text_key, parent_keys, ['line\n'])
750
1098
 
751
1099
    def test_insert_from_broken_repo(self):
752
1100
        """Inserting a data stream from a broken repository won't silently
754
1102
        """
755
1103
        broken_repo = self.make_broken_repository()
756
1104
        empty_repo = self.make_repository('empty-repo')
757
 
        stream = broken_repo.get_data_stream(['rev1a', 'rev2', 'rev3'])
758
 
        empty_repo.lock_write()
 
1105
        try:
 
1106
            empty_repo.fetch(broken_repo)
 
1107
        except (errors.RevisionNotPresent, errors.BzrCheckError):
 
1108
            # Test successful: compression parent not being copied leads to
 
1109
            # error.
 
1110
            return
 
1111
        empty_repo.lock_read()
759
1112
        self.addCleanup(empty_repo.unlock)
760
 
        empty_repo.start_write_group()
761
 
        try:
762
 
            self.assertRaises(
763
 
                errors.KnitCorrupt, empty_repo.insert_data_stream, stream)
764
 
        finally:
765
 
            empty_repo.abort_write_group()
766
 
 
767
 
 
768
 
class TestKnitPackNoSubtrees(TestCaseWithTransport):
769
 
 
770
 
    def get_format(self):
771
 
        return bzrdir.format_registry.make_bzrdir('pack-0.92')
772
 
 
773
 
    def test_disk_layout(self):
774
 
        format = self.get_format()
775
 
        repo = self.make_repository('.', format=format)
776
 
        # in case of side effects of locking.
777
 
        repo.lock_write()
778
 
        repo.unlock()
779
 
        t = repo.bzrdir.get_repository_transport(None)
780
 
        self.check_format(t)
781
 
        # XXX: no locks left when unlocked at the moment
782
 
        # self.assertEqualDiff('', t.get('lock').read())
783
 
        self.check_databases(t)
784
 
 
785
 
    def check_format(self, t):
786
 
        self.assertEqualDiff(
787
 
            "Bazaar pack repository format 1 (needs bzr 0.92)\n",
788
 
                             t.get('format').read())
789
 
 
790
 
    def assertHasKndx(self, t, knit_name):
791
 
        """Assert that knit_name exists on t."""
792
 
        self.assertEqualDiff('# bzr knit index 8\n',
793
 
                             t.get(knit_name + '.kndx').read())
794
 
 
795
 
    def assertHasNoKndx(self, t, knit_name):
796
 
        """Assert that knit_name has no index on t."""
797
 
        self.assertFalse(t.has(knit_name + '.kndx'))
798
 
 
799
 
    def assertHasNoKnit(self, t, knit_name):
800
 
        """Assert that knit_name exists on t."""
801
 
        # no default content
802
 
        self.assertFalse(t.has(knit_name + '.knit'))
803
 
 
804
 
    def check_databases(self, t):
805
 
        """check knit content for a repository."""
806
 
        # check conversion worked
807
 
        self.assertHasNoKndx(t, 'inventory')
808
 
        self.assertHasNoKnit(t, 'inventory')
809
 
        self.assertHasNoKndx(t, 'revisions')
810
 
        self.assertHasNoKnit(t, 'revisions')
811
 
        self.assertHasNoKndx(t, 'signatures')
812
 
        self.assertHasNoKnit(t, 'signatures')
813
 
        self.assertFalse(t.has('knits'))
814
 
        # revision-indexes file-container directory
815
 
        self.assertEqual([],
816
 
            list(GraphIndex(t, 'pack-names', None).iter_all_entries()))
817
 
        self.assertTrue(S_ISDIR(t.stat('packs').st_mode))
818
 
        self.assertTrue(S_ISDIR(t.stat('upload').st_mode))
819
 
        self.assertTrue(S_ISDIR(t.stat('indices').st_mode))
820
 
        self.assertTrue(S_ISDIR(t.stat('obsolete_packs').st_mode))
821
 
 
822
 
    def test_shared_disk_layout(self):
823
 
        format = self.get_format()
824
 
        repo = self.make_repository('.', shared=True, format=format)
825
 
        # we want:
826
 
        t = repo.bzrdir.get_repository_transport(None)
827
 
        self.check_format(t)
828
 
        # XXX: no locks left when unlocked at the moment
829
 
        # self.assertEqualDiff('', t.get('lock').read())
830
 
        # We should have a 'shared-storage' marker file.
831
 
        self.assertEqualDiff('', t.get('shared-storage').read())
832
 
        self.check_databases(t)
833
 
 
834
 
    def test_shared_no_tree_disk_layout(self):
835
 
        format = self.get_format()
836
 
        repo = self.make_repository('.', shared=True, format=format)
837
 
        repo.set_make_working_trees(False)
838
 
        # we want:
839
 
        t = repo.bzrdir.get_repository_transport(None)
840
 
        self.check_format(t)
841
 
        # XXX: no locks left when unlocked at the moment
842
 
        # self.assertEqualDiff('', t.get('lock').read())
843
 
        # We should have a 'shared-storage' marker file.
844
 
        self.assertEqualDiff('', t.get('shared-storage').read())
845
 
        # We should have a marker for the no-working-trees flag.
846
 
        self.assertEqualDiff('', t.get('no-working-trees').read())
847
 
        # The marker should go when we toggle the setting.
848
 
        repo.set_make_working_trees(True)
849
 
        self.assertFalse(t.has('no-working-trees'))
850
 
        self.check_databases(t)
851
 
 
852
 
    def test_adding_revision_creates_pack_indices(self):
853
 
        format = self.get_format()
854
 
        tree = self.make_branch_and_tree('.', format=format)
855
 
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
856
 
        self.assertEqual([],
857
 
            list(GraphIndex(trans, 'pack-names', None).iter_all_entries()))
858
 
        tree.commit('foobarbaz')
859
 
        index = GraphIndex(trans, 'pack-names', None)
860
 
        index_nodes = list(index.iter_all_entries())
861
 
        self.assertEqual(1, len(index_nodes))
862
 
        node = index_nodes[0]
863
 
        name = node[1][0]
864
 
        # the pack sizes should be listed in the index
865
 
        pack_value = node[2]
866
 
        sizes = [int(digits) for digits in pack_value.split(' ')]
867
 
        for size, suffix in zip(sizes, ['.rix', '.iix', '.tix', '.six']):
868
 
            stat = trans.stat('indices/%s%s' % (name, suffix))
869
 
            self.assertEqual(size, stat.st_size)
870
 
 
871
 
    def test_pulling_nothing_leads_to_no_new_names(self):
872
 
        format = self.get_format()
873
 
        tree1 = self.make_branch_and_tree('1', format=format)
874
 
        tree2 = self.make_branch_and_tree('2', format=format)
875
 
        tree1.branch.repository.fetch(tree2.branch.repository)
876
 
        trans = tree1.branch.repository.bzrdir.get_repository_transport(None)
877
 
        self.assertEqual([],
878
 
            list(GraphIndex(trans, 'pack-names', None).iter_all_entries()))
879
 
 
880
 
    def test_commit_across_pack_shape_boundary_autopacks(self):
881
 
        format = self.get_format()
882
 
        tree = self.make_branch_and_tree('.', format=format)
883
 
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
884
 
        # This test could be a little cheaper by replacing the packs
885
 
        # attribute on the repository to allow a different pack distribution
886
 
        # and max packs policy - so we are checking the policy is honoured
887
 
        # in the test. But for now 11 commits is not a big deal in a single
888
 
        # test.
889
 
        for x in range(9):
890
 
            tree.commit('commit %s' % x)
891
 
        # there should be 9 packs:
892
 
        index = GraphIndex(trans, 'pack-names', None)
893
 
        self.assertEqual(9, len(list(index.iter_all_entries())))
894
 
        # insert some files in obsolete_packs which should be removed by pack.
895
 
        trans.put_bytes('obsolete_packs/foo', '123')
896
 
        trans.put_bytes('obsolete_packs/bar', '321')
897
 
        # committing one more should coalesce to 1 of 10.
898
 
        tree.commit('commit triggering pack')
899
 
        index = GraphIndex(trans, 'pack-names', None)
900
 
        self.assertEqual(1, len(list(index.iter_all_entries())))
901
 
        # packing should not damage data
902
 
        tree = tree.bzrdir.open_workingtree()
903
 
        check_result = tree.branch.repository.check(
904
 
            [tree.branch.last_revision()])
905
 
        # We should have 50 (10x5) files in the obsolete_packs directory.
906
 
        obsolete_files = list(trans.list_dir('obsolete_packs'))
907
 
        self.assertFalse('foo' in obsolete_files)
908
 
        self.assertFalse('bar' in obsolete_files)
909
 
        self.assertEqual(50, len(obsolete_files))
910
 
        # XXX: Todo check packs obsoleted correctly - old packs and indices
911
 
        # in the obsolete_packs directory.
912
 
        large_pack_name = list(index.iter_all_entries())[0][1][0]
913
 
        # finally, committing again should not touch the large pack.
914
 
        tree.commit('commit not triggering pack')
915
 
        index = GraphIndex(trans, 'pack-names', None)
916
 
        self.assertEqual(2, len(list(index.iter_all_entries())))
917
 
        pack_names = [node[1][0] for node in index.iter_all_entries()]
918
 
        self.assertTrue(large_pack_name in pack_names)
919
 
 
920
 
    def test_pack_after_two_commits_packs_everything(self):
921
 
        format = self.get_format()
922
 
        tree = self.make_branch_and_tree('.', format=format)
923
 
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
924
 
        tree.commit('start')
925
 
        tree.commit('more work')
926
 
        tree.branch.repository.pack()
927
 
        # there should be 1 pack:
928
 
        index = GraphIndex(trans, 'pack-names', None)
929
 
        self.assertEqual(1, len(list(index.iter_all_entries())))
930
 
        self.assertEqual(2, len(tree.branch.repository.all_revision_ids()))
931
 
 
932
 
    def test_pack_layout(self):
933
 
        format = self.get_format()
934
 
        tree = self.make_branch_and_tree('.', format=format)
935
 
        trans = tree.branch.repository.bzrdir.get_repository_transport(None)
936
 
        tree.commit('start', rev_id='1')
937
 
        tree.commit('more work', rev_id='2')
938
 
        tree.branch.repository.pack()
939
 
        tree.lock_read()
940
 
        self.addCleanup(tree.unlock)
941
 
        pack = tree.branch.repository._pack_collection.get_pack_by_name(
942
 
            tree.branch.repository._pack_collection.names()[0])
943
 
        # revision access tends to be tip->ancestor, so ordering that way on 
944
 
        # disk is a good idea.
945
 
        for _1, key, val, refs in pack.revision_index.iter_all_entries():
946
 
            if key == ('1',):
947
 
                pos_1 = int(val[1:].split()[0])
948
 
            else:
949
 
                pos_2 = int(val[1:].split()[0])
950
 
        self.assertTrue(pos_2 < pos_1)
951
 
 
952
 
    def test_pack_repositories_support_multiple_write_locks(self):
953
 
        format = self.get_format()
954
 
        self.make_repository('.', shared=True, format=format)
955
 
        r1 = repository.Repository.open('.')
956
 
        r2 = repository.Repository.open('.')
957
 
        r1.lock_write()
958
 
        self.addCleanup(r1.unlock)
959
 
        r2.lock_write()
960
 
        r2.unlock()
961
 
 
962
 
    def _add_text(self, repo, fileid):
963
 
        """Add a text to the repository within a write group."""
964
 
        vf =repo.weave_store.get_weave(fileid, repo.get_transaction())
965
 
        vf.add_lines('samplerev+' + fileid, [], [])
966
 
 
967
 
    def test_concurrent_writers_merge_new_packs(self):
968
 
        format = self.get_format()
969
 
        self.make_repository('.', shared=True, format=format)
970
 
        r1 = repository.Repository.open('.')
971
 
        r2 = repository.Repository.open('.')
972
 
        r1.lock_write()
973
 
        try:
974
 
            # access enough data to load the names list
975
 
            list(r1.all_revision_ids())
976
 
            r2.lock_write()
977
 
            try:
978
 
                # access enough data to load the names list
979
 
                list(r2.all_revision_ids())
980
 
                r1.start_write_group()
981
 
                try:
982
 
                    r2.start_write_group()
983
 
                    try:
984
 
                        self._add_text(r1, 'fileidr1')
985
 
                        self._add_text(r2, 'fileidr2')
986
 
                    except:
987
 
                        r2.abort_write_group()
988
 
                        raise
989
 
                except:
990
 
                    r1.abort_write_group()
991
 
                    raise
992
 
                # both r1 and r2 have open write groups with data in them
993
 
                # created while the other's write group was open.
994
 
                # Commit both which requires a merge to the pack-names.
995
 
                try:
996
 
                    r1.commit_write_group()
997
 
                except:
998
 
                    r1.abort_write_group()
999
 
                    r2.abort_write_group()
1000
 
                    raise
1001
 
                r2.commit_write_group()
1002
 
                # tell r1 to reload from disk
1003
 
                r1._pack_collection.reset()
1004
 
                # Now both repositories should know about both names
1005
 
                r1._pack_collection.ensure_loaded()
1006
 
                r2._pack_collection.ensure_loaded()
1007
 
                self.assertEqual(r1._pack_collection.names(), r2._pack_collection.names())
1008
 
                self.assertEqual(2, len(r1._pack_collection.names()))
1009
 
            finally:
1010
 
                r2.unlock()
1011
 
        finally:
1012
 
            r1.unlock()
1013
 
 
1014
 
    def test_concurrent_writer_second_preserves_dropping_a_pack(self):
1015
 
        format = self.get_format()
1016
 
        self.make_repository('.', shared=True, format=format)
1017
 
        r1 = repository.Repository.open('.')
1018
 
        r2 = repository.Repository.open('.')
1019
 
        # add a pack to drop
1020
 
        r1.lock_write()
1021
 
        try:
1022
 
            r1.start_write_group()
1023
 
            try:
1024
 
                self._add_text(r1, 'fileidr1')
1025
 
            except:
1026
 
                r1.abort_write_group()
1027
 
                raise
1028
 
            else:
1029
 
                r1.commit_write_group()
1030
 
            r1._pack_collection.ensure_loaded()
1031
 
            name_to_drop = r1._pack_collection.all_packs()[0].name
1032
 
        finally:
1033
 
            r1.unlock()
1034
 
        r1.lock_write()
1035
 
        try:
1036
 
            # access enough data to load the names list
1037
 
            list(r1.all_revision_ids())
1038
 
            r2.lock_write()
1039
 
            try:
1040
 
                # access enough data to load the names list
1041
 
                list(r2.all_revision_ids())
1042
 
                r1._pack_collection.ensure_loaded()
1043
 
                try:
1044
 
                    r2.start_write_group()
1045
 
                    try:
1046
 
                        # in r1, drop the pack
1047
 
                        r1._pack_collection._remove_pack_from_memory(
1048
 
                            r1._pack_collection.get_pack_by_name(name_to_drop))
1049
 
                        # in r2, add a pack
1050
 
                        self._add_text(r2, 'fileidr2')
1051
 
                    except:
1052
 
                        r2.abort_write_group()
1053
 
                        raise
1054
 
                except:
1055
 
                    r1._pack_collection.reset()
1056
 
                    raise
1057
 
                # r1 has a changed names list, and r2 an open write groups with
1058
 
                # changes.
1059
 
                # save r1, and then commit the r2 write group, which requires a
1060
 
                # merge to the pack-names, which should not reinstate
1061
 
                # name_to_drop
1062
 
                try:
1063
 
                    r1._pack_collection._save_pack_names()
1064
 
                    r1._pack_collection.reset()
1065
 
                except:
1066
 
                    r2.abort_write_group()
1067
 
                    raise
1068
 
                try:
1069
 
                    r2.commit_write_group()
1070
 
                except:
1071
 
                    r2.abort_write_group()
1072
 
                    raise
1073
 
                # Now both repositories should now about just one name.
1074
 
                r1._pack_collection.ensure_loaded()
1075
 
                r2._pack_collection.ensure_loaded()
1076
 
                self.assertEqual(r1._pack_collection.names(), r2._pack_collection.names())
1077
 
                self.assertEqual(1, len(r1._pack_collection.names()))
1078
 
                self.assertFalse(name_to_drop in r1._pack_collection.names())
1079
 
            finally:
1080
 
                r2.unlock()
1081
 
        finally:
1082
 
            r1.unlock()
1083
 
 
1084
 
    def test_lock_write_does_not_physically_lock(self):
1085
 
        repo = self.make_repository('.', format=self.get_format())
1086
 
        repo.lock_write()
1087
 
        self.addCleanup(repo.unlock)
1088
 
        self.assertFalse(repo.get_physical_lock_status())
1089
 
 
1090
 
    def prepare_for_break_lock(self):
1091
 
        # Setup the global ui factory state so that a break-lock method call
1092
 
        # will find usable input in the input stream.
1093
 
        old_factory = bzrlib.ui.ui_factory
1094
 
        def restoreFactory():
1095
 
            bzrlib.ui.ui_factory = old_factory
1096
 
        self.addCleanup(restoreFactory)
1097
 
        bzrlib.ui.ui_factory = bzrlib.ui.SilentUIFactory()
1098
 
        bzrlib.ui.ui_factory.stdin = StringIO("y\n")
1099
 
 
1100
 
    def test_break_lock_breaks_physical_lock(self):
1101
 
        repo = self.make_repository('.', format=self.get_format())
1102
 
        repo._pack_collection.lock_names()
1103
 
        repo2 = repository.Repository.open('.')
1104
 
        self.assertTrue(repo.get_physical_lock_status())
1105
 
        self.prepare_for_break_lock()
1106
 
        repo2.break_lock()
1107
 
        self.assertFalse(repo.get_physical_lock_status())
1108
 
 
1109
 
    def test_broken_physical_locks_error_on__unlock_names_lock(self):
1110
 
        repo = self.make_repository('.', format=self.get_format())
1111
 
        repo._pack_collection.lock_names()
1112
 
        self.assertTrue(repo.get_physical_lock_status())
1113
 
        repo2 = repository.Repository.open('.')
1114
 
        self.prepare_for_break_lock()
1115
 
        repo2.break_lock()
1116
 
        self.assertRaises(errors.LockBroken, repo._pack_collection._unlock_names)
1117
 
 
1118
 
    def test_fetch_without_find_ghosts_ignores_ghosts(self):
1119
 
        # we want two repositories at this point:
1120
 
        # one with a revision that is a ghost in the other
1121
 
        # repository.
1122
 
        # 'ghost' is present in has_ghost, 'ghost' is absent in 'missing_ghost'.
1123
 
        # 'references' is present in both repositories, and 'tip' is present
1124
 
        # just in has_ghost.
1125
 
        # has_ghost       missing_ghost
1126
 
        #------------------------------
1127
 
        # 'ghost'             -
1128
 
        # 'references'    'references'
1129
 
        # 'tip'               -
1130
 
        # In this test we fetch 'tip' which should not fetch 'ghost'
1131
 
        has_ghost = self.make_repository('has_ghost', format=self.get_format())
1132
 
        missing_ghost = self.make_repository('missing_ghost',
1133
 
            format=self.get_format())
1134
 
 
1135
 
        def add_commit(repo, revision_id, parent_ids):
1136
 
            repo.lock_write()
1137
 
            repo.start_write_group()
1138
 
            inv = inventory.Inventory(revision_id=revision_id)
1139
 
            inv.root.revision = revision_id
1140
 
            root_id = inv.root.file_id
1141
 
            sha1 = repo.add_inventory(revision_id, inv, [])
1142
 
            vf = repo.weave_store.get_weave_or_empty(root_id,
1143
 
                repo.get_transaction())
1144
 
            vf.add_lines(revision_id, [], [])
1145
 
            rev = bzrlib.revision.Revision(timestamp=0,
1146
 
                                           timezone=None,
1147
 
                                           committer="Foo Bar <foo@example.com>",
1148
 
                                           message="Message",
1149
 
                                           inventory_sha1=sha1,
1150
 
                                           revision_id=revision_id)
1151
 
            rev.parent_ids = parent_ids
1152
 
            repo.add_revision(revision_id, rev)
1153
 
            repo.commit_write_group()
1154
 
            repo.unlock()
1155
 
        add_commit(has_ghost, 'ghost', [])
1156
 
        add_commit(has_ghost, 'references', ['ghost'])
1157
 
        add_commit(missing_ghost, 'references', ['ghost'])
1158
 
        add_commit(has_ghost, 'tip', ['references'])
1159
 
        missing_ghost.fetch(has_ghost, 'tip')
1160
 
        # missing ghost now has tip and not ghost.
1161
 
        rev = missing_ghost.get_revision('tip')
1162
 
        inv = missing_ghost.get_inventory('tip')
1163
 
        self.assertRaises(errors.NoSuchRevision,
1164
 
            missing_ghost.get_revision, 'ghost')
1165
 
        self.assertRaises(errors.RevisionNotPresent,
1166
 
            missing_ghost.get_inventory, 'ghost')
1167
 
 
1168
 
 
1169
 
class TestKnitPackSubtrees(TestKnitPackNoSubtrees):
1170
 
 
1171
 
    def get_format(self):
1172
 
        return bzrdir.format_registry.make_bzrdir(
1173
 
            'pack-0.92-subtree')
1174
 
 
1175
 
    def check_format(self, t):
1176
 
        self.assertEqualDiff(
1177
 
            "Bazaar pack repository format 1 with subtree support (needs bzr 0.92)\n",
1178
 
            t.get('format').read())
 
1113
        text = empty_repo.texts.get_record_stream(
 
1114
            [('file2-id', 'rev3')], 'topological', True).next()
 
1115
        self.assertEqual('line\n', text.get_bytes_as('fulltext'))
1179
1116
 
1180
1117
 
1181
1118
class TestRepositoryPackCollection(TestCaseWithTransport):
1183
1120
    def get_format(self):
1184
1121
        return bzrdir.format_registry.make_bzrdir('pack-0.92')
1185
1122
 
 
1123
    def get_packs(self):
 
1124
        format = self.get_format()
 
1125
        repo = self.make_repository('.', format=format)
 
1126
        return repo._pack_collection
 
1127
 
 
1128
    def make_packs_and_alt_repo(self, write_lock=False):
 
1129
        """Create a pack repo with 3 packs, and access it via a second repo."""
 
1130
        tree = self.make_branch_and_tree('.', format=self.get_format())
 
1131
        tree.lock_write()
 
1132
        self.addCleanup(tree.unlock)
 
1133
        rev1 = tree.commit('one')
 
1134
        rev2 = tree.commit('two')
 
1135
        rev3 = tree.commit('three')
 
1136
        r = repository.Repository.open('.')
 
1137
        if write_lock:
 
1138
            r.lock_write()
 
1139
        else:
 
1140
            r.lock_read()
 
1141
        self.addCleanup(r.unlock)
 
1142
        packs = r._pack_collection
 
1143
        packs.ensure_loaded()
 
1144
        return tree, r, packs, [rev1, rev2, rev3]
 
1145
 
 
1146
    def test__clear_obsolete_packs(self):
 
1147
        packs = self.get_packs()
 
1148
        obsolete_pack_trans = packs.transport.clone('obsolete_packs')
 
1149
        obsolete_pack_trans.put_bytes('a-pack.pack', 'content\n')
 
1150
        obsolete_pack_trans.put_bytes('a-pack.rix', 'content\n')
 
1151
        obsolete_pack_trans.put_bytes('a-pack.iix', 'content\n')
 
1152
        obsolete_pack_trans.put_bytes('another-pack.pack', 'foo\n')
 
1153
        obsolete_pack_trans.put_bytes('not-a-pack.rix', 'foo\n')
 
1154
        res = packs._clear_obsolete_packs()
 
1155
        self.assertEqual(['a-pack', 'another-pack'], sorted(res))
 
1156
        self.assertEqual([], obsolete_pack_trans.list_dir('.'))
 
1157
 
 
1158
    def test__clear_obsolete_packs_preserve(self):
 
1159
        packs = self.get_packs()
 
1160
        obsolete_pack_trans = packs.transport.clone('obsolete_packs')
 
1161
        obsolete_pack_trans.put_bytes('a-pack.pack', 'content\n')
 
1162
        obsolete_pack_trans.put_bytes('a-pack.rix', 'content\n')
 
1163
        obsolete_pack_trans.put_bytes('a-pack.iix', 'content\n')
 
1164
        obsolete_pack_trans.put_bytes('another-pack.pack', 'foo\n')
 
1165
        obsolete_pack_trans.put_bytes('not-a-pack.rix', 'foo\n')
 
1166
        res = packs._clear_obsolete_packs(preserve=set(['a-pack']))
 
1167
        self.assertEqual(['a-pack', 'another-pack'], sorted(res))
 
1168
        self.assertEqual(['a-pack.iix', 'a-pack.pack', 'a-pack.rix'],
 
1169
                         sorted(obsolete_pack_trans.list_dir('.')))
 
1170
 
1186
1171
    def test__max_pack_count(self):
1187
1172
        """The maximum pack count is a function of the number of revisions."""
1188
 
        format = self.get_format()
1189
 
        repo = self.make_repository('.', format=format)
1190
 
        packs = repo._pack_collection
1191
1173
        # no revisions - one pack, so that we can have a revision free repo
1192
1174
        # without it blowing up
 
1175
        packs = self.get_packs()
1193
1176
        self.assertEqual(1, packs._max_pack_count(0))
1194
1177
        # after that the sum of the digits, - check the first 1-9
1195
1178
        self.assertEqual(1, packs._max_pack_count(1))
1210
1193
        # check some arbitrary big numbers
1211
1194
        self.assertEqual(25, packs._max_pack_count(112894))
1212
1195
 
 
1196
    def test_repr(self):
 
1197
        packs = self.get_packs()
 
1198
        self.assertContainsRe(repr(packs),
 
1199
            'RepositoryPackCollection(.*Repository(.*))')
 
1200
 
 
1201
    def test__obsolete_packs(self):
 
1202
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
 
1203
        names = packs.names()
 
1204
        pack = packs.get_pack_by_name(names[0])
 
1205
        # Schedule this one for removal
 
1206
        packs._remove_pack_from_memory(pack)
 
1207
        # Simulate a concurrent update by renaming the .pack file and one of
 
1208
        # the indices
 
1209
        packs.transport.rename('packs/%s.pack' % (names[0],),
 
1210
                               'obsolete_packs/%s.pack' % (names[0],))
 
1211
        packs.transport.rename('indices/%s.iix' % (names[0],),
 
1212
                               'obsolete_packs/%s.iix' % (names[0],))
 
1213
        # Now trigger the obsoletion, and ensure that all the remaining files
 
1214
        # are still renamed
 
1215
        packs._obsolete_packs([pack])
 
1216
        self.assertEqual([n + '.pack' for n in names[1:]],
 
1217
                         sorted(packs._pack_transport.list_dir('.')))
 
1218
        # names[0] should not be present in the index anymore
 
1219
        self.assertEqual(names[1:],
 
1220
            sorted(set([osutils.splitext(n)[0] for n in
 
1221
                        packs._index_transport.list_dir('.')])))
 
1222
 
1213
1223
    def test_pack_distribution_zero(self):
1214
 
        format = self.get_format()
1215
 
        repo = self.make_repository('.', format=format)
1216
 
        packs = repo._pack_collection
 
1224
        packs = self.get_packs()
1217
1225
        self.assertEqual([0], packs.pack_distribution(0))
1218
1226
 
1219
1227
    def test_ensure_loaded_unlocked(self):
1220
 
        format = self.get_format()
1221
 
        repo = self.make_repository('.', format=format)
 
1228
        packs = self.get_packs()
1222
1229
        self.assertRaises(errors.ObjectNotLocked,
1223
 
                          repo._pack_collection.ensure_loaded)
 
1230
                          packs.ensure_loaded)
1224
1231
 
1225
1232
    def test_pack_distribution_one_to_nine(self):
1226
 
        format = self.get_format()
1227
 
        repo = self.make_repository('.', format=format)
1228
 
        packs = repo._pack_collection
 
1233
        packs = self.get_packs()
1229
1234
        self.assertEqual([1],
1230
1235
            packs.pack_distribution(1))
1231
1236
        self.assertEqual([1, 1],
1247
1252
 
1248
1253
    def test_pack_distribution_stable_at_boundaries(self):
1249
1254
        """When there are multi-rev packs the counts are stable."""
1250
 
        format = self.get_format()
1251
 
        repo = self.make_repository('.', format=format)
1252
 
        packs = repo._pack_collection
 
1255
        packs = self.get_packs()
1253
1256
        # in 10s:
1254
1257
        self.assertEqual([10], packs.pack_distribution(10))
1255
1258
        self.assertEqual([10, 1], packs.pack_distribution(11))
1264
1267
        self.assertEqual([100, 100, 10, 1], packs.pack_distribution(211))
1265
1268
 
1266
1269
    def test_plan_pack_operations_2009_revisions_skip_all_packs(self):
1267
 
        format = self.get_format()
1268
 
        repo = self.make_repository('.', format=format)
1269
 
        packs = repo._pack_collection
 
1270
        packs = self.get_packs()
1270
1271
        existing_packs = [(2000, "big"), (9, "medium")]
1271
1272
        # rev count - 2009 -> 2x1000 + 9x1
1272
1273
        pack_operations = packs.plan_autopack_combinations(
1274
1275
        self.assertEqual([], pack_operations)
1275
1276
 
1276
1277
    def test_plan_pack_operations_2010_revisions_skip_all_packs(self):
1277
 
        format = self.get_format()
1278
 
        repo = self.make_repository('.', format=format)
1279
 
        packs = repo._pack_collection
 
1278
        packs = self.get_packs()
1280
1279
        existing_packs = [(2000, "big"), (9, "medium"), (1, "single")]
1281
1280
        # rev count - 2010 -> 2x1000 + 1x10
1282
1281
        pack_operations = packs.plan_autopack_combinations(
1284
1283
        self.assertEqual([], pack_operations)
1285
1284
 
1286
1285
    def test_plan_pack_operations_2010_combines_smallest_two(self):
1287
 
        format = self.get_format()
1288
 
        repo = self.make_repository('.', format=format)
1289
 
        packs = repo._pack_collection
 
1286
        packs = self.get_packs()
1290
1287
        existing_packs = [(1999, "big"), (9, "medium"), (1, "single2"),
1291
1288
            (1, "single1")]
1292
1289
        # rev count - 2010 -> 2x1000 + 1x10 (3)
1293
1290
        pack_operations = packs.plan_autopack_combinations(
1294
1291
            existing_packs, [1000, 1000, 10])
1295
 
        self.assertEqual([[2, ["single2", "single1"]], [0, []]], pack_operations)
 
1292
        self.assertEqual([[2, ["single2", "single1"]]], pack_operations)
 
1293
 
 
1294
    def test_plan_pack_operations_creates_a_single_op(self):
 
1295
        packs = self.get_packs()
 
1296
        existing_packs = [(50, 'a'), (40, 'b'), (30, 'c'), (10, 'd'),
 
1297
                          (10, 'e'), (6, 'f'), (4, 'g')]
 
1298
        # rev count 150 -> 1x100 and 5x10
 
1299
        # The two size 10 packs do not need to be touched. The 50, 40, 30 would
 
1300
        # be combined into a single 120 size pack, and the 6 & 4 would
 
1301
        # becombined into a size 10 pack. However, if we have to rewrite them,
 
1302
        # we save a pack file with no increased I/O by putting them into the
 
1303
        # same file.
 
1304
        distribution = packs.pack_distribution(150)
 
1305
        pack_operations = packs.plan_autopack_combinations(existing_packs,
 
1306
                                                           distribution)
 
1307
        self.assertEqual([[130, ['a', 'b', 'c', 'f', 'g']]], pack_operations)
1296
1308
 
1297
1309
    def test_all_packs_none(self):
1298
1310
        format = self.get_format()
1336
1348
        tree.lock_read()
1337
1349
        self.addCleanup(tree.unlock)
1338
1350
        packs = tree.branch.repository._pack_collection
 
1351
        packs.reset()
1339
1352
        packs.ensure_loaded()
1340
1353
        name = packs.names()[0]
1341
1354
        pack_1 = packs.get_pack_by_name(name)
1342
1355
        # the pack should be correctly initialised
1343
 
        rev_index = GraphIndex(packs._index_transport, name + '.rix',
1344
 
            packs._names[name][0])
1345
 
        inv_index = GraphIndex(packs._index_transport, name + '.iix',
1346
 
            packs._names[name][1])
1347
 
        txt_index = GraphIndex(packs._index_transport, name + '.tix',
1348
 
            packs._names[name][2])
1349
 
        sig_index = GraphIndex(packs._index_transport, name + '.six',
1350
 
            packs._names[name][3])
 
1356
        sizes = packs._names[name]
 
1357
        rev_index = GraphIndex(packs._index_transport, name + '.rix', sizes[0])
 
1358
        inv_index = GraphIndex(packs._index_transport, name + '.iix', sizes[1])
 
1359
        txt_index = GraphIndex(packs._index_transport, name + '.tix', sizes[2])
 
1360
        sig_index = GraphIndex(packs._index_transport, name + '.six', sizes[3])
1351
1361
        self.assertEqual(pack_repo.ExistingPack(packs._pack_transport,
1352
1362
            name, rev_index, inv_index, txt_index, sig_index), pack_1)
1353
1363
        # and the same instance should be returned on successive calls.
1354
1364
        self.assertTrue(pack_1 is packs.get_pack_by_name(name))
1355
1365
 
 
1366
    def test_reload_pack_names_new_entry(self):
 
1367
        tree, r, packs, revs = self.make_packs_and_alt_repo()
 
1368
        names = packs.names()
 
1369
        # Add a new pack file into the repository
 
1370
        rev4 = tree.commit('four')
 
1371
        new_names = tree.branch.repository._pack_collection.names()
 
1372
        new_name = set(new_names).difference(names)
 
1373
        self.assertEqual(1, len(new_name))
 
1374
        new_name = new_name.pop()
 
1375
        # The old collection hasn't noticed yet
 
1376
        self.assertEqual(names, packs.names())
 
1377
        self.assertTrue(packs.reload_pack_names())
 
1378
        self.assertEqual(new_names, packs.names())
 
1379
        # And the repository can access the new revision
 
1380
        self.assertEqual({rev4:(revs[-1],)}, r.get_parent_map([rev4]))
 
1381
        self.assertFalse(packs.reload_pack_names())
 
1382
 
 
1383
    def test_reload_pack_names_added_and_removed(self):
 
1384
        tree, r, packs, revs = self.make_packs_and_alt_repo()
 
1385
        names = packs.names()
 
1386
        # Now repack the whole thing
 
1387
        tree.branch.repository.pack()
 
1388
        new_names = tree.branch.repository._pack_collection.names()
 
1389
        # The other collection hasn't noticed yet
 
1390
        self.assertEqual(names, packs.names())
 
1391
        self.assertTrue(packs.reload_pack_names())
 
1392
        self.assertEqual(new_names, packs.names())
 
1393
        self.assertEqual({revs[-1]:(revs[-2],)}, r.get_parent_map([revs[-1]]))
 
1394
        self.assertFalse(packs.reload_pack_names())
 
1395
 
 
1396
    def test_reload_pack_names_preserves_pending(self):
 
1397
        # TODO: Update this to also test for pending-deleted names
 
1398
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
 
1399
        # We will add one pack (via start_write_group + insert_record_stream),
 
1400
        # and remove another pack (via _remove_pack_from_memory)
 
1401
        orig_names = packs.names()
 
1402
        orig_at_load = packs._packs_at_load
 
1403
        to_remove_name = iter(orig_names).next()
 
1404
        r.start_write_group()
 
1405
        self.addCleanup(r.abort_write_group)
 
1406
        r.texts.insert_record_stream([versionedfile.FulltextContentFactory(
 
1407
            ('text', 'rev'), (), None, 'content\n')])
 
1408
        new_pack = packs._new_pack
 
1409
        self.assertTrue(new_pack.data_inserted())
 
1410
        new_pack.finish()
 
1411
        packs.allocate(new_pack)
 
1412
        packs._new_pack = None
 
1413
        removed_pack = packs.get_pack_by_name(to_remove_name)
 
1414
        packs._remove_pack_from_memory(removed_pack)
 
1415
        names = packs.names()
 
1416
        all_nodes, deleted_nodes, new_nodes, _ = packs._diff_pack_names()
 
1417
        new_names = set([x[0][0] for x in new_nodes])
 
1418
        self.assertEqual(names, sorted([x[0][0] for x in all_nodes]))
 
1419
        self.assertEqual(set(names) - set(orig_names), new_names)
 
1420
        self.assertEqual(set([new_pack.name]), new_names)
 
1421
        self.assertEqual([to_remove_name],
 
1422
                         sorted([x[0][0] for x in deleted_nodes]))
 
1423
        packs.reload_pack_names()
 
1424
        reloaded_names = packs.names()
 
1425
        self.assertEqual(orig_at_load, packs._packs_at_load)
 
1426
        self.assertEqual(names, reloaded_names)
 
1427
        all_nodes, deleted_nodes, new_nodes, _ = packs._diff_pack_names()
 
1428
        new_names = set([x[0][0] for x in new_nodes])
 
1429
        self.assertEqual(names, sorted([x[0][0] for x in all_nodes]))
 
1430
        self.assertEqual(set(names) - set(orig_names), new_names)
 
1431
        self.assertEqual(set([new_pack.name]), new_names)
 
1432
        self.assertEqual([to_remove_name],
 
1433
                         sorted([x[0][0] for x in deleted_nodes]))
 
1434
 
 
1435
    def test_autopack_obsoletes_new_pack(self):
 
1436
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
 
1437
        packs._max_pack_count = lambda x: 1
 
1438
        packs.pack_distribution = lambda x: [10]
 
1439
        r.start_write_group()
 
1440
        r.revisions.insert_record_stream([versionedfile.FulltextContentFactory(
 
1441
            ('bogus-rev',), (), None, 'bogus-content\n')])
 
1442
        # This should trigger an autopack, which will combine everything into a
 
1443
        # single pack file.
 
1444
        new_names = r.commit_write_group()
 
1445
        names = packs.names()
 
1446
        self.assertEqual(1, len(names))
 
1447
        self.assertEqual([names[0] + '.pack'],
 
1448
                         packs._pack_transport.list_dir('.'))
 
1449
 
 
1450
    def test_autopack_reloads_and_stops(self):
 
1451
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
 
1452
        # After we have determined what needs to be autopacked, trigger a
 
1453
        # full-pack via the other repo which will cause us to re-evaluate and
 
1454
        # decide we don't need to do anything
 
1455
        orig_execute = packs._execute_pack_operations
 
1456
        def _munged_execute_pack_ops(*args, **kwargs):
 
1457
            tree.branch.repository.pack()
 
1458
            return orig_execute(*args, **kwargs)
 
1459
        packs._execute_pack_operations = _munged_execute_pack_ops
 
1460
        packs._max_pack_count = lambda x: 1
 
1461
        packs.pack_distribution = lambda x: [10]
 
1462
        self.assertFalse(packs.autopack())
 
1463
        self.assertEqual(1, len(packs.names()))
 
1464
        self.assertEqual(tree.branch.repository._pack_collection.names(),
 
1465
                         packs.names())
 
1466
 
 
1467
    def test__save_pack_names(self):
 
1468
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
 
1469
        names = packs.names()
 
1470
        pack = packs.get_pack_by_name(names[0])
 
1471
        packs._remove_pack_from_memory(pack)
 
1472
        packs._save_pack_names(obsolete_packs=[pack])
 
1473
        cur_packs = packs._pack_transport.list_dir('.')
 
1474
        self.assertEqual([n + '.pack' for n in names[1:]], sorted(cur_packs))
 
1475
        # obsolete_packs will also have stuff like .rix and .iix present.
 
1476
        obsolete_packs = packs.transport.list_dir('obsolete_packs')
 
1477
        obsolete_names = set([osutils.splitext(n)[0] for n in obsolete_packs])
 
1478
        self.assertEqual([pack.name], sorted(obsolete_names))
 
1479
 
 
1480
    def test__save_pack_names_already_obsoleted(self):
 
1481
        tree, r, packs, revs = self.make_packs_and_alt_repo(write_lock=True)
 
1482
        names = packs.names()
 
1483
        pack = packs.get_pack_by_name(names[0])
 
1484
        packs._remove_pack_from_memory(pack)
 
1485
        # We are going to simulate a concurrent autopack by manually obsoleting
 
1486
        # the pack directly.
 
1487
        packs._obsolete_packs([pack])
 
1488
        packs._save_pack_names(clear_obsolete_packs=True,
 
1489
                               obsolete_packs=[pack])
 
1490
        cur_packs = packs._pack_transport.list_dir('.')
 
1491
        self.assertEqual([n + '.pack' for n in names[1:]], sorted(cur_packs))
 
1492
        # Note that while we set clear_obsolete_packs=True, it should not
 
1493
        # delete a pack file that we have also scheduled for obsoletion.
 
1494
        obsolete_packs = packs.transport.list_dir('obsolete_packs')
 
1495
        obsolete_names = set([osutils.splitext(n)[0] for n in obsolete_packs])
 
1496
        self.assertEqual([pack.name], sorted(obsolete_names))
 
1497
 
 
1498
 
1356
1499
 
1357
1500
class TestPack(TestCaseWithTransport):
1358
1501
    """Tests for the Pack object."""
1412
1555
        pack_transport = self.get_transport('pack')
1413
1556
        index_transport = self.get_transport('index')
1414
1557
        upload_transport.mkdir('.')
1415
 
        pack = pack_repo.NewPack(upload_transport, index_transport,
1416
 
            pack_transport)
1417
 
        self.assertIsInstance(pack.revision_index, InMemoryGraphIndex)
1418
 
        self.assertIsInstance(pack.inventory_index, InMemoryGraphIndex)
1419
 
        self.assertIsInstance(pack._hash, type(md5.new()))
 
1558
        collection = pack_repo.RepositoryPackCollection(
 
1559
            repo=None,
 
1560
            transport=self.get_transport('.'),
 
1561
            index_transport=index_transport,
 
1562
            upload_transport=upload_transport,
 
1563
            pack_transport=pack_transport,
 
1564
            index_builder_class=BTreeBuilder,
 
1565
            index_class=BTreeGraphIndex,
 
1566
            use_chk_index=False)
 
1567
        pack = pack_repo.NewPack(collection)
 
1568
        self.addCleanup(pack.abort) # Make sure the write stream gets closed
 
1569
        self.assertIsInstance(pack.revision_index, BTreeBuilder)
 
1570
        self.assertIsInstance(pack.inventory_index, BTreeBuilder)
 
1571
        self.assertIsInstance(pack._hash, type(osutils.md5()))
1420
1572
        self.assertTrue(pack.upload_transport is upload_transport)
1421
1573
        self.assertTrue(pack.index_transport is index_transport)
1422
1574
        self.assertTrue(pack.pack_transport is pack_transport)
1429
1581
class TestPacker(TestCaseWithTransport):
1430
1582
    """Tests for the packs repository Packer class."""
1431
1583
 
1432
 
    # To date, this class has been factored out and nothing new added to it;
1433
 
    # thus there are not yet any tests.
 
1584
    def test_pack_optimizes_pack_order(self):
 
1585
        builder = self.make_branch_builder('.', format="1.9")
 
1586
        builder.start_series()
 
1587
        builder.build_snapshot('A', None, [
 
1588
            ('add', ('', 'root-id', 'directory', None)),
 
1589
            ('add', ('f', 'f-id', 'file', 'content\n'))])
 
1590
        builder.build_snapshot('B', ['A'],
 
1591
            [('modify', ('f-id', 'new-content\n'))])
 
1592
        builder.build_snapshot('C', ['B'],
 
1593
            [('modify', ('f-id', 'third-content\n'))])
 
1594
        builder.build_snapshot('D', ['C'],
 
1595
            [('modify', ('f-id', 'fourth-content\n'))])
 
1596
        b = builder.get_branch()
 
1597
        b.lock_read()
 
1598
        builder.finish_series()
 
1599
        self.addCleanup(b.unlock)
 
1600
        # At this point, we should have 4 pack files available
 
1601
        # Because of how they were built, they correspond to
 
1602
        # ['D', 'C', 'B', 'A']
 
1603
        packs = b.repository._pack_collection.packs
 
1604
        packer = pack_repo.Packer(b.repository._pack_collection,
 
1605
                                  packs, 'testing',
 
1606
                                  revision_ids=['B', 'C'])
 
1607
        # Now, when we are copying the B & C revisions, their pack files should
 
1608
        # be moved to the front of the stack
 
1609
        # The new ordering moves B & C to the front of the .packs attribute,
 
1610
        # and leaves the others in the original order.
 
1611
        new_packs = [packs[1], packs[2], packs[0], packs[3]]
 
1612
        new_pack = packer.pack()
 
1613
        self.assertEqual(new_packs, packer.packs)
 
1614
 
 
1615
 
 
1616
class TestOptimisingPacker(TestCaseWithTransport):
 
1617
    """Tests for the OptimisingPacker class."""
 
1618
 
 
1619
    def get_pack_collection(self):
 
1620
        repo = self.make_repository('.')
 
1621
        return repo._pack_collection
 
1622
 
 
1623
    def test_open_pack_will_optimise(self):
 
1624
        packer = pack_repo.OptimisingPacker(self.get_pack_collection(),
 
1625
                                            [], '.test')
 
1626
        new_pack = packer.open_pack()
 
1627
        self.addCleanup(new_pack.abort) # ensure cleanup
 
1628
        self.assertIsInstance(new_pack, pack_repo.NewPack)
 
1629
        self.assertTrue(new_pack.revision_index._optimize_for_size)
 
1630
        self.assertTrue(new_pack.inventory_index._optimize_for_size)
 
1631
        self.assertTrue(new_pack.text_index._optimize_for_size)
 
1632
        self.assertTrue(new_pack.signature_index._optimize_for_size)
 
1633
 
 
1634
 
 
1635
class TestCrossFormatPacks(TestCaseWithTransport):
 
1636
 
 
1637
    def log_pack(self, hint=None):
 
1638
        self.calls.append(('pack', hint))
 
1639
        self.orig_pack(hint=hint)
 
1640
        if self.expect_hint:
 
1641
            self.assertTrue(hint)
 
1642
 
 
1643
    def run_stream(self, src_fmt, target_fmt, expect_pack_called):
 
1644
        self.expect_hint = expect_pack_called
 
1645
        self.calls = []
 
1646
        source_tree = self.make_branch_and_tree('src', format=src_fmt)
 
1647
        source_tree.lock_write()
 
1648
        self.addCleanup(source_tree.unlock)
 
1649
        tip = source_tree.commit('foo')
 
1650
        target = self.make_repository('target', format=target_fmt)
 
1651
        target.lock_write()
 
1652
        self.addCleanup(target.unlock)
 
1653
        source = source_tree.branch.repository._get_source(target._format)
 
1654
        self.orig_pack = target.pack
 
1655
        target.pack = self.log_pack
 
1656
        search = target.search_missing_revision_ids(
 
1657
            source_tree.branch.repository, tip)
 
1658
        stream = source.get_stream(search)
 
1659
        from_format = source_tree.branch.repository._format
 
1660
        sink = target._get_sink()
 
1661
        sink.insert_stream(stream, from_format, [])
 
1662
        if expect_pack_called:
 
1663
            self.assertLength(1, self.calls)
 
1664
        else:
 
1665
            self.assertLength(0, self.calls)
 
1666
 
 
1667
    def run_fetch(self, src_fmt, target_fmt, expect_pack_called):
 
1668
        self.expect_hint = expect_pack_called
 
1669
        self.calls = []
 
1670
        source_tree = self.make_branch_and_tree('src', format=src_fmt)
 
1671
        source_tree.lock_write()
 
1672
        self.addCleanup(source_tree.unlock)
 
1673
        tip = source_tree.commit('foo')
 
1674
        target = self.make_repository('target', format=target_fmt)
 
1675
        target.lock_write()
 
1676
        self.addCleanup(target.unlock)
 
1677
        source = source_tree.branch.repository
 
1678
        self.orig_pack = target.pack
 
1679
        target.pack = self.log_pack
 
1680
        target.fetch(source)
 
1681
        if expect_pack_called:
 
1682
            self.assertLength(1, self.calls)
 
1683
        else:
 
1684
            self.assertLength(0, self.calls)
 
1685
 
 
1686
    def test_sink_format_hint_no(self):
 
1687
        # When the target format says packing makes no difference, pack is not
 
1688
        # called.
 
1689
        self.run_stream('1.9', 'rich-root-pack', False)
 
1690
 
 
1691
    def test_sink_format_hint_yes(self):
 
1692
        # When the target format says packing makes a difference, pack is
 
1693
        # called.
 
1694
        self.run_stream('1.9', '2a', True)
 
1695
 
 
1696
    def test_sink_format_same_no(self):
 
1697
        # When the formats are the same, pack is not called.
 
1698
        self.run_stream('2a', '2a', False)
 
1699
 
 
1700
    def test_IDS_format_hint_no(self):
 
1701
        # When the target format says packing makes no difference, pack is not
 
1702
        # called.
 
1703
        self.run_fetch('1.9', 'rich-root-pack', False)
 
1704
 
 
1705
    def test_IDS_format_hint_yes(self):
 
1706
        # When the target format says packing makes a difference, pack is
 
1707
        # called.
 
1708
        self.run_fetch('1.9', '2a', True)
 
1709
 
 
1710
    def test_IDS_format_same_no(self):
 
1711
        # When the formats are the same, pack is not called.
 
1712
        self.run_fetch('2a', '2a', False)